From 243527b3571a6132e2fbd75b2ea147acee7e1010 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 11:00:22 +0200 Subject: [PATCH 01/25] Outputting into 3 diff folders depending on backend and pre-stored in separate folders template files need to be prunned and make it compiled --- .../madmatrix/backend/cpu/Bridge.h | 633 +++++++++ .../madmatrix/backend/cpu/BridgeKernels.cc | 164 +++ .../madmatrix/backend/cpu/BridgeKernels.h | 146 +++ .../backend/cpu/CommonRandomNumberKernel.cc | 39 + .../backend/cpu/CrossSectionKernels.cc | 238 ++++ .../backend/cpu/CrossSectionKernels.h | 139 ++ .../backend/cpu/CurandRandomNumberKernel.cc | 136 ++ .../madmatrix/backend/cpu/EventStatistics.h | 175 +++ .../madmatrix/backend/cpu/GpuAbstraction.h | 168 +++ .../madmatrix/backend/cpu/GpuRuntime.h | 102 ++ .../backend/cpu/HiprandRandomNumberKernel.cc | 146 +++ .../madmatrix/backend/cpu/MadgraphTest.h | 339 +++++ .../backend/cpu/MatrixElementKernels.cc | 509 ++++++++ .../backend/cpu/MatrixElementKernels.h | 250 ++++ .../backend/cpu/MemoryAccessAmplitudes.h | 165 +++ .../backend/cpu/MemoryAccessChannelIds.h | 126 ++ .../backend/cpu/MemoryAccessCouplings.h | 276 ++++ .../backend/cpu/MemoryAccessCouplingsFixed.h | 89 ++ .../backend/cpu/MemoryAccessDenominators.h | 31 + .../madmatrix/backend/cpu/MemoryAccessGs.h | 171 +++ .../backend/cpu/MemoryAccessHelpers.h | 158 +++ .../backend/cpu/MemoryAccessIflavorVec.h | 125 ++ .../backend/cpu/MemoryAccessMatrixElements.h | 147 +++ .../backend/cpu/MemoryAccessMomenta.h | 276 ++++ .../backend/cpu/MemoryAccessNumerators.h | 31 + .../backend/cpu/MemoryAccessRandomNumbers.h | 145 +++ .../backend/cpu/MemoryAccessVectors.h | 138 ++ .../backend/cpu/MemoryAccessWavefunctions.h | 170 +++ .../backend/cpu/MemoryAccessWeights.h | 150 +++ .../madmatrix/backend/cpu/MemoryBuffers.h | 603 +++++++++ .../backend/cpu/RamboSamplingKernels.cc | 246 ++++ .../backend/cpu/RamboSamplingKernels.h | 173 +++ .../backend/cpu/RandomNumberKernels.h | 192 +++ .../madmatrix/backend/cpu/check_sa.cc | 1159 +++++++++++++++++ .../madmatrix/backend/cpu/color_sum.cc | 434 ++++++ .../madmatrix/backend/cpu/color_sum.h | 105 ++ .../madmatrix/backend/cpu/coloramps.h | 68 + .../madmatrix/backend/cpu/constexpr_math.h | 335 +++++ .../madmatrix/backend/cpu/cpp_hel_amps_h.inc | 90 ++ .../backend/cpu/cpp_model_parameters_cc.inc | 101 ++ .../backend/cpu/cpp_model_parameters_h.inc | 264 ++++ .../madmatrix/backend/cpu/fbridge.cc | 161 +++ .../madmatrix/backend/cpu/fbridge.h | 52 + .../madmatrix/backend/cpu/fsampler.cc | 166 +++ .../madmatrix/backend/cpu/massless_rambo.h | 200 +++ .../madmatrix/backend/cpu/mgOnGpuConfig.h | 296 +++++ .../madmatrix/backend/cpu/mgOnGpuCxtypes.h | 745 +++++++++++ .../madmatrix/backend/cpu/mgOnGpuFptypes.h | 102 ++ .../madmatrix/backend/cpu/mgOnGpuVectors.h | 932 +++++++++++++ .../madmatrix/backend/cpu/runTest.cc | 432 ++++++ .../madmatrix/backend/cpu/testmisc.cc | 517 ++++++++ .../madmatrix/backend/cpu/testxxx.cc | 456 +++++++ .../madmatrix/backend/cpu/umami.cc | 667 ++++++++++ .../madmatrix/backend/gpu/Bridge.h | 633 +++++++++ .../madmatrix/backend/gpu/BridgeKernels.cc | 164 +++ .../madmatrix/backend/gpu/BridgeKernels.h | 146 +++ .../backend/gpu/CommonRandomNumberKernel.cc | 39 + .../backend/gpu/CrossSectionKernels.cc | 238 ++++ .../backend/gpu/CrossSectionKernels.h | 139 ++ .../backend/gpu/CurandRandomNumberKernel.cc | 136 ++ .../madmatrix/backend/gpu/EventStatistics.h | 175 +++ .../madmatrix/backend/gpu/GpuAbstraction.h | 168 +++ .../madmatrix/backend/gpu/GpuRuntime.h | 102 ++ .../backend/gpu/HiprandRandomNumberKernel.cc | 146 +++ .../madmatrix/backend/gpu/MadgraphTest.h | 339 +++++ .../backend/gpu/MatrixElementKernels.cc | 509 ++++++++ .../backend/gpu/MatrixElementKernels.h | 250 ++++ .../backend/gpu/MemoryAccessAmplitudes.h | 165 +++ .../backend/gpu/MemoryAccessChannelIds.h | 126 ++ .../backend/gpu/MemoryAccessCouplings.h | 276 ++++ .../backend/gpu/MemoryAccessCouplingsFixed.h | 89 ++ .../backend/gpu/MemoryAccessDenominators.h | 31 + .../madmatrix/backend/gpu/MemoryAccessGs.h | 171 +++ .../backend/gpu/MemoryAccessHelpers.h | 158 +++ .../backend/gpu/MemoryAccessIflavorVec.h | 125 ++ .../backend/gpu/MemoryAccessMatrixElements.h | 147 +++ .../backend/gpu/MemoryAccessMomenta.h | 276 ++++ .../backend/gpu/MemoryAccessNumerators.h | 31 + .../backend/gpu/MemoryAccessRandomNumbers.h | 145 +++ .../backend/gpu/MemoryAccessVectors.h | 138 ++ .../backend/gpu/MemoryAccessWavefunctions.h | 170 +++ .../backend/gpu/MemoryAccessWeights.h | 150 +++ .../madmatrix/backend/gpu/MemoryBuffers.h | 603 +++++++++ .../backend/gpu/RamboSamplingKernels.cc | 246 ++++ .../backend/gpu/RamboSamplingKernels.h | 173 +++ .../backend/gpu/RandomNumberKernels.h | 192 +++ .../madmatrix/backend/gpu/check_sa.cc | 1159 +++++++++++++++++ .../madmatrix/backend/gpu/color_sum.cc | 434 ++++++ .../madmatrix/backend/gpu/color_sum.h | 105 ++ .../madmatrix/backend/gpu/coloramps.h | 68 + .../madmatrix/backend/gpu/constexpr_math.h | 335 +++++ .../madmatrix/backend/gpu/cpp_hel_amps_h.inc | 90 ++ .../backend/gpu/cpp_model_parameters_cc.inc | 101 ++ .../backend/gpu/cpp_model_parameters_h.inc | 264 ++++ .../madmatrix/backend/gpu/fbridge.cc | 161 +++ .../madmatrix/backend/gpu/fbridge.h | 52 + .../madmatrix/backend/gpu/fsampler.cc | 166 +++ .../madmatrix/backend/gpu/massless_rambo.h | 200 +++ .../madmatrix/backend/gpu/mgOnGpuConfig.h | 296 +++++ .../madmatrix/backend/gpu/mgOnGpuCxtypes.h | 745 +++++++++++ .../madmatrix/backend/gpu/mgOnGpuFptypes.h | 102 ++ .../madmatrix/backend/gpu/mgOnGpuVectors.h | 932 +++++++++++++ .../madmatrix/backend/gpu/runTest.cc | 432 ++++++ .../madmatrix/backend/gpu/testmisc.cc | 517 ++++++++ .../madmatrix/backend/gpu/testxxx.cc | 456 +++++++ .../madmatrix/backend/gpu/umami.cc | 667 ++++++++++ .../madmatrix/backend/simd/Bridge.h | 633 +++++++++ .../madmatrix/backend/simd/BridgeKernels.cc | 164 +++ .../madmatrix/backend/simd/BridgeKernels.h | 146 +++ .../backend/simd/CommonRandomNumberKernel.cc | 39 + .../backend/simd/CrossSectionKernels.cc | 238 ++++ .../backend/simd/CrossSectionKernels.h | 139 ++ .../backend/simd/CurandRandomNumberKernel.cc | 136 ++ .../madmatrix/backend/simd/EventStatistics.h | 175 +++ .../madmatrix/backend/simd/GpuAbstraction.h | 168 +++ .../madmatrix/backend/simd/GpuRuntime.h | 102 ++ .../backend/simd/HiprandRandomNumberKernel.cc | 146 +++ .../madmatrix/backend/simd/MadgraphTest.h | 339 +++++ .../backend/simd/MatrixElementKernels.cc | 509 ++++++++ .../backend/simd/MatrixElementKernels.h | 250 ++++ .../backend/simd/MemoryAccessAmplitudes.h | 165 +++ .../backend/simd/MemoryAccessChannelIds.h | 126 ++ .../backend/simd/MemoryAccessCouplings.h | 276 ++++ .../backend/simd/MemoryAccessCouplingsFixed.h | 89 ++ .../backend/simd/MemoryAccessDenominators.h | 31 + .../madmatrix/backend/simd/MemoryAccessGs.h | 171 +++ .../backend/simd/MemoryAccessHelpers.h | 158 +++ .../backend/simd/MemoryAccessIflavorVec.h | 125 ++ .../backend/simd/MemoryAccessMatrixElements.h | 147 +++ .../backend/simd/MemoryAccessMomenta.h | 276 ++++ .../backend/simd/MemoryAccessNumerators.h | 31 + .../backend/simd/MemoryAccessRandomNumbers.h | 145 +++ .../backend/simd/MemoryAccessVectors.h | 138 ++ .../backend/simd/MemoryAccessWavefunctions.h | 170 +++ .../backend/simd/MemoryAccessWeights.h | 150 +++ .../madmatrix/backend/simd/MemoryBuffers.h | 603 +++++++++ .../backend/simd/RamboSamplingKernels.cc | 246 ++++ .../backend/simd/RamboSamplingKernels.h | 173 +++ .../backend/simd/RandomNumberKernels.h | 192 +++ .../madmatrix/backend/simd/check_sa.cc | 1159 +++++++++++++++++ .../madmatrix/backend/simd/color_sum.cc | 434 ++++++ .../madmatrix/backend/simd/color_sum.h | 105 ++ .../madmatrix/backend/simd/coloramps.h | 68 + .../madmatrix/backend/simd/constexpr_math.h | 335 +++++ .../madmatrix/backend/simd/cpp_hel_amps_h.inc | 90 ++ .../backend/simd/cpp_model_parameters_cc.inc | 101 ++ .../backend/simd/cpp_model_parameters_h.inc | 264 ++++ .../madmatrix/backend/simd/fbridge.cc | 161 +++ .../madmatrix/backend/simd/fbridge.h | 52 + .../madmatrix/backend/simd/fsampler.cc | 166 +++ .../madmatrix/backend/simd/massless_rambo.h | 200 +++ .../madmatrix/backend/simd/mgOnGpuConfig.h | 296 +++++ .../madmatrix/backend/simd/mgOnGpuCxtypes.h | 745 +++++++++++ .../madmatrix/backend/simd/mgOnGpuFptypes.h | 102 ++ .../madmatrix/backend/simd/mgOnGpuVectors.h | 932 +++++++++++++ .../madmatrix/backend/simd/runTest.cc | 432 ++++++ .../madmatrix/backend/simd/testmisc.cc | 517 ++++++++ .../madmatrix/backend/simd/testxxx.cc | 456 +++++++ .../madmatrix/backend/simd/umami.cc | 667 ++++++++++ madmatrix/output.py | 30 +- madmatrix/trex.py | 3 +- 161 files changed, 41665 insertions(+), 2 deletions(-) create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/Bridge.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuAbstraction.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_hel_amps_h.inc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_cc.inc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_h.inc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/runTest.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/Bridge.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_hel_amps_h.inc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_cc.inc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_h.inc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/runTest.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/Bridge.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/GpuAbstraction.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_hel_amps_h.inc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_cc.inc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_h.inc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/runTest.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/Bridge.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/Bridge.h new file mode 100644 index 0000000000..8417e3ad15 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/Bridge.h @@ -0,0 +1,633 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Nov 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2021-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef BRIDGE_H +#define BRIDGE_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" // for CPPProcess +#include "CrossSectionKernels.h" // for flagAbnormalMEs +#include "MatrixElementKernels.h" // for MatrixElementKernelHost, MatrixElementKernelDevice +#include "MemoryAccessMomenta.h" // for MemoryAccessMomenta::neppM +#include "MemoryBuffers.h" // for HostBufferMomenta, DeviceBufferMomenta etc + +//#ifdef __HIPCC__ +//#include // see +//https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 #else #include +// // bypass this completely to ease portability on LUMI #803 #endif + +#include // bypass std::filesystem #803 + +#include +#include +#include +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + /** + * A base class for a class whose pointer is passed between Fortran and C++. + * This is not really necessary, but it allows minimal type checks on all such + * pointers. + */ + struct CppObjectInFortran + { + CppObjectInFortran() {} + virtual ~CppObjectInFortran() {} + }; + + //-------------------------------------------------------------------------- + /** + * A templated class for calling the CUDA/C++ matrix element calculations of the + * event generation workflow. The FORTRANFPTYPE template parameter indicates the + * precision of the Fortran momenta from MadEvent (float or double). The + * precision of the matrix element calculation is hardcoded in the fptype + * typedef in CUDA/C++. + * + * The Fortran momenta passed in are in the form of + * DOUBLE PRECISION P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) + * where the dimensions are , , + * . In memory, this is stored in a way that C reads as an array + * P_MULTI[nevtF][nparF][np4F]. The CUDA/C++ momenta are stored as an + * array[npagM][npar][np4][neppM] with nevt=npagM*neppM. The Bridge is + * configured to store nevt==nevtF events in CUDA/C++. It also checks that + * Fortran and C++ parameters match, nparF==npar and np4F==np4. + * + * The cpu/gpu sequences take FORTRANFPTYPE* (not fptype*) momenta/MEs. + * This allows mixing double in MadEvent Fortran with float in CUDA/C++ + * sigmaKin. In the fcheck_sa.f test, Fortran uses double while CUDA/C++ may use + * double or float. In the check_sa "--bridge" test, everything is implemented + * in fptype (double or float). + */ + template + class Bridge final : public CppObjectInFortran + { + public: + /** + * Constructor + * + * @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran array + * loops (VECSIZE_USED <= VECSIZE_MEMMAX) + * @param nparF (NEXTERNAL, nexternal.inc) number of external particles in + * Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + * @param np4F number of momenta components, usually 4, in Fortran arrays + * (KEPT FOR SANITY CHECKS ONLY) + */ + Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F ); + + /** + * Destructor + */ + virtual ~Bridge() {} + + // Delete copy/move constructors and assignment operators + Bridge( const Bridge& ) = delete; + Bridge( Bridge&& ) = delete; + Bridge& operator=( const Bridge& ) = delete; + Bridge& operator=( Bridge&& ) = delete; + +#ifdef MGONGPUCPP_GPUIMPL + /** + * Set the gpublocks and gputhreads for the gpusequence - throws if evnt != + * gpublocks*gputhreads (this is needed for BridgeKernel tests rather than for + * actual production use in Fortran) + * + * @param gpublocks number of gpublocks + * @param gputhreads number of gputhreads + */ + void set_gpugrid( const int gpublocks, const int gputhreads ); + + /** + * Sequence to be executed for the Cuda matrix element calculation + * + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant + * alphas) + * @param iflavorVec the index of the flavor combination + * @param rndhel the pointer to the input random numbers for helicity + * selection + * @param rndcol the pointer to the input random numbers for color selection + * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 + * to n + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void gpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); +#else + /** + * Sequence to be executed for the vectorized CPU matrix element calculation + * + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant + * alphas) + * @param iflavorVec the index of the flavor combination + * @param rndhel the pointer to the input random numbers for helicity + * selection + * @param rndcol the pointer to the input random numbers for color selection + * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 + * to n + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void cpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); +#endif + + // Return the number of good helicities (-1 initially when they have not yet + // been calculated) + int nGoodHel() const { return m_nGoodHel; } + + // Return the total number of helicities (expose cudacpp ncomb in the Bridge + // interface to Fortran) + constexpr int nTotHel() const { return CPPProcess::ncomb; } + + private: + unsigned int m_nevt; // number of events + int m_nGoodHel; // the number of good helicities (-1 initially when they have + // not yet been calculated) + +#ifdef MGONGPUCPP_GPUIMPL + int m_gputhreads; // number of gpu threads (default set from number of + // events, can be modified) + int m_gpublocks; // number of gpu blocks (default set from number of events, + // can be modified) + DeviceBuffer m_devMomentaF; + DeviceBufferMomenta m_devMomentaC; + DeviceBufferGs m_devGs; + DeviceBufferIflavorVec m_devIflavorVec; + DeviceBufferRndNumHelicity m_devRndHel; + DeviceBufferRndNumColor m_devRndCol; + DeviceBufferMatrixElements m_devMEs; + DeviceBufferSelectedHelicity m_devSelHel; + DeviceBufferSelectedColor m_devSelCol; + DeviceBufferChannelIds m_devChannelIds; + PinnedHostBufferIflavorVec m_hstIflavorVec; + PinnedHostBufferGs m_hstGs; + PinnedHostBufferRndNumHelicity m_hstRndHel; + PinnedHostBufferRndNumColor m_hstRndCol; + PinnedHostBufferMatrixElements m_hstMEs; + PinnedHostBufferSelectedHelicity m_hstSelHel; + PinnedHostBufferSelectedColor m_hstSelCol; + PinnedHostBufferChannelIds m_hstChannelIds; + std::unique_ptr m_pmek; + // static constexpr int s_gputhreadsmin = 16; // minimum number of gpu threads + // (TEST VALUE FOR MADEVENT) + static constexpr int s_gputhreadsmin = + 32; // minimum number of gpu threads (DEFAULT) +#else + HostBufferMomenta m_hstMomentaC; + HostBufferGs m_hstGs; + HostBufferIflavorVec m_hstIflavorVec; + HostBufferRndNumHelicity m_hstRndHel; + HostBufferRndNumColor m_hstRndCol; + HostBufferMatrixElements m_hstMEs; + HostBufferSelectedHelicity m_hstSelHel; + HostBufferSelectedColor m_hstSelCol; + HostBufferChannelIds m_hstChannelIds; + std::unique_ptr m_pmek; +#endif + }; + + //-------------------------------------------------------------------------- + // + // Forward declare transposition methods + // + +#ifdef MGONGPUCPP_GPUIMPL + + template + __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); + +#endif // MGONGPUCPP_GPUIMPL + + template + void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); + + template + void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ); + + //-------------------------------------------------------------------------- + // + // Implementations of member functions of class Bridge + // + + template + Bridge::Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F) + : m_nevt( nevtF ), m_nGoodHel( -1 ) +#ifdef MGONGPUCPP_GPUIMPL + , m_gputhreads( 256 ) // default number of gpu threads + , m_gpublocks( m_nevt / m_gputhreads ) // this ensures m_nevt <= m_gpublocks*m_gputhreads + , m_devMomentaF( m_nevt ) + , m_devMomentaC( m_nevt ) + , m_devIflavorVec( m_nevt ) + , m_devGs( m_nevt ) + , m_devRndHel( m_nevt ) + , m_devRndCol( m_nevt ) + , m_devMEs( m_nevt ) + , m_devSelHel( m_nevt ) + , m_devSelCol( m_nevt ) + , m_devChannelIds( m_nevt ) +#else + , m_hstMomentaC( m_nevt ) +#endif + , m_hstGs( m_nevt ) + , m_hstIflavorVec( m_nevt ) + , m_hstRndHel( m_nevt ) + , m_hstRndCol( m_nevt ) + , m_hstMEs( m_nevt ) + , m_hstSelHel( m_nevt ) + , m_hstSelCol( m_nevt ) + , m_hstChannelIds( m_nevt ) + , m_pmek( nullptr ) + { + if( nparF != CPPProcess::npar ) + throw std::runtime_error( "Bridge constructor: npar mismatch" ); + if( np4F != CPPProcess::np4 ) + throw std::runtime_error( "Bridge constructor: np4 mismatch" ); +#ifdef MGONGPUCPP_GPUIMPL + if( ( m_nevt < s_gputhreadsmin ) || ( m_nevt % s_gputhreadsmin != 0 ) ) + throw std::runtime_error( + "Bridge constructor: nevt should be a multiple of " + + std::to_string( s_gputhreadsmin ) ); + while( m_nevt != m_gpublocks * m_gputhreads ) + { + m_gputhreads /= 2; + if( m_gputhreads < s_gputhreadsmin ) + throw std::logic_error( + "Bridge constructor: FIXME! cannot choose gputhreads" ); // this + // should + // never + // happen! + m_gpublocks = m_nevt / m_gputhreads; + } +#ifdef MGONGPUCPP_VERBOSE + std::cout << "WARNING! Instantiate device Bridge (nevt=" << m_nevt + << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads + << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" + << std::endl; +#endif + m_pmek.reset( new MatrixElementKernelDevice( + m_devMomentaC, m_devGs, m_devIflavorVec, m_devRndHel, m_devRndCol, m_devChannelIds, m_devMEs, m_devSelHel, m_devSelCol, m_gpublocks, m_gputhreads) ); +#else +#ifdef MGONGPUCPP_VERBOSE + std::cout << "WARNING! Instantiate host Bridge (nevt=" << m_nevt << ")" + << std::endl; +#endif + m_pmek.reset( new MatrixElementKernelHost( + m_hstMomentaC, m_hstGs, m_hstIflavorVec, m_hstRndHel, m_hstRndCol, m_hstChannelIds, m_hstMEs, m_hstSelHel, m_hstSelCol, m_nevt ) ); +#endif // MGONGPUCPP_GPUIMPL + // Create a process object, read param card and set parameters + // FIXME: the process instance can happily go out of scope because it is only + // needed to read parameters? + // FIXME: the CPPProcess should really be a singleton? what if fbridgecreate + // is called from several Fortran threads? + CPPProcess process( /*verbose=*/false ); + std::string paramCard = + "../Cards/param_card.dat"; // ZW: change default param_card.dat location + // to one dir down + /* +#ifdef __HIPCC__ + if( !std::experimental::filesystem::exists( paramCard ) ) paramCard = "../" + +paramCard; #else if( !std::filesystem::exists( paramCard ) ) paramCard = "../" + +paramCard; #endif + */ + // struct stat dummybuffer; // bypass std::filesystem #803 + // if( !( stat( paramCard.c_str(), &dummyBuffer ) == 0 ) ) paramCard = "../" + + // paramCard; // + auto fileExists = []( std::string& fileName ) + { + struct stat buffer; + return stat( fileName.c_str(), &buffer ) == 0; + }; + size_t paramCardCheck = 2; // ZW: check for paramCard up to 2 directories up + for( size_t k = 0; k < paramCardCheck; ++k ) + { + if( fileExists( paramCard ) ) break; // bypass std::filesystem #803 + paramCard = "../" + paramCard; + } + process.initProc( paramCard ); + } + +#ifdef MGONGPUCPP_GPUIMPL + template + void Bridge::set_gpugrid( const int gpublocks, + const int gputhreads ) + { + if( m_nevt != gpublocks * gputhreads ) + throw std::runtime_error( + "Bridge: gpublocks*gputhreads must equal m_nevt in set_gpugrid" ); + m_gpublocks = gpublocks; + m_gputhreads = gputhreads; +#ifdef MGONGPUCPP_VERBOSE + std::cout << "WARNING! Set grid in Bridge (nevt=" << m_nevt + << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads + << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" + << std::endl; +#endif + m_pmek->setGrid( m_gpublocks, m_gputhreads ); + } +#endif + +#ifdef MGONGPUCPP_GPUIMPL + template + void Bridge::gpu_sequence( const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool goodHelOnly ) + { + constexpr int neppM = MemoryAccessMomenta::neppM; + if constexpr( neppM == 1 && std::is_same_v ) + { + gpuMemcpy( m_devMomentaC.data(), momenta, m_devMomentaC.bytes(), gpuMemcpyHostToDevice ); + } + else + { + gpuMemcpy( m_devMomentaF.data(), momenta, m_devMomentaF.bytes(), gpuMemcpyHostToDevice ); + const int thrPerEvt = + CPPProcess::npar * + CPPProcess::np4; // AV: transpose alg does 1 element per thread (NOT 1 + // event per thread) + // const int thrPerEvt = 1; // AV: try new alg with 1 event per thread... + // this seems slower + gpuLaunchKernel( dev_transposeMomentaF2C, m_gpublocks * thrPerEvt, m_gputhreads, m_devMomentaF.data(), m_devMomentaC.data(), m_nevt ); + } + if constexpr( std::is_same_v ) + { + memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); + } + else + { + std::copy( gs, gs + m_nevt, m_hstGs.data() ); + std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); + std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); + } + const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); + if( useChannelIds ) + memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); + // else ... // no need to initialize m_hstChannel: it is allocated with + // gpuMallocHost and NOT initialized in PinnedHostBufferBase, but it is NOT + // used later on + // initialise iflavorVec + memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); + copyDeviceFromHost( m_devGs, m_hstGs ); + copyDeviceFromHost( m_devRndHel, m_hstRndHel ); + copyDeviceFromHost( m_devRndCol, m_hstRndCol ); + if( useChannelIds ) copyDeviceFromHost( m_devChannelIds, m_hstChannelIds ); + copyDeviceFromHost( m_devIflavorVec, m_hstIflavorVec ); + if( m_nGoodHel < 0 ) + { + m_nGoodHel = m_pmek->computeGoodHelicities(); + if( m_nGoodHel < 0 ) + throw std::runtime_error( + "Bridge gpu_sequence: computeGoodHelicities returned nGoodHel<0" ); + } + if( goodHelOnly ) return; + m_pmek->computeMatrixElements( useChannelIds ); + copyHostFromDevice( m_hstMEs, m_devMEs ); +#ifdef MGONGPUCPP_VERBOSE + flagAbnormalMEs( m_hstMEs.data(), m_nevt ); +#endif + copyHostFromDevice( m_hstSelHel, m_devSelHel ); + copyHostFromDevice( m_hstSelCol, m_devSelCol ); + if constexpr( std::is_same_v ) + { + memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); + memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); + memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); + } + else + { + std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); + std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); + std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); + } + } +#endif + +#ifndef MGONGPUCPP_GPUIMPL + template + void Bridge::cpu_sequence( const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool goodHelOnly ) + { + hst_transposeMomentaF2C( momenta, m_hstMomentaC.data(), m_nevt ); + if constexpr( std::is_same_v ) + { + memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); + } + else + { + std::copy( gs, gs + m_nevt, m_hstGs.data() ); + std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); + std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); + } + const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); + if( useChannelIds ) + memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); + // else ... // no need to initialize m_hstChannel: it is allocated and default + // initialized in HostBufferBase (and it is not used later on anyway) + // initialise iflavorVec + memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); + if( m_nGoodHel < 0 ) + { + m_nGoodHel = m_pmek->computeGoodHelicities(); + if( m_nGoodHel < 0 ) + throw std::runtime_error( + "Bridge cpu_sequence: computeGoodHelicities returned nGoodHel<0" ); + } + if( goodHelOnly ) return; + m_pmek->computeMatrixElements( useChannelIds ); +#ifdef MGONGPUCPP_VERBOSE + flagAbnormalMEs( m_hstMEs.data(), m_nevt ); +#endif + if constexpr( std::is_same_v ) + { + memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); + memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); + memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); + } + else + { + std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); + std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); + std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); + } + } +#endif + + //-------------------------------------------------------------------------- + // + // Implementations of transposition methods + // - FORTRAN arrays: P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) ==> + // p_multi[nevtF][nparF][np4F] in C++ (AOS) + // - C++ array: momenta[npagM][npar][np4][neppM] with nevt=npagM*neppM (AOSOA) + // + +#ifdef MGONGPUCPP_GPUIMPL + template + __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool oldImplementation = true; // default: use old implementation + if constexpr( oldImplementation ) + { + // SR initial implementation + constexpr int part = CPPProcess::npar; + constexpr int mome = CPPProcess::np4; + constexpr int strd = MemoryAccessMomenta::neppM; + int pos = blockDim.x * blockIdx.x + threadIdx.x; + int arrlen = nevt * part * mome; + if( pos < arrlen ) + { + int page_i = pos / ( strd * mome * part ); + int rest_1 = pos % ( strd * mome * part ); + int part_i = rest_1 / ( strd * mome ); + int rest_2 = rest_1 % ( strd * mome ); + int mome_i = rest_2 / strd; + int strd_i = rest_2 % strd; + int inpos = ( page_i * strd + strd_i ) // event number + * ( part * mome ) // event size (pos of event) + + part_i * mome // particle inside event + + mome_i; // momentum inside particle + out[pos] = in[inpos]; // F2C (Fortran to C) + } + } + else + { + // AV attempt another implementation with 1 event per thread: this seems + // slower... F-style: AOS[nevtF][nparF][np4F] C-style: + // AOSOA[npagM][npar][np4][neppM] with nevt=npagM*neppM + constexpr int npar = CPPProcess::npar; + constexpr int np4 = CPPProcess::np4; + constexpr int neppM = MemoryAccessMomenta::neppM; + assert( nevt % neppM == + 0 ); // number of events is not a multiple of neppM??? + int ievt = blockDim.x * blockIdx.x + threadIdx.x; + int ipagM = ievt / neppM; + int ieppM = ievt % neppM; + for( int ip4 = 0; ip4 < np4; ip4++ ) + for( int ipar = 0; ipar < npar; ipar++ ) + { + int cpos = ipagM * npar * np4 * neppM + ipar * np4 * neppM + + ip4 * neppM + ieppM; + int fpos = ievt * npar * np4 + ipar * np4 + ip4; + out[cpos] = in[fpos]; // F2C (Fortran to C) + } + } + } +#endif + + template + void hst_transposeMomenta( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool oldImplementation = false; // default: use new implementation + if constexpr( oldImplementation ) + { + // SR initial implementation + constexpr unsigned int part = CPPProcess::npar; + constexpr unsigned int mome = CPPProcess::np4; + constexpr unsigned int strd = MemoryAccessMomenta::neppM; + unsigned int arrlen = nevt * part * mome; + for( unsigned int pos = 0; pos < arrlen; ++pos ) + { + unsigned int page_i = pos / ( strd * mome * part ); + unsigned int rest_1 = pos % ( strd * mome * part ); + unsigned int part_i = rest_1 / ( strd * mome ); + unsigned int rest_2 = rest_1 % ( strd * mome ); + unsigned int mome_i = rest_2 / strd; + unsigned int strd_i = rest_2 % strd; + unsigned int inpos = ( page_i * strd + strd_i ) // event number + * ( part * mome ) // event size (pos of event) + + part_i * mome // particle inside event + + mome_i; // momentum inside particle + if constexpr( F2C ) // needs c++17 and cuda >=11.2 (#333) + out[pos] = in[inpos]; // F2C (Fortran to C) + else + out[inpos] = in[pos]; // C2F (C to Fortran) + } + } + else + { + // AV attempt another implementation: this is slightly faster (better c++ + // pipelining?) [NB! this is not a transposition, it is an AOS to AOSOA + // conversion: if neppM=1, a memcpy is enough] F-style: + // AOS[nevtF][nparF][np4F] C-style: AOSOA[npagM][npar][np4][neppM] with + // nevt=npagM*neppM + constexpr unsigned int npar = CPPProcess::npar; + constexpr unsigned int np4 = CPPProcess::np4; + constexpr unsigned int neppM = MemoryAccessMomenta::neppM; + if constexpr( neppM == 1 && std::is_same_v ) + { + memcpy( out, in, nevt * npar * np4 * sizeof( Tin ) ); + } + else + { + const unsigned int npagM = nevt / neppM; + assert( nevt % neppM == + 0 ); // number of events is not a multiple of neppM??? + for( unsigned int ipagM = 0; ipagM < npagM; ipagM++ ) + for( unsigned int ip4 = 0; ip4 < np4; ip4++ ) + for( unsigned int ipar = 0; ipar < npar; ipar++ ) + for( unsigned int ieppM = 0; ieppM < neppM; ieppM++ ) + { + unsigned int ievt = ipagM * neppM + ieppM; + unsigned int cpos = ipagM * npar * np4 * neppM + + ipar * np4 * neppM + ip4 * neppM + ieppM; + unsigned int fpos = ievt * npar * np4 + ipar * np4 + ip4; + if constexpr( F2C ) + out[cpos] = in[fpos]; // F2C (Fortran to C) + else + out[fpos] = in[cpos]; // C2F (C to Fortran) + } + } + } + } + + template + void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool F2C = true; + hst_transposeMomenta( in, out, nevt ); + } + + template + void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool F2C = false; + hst_transposeMomenta( in, out, nevt ); + } + + //-------------------------------------------------------------------------- +} // namespace mg5amcGpu +#endif // BRIDGE_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.cc new file mode 100644 index 0000000000..65c97616d8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.cc @@ -0,0 +1,164 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "BridgeKernels.h" + +#include "GpuAbstraction.h" +#include "MemoryAccessMomenta.h" + +#include + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) + constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- + + //-------------------------------------------------------------------------- + + BridgeKernelBase::BridgeKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( nevt ) + , m_bridge( nevt, npar, np4 ) + { + if( m_momenta.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: momenta must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: matrixElements must be a host array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +#ifndef MGONGPUCPP_GPUIMPL +namespace mg5amcCpu +{ + + //-------------------------------------------------------------------------- + + BridgeKernelHost::BridgeKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: Gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt) + : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, nevt) + , m_fortranMomenta( nevt ) + { + } + + //-------------------------------------------------------------------------- + + void BridgeKernelHost::transposeInputMomentaC2F() + { + hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); + } + + //-------------------------------------------------------------------------- + + int BridgeKernelHost::computeGoodHelicities() + { + constexpr bool goodHelOnly = true; + constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering + m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + return m_bridge.nGoodHel(); + } + + //-------------------------------------------------------------------------- + + void BridgeKernelHost::computeMatrixElements( const bool useChannelIds ) + { + constexpr bool goodHelOnly = false; + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + + //-------------------------------------------------------------------------- + + BridgeKernelDevice::BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: Gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads) + : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, gpublocks * gputhreads) + , m_fortranMomenta( nevt() ) + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + if( m_gpublocks == 0 ) throw std::runtime_error( "BridgeKernelDevice: gpublocks must be > 0" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "BridgeKernelDevice: gputhreads must be > 0" ); + m_bridge.set_gpugrid( gpublocks, gputhreads ); + } + + //-------------------------------------------------------------------------- + + void BridgeKernelDevice::transposeInputMomentaC2F() + { + hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); + } + + //-------------------------------------------------------------------------- + + int BridgeKernelDevice::computeGoodHelicities() + { + constexpr bool goodHelOnly = true; + constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering + m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + return m_bridge.nGoodHel(); + } + + //-------------------------------------------------------------------------- + + void BridgeKernelDevice::computeMatrixElements( const bool useChannelIds ) + { + constexpr bool goodHelOnly = false; + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.h new file mode 100644 index 0000000000..2d873486a8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.h @@ -0,0 +1,146 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally originally by: A. Valassi (Jan 2022). +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef BRIDGEKERNELS_H +#define BRIDGEKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "Bridge.h" +#include "MatrixElementKernels.h" +#include "MemoryBuffers.h" + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // A Bridge wrapper base class encapsulating matrix element calculations on a CPU host + class BridgeKernelBase : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + BridgeKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~BridgeKernelBase() {} + + // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge + virtual void transposeInputMomentaC2F() = 0; + + protected: + + // The wrapped bridge + Bridge m_bridge; + }; + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + // A Bridge wrapper class encapsulating matrix element calculations on a CPU host + class BridgeKernelHost final : public BridgeKernelBase + { + public: + + // Constructor from existing input and output buffers + BridgeKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~BridgeKernelHost() {} + + // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge + void transposeInputMomentaC2F() override final; + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // The buffer for the input momenta, transposed to Fortran array indexing + HostBufferMomenta m_fortranMomenta; + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A Bridge wrapper class encapsulating matrix element calculations on a GPU device + class BridgeKernelDevice : public BridgeKernelBase + { + public: + + // Constructor from existing input and output buffers + BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads); + + // Destructor + virtual ~BridgeKernelDevice() {} + + // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge + void transposeInputMomentaC2F() override final; + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The buffer for the input momenta, transposed to Fortran array indexing + PinnedHostBufferMomenta m_fortranMomenta; + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + }; +#endif + + //-------------------------------------------------------------------------- +} +#endif // BRIDGEKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc new file mode 100644 index 0000000000..16671b5686 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc @@ -0,0 +1,39 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "CommonRandomNumbers.h" +#include "GpuAbstraction.h" +#include "MemoryBuffers.h" +#include "RandomNumberKernels.h" + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + CommonRandomNumberKernel::CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ) + : RandomNumberKernelBase( rnarray ) + , m_seed( 20211220 ) + { + if( m_rnarray.isOnDevice() ) + throw std::runtime_error( "CommonRandomNumberKernel on host with a device random number array" ); + } + + //-------------------------------------------------------------------------- + + void CommonRandomNumberKernel::generateRnarray() + { + std::vector rnd = CommonRandomNumbers::generate( m_rnarray.size(), m_seed ); // NB: generate as double (HARDCODED) + std::copy( rnd.begin(), rnd.end(), m_rnarray.data() ); // NB: copy may imply a double-to-float conversion + } + + //-------------------------------------------------------------------------- +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc new file mode 100644 index 0000000000..7a3e0e0519 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc @@ -0,0 +1,238 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "CrossSectionKernels.h" + +#include "GpuAbstraction.h" +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessWeights.h" +#include "MemoryBuffers.h" + +#include + +// ****************************************************************************************** +// *** NB: Disabling fast math is essential here, otherwise results are undefined *** +// *** NB: This file CrossSectionKernels.cc IS BUILT WITH -fno-fast-math in the Makefile! *** +// *** NB: Attempts with __attribute__((optimize("-fno-fast-math"))) were unsatisfactory *** +// ****************************************************************************************** + +inline bool +fp_is_nan( const fptype& fp ) +{ + //#pragma clang diagnostic push + //#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) + return std::isnan( fp ); // always false for clang in fast math mode (tautological compare)? + //#pragma clang diagnostic pop +} + +inline bool +fp_is_abnormal( const fptype& fp ) +{ + if( fp_is_nan( fp ) ) return true; + if( fp != fp ) return true; + return false; +} + +inline bool +fp_is_zero( const fptype& fp ) +{ + if( fp == 0 ) return true; + return false; +} + +// See https://en.cppreference.com/w/cpp/numeric/math/FP_categories +inline const char* +fp_show_class( const fptype& fp ) +{ + switch( std::fpclassify( fp ) ) + { + case FP_INFINITE: return "Inf"; + case FP_NAN: return "NaN"; + case FP_NORMAL: return "normal"; + case FP_SUBNORMAL: return "subnormal"; + case FP_ZERO: return "zero"; + default: return "unknown"; + } +} + +inline void +debug_me_is_abnormal( const fptype& me, size_t ievtALL ) +{ + std::cout << "DEBUG[" << ievtALL << "]" + << " ME=" << me + << " fpisabnormal=" << fp_is_abnormal( me ) + << " fpclass=" << fp_show_class( me ) + << " (me==me)=" << ( me == me ) + << " (me==me+1)=" << ( me == me + 1 ) + << " isnan=" << fp_is_nan( me ) + << " isfinite=" << std::isfinite( me ) + << " isnormal=" << std::isnormal( me ) + << " is0=" << ( me == 0 ) + << " is1=" << ( me == 1 ) + << " abs(ME)=" << std::abs( me ) + << " isnan=" << fp_is_nan( std::abs( me ) ) + << std::endl; +} + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ) + { + for( unsigned int ievt = 0; ievt < nevt; ievt++ ) + { + if( fp_is_abnormal( hstMEs[ievt] ) ) + { + std::cout << "WARNING! flagging abnormal ME for ievt=" << ievt << std::endl; + hstMEs[ievt] = std::sqrt( -1. ); + } + } + } + + //-------------------------------------------------------------------------- + + CrossSectionKernelHost::CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t nevt ) + : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) + , NumberOfEvents( nevt ) + { + if( m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: samplingWeights must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: matrixElements must be a host array" ); + if( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with samplingWeights" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelHost::updateEventStatistics( const bool debug ) + { + EventStatistics stats; // new statistics for the new nevt events + // FIRST PASS: COUNT ALL/ABN/ZERO EVENTS, COMPUTE MIN/MAX, COMPUTE REFS AS MEANS OF SIMPLE SUMS + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + const size_t ievtALL = m_iter * nevt() + ievt; + // The following events are abnormal in a run with "-p 2048 256 12 -d" + // - check.exe/commonrand: ME[310744,451171,3007871,3163868,4471038,5473927] with fast math + // - check.exe/curand: ME[578162,1725762,2163579,5407629,5435532,6014690] with fast math + // - gcheck.exe/curand: ME[596016,1446938] with fast math + // Debug NaN/abnormal issues + //if ( ievtALL == 310744 ) // this ME is abnormal both with and without fast math + // debug_me_is_abnormal( me, ievtALL ); + //if ( ievtALL == 5473927 ) // this ME is abnormal only with fast math + // debug_me_is_abnormal( me, ievtALL ); + stats.nevtALL++; + if( fp_is_abnormal( me ) ) + { + if( debug ) // only printed out with "-p -d" (matrixelementALL is not filled without -p) + std::cout << "WARNING! ME[" << ievtALL << "] is NaN/abnormal" << std::endl; + stats.nevtABN++; + continue; + } + if( fp_is_zero( me ) ) stats.nevtZERO++; + stats.minME = std::min( stats.minME, (double)me ); + stats.maxME = std::max( stats.maxME, (double)me ); + stats.minWG = std::min( stats.minWG, (double)wg ); + stats.maxWG = std::max( stats.maxWG, (double)wg ); + stats.sumMEdiff += me; // NB stats.refME is 0 here + stats.sumWGdiff += wg; // NB stats.refWG is 0 here + } + stats.refME = stats.meanME(); // draft ref + stats.refWG = stats.meanWG(); // draft ref + stats.sumMEdiff = 0; + stats.sumWGdiff = 0; + // SECOND PASS: IMPROVE MEANS FROM SUMS OF DIFFS TO PREVIOUS REF, UPDATE REF + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + if( fp_is_abnormal( me ) ) continue; + stats.sumMEdiff += ( me - stats.refME ); + stats.sumWGdiff += ( wg - stats.refWG ); + } + stats.refME = stats.meanME(); // final ref + stats.refWG = stats.meanWG(); // final ref + stats.sumMEdiff = 0; + stats.sumWGdiff = 0; + // THIRD PASS: COMPUTE STDDEV FROM SQUARED SUMS OF DIFFS TO REF + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + if( fp_is_abnormal( me ) ) continue; + stats.sqsMEdiff += std::pow( me - stats.refME, 2 ); + stats.sqsWGdiff += std::pow( wg - stats.refWG, 2 ); + } + // FOURTH PASS: UPDATE THE OVERALL STATS BY ADDING THE NEW STATS + m_stats += stats; + // Increment the iterations counter + m_iter++; + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + + /* + //-------------------------------------------------------------------------- + + CrossSectionKernelDevice::CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t gpublocks, + const size_t gputhreads ) + : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) + , NumberOfEvents( gpublocks*gputhreads ) + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + if ( ! m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: samplingWeights must be a device array" ); + if ( ! m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: matrixElements must be a device array" ); + if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0" ); + if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0" ); + if ( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with samplingWeights" ); + if ( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelDevice::setGrid( const size_t gpublocks, const size_t gputhreads ) + { + if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0 in setGrid" ); + if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0 in setGrid" ); + if ( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch in setGrid" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelDevice::updateEventStatistics( const bool debug ) + { + // Increment the iterations counter + m_iter++; + } + + //-------------------------------------------------------------------------- + */ + +} +#endif + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h new file mode 100644 index 0000000000..d1e29b7999 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h @@ -0,0 +1,139 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef CROSSSECTIONKERNELS_H +#define CROSSSECTIONKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "EventStatistics.h" +#include "MemoryBuffers.h" + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // Helper function for Bridge.h: must be compiled without fast math + // Iterate through all output MEs and replace any NaN/abnormal ones by sqrt(-1) + void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ); + + //-------------------------------------------------------------------------- + + // A base class encapsulating the calculation of event statistics on a CPU host or on a GPU device + class CrossSectionKernelBase //: virtual public ICrossSectionKernel + { + protected: + + // Constructor from existing input and output buffers + CrossSectionKernelBase( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats ) // output: event statistics + : m_samplingWeights( samplingWeights ) + , m_matrixElements( matrixElements ) + , m_stats( stats ) + , m_iter( 0 ) + { + // NB: do not initialise EventStatistics (you may be asked to update an existing result) + } + + public: + + // Destructor + virtual ~CrossSectionKernelBase() {} + + // Update event statistics + virtual void updateEventStatistics( const bool debug = false ) = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The buffer for the sampling weights + const BufferWeights& m_samplingWeights; + + // The buffer for the output matrix elements + const BufferMatrixElements& m_matrixElements; + + // The event statistics + EventStatistics& m_stats; + + // The number of iterations processed so far + size_t m_iter; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating the calculation of event statistics on a CPU host + class CrossSectionKernelHost final : public CrossSectionKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t nevt ); + + // Destructor + virtual ~CrossSectionKernelHost() {} + + // Update event statistics + void updateEventStatistics( const bool debug = false ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + }; + + //-------------------------------------------------------------------------- + + /* +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating the calculation of event statistics on a GPU device + class CrossSectionKernelDevice : public CrossSectionKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t gpublocks, + const size_t gputhreads ); + + // Destructor + virtual ~CrossSectionKernelDevice(){} + + // Reset gpublocks and gputhreads + void setGrid( const size_t gpublocks, const size_t gputhreads ); + + // Update event statistics + void updateEventStatistics( const bool debug=false ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + + }; +#endif + */ + + //-------------------------------------------------------------------------- +} +#endif // CROSSSECTIONKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc new file mode 100644 index 0000000000..0c8e6e6445 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc @@ -0,0 +1,136 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "GpuRuntime.h" +#include "MemoryBuffers.h" +#include "RandomNumberKernels.h" + +#include + +#ifndef MGONGPU_HAS_NO_CURAND /* clang-format off */ +// NB This must come AFTER mgOnGpuConfig.h which contains our definition of __global__ when MGONGPUCPP_GPUIMPL is not defined +#include "curand.h" +#define checkCurand( code ){ assertCurand( code, __FILE__, __LINE__ ); } +inline void assertCurand( curandStatus_t code, const char *file, int line, bool abort = true ) +{ + if ( code != CURAND_STATUS_SUCCESS ) + { + printf( "CurandAssert: %s:%d code=%d\n", file, line, code ); + if ( abort ) assert( code == CURAND_STATUS_SUCCESS ); + } +} +#endif /* clang-format on */ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- +#ifndef MGONGPU_HAS_NO_CURAND + CurandRandomNumberKernel::CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) + : RandomNumberKernelBase( rnarray ) + , m_isOnDevice( onDevice ) + { + if( m_isOnDevice ) + { +#ifdef MGONGPUCPP_GPUIMPL + if( !m_rnarray.isOnDevice() ) + throw std::runtime_error( "CurandRandomNumberKernel on device with a host random number array" ); +#else + throw std::runtime_error( "CurandRandomNumberKernel does not support CurandDevice on CPU host" ); +#endif + } + else + { + if( m_rnarray.isOnDevice() ) + throw std::runtime_error( "CurandRandomNumberKernel on host with a device random number array" ); + } + createGenerator(); + } + + //-------------------------------------------------------------------------- + + CurandRandomNumberKernel::~CurandRandomNumberKernel() + { + destroyGenerator(); + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::seedGenerator( const unsigned int seed ) + { + if( m_isOnDevice ) + { + destroyGenerator(); // workaround for #429 + createGenerator(); // workaround for #429 + } + //printf( "seedGenerator: seed %d\n", seed ); + checkCurand( curandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::createGenerator() + { + // [NB Timings are for GenRnGen host|device (cpp|cuda) generation of 256*32*1 events with nproc=1: rn(0) is host=0.0012s] + const curandRngType_t type = CURAND_RNG_PSEUDO_MTGP32; // 0.00082s | 0.00064s (FOR FAST TESTS) + //const curandRngType_t type = CURAND_RNG_PSEUDO_XORWOW; // 0.049s | 0.0016s + //const curandRngType_t type = CURAND_RNG_PSEUDO_MRG32K3A; // 0.71s | 0.0012s (better but slower, especially in c++) + //const curandRngType_t type = CURAND_RNG_PSEUDO_MT19937; // 21s | 0.021s + //const curandRngType_t type = CURAND_RNG_PSEUDO_PHILOX4_32_10; // 0.024s | 0.00026s (used to segfault?) + if( m_isOnDevice ) + { + checkCurand( curandCreateGenerator( &m_rnGen, type ) ); + } + else + { + checkCurand( curandCreateGeneratorHost( &m_rnGen, type ) ); + } + //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_LEGACY ) ); // fails with code=104 (see #429) + checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_BEST ) ); + //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_DYNAMIC ) ); // fails with code=104 (see #429) + //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_SEEDED ) ); // fails with code=104 (see #429) + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::destroyGenerator() + { + checkCurand( curandDestroyGenerator( m_rnGen ) ); + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::generateRnarray() + { +#if defined MGONGPU_FPTYPE_DOUBLE + checkCurand( curandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#elif defined MGONGPU_FPTYPE_FLOAT + checkCurand( curandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#endif + /* + printf( "\nCurandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); + fptype* data = m_rnarray.data(); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) + { + data = new fptype[m_rnarray.size()](); + checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); + } +#endif + for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) + printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) delete[] data; +#endif + */ + } + + //-------------------------------------------------------------------------- +#endif +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h new file mode 100644 index 0000000000..4a1fd8464d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h @@ -0,0 +1,175 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef EventStatistics_H +#define EventStatistics_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" // for npar (meGeVexponent) + +#include +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // The EventStatistics struct is used to accumulate running aggregates of event statistics. + // This will eventually include the process cross section and the process maximum weight: + // one important case of EventStatistics will then be the "gridpack" result set, which is + // the output of the "integration" step and the input to "unweighted event generation" step. + // The current implementation only includes statistics for matrix elements (ME) and sampling weights (WG); + // in first approximation, the process cross section and maximum weight are just the mean ME and maximum ME, + // but eventually the sampling weights WG (e.g. from Rambo) must also be taken into account in the calculation. + // The implementation uses differences to reference values to improve numerical precision. + struct EventStatistics + { + public: + size_t nevtALL; // total number of events used + size_t nevtABN; // number of events used, where ME is abnormal (nevtABN <= nevtALL) + size_t nevtZERO; // number of not-abnormal events used, where ME is zero (nevtZERO <= nevtOK) + double minME; // minimum matrix element + double maxME; // maximum matrix element + double minWG; // minimum sampling weight + double maxWG; // maximum sampling weight + double refME; // "reference" matrix element (normally the current mean) + double refWG; // "reference" sampling weight (normally the current mean) + double sumMEdiff; // sum of diff to ref for matrix element + double sumWGdiff; // sum of diff to ref for sampling weight + double sqsMEdiff; // squared sum of diff to ref for matrix element + double sqsWGdiff; // squared sum of diff to ref for sampling weight + std::string tag; // a text tag for printouts + // Number of events used, where ME is not abnormal + size_t nevtOK() const { return nevtALL - nevtABN; } + // Mean matrix element + // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] + double meanME() const + { + return refME + ( nevtOK() > 0 ? sumMEdiff / nevtOK() : 0 ); + } + // Mean sampling weight + // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] + double meanWG() const + { + return refWG + ( nevtOK() > 0 ? sumWGdiff / nevtOK() : 0 ); + } + // Variance matrix element + // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] + double varME() const { return ( sqsMEdiff - std::pow( sumMEdiff, 2 ) / nevtOK() ) / nevtOK(); } + // Variance sampling weight + // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] + double varWG() const { return ( sqsWGdiff - std::pow( sumWGdiff, 2 ) / nevtOK() ) / nevtOK(); } + // Standard deviation matrix element + double stdME() const { return std::sqrt( varME() ); } + // Standard deviation sampling weight + double stdWG() const { return std::sqrt( varWG() ); } + // Update reference matrix element + void updateRefME( const double newRef ) + { + const double deltaRef = refME - newRef; + sqsMEdiff += deltaRef * ( 2 * sumMEdiff + nevtOK() * deltaRef ); + sumMEdiff += deltaRef * nevtOK(); + refME = newRef; + } + // Update reference sampling weight + void updateRefWG( const double newRef ) + { + const double deltaRef = refWG - newRef; + sqsWGdiff += deltaRef * ( 2 * sumWGdiff + nevtOK() * deltaRef ); + sumWGdiff += deltaRef * nevtOK(); + refWG = newRef; + } + // Constructor + EventStatistics() + : nevtALL( 0 ) + , nevtABN( 0 ) + , nevtZERO( 0 ) + , minME( std::numeric_limits::max() ) + , maxME( std::numeric_limits::lowest() ) + , minWG( std::numeric_limits::max() ) + , maxWG( std::numeric_limits::lowest() ) + , refME( 0 ) + , refWG( 0 ) + , sumMEdiff( 0 ) + , sumWGdiff( 0 ) + , sqsMEdiff( 0 ) + , sqsWGdiff( 0 ) + , tag( "" ) {} + // Combine two EventStatistics +#ifdef __clang__ + // Disable optimizations for this function in HIP (work around FPE crash #1003: originally using #if __HIP_CLANG_ONLY__) + // Disable optimizations for this function in clang tout court (work around FPE crash #1005: now using #ifdef __clang__) + // See https://clang.llvm.org/docs/LanguageExtensions.html#extensions-for-selectively-disabling-optimization + __attribute__( ( optnone ) ) +#endif + EventStatistics& + operator+=( const EventStatistics& stats ) + { + EventStatistics s1 = *this; // temporary copy + EventStatistics s2 = stats; // temporary copy + EventStatistics& sum = *this; + sum.nevtALL = s1.nevtALL + s2.nevtALL; + sum.nevtABN = s1.nevtABN + s2.nevtABN; + sum.nevtZERO = s1.nevtZERO + s2.nevtZERO; + sum.minME = std::min( s1.minME, s2.minME ); + sum.maxME = std::max( s1.maxME, s2.maxME ); + sum.minWG = std::min( s1.minWG, s2.minWG ); + sum.maxWG = std::max( s1.maxWG, s2.maxWG ); + sum.refME = ( s1.meanME() * s1.nevtOK() + s2.meanME() * s2.nevtOK() ) / sum.nevtOK(); // new mean ME + s1.updateRefME( sum.refME ); + s2.updateRefME( sum.refME ); + sum.sumMEdiff = s1.sumMEdiff + s2.sumMEdiff; + sum.sqsMEdiff = s1.sqsMEdiff + s2.sqsMEdiff; + sum.refWG = ( s1.meanWG() * s1.nevtOK() + s2.meanWG() * s2.nevtOK() ) / sum.nevtOK(); // new mean WG + s1.updateRefWG( sum.refWG ); + s2.updateRefWG( sum.refWG ); + sum.sumWGdiff = s1.sumWGdiff + s2.sumWGdiff; + sum.sqsWGdiff = s1.sqsWGdiff + s2.sqsWGdiff; + return sum; + } + // Printout + void printout( std::ostream& out ) const + { + const EventStatistics& s = *this; + constexpr int meGeVexponent = -( 2 * CPPProcess::npar - 8 ); + out << s.tag << "NumMatrixElems(notAbnormal) = " << s.nevtOK() << std::endl + << std::scientific // fixed format: affects all floats (default precision: 6) + << s.tag << "MeanMatrixElemValue = ( " << s.meanME() + << " +- " << s.stdME() / std::sqrt( s.nevtOK() ) << " ) GeV^" << meGeVexponent << std::endl // standard error + << s.tag << "[Min,Max]MatrixElemValue = [ " << s.minME + << " , " << s.maxME << " ] GeV^" << meGeVexponent << std::endl + << s.tag << "StdDevMatrixElemValue = ( " << s.stdME() + << std::string( 16, ' ' ) << " ) GeV^" << meGeVexponent << std::endl + << s.tag << "MeanWeight = ( " << s.meanWG() + << " +- " << s.stdWG() / std::sqrt( s.nevtOK() ) << std::endl // standard error + << s.tag << "[Min,Max]Weight = [ " << s.minWG + << " , " << s.maxWG << " ]" << std::endl + << s.tag << "StdDevWeight = ( " << s.stdWG() + << std::string( 16, ' ' ) << " )" << std::endl + << std::defaultfloat; // default format: affects all floats + } + }; + + //-------------------------------------------------------------------------- + + inline std::ostream& operator<<( std::ostream& out, const EventStatistics& s ) + { + s.printout( out ); + return out; + } + + //-------------------------------------------------------------------------- +} + +#endif // EventStatistics_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuAbstraction.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuAbstraction.h new file mode 100644 index 0000000000..6f709dfe9a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuAbstraction.h @@ -0,0 +1,168 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: J. Teig (Jul 2023) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2020-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MG5AMC_GPUABSTRACTION_H +#define MG5AMC_GPUABSTRACTION_H 1 + +#include "mgOnGpuConfig.h" + +#include + +//-------------------------------------------------------------------------- + +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) + +#ifndef MGONGPU_HAS_NO_BLAS +#include "cublas_v2.h" +#endif + +#define gpuError_t cudaError_t +#define gpuPeekAtLastError cudaPeekAtLastError +#define gpuGetErrorString cudaGetErrorString +#define gpuSuccess cudaSuccess + +#define gpuMallocHost( ptr, size ) checkGpu( cudaMallocHost( ptr, size ) ) +#define gpuMalloc( ptr, size ) checkGpu( cudaMalloc( ptr, size ) ) + +#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( cudaMemcpy( dstData, srcData, srcBytes, func ) ) +#define gpuMemset( data, value, bytes ) checkGpu( cudaMemset( data, value, bytes ) ) +#define gpuMemcpyHostToDevice cudaMemcpyHostToDevice +#define gpuMemcpyDeviceToHost cudaMemcpyDeviceToHost +#define gpuMemcpyDeviceToDevice cudaMemcpyDeviceToDevice +#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( cudaMemcpyToSymbol( type1, type2, size ) ) + +#define gpuFree( ptr ) checkGpu( cudaFree( ptr ) ) +#define gpuFreeHost( ptr ) checkGpu( cudaFreeHost( ptr ) ) + +#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( cudaGetSymbolAddress( devPtr, symbol ) ) + +#define gpuSetDevice cudaSetDevice +#define gpuDeviceSynchronize cudaDeviceSynchronize +#define gpuDeviceReset cudaDeviceReset + +#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) +//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> +#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) +#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) + +#define gpuStream_t cudaStream_t +#define gpuStreamCreate( pStream ) checkGpu( cudaStreamCreate( pStream ) ) +#define gpuStreamDestroy( stream ) checkGpu( cudaStreamDestroy( stream ) ) +#define gpuMallocAsync( ptr, size, stream ) checkGpu( cudaMallocAsync( ptr, size, stream ) ) +#define gpuFreeAsync( ptr, stream ) checkGpu( cudaFreeAsync( ptr, stream ) ) +#define gpuStreamSynchronize( stream ) checkGpu( cudaStreamSynchronize( stream ) ) + +#define gpuBlasStatus_t cublasStatus_t +#define GPUBLAS_STATUS_SUCCESS CUBLAS_STATUS_SUCCESS +#ifndef MGONGPU_HAS_NO_BLAS +#define gpuBlasHandle_t cublasHandle_t +#else +#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds +#endif +#define gpuBlasCreate cublasCreate +#define gpuBlasDestroy cublasDestroy +#define gpuBlasSetStream cublasSetStream + +#define gpuBlasSaxpy cublasSaxpy +#define gpuBlasSdot cublasSdot +#define gpuBlasSgemv cublasSgemv +#define gpuBlasSgemm cublasSgemm +#define gpuBlasSgemmStridedBatched cublasSgemmStridedBatched +#define gpuBlasDaxpy cublasDaxpy +#define gpuBlasDdot cublasDdot +#define gpuBlasDgemv cublasDgemv +#define gpuBlasDgemm cublasDgemm +#define gpuBlasDgemmStridedBatched cublasDgemmStridedBatched +#define GPUBLAS_OP_N CUBLAS_OP_N +#define GPUBLAS_OP_T CUBLAS_OP_T + +//-------------------------------------------------------------------------- + +#elif defined __HIPCC__ + +#ifndef MGONGPU_HAS_NO_BLAS +#include "hipblas/hipblas.h" +#endif + +#define gpuError_t hipError_t +#define gpuPeekAtLastError hipPeekAtLastError +#define gpuGetErrorString hipGetErrorString +#define gpuSuccess hipSuccess + +#define gpuMallocHost( ptr, size ) checkGpu( hipHostMalloc( ptr, size ) ) // HostMalloc better +#define gpuMalloc( ptr, size ) checkGpu( hipMalloc( ptr, size ) ) + +#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( hipMemcpy( dstData, srcData, srcBytes, func ) ) +#define gpuMemset( data, value, bytes ) checkGpu( hipMemset( data, value, bytes ) ) +#define gpuMemcpyHostToDevice hipMemcpyHostToDevice +#define gpuMemcpyDeviceToHost hipMemcpyDeviceToHost +#define gpuMemcpyDeviceToDevice hipMemcpyDeviceToDevice +#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( hipMemcpyToSymbol( type1, type2, size ) ) + +#define gpuFree( ptr ) checkGpu( hipFree( ptr ) ) +#define gpuFreeHost( ptr ) checkGpu( hipHostFree( ptr ) ) + +#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( hipGetSymbolAddress( devPtr, symbol ) ) + +#define gpuSetDevice hipSetDevice +#define gpuDeviceSynchronize hipDeviceSynchronize +#define gpuDeviceReset hipDeviceReset + +#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) +//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> +#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) +#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) + +#define gpuStream_t hipStream_t +#define gpuStreamCreate( pStream ) checkGpu( hipStreamCreate( pStream ) ) +#define gpuStreamDestroy( stream ) checkGpu( hipStreamDestroy( stream ) ) +#define gpuMallocAsync( ptr, size, stream ) checkGpu( hipMallocAsync( ptr, size, stream ) ) +#define gpuFreeAsync( ptr, stream ) checkGpu( hipFreeAsync( ptr, stream ) ) +#define gpuStreamSynchronize( stream ) checkGpu( hipStreamSynchronize( stream ) ) + +#define gpuBlasStatus_t hipblasStatus_t +#define GPUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS +#ifndef MGONGPU_HAS_NO_BLAS +#define gpuBlasHandle_t hipblasHandle_t +#else +#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds +#endif +#define gpuBlasCreate hipblasCreate +#define gpuBlasDestroy hipblasDestroy +#define gpuBlasSetStream hipblasSetStream + +#define gpuBlasSaxpy hipblasSaxpy +#define gpuBlasSdot hipblasSdot +#define gpuBlasSgemv hipblasSgemv +#define gpuBlasSgemm hipblasSgemm +#define gpuBlasSgemmStridedBatched hipblasSgemmStridedBatched +#define gpuBlasDaxpy hipblasDaxpy +#define gpuBlasDdot hipblasDdot +#define gpuBlasDgemv hipblasDgemv +#define gpuBlasDgemm hipblasDgemm +#define gpuBlasDgemmStridedBatched hipblasDgemmStridedBatched +#define GPUBLAS_OP_N HIPBLAS_OP_N +#define GPUBLAS_OP_T HIPBLAS_OP_T + +#endif + +//-------------------------------------------------------------------------- + +#ifdef MGONGPU_FPTYPE2_FLOAT +#define gpuBlasTaxpy gpuBlasSaxpy +#define gpuBlasTdot gpuBlasSdot +#define gpuBlasTgemv gpuBlasSgemv +#define gpuBlasTgemm gpuBlasSgemm +#define gpuBlasTgemmStridedBatched gpuBlasSgemmStridedBatched +#else +#define gpuBlasTaxpy gpuBlasDaxpy +#define gpuBlasTdot gpuBlasDdot +#define gpuBlasTgemv gpuBlasDgemv +#define gpuBlasTgemm gpuBlasDgemm +#define gpuBlasTgemmStridedBatched gpuBlasDgemmStridedBatched +#endif + +#endif // MG5AMC_GPUABSTRACTION_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h new file mode 100644 index 0000000000..2c8ae1a804 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h @@ -0,0 +1,102 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: J. Teig (Jun 2023, based on earlier work by S. Roiser) for the MG5aMC CUDACPP plugin. +// Further modified by: O. Mattelaer, S. Roiser, J. Teig, A. Valassi, Z. Wettersten (2020-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MG5AMC_GPURUNTIME_H +#define MG5AMC_GPURUNTIME_H 1 + +// MG5AMC on GPU uses the CUDA runtime API, not the lower level CUDA driver API +// See https://docs.nvidia.com/cuda/cuda-runtime-api/driver-vs-runtime-api.html#driver-vs-runtime-api + +#include "GpuAbstraction.h" + +#include + +//-------------------------------------------------------------------------- + +// See https://stackoverflow.com/a/14038590 +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ +#define checkGpu( code ) { assertGpu( code, __FILE__, __LINE__ ); } +inline void assertGpu( gpuError_t code, const char* file, int line, bool abort = true ) +{ + if( code != gpuSuccess ) + { + printf( "ERROR! assertGpu: '%s' (%d) in %s:%d\n", gpuGetErrorString( code ), code, file, line ); + if( abort ) assert( code == gpuSuccess ); + } +} +#endif /* clang-format on */ + +//-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ +#ifndef MGONGPU_HAS_NO_BLAS +#define checkGpuBlas( code ){ assertGpuBlas( code, __FILE__, __LINE__ ); } +inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, bool abort = true ) +{ + if ( code != GPUBLAS_STATUS_SUCCESS ) + { + printf( "ERROR! assertGpuBlas: '%d' in %s:%d\n", code, file, line ); + if( abort ) assert( code == GPUBLAS_STATUS_SUCCESS ); + } +} +#endif +#endif /* clang-format on */ + +//-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + // Instantiate a GpuRuntime at the beginnining of the application's main to + // invoke gpuSetDevice(0) in the constructor and book a gpuDeviceReset() call in the destructor + // *** FIXME! This will all need to be designed differently when going to multi-GPU nodes! *** + struct GpuRuntime final + { + GpuRuntime( const bool debug = true ) + : m_debug( debug ) { setUp( m_debug ); } + ~GpuRuntime() { tearDown( m_debug ); } + GpuRuntime( const GpuRuntime& ) = delete; + GpuRuntime( GpuRuntime&& ) = delete; + GpuRuntime& operator=( const GpuRuntime& ) = delete; + GpuRuntime& operator=( GpuRuntime&& ) = delete; + bool m_debug; + + // Set up CUDA application + // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** + // Calling cudaSetDevice on startup is useful to properly book-keep the time spent in CUDA initialization + static void setUp( const bool debug = false ) // ZW: changed debug default to false + { + // ** NB: it is useful to call cudaSetDevice, or cudaFree, to properly book-keep the time spent in CUDA initialization + // ** NB: otherwise, the first CUDA operation (eg a cudaMemcpyToSymbol in CPPProcess ctor) appears to take much longer! + /* + // [We initially added cudaFree(0) to "ease profile analysis" only because it shows up as a big recognizable block!] + // No explicit initialization is needed: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#initialization + // It is not clear what cudaFree(0) does at all: https://stackoverflow.com/questions/69967813/ + if ( debug ) std::cout << "__CudaRuntime: calling cudaFree(0)" << std::endl; + checkCuda( cudaFree( 0 ) ); // SLOW! + */ + // Replace cudaFree(0) by cudaSetDevice(0), even if it is not really needed either + // (but see https://developer.nvidia.com/blog/cuda-pro-tip-always-set-current-device-avoid-multithreading-bugs) + if( debug ) std::cout << "__GpuRuntime: calling GpuSetDevice(0)" << std::endl; + checkGpu( gpuSetDevice( 0 ) ); // SLOW! + } + + // Tear down CUDA application (call cudaDeviceReset) + // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** + // Calling cudaDeviceReset on shutdown is only needed for checking memory leaks in cuda-memcheck + // See https://docs.nvidia.com/cuda/cuda-memcheck/index.html#leak-checking + static void tearDown( const bool debug = false ) // ZW: changed debug default to false + { + if( debug ) std::cout << "__GpuRuntime: calling GpuDeviceReset()" << std::endl; + checkGpu( gpuDeviceReset() ); + } + }; +} +#endif + +//-------------------------------------------------------------------------- + +#endif // MG5AMC_GPURUNTIME_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc new file mode 100644 index 0000000000..b806da2e09 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc @@ -0,0 +1,146 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "mgOnGpuConfig.h" + +#include "GpuRuntime.h" +#include "MemoryBuffers.h" +#include "RandomNumberKernels.h" + +#include + +#ifndef MGONGPU_HAS_NO_HIPRAND /* clang-format off */ +#ifndef __HIP_PLATFORM_AMD__ +#define __HIP_PLATFORM_AMD__ 1 // enable hiprand for AMD (rocrand) +#endif +#include +#define checkHiprand( code ){ assertHiprand( code, __FILE__, __LINE__ ); } +inline void assertHiprand( hiprandStatus_t code, const char *file, int line, bool abort = true ) +{ + if ( code != HIPRAND_STATUS_SUCCESS ) + { + printf( "HiprandAssert: %s:%d code=%d\n", file, line, code ); + if ( abort ) assert( code == HIPRAND_STATUS_SUCCESS ); + } +} +#endif /* clang-format on */ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- +#ifndef MGONGPU_HAS_NO_HIPRAND + HiprandRandomNumberKernel::HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) + : RandomNumberKernelBase( rnarray ) + , m_isOnDevice( onDevice ) + { + if( m_isOnDevice ) + { +#ifdef MGONGPUCPP_GPUIMPL + if( !m_rnarray.isOnDevice() ) + throw std::runtime_error( "HiprandRandomNumberKernel on device with a host random number array" ); +#else + throw std::runtime_error( "HiprandRandomNumberKernel does not support HiprandDevice on CPU host" ); +#endif + } + else + { + if( m_rnarray.isOnDevice() ) + throw std::runtime_error( "HiprandRandomNumberKernel on host with a device random number array" ); + } + createGenerator(); + } + + //-------------------------------------------------------------------------- + + HiprandRandomNumberKernel::~HiprandRandomNumberKernel() + { + destroyGenerator(); + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::seedGenerator( const unsigned int seed ) + { + if( m_isOnDevice ) + { + destroyGenerator(); // workaround for #429 + createGenerator(); // workaround for #429 + } + //printf( "seedGenerator: seed %d\n", seed ); + checkHiprand( hiprandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::createGenerator() + { + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_DEFAULT; + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_XORWOW; + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MRG32K3A; + const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MTGP32; // same as curand; not implemented yet (code=1000) in host code + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MT19937; + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_PHILOX4_32_10; + if( m_isOnDevice ) + { + checkHiprand( hiprandCreateGenerator( &m_rnGen, type ) ); + } + else + { + // See https://github.com/ROCm/hipRAND/issues/76 + throw std::runtime_error( "HiprandRandomNumberKernel on host is not supported yet (hiprandCreateGeneratorHost is not implemented yet)" ); + //checkHiprand( hiprandCreateGeneratorHost( &m_rnGen, type ) ); // ALWAYS FAILS WITH CODE=1000 + } + // FIXME: hiprand ordering is not implemented yet + // See https://github.com/ROCm/hipRAND/issues/75 + /* + //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_LEGACY ) ); + checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_BEST ) ); + //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_DYNAMIC ) ); + //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_SEEDED ) ); + */ + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::destroyGenerator() + { + checkHiprand( hiprandDestroyGenerator( m_rnGen ) ); + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::generateRnarray() + { +#if defined MGONGPU_FPTYPE_DOUBLE + checkHiprand( hiprandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#elif defined MGONGPU_FPTYPE_FLOAT + checkHiprand( hiprandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#endif + /* + printf( "\nHiprandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); + fptype* data = m_rnarray.data(); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) + { + data = new fptype[m_rnarray.size()](); + checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); + } +#endif + for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) + printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) delete[] data; +#endif + */ + } + + //-------------------------------------------------------------------------- +#endif +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h new file mode 100644 index 0000000000..12b7d45adf --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h @@ -0,0 +1,339 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Hageboeck (Dec 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MADGRAPHTEST_H_ +#define MADGRAPHTEST_H_ 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" + +#include + +#include +#include +//#ifdef __HIPCC__ +//#include // see https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 +//#else +//#include // bypass this completely to ease portability on LUMI #803 +//#endif +#include +#include +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +using mg5amcGpu::CPPProcess; +#else +using mg5amcCpu::CPPProcess; +#endif + +namespace +{ + struct ReferenceData + { + std::vector>> momenta; + std::vector MEs; + std::vector ChanIds; + std::vector SelHels; + std::vector SelCols; + }; + + /// Read batches of reference data from a file and store them in a map. + std::map readReferenceData( const std::string& refFileName ) + { + std::cout << "INFO: Opening reference file " << refFileName << std::endl; + std::ifstream referenceFile( refFileName.c_str() ); + EXPECT_TRUE( referenceFile.is_open() ) << refFileName; + std::map referenceData; + unsigned int evtNo; + unsigned int batchNo; + for( std::string line; std::getline( referenceFile, line ); ) + { + std::stringstream lineStr( line ); + if( line.empty() || line[0] == '#' ) + { + continue; + } + else if( line.find( "Event" ) != std::string::npos ) + { + std::string dummy; + lineStr >> dummy >> evtNo >> dummy >> batchNo; + } + else if( line.find( "ME" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].MEs.size() ) + referenceData[batchNo].MEs.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].MEs[evtNo]; + } + else if( line.find( "ChanId" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].ChanIds.size() ) + referenceData[batchNo].ChanIds.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].ChanIds[evtNo]; + referenceData[batchNo].ChanIds[evtNo] = 0; // disable ChanId comparison if multichannel is not supported #976 + } + else if( line.find( "SelHel" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].SelHels.size() ) + referenceData[batchNo].SelHels.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].SelHels[evtNo]; + } + else if( line.find( "SelCol" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].SelCols.size() ) + referenceData[batchNo].SelCols.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].SelCols[evtNo]; + } + else + { + unsigned int particleIndex; + lineStr >> particleIndex; + if( evtNo <= referenceData[batchNo].momenta.size() ) + referenceData[batchNo].momenta.resize( evtNo + 1 ); + if( particleIndex <= referenceData[batchNo].momenta[evtNo].size() ) + referenceData[batchNo].momenta[evtNo].resize( particleIndex + 1 ); + auto& fourVec = referenceData[batchNo].momenta[evtNo][particleIndex]; + for( unsigned int i = 0; i < fourVec.size(); ++i ) + { + EXPECT_TRUE( lineStr.good() ); + lineStr >> fourVec[i]; + } + EXPECT_TRUE( lineStr.eof() ); + } + } + return referenceData; + } + +} + +/** + * Test driver providing a common interface for testing different implementations. + * Users need to implement: + * - Functions to retrieve matrix element and 4-momenta. These are used in the tests. + * - Driver functions that run the madgraph workflow. + */ +class TestDriverBase +{ + std::string m_refFileName; +public: + const unsigned int nparticle; + static constexpr unsigned int niter = 2; + static constexpr unsigned int gpublocks = 2; + static constexpr unsigned int gputhreads = 128; + static constexpr unsigned int nevt = gpublocks * gputhreads; + + TestDriverBase( unsigned int npart, const std::string& refFileName ) + : m_refFileName( refFileName ) + , nparticle( npart ) + { + } + TestDriverBase() = delete; + virtual ~TestDriverBase() {} + const std::string& getRefFileName() { return m_refFileName; } + + // ------------------------------------------------ + // Interface for retrieving info from madgraph + // ------------------------------------------------ + virtual fptype getMomentum( std::size_t evtNo, unsigned int particleNo, unsigned int component ) const = 0; + virtual fptype getMatrixElement( std::size_t evtNo ) const = 0; + virtual int getChannelId( std::size_t ievt ) const = 0; + virtual int getSelectedHelicity( std::size_t ievt ) const = 0; + virtual int getSelectedColor( std::size_t ievt ) const = 0; + + // ------------------------------------------------ + // Interface for steering madgraph run + // ------------------------------------------------ + virtual void prepareRandomNumbers( unsigned int iiter ) = 0; + virtual void prepareMomenta( fptype energy ) = 0; + virtual void runSigmaKin( std::size_t iiter ) = 0; + + /// Print the requested event into the stream. If the reference data has enough events, it will be printed as well. + void dumpParticles( std::ostream& stream, std::size_t ievt, unsigned int numParticles, unsigned int nDigit, const ReferenceData& referenceData ) const + { + const auto width = nDigit + 8; + for( unsigned int ipar = 0; ipar < numParticles; ipar++ ) + { + // NB: 'setw' affects only the next field (of any type) + stream << std::scientific // fixed format: affects all floats (default nDigit: 6) + << std::setprecision( nDigit ) + << std::setw( 4 ) << ipar + << std::setw( width ) << getMomentum( ievt, ipar, 0 ) + << std::setw( width ) << getMomentum( ievt, ipar, 1 ) + << std::setw( width ) << getMomentum( ievt, ipar, 2 ) + << std::setw( width ) << getMomentum( ievt, ipar, 3 ) + << "\n"; + if( ievt < referenceData.momenta.size() ) + { + stream << "ref" << ipar; + stream << std::setw( width ) << referenceData.momenta[ievt][ipar][0] + << std::setw( width ) << referenceData.momenta[ievt][ipar][1] + << std::setw( width ) << referenceData.momenta[ievt][ipar][2] + << std::setw( width ) << referenceData.momenta[ievt][ipar][3] + << "\n\n"; + } + stream << std::flush << std::defaultfloat; // default format: affects all floats + } + } +}; + +/** + * Test class that's defining all tests to run with a Madgraph workflow. + */ +class MadgraphTest +{ +public: + MadgraphTest( TestDriverBase& testDriverRef ) + : testDriver( &testDriverRef ) {} + ~MadgraphTest() {} + void CompareMomentaAndME( testing::Test& googleTest ) const; // NB: googleTest is ONLY needed for the HasFailure method... +private: + TestDriverBase* testDriver; // non-owning pointer +}; + +void +MadgraphTest::CompareMomentaAndME( testing::Test& googleTest ) const +{ + const fptype toleranceMomenta = std::is_same::value ? 1.E-10 : 4.E-2; // see #735 +#ifdef __APPLE__ + const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-2; // see #583 +#else + //const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 2.E-3; // fails smeft/hip #843 + const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-3; +#endif + constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) + // Dump events to a new reference file? + const char* dumpEventsC = getenv( "CUDACPP_RUNTEST_DUMPEVENTS" ); + const bool dumpEvents = ( dumpEventsC != 0 ) && ( std::string( dumpEventsC ) != "" ); + const std::string refFileName = testDriver->getRefFileName(); + /* +#ifdef __HIPCC__ + const std::string dumpFileName = std::experimental::filesystem::path( refFileName ).filename(); +#else + const std::string dumpFileName = std::filesystem::path( refFileName ).filename(); +#endif + */ + const std::string dumpFileName = refFileName; // bypass std::filesystem #803 + std::ofstream dumpFile; + if( dumpEvents ) + { + dumpFile.open( dumpFileName, std::ios::trunc ); + } + // Read reference data + std::map referenceData; + if( !dumpEvents ) + { + referenceData = readReferenceData( refFileName ); + } + ASSERT_FALSE( googleTest.HasFailure() ); // It doesn't make any sense to continue if we couldn't read the reference file. + // ************************************** + // *** START MAIN LOOP ON #ITERATIONS *** + // ************************************** + for( unsigned int iiter = 0; iiter < testDriver->niter; ++iiter ) + { + testDriver->prepareRandomNumbers( iiter ); + testDriver->prepareMomenta( energy ); + testDriver->runSigmaKin( iiter ); + // --- Run checks on all events produced in this iteration + for( std::size_t ievt = 0; ievt < testDriver->nevt && !googleTest.HasFailure(); ++ievt ) + { + if( dumpEvents ) + { + ASSERT_TRUE( dumpFile.is_open() ) << dumpFileName; + dumpFile << "Event " << std::setw( 8 ) << ievt << " " + << "Batch " << std::setw( 4 ) << iiter << "\n"; + testDriver->dumpParticles( dumpFile, ievt, testDriver->nparticle, 15, ReferenceData() ); + // Dump matrix element + dumpFile << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) + << testDriver->getMatrixElement( ievt ) << "\n" + << std::defaultfloat; + // Dump channelId + dumpFile << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n"; + // Dump selected helicity and color + dumpFile << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n"; + dumpFile << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" + << std::endl; // leave one line between events + continue; + } + // Check that we have the required reference data + ASSERT_GT( referenceData.size(), iiter ) + << "Don't have enough reference data for iteration " << iiter << ". Ref file:" << refFileName; + ASSERT_GT( referenceData[iiter].MEs.size(), ievt ) + << "Don't have enough reference MEs for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].ChanIds.size(), ievt ) + << "Don't have enough reference ChanIds for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].SelHels.size(), ievt ) + << "Don't have enough reference SelHels for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].SelCols.size(), ievt ) + << "Don't have enough reference SelCols for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].momenta.size(), ievt ) + << "Don't have enough reference momenta for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GE( referenceData[iiter].momenta[ievt].size(), testDriver->nparticle ) + << "Don't have enough reference particles for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + // This trace will help to understand the event that is being checked. + // It will only be printed in case of failures: + std::stringstream eventTrace; + eventTrace << "In comparing event " << ievt << " from iteration " << iiter << "\n"; + testDriver->dumpParticles( eventTrace, ievt, testDriver->nparticle, 15, referenceData[iiter] ); + eventTrace << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) + << testDriver->getMatrixElement( ievt ) << "\n" + << std::setw( 4 ) << "r.ME" << std::scientific << std::setw( 15 + 8 ) + << referenceData[iiter].MEs[ievt] << std::endl + << std::defaultfloat; + eventTrace << std::setw( 8 ) << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n" + << std::setw( 8 ) << "r.ChanId" << std::setw( 8 ) << referenceData[iiter].ChanIds[ievt] << std::endl; + eventTrace << std::setw( 8 ) << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n" + << std::setw( 8 ) << "r.SelHel" << std::setw( 8 ) << referenceData[iiter].SelHels[ievt] << std::endl; + eventTrace << std::setw( 8 ) << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" + << std::setw( 8 ) << "r.SelCol" << std::setw( 8 ) << referenceData[iiter].SelCols[ievt] << std::endl; + SCOPED_TRACE( eventTrace.str() ); + // Compare Momenta + for( unsigned int ipar = 0; ipar < testDriver->nparticle; ++ipar ) + { + std::stringstream momentumErrors; + for( unsigned int icomp = 0; icomp < CPPProcess::np4; ++icomp ) + { + const fptype pMadg = testDriver->getMomentum( ievt, ipar, icomp ); + const fptype pOrig = referenceData[iiter].momenta[ievt][ipar][icomp]; + //const fptype relDelta = fabs( ( pMadg - pOrig ) / pOrig ); // computing relDelta may lead to FPEs + const fptype delta = fabs( pMadg - pOrig ); + if( delta > toleranceMomenta * fabs( pOrig ) ) // better than "relDelta > toleranceMomenta" + { + momentumErrors << std::setprecision( 15 ) << std::scientific << "\nparticle " << ipar << "\tcomponent " << icomp + << "\n\t madGraph: " << std::setw( 22 ) << pMadg + << "\n\t reference: " << std::setw( 22 ) << pOrig + << "\n\t relative delta exceeds tolerance of " << toleranceMomenta; + } + } + ASSERT_TRUE( momentumErrors.str().empty() ) << momentumErrors.str(); + } + // Compare ME: + EXPECT_NEAR( testDriver->getMatrixElement( ievt ), + referenceData[iiter].MEs[ievt], + toleranceMEs * referenceData[iiter].MEs[ievt] ); + // Compare channelId + EXPECT_EQ( testDriver->getChannelId( ievt ), + referenceData[iiter].ChanIds[ievt] ); + // Compare selected helicity and color + EXPECT_EQ( testDriver->getSelectedHelicity( ievt ), + referenceData[iiter].SelHels[ievt] ); + EXPECT_EQ( testDriver->getSelectedColor( ievt ), + referenceData[iiter].SelCols[ievt] ); + } + } + if( dumpEvents ) + { + std::cout << "Event dump written to " << dumpFileName << std::endl; + } +} + +#endif /* MADGRAPHTEST_H_ */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc new file mode 100644 index 0000000000..872e4795e3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc @@ -0,0 +1,509 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "MatrixElementKernels.h" + +#include "CPPProcess.h" +#include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" + +#include // for fetestexcept +#include +#include + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + MatrixElementKernelBase::MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol ) // output: color selection + : m_momenta( momenta ) + , m_gs( gs ) + , m_iflavorVec( iflavorVec ) + , m_rndhel( rndhel ) + , m_rndcol( rndcol ) + , m_channelIds( channelIds ) + , m_matrixElements( matrixElements ) + , m_selhel( selhel ) + , m_selcol( selcol ) +#ifdef MGONGPU_CHANNELID_DEBUG + , m_nevtProcessedByChannel() + , m_tag() +#endif + { + //std::cout << "DEBUG: MatrixElementKernelBase ctor " << this << std::endl; +#ifdef MGONGPU_CHANNELID_DEBUG + for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + m_nevtProcessedByChannel[channelId] = 0; +#endif + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelBase::~MatrixElementKernelBase() + { + //std::cout << "DEBUG: MatrixElementKernelBase dtor " << this << std::endl; +#ifdef MGONGPU_CHANNELID_DEBUG + MatrixElementKernelBase::dumpNevtProcessedByChannel(); +#endif +#ifdef MGONGPUCPP_VERBOSE + MatrixElementKernelBase::dumpSignallingFPEs(); +#endif + } + + //-------------------------------------------------------------------------- + +#ifdef MGONGPU_CHANNELID_DEBUG + void MatrixElementKernelBase::updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ) + { + if( pHstChannelIds != nullptr ) + { + //std::cout << "DEBUG " << this << ": not nullptr " << nevt << std::endl; + for( unsigned int ievt = 0; ievt < nevt; ievt++ ) + { + const size_t channelId = pHstChannelIds[ievt]; // Fortran indexing + //assert( channelId > 0 ); + //assert( channelId < CPPProcess::ndiagrams ); + m_nevtProcessedByChannel[channelId]++; + } + } + else + { + //std::cout << "DEBUG " << this << ": nullptr " << std::endl; + m_nevtProcessedByChannel[0] += nevt; + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPU_CHANNELID_DEBUG + void MatrixElementKernelBase::dumpNevtProcessedByChannel() + { + size_t nevtProcessed = 0; + for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + nevtProcessed += m_nevtProcessedByChannel[channelId]; + std::ostringstream sstr; + sstr << " {"; + for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + { + if( m_nevtProcessedByChannel[channelId] > 0 ) + { + if( sstr.str() != " {" ) sstr << ","; + if( channelId == 0 ) + sstr << " no-multichannel"; + else + sstr << " " << channelId; + sstr << " : " << m_nevtProcessedByChannel[channelId]; + } + } + sstr << " }"; + std::cout << "DEBUG: MEK " << this; + if( m_tag != "" ) std::cout << " " << m_tag; + std::cout << " processed " << nevtProcessed << " events across " << CPPProcess::ndiagrams << " channels" << sstr.str() << std::endl; + } +#endif + + //-------------------------------------------------------------------------- + + void MatrixElementKernelBase::dumpSignallingFPEs() + { + // New strategy for issue #831: add a final report of FPEs + // Note: normally only underflow will be reported here (inexact is switched off because it would almost always signal; + // divbyzero, invalid and overflow are configured by feenablexcept to send a SIGFPE signal, and are normally fixed in the code) + // Note: this is now called in the individual destructors of MEK classes rather than in that of MatrixElementKernelBase(#837) + std::string fpes; + if( std::fetestexcept( FE_DIVBYZERO ) ) fpes += " FE_DIVBYZERO"; + if( std::fetestexcept( FE_INVALID ) ) fpes += " FE_INVALID"; + if( std::fetestexcept( FE_OVERFLOW ) ) fpes += " FE_OVERFLOW"; + if( std::fetestexcept( FE_UNDERFLOW ) ) fpes += " FE_UNDERFLOW"; + //if( std::fetestexcept( FE_INEXACT ) ) fpes += " FE_INEXACT"; // do not print this out: this would almost always signal! + if( fpes == "" ) + std::cout << "INFO: No Floating Point Exceptions have been reported" << std::endl; + else + std::cerr << "INFO: The following Floating Point Exceptions have been reported:" << fpes << std::endl; + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +#ifndef MGONGPUCPP_GPUIMPL +namespace mg5amcCpu +{ + + //-------------------------------------------------------------------------- + + MatrixElementKernelHost::MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt ) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( nevt ) + , m_couplings( nevt ) + , m_numerators( nevt * CPPProcess::ndiagrams ) + , m_denominators( nevt ) + { + //std::cout << "DEBUG: MatrixElementKernelHost::ctor " << this << std::endl; + if( m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: momenta must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: matrixElements must be a host array" ); + if( m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: channelIds must be a device array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with matrixElements" ); + if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with channelIds" ); + if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with iflavorVec" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MatrixElementKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Fail gently and avoid "Illegal instruction (core dumped)" if the host does not support the SIMD used in the ME calculation + // Note: this prevents a crash on pmpe04 but not on some github CI nodes? + // [NB: SIMD vectorization in mg5amc C++ code is only used in the ME calculation below MatrixElementKernelHost!] + if( !MatrixElementKernelHost::hostSupportsSIMD() ) + throw std::runtime_error( "Host does not support the SIMD implementation of MatrixElementKernelsHost" ); + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelHost::~MatrixElementKernelHost() + { + //std::cout << "DEBUG: MatrixElementKernelBase::dtor " << this << std::endl; + } + + //-------------------------------------------------------------------------- + + int MatrixElementKernelHost::computeGoodHelicities() + { + HostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); + // ... 0d1. Compute good helicity mask on the host + computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); + sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_numerators.data(), m_denominators.data(), hstIsGoodHel.data(), nevt() ); + // ... 0d2. Copy good helicity list to static memory on the host + // [FIXME! REMOVE THIS STATIC THAT BREAKS MULTITHREADING?] + return sigmaKin_setGoodHel( hstIsGoodHel.data() ); + } + + //-------------------------------------------------------------------------- + + void MatrixElementKernelHost::computeMatrixElements( const bool useChannelIds ) + { + computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_numerators.data(), m_denominators.data(), nullptr, true, nevt() ); +#ifdef MGONGPU_CHANNELID_DEBUG + //std::cout << "DEBUG: MatrixElementKernelHost::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; + MatrixElementKernelBase::updateNevtProcessedByChannel( pChannelIds, nevt() ); +#endif + } + + //-------------------------------------------------------------------------- + + // Does this host system support the SIMD used in the matrix element calculation? + bool MatrixElementKernelHost::hostSupportsSIMD( const bool verbose ) + { +#if defined __AVX512VL__ + bool known = true; + bool ok = __builtin_cpu_supports( "avx512vl" ); + const std::string tag = "skylake-avx512 (AVX512VL)"; +#elif defined __AVX2__ + bool known = true; + bool ok = __builtin_cpu_supports( "avx2" ); + const std::string tag = "haswell (AVX2)"; +#elif defined __SSE4_2__ +#ifdef __PPC__ + // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html + bool known = true; + bool ok = __builtin_cpu_supports( "vsx" ); + const std::string tag = "powerpc vsx (128bit as in SSE4.2)"; +#elif defined( __x86_64__ ) || defined( __i386__ ) + bool known = true; + bool ok = __builtin_cpu_supports( "sse4.2" ); + const std::string tag = "nehalem (SSE4.2)"; +#else // AV FIXME! Added by OM for Mac, should identify the correct __xxx__ flag that should be targeted + // DM now we have an explicit NEON target for ARM + bool known = false; // __builtin_cpu_supports is not supported + bool ok = true; // this is just an assumption! + const std::string tag = "simd arch not defined"; +#endif +#elif defined __ARM_NEON // consider using __BUILTIN_CPU_SUPPORTS__ + bool known = false; // __builtin_cpu_supports is not supported + // See https://stackoverflow.com/q/62783908 + // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu + bool ok = true; // this is just an assumption! + const std::string tag = "arm neon (128bit as in SSE4.2)"; +#else + bool known = true; + bool ok = true; + const std::string tag = "none"; +#endif + if( verbose ) + { + if( tag == "none" ) + std::cout << "INFO: The application does not require the host to support any AVX feature" << std::endl; + else if( ok && known ) + std::cout << "INFO: The application is built for " << tag << " and the host supports it" << std::endl; + else if( ok ) + std::cout << "WARNING: The application is built for " << tag << " but it is unknown if the host supports it" << std::endl; + else + std::cout << "ERROR! The application is built for " << tag << " but the host does not support it" << std::endl; + } + return ok; + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + + //-------------------------------------------------------------------------- + + MatrixElementKernelDevice::MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( gpublocks * gputhreads ) + , m_couplings( this->nevt() ) + , m_pHelMEs() + , m_pHelJamps() + , m_pHelNumerators() + , m_pHelDenominators() + , m_colJamp2s( CPPProcess::ncolor * this->nevt() ) +#ifdef MGONGPU_CHANNELID_DEBUG + , m_hstChannelIds( this->nevt() ) +#endif +#ifndef MGONGPU_HAS_NO_BLAS + , m_blasColorSum( false ) + , m_blasTf32Tensor( false ) + , m_pHelBlasTmp() + , m_blasHandle() +#endif + , m_helStreams() + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + //std::cout << "DEBUG: MatrixElementKernelDevice::ctor " << this << std::endl; + if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: momenta must be a device array" ); + if( !m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: matrixElements must be a device array" ); + if( !m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: channelIds must be a device array" ); // FIXME?! + if( !m_iflavorVec.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: iflavorVec must be a device array" ); + if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with matrixElements" ); + if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with channelIds" ); + if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with iflavorVec" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( m_gputhreads % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MatrixElementKernelHost: gputhreads should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Create the "one-helicity" jamp buffer that will be used for helicity filtering + m_pHelJamps.reset( new DeviceBufferSimple( CPPProcess::ncolor * mgOnGpu::nx2 * this->nevt() ) ); + // Create the "one-helicity" numerator and denominator buffers that will be used for helicity filtering + m_pHelNumerators.reset( new DeviceBufferSimple( this->nevt() * CPPProcess::ndiagrams ) ); + m_pHelDenominators.reset( new DeviceBufferSimple( this->nevt() ) ); + // Decide at runtime whether to use BLAS for color sums + // Decide at runtime whether TF32TENSOR math should be used in cuBLAS + static bool first = true; + if( first ) + { + first = false; + // Analyse environment variable CUDACPP_RUNTIME_BLASCOLORSUM + const char* blasEnv = getenv( "CUDACPP_RUNTIME_BLASCOLORSUM" ); + if( blasEnv && std::string( blasEnv ) != "" ) + { +#ifndef MGONGPU_HAS_NO_BLAS + m_blasColorSum = true; // fixme? eventually set default=true and decode "Y" and "N" choices? + std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty: enable BLAS" << std::endl; +#else + throw std::runtime_error( "Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty, but BLAS was disabled at build time" ); +#endif + } + else + { +#ifndef MGONGPU_HAS_NO_BLAS + std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is empty or not set: disable BLAS" << std::endl; +#else + std::cout << "INFO: BLAS was disabled at build time" << std::endl; +#endif + } +#ifndef MGONGPU_HAS_NO_BLAS +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) + // Analyse environment variable CUDACPP_RUNTIME_CUBLASTF32TENSOR + const char* blasEnv2 = getenv( "CUDACPP_RUNTIME_CUBLASTF32TENSOR" ); + if( blasEnv2 && std::string( blasEnv2 ) != "" ) + { + if( m_blasColorSum ) + { +#ifdef MGONGPU_FPTYPE2_FLOAT + m_blasTf32Tensor = true; + std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty: enable CUBLAS_TF32_TENSOR_OP_MATH" << std::endl; +#else + std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but color sums use FP64" << std::endl; +#endif + } + else + std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but BLAS was disabled at runtime" << std::endl; + } +#ifdef MGONGPU_FPTYPE2_FLOAT + else + { + if( m_blasColorSum ) + std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is empty or not set: keep cuBLAS math defaults" << std::endl; + } +#endif +#endif +#endif + } + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelDevice::~MatrixElementKernelDevice() + { + //std::cout << "DEBUG: MatrixElementKernelDevice::dtor " << this << std::endl; +#ifndef MGONGPU_HAS_NO_BLAS + if( m_blasHandle ) gpuBlasDestroy( m_blasHandle ); +#endif + for( int ihel = 0; ihel < CPPProcess::ncomb; ihel++ ) + { + if( m_helStreams[ihel] ) gpuStreamDestroy( m_helStreams[ihel] ); // do not destroy if nullptr + } + } + + //-------------------------------------------------------------------------- + + // FIXME! The relevance of this function should be reassessed (#543 and #902) + void MatrixElementKernelDevice::setGrid( const int /*gpublocks*/, const int /*gputhreads*/ ) + { + if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0 in setGrid" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0 in setGrid" ); + if( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch in setGrid" ); + } + + //-------------------------------------------------------------------------- + + int MatrixElementKernelDevice::computeGoodHelicities() + { + PinnedHostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); + // ... 0d1. Compute good helicity mask (a host variable) on the device + gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); + const int nevt = m_gpublocks * m_gputhreads; + sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_pHelJamps->data(), m_pHelNumerators->data(), m_pHelDenominators->data(), hstIsGoodHel.data(), nevt ); + // ... 0d3. Set good helicity list in host static memory + int nGoodHel = sigmaKin_setGoodHel( hstIsGoodHel.data() ); + assert( nGoodHel > 0 ); // SANITY CHECK: there should be at least one good helicity + // Create one GPU stream for each good helicity + for( int ighel = 0; ighel < nGoodHel; ighel++ ) + gpuStreamCreate( &m_helStreams[ighel] ); +#ifndef MGONGPU_HAS_NO_BLAS + // Create one cuBLAS/hipBLAS handle for each good helicity (attached to the default stream) + if( m_blasColorSum ) + { + checkGpuBlas( gpuBlasCreate( &m_blasHandle ) ); +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) + if( m_blasTf32Tensor ) + checkGpuBlas( cublasSetMathMode( m_blasHandle, CUBLAS_TF32_TENSOR_OP_MATH ) ); // enable TF32 tensor cores +#endif + } +#endif + // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) + m_pHelMEs.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); + // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) + // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) + m_pHelJamps.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); + // ... Create the "many-helicity" super-buffers of nGoodHel numerator and denominator buffers (dynamically allocated) + // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) + m_pHelNumerators.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ndiagrams * nevt ) ); + m_pHelDenominators.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); +#ifndef MGONGPU_HAS_NO_BLAS + // Create the "many-helicity" super-buffers of real/imag ncolor*nevt temporary buffers for cuBLAS/hipBLAS intermediate results in color_sum_blas +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Mixed precision mode: need two fptype2[ncolor*2*nevt] buffers and one fptype2[nevt] buffer per good helicity + if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ( 2 * CPPProcess::ncolor * mgOnGpu::nx2 + 1 ) * nevt ) ); +#else + // Standard single/double precision mode: need one fptype2[ncolor*2*nevt] buffer per good helicity + if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); +#endif +#endif + // Return the number of good helicities + return nGoodHel; + } + + //-------------------------------------------------------------------------- + + void MatrixElementKernelDevice::computeMatrixElements( const bool useChannelIds ) + { + gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); +#ifndef MGONGPU_HAS_NO_BLAS + fptype2* ghelAllBlasTmp = ( m_blasColorSum ? m_pHelBlasTmp->data() : nullptr ); + gpuBlasHandle_t* pBlasHandle = ( m_blasColorSum ? &m_blasHandle : nullptr ); +#else + fptype2* ghelAllBlasTmp = nullptr; + gpuBlasHandle_t* pBlasHandle = nullptr; +#endif + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_colJamp2s.data(), m_pHelNumerators->data(), m_pHelDenominators->data(), nullptr, true, m_pHelMEs->data(), m_pHelJamps->data(), ghelAllBlasTmp, pBlasHandle, m_helStreams, false, m_gpublocks, m_gputhreads ); +#ifdef MGONGPU_CHANNELID_DEBUG + //std::cout << "DEBUG: MatrixElementKernelDevice::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; + copyHostFromDevice( m_hstChannelIds, m_channelIds ); // FIXME?! + const unsigned int* pHstChannelIds = ( useChannelIds ? m_hstChannelIds.data() : nullptr ); + MatrixElementKernelBase::updateNevtProcessedByChannel( pHstChannelIds, nevt() ); +#endif + checkGpu( gpuPeekAtLastError() ); // is this needed? + checkGpu( gpuDeviceSynchronize() ); // probably not needed? but it avoids errors in sigmaKin above from appearing later on in random places... + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h new file mode 100644 index 0000000000..01041e43b7 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h @@ -0,0 +1,250 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MATRIXELEMENTKERNELS_H +#define MATRIXELEMENTKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "GpuAbstraction.h" +#include "MemoryBuffers.h" + +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // A base class encapsulating matrix element calculations on a CPU host or on a GPU device + class MatrixElementKernelBase //: virtual public IMatrixElementKernel + { + protected: + + // Constructor from existing input and output buffers + MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol); // output: color selection + + public: + + // Destructor + virtual ~MatrixElementKernelBase(); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + virtual int computeGoodHelicities() = 0; + + // Compute matrix elements + virtual void computeMatrixElements( const bool useChannelIds ) = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + // Dump signalling FPEs (#831 and #837) + static void dumpSignallingFPEs(); + +#ifdef MGONGPU_CHANNELID_DEBUG + // Add a MEK identifier for the channelId debug printout + void setTagForNevtProcessedByChannel( const std::string& tag ) { m_tag = tag; } + + protected: + // Update number of events processed by channel + void updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ); + + // Dump number of events processed by channel + void dumpNevtProcessedByChannel(); +#endif + + protected: + + // The buffer for the input momenta + const BufferMomenta& m_momenta; + + // The buffer for the gs to calculate the alphaS values + const BufferGs& m_gs; + + // The buffer for the flavor indices for the flavor combination + const BufferIflavorVec& m_iflavorVec; + + // The buffer for the random numbers for helicity selection + const BufferRndNumHelicity& m_rndhel; + + // The buffer for the random numbers for color selection + const BufferRndNumColor& m_rndcol; + + // The buffer for the channel ids for single-diagram enhancement + const BufferChannelIds& m_channelIds; + + // The buffer for the output matrix elements + BufferMatrixElements& m_matrixElements; + + // The buffer for the output helicity selection + BufferSelectedHelicity& m_selhel; + + // The buffer for the output color selection + BufferSelectedColor& m_selcol; + +#ifdef MGONGPU_CHANNELID_DEBUG + // The events-per-channel counter for debugging + std::map m_nevtProcessedByChannel; + + // The tag for events-per-channel debugging + std::string m_tag; +#endif + }; + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating matrix element calculations on a CPU host + class MatrixElementKernelHost final : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~MatrixElementKernelHost(); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // Does this host system support the SIMD used in the matrix element calculation? + // [NB: this is private, SIMD vectorization in mg5amc C++ code is currently only used in the ME calculations below MatrixElementKernelHost!] + static bool hostSupportsSIMD( const bool verbose = false ); // ZW: default verbose false + + private: + + // The buffer for the event-by-event couplings that depends on alphas QCD + HostBufferCouplings m_couplings; + + // The buffer for the event-by-event numerators of multichannel factors + HostBufferNumerators m_numerators; + + // The buffer for the event-by-event denominators of multichannel factors + HostBufferDenominators m_denominators; + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating matrix element calculations on a GPU device + class MatrixElementKernelDevice : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads ); + + // Destructor + virtual ~MatrixElementKernelDevice(); + + // Reset gpublocks and gputhreads + void setGrid( const int gpublocks, const int gputhreads ); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The buffer for the event-by-event couplings that depends on alphas QCD + DeviceBufferCouplings m_couplings; + + // The super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelMEs; + + // The super-buffer of nGoodHel jamp buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelJamps; + + // The super-buffer of nGoodHel numerator buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelNumerators; + + // The super-buffer of nGoodHel denominator buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelDenominators; + + // The super-buffer of ncolor jamp2 buffers + DeviceBufferSimple m_colJamp2s; + +#ifdef MGONGPU_CHANNELID_DEBUG + // The **host** buffer for the channelId array + // FIXME? MEKD should accept a host buffer as an argument instead of a device buffer, so that a second copy can be avoided? + PinnedHostBufferChannelIds m_hstChannelIds; +#endif + +#ifndef MGONGPU_HAS_NO_BLAS + // Decide at runtime whether to use BLAS for color sums + bool m_blasColorSum; + + // Decide at runtime whether TF32TENSOR math should be used in cuBLAS + bool m_blasTf32Tensor; + + // The super-buffer of nGoodHel cuBLAS/hipBLAS temporary buffers + std::unique_ptr m_pHelBlasTmp; + + // The cuBLAS/hipBLAS handle (a single one for all good helicities) + gpuBlasHandle_t m_blasHandle; +#endif + + // The array of GPU streams (one for each good helicity) + gpuStream_t m_helStreams[CPPProcess::ncomb]; // reserve ncomb streams (but only nGoodHel <= ncomb will be used) + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + }; +#endif + + //-------------------------------------------------------------------------- +} +#endif // MATRIXELEMENTKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h new file mode 100644 index 0000000000..f41e40bf82 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h @@ -0,0 +1,165 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessAmplitudes_H +#define MemoryAccessAmplitudes_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" + +#define MGONGPU_TRIVIAL_AMPLITUDES 1 + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + +#ifndef MGONGPU_TRIVIAL_AMPLITUDES + + // A class describing the internal layout of memory buffers for amplitudes + // This implementation uses an AOSOA[npagA][nx2][neppA] where nevt=npagA*neppA + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessAmplitudesBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the amplitude AOSOA memory buffer layout + static constexpr int neppA = 1; // AOS (just a test...) + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagA = ievt / neppA; // #event "A-page" + const int ieppA = ievt % neppA; // #event in the current event A-page + constexpr int ix2 = 0; + return &( buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA] ); // AOSOA[ipagA][ix2][ieppA] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ix2 ) + { + constexpr int ipagA = 0; + constexpr int ieppA = 0; + return buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA]; // AOSOA[ipagA][ix2][ieppA] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessAmplitudes : public MemoryAccessAmplitudesBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + +#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessAmplitudes + { + public: + +#ifndef MGONGPU_TRIVIAL_AMPLITUDES + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2 = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2Const = + KernelAccessHelper::template kernelAccessFieldConst; + +#else + + static __host__ __device__ inline cxtype_sv* + kernelAccess( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + static __host__ __device__ inline const cxtype_sv* + kernelAccessConst( const fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + +#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessAmplitudes HostAccessAmplitudes; + typedef KernelAccessAmplitudes DeviceAccessAmplitudes; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h new file mode 100644 index 0000000000..331e7f12c3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h @@ -0,0 +1,126 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Dec 2023, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessChannelIds_H +#define MemoryAccessChannelIds_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for channel ids + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessChannelIdsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] + static __host__ __device__ inline unsigned int* + ieventAccessRecord( unsigned int* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline unsigned int& + decodeRecord( unsigned int* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessChannelIds : public MemoryAccessChannelIdsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] + static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessChannelIds + { + public: + + // Expose selected functions from MemoryAccessChannelIds + static constexpr auto ieventAccessRecordConst = MemoryAccessChannelIds::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] + static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] + static __host__ __device__ inline const uint_sv& + kernelAccessConst( const unsigned int* buffer ) + { + const unsigned int& out = kernelAccessConst_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessChannelIds HostAccessChannelIds; + typedef KernelAccessChannelIds DeviceAccessChannelIds; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h new file mode 100644 index 0000000000..fd3fb80c65 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h @@ -0,0 +1,276 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessCouplings_H +#define MemoryAccessCouplings_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM +#include "MemoryBuffers.h" // for HostBufferCouplings::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for couplings + // This implementation uses an AOSOA[npagC][ndcoup][nx2][neppC] "super-buffer" where nevt=npagC*neppC + // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessCouplingsBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the coupling AOSOA memory buffer layout + static constexpr int neppC = MemoryAccessMomentaBase::neppM; // use the same AOSOA striding as for momenta + + // SANITY CHECK: check that neppC is a power of two + static_assert( ispoweroftwo( neppC ), "neppC is not a power of 2" ); + + //-------------------------------------------------------------------------- + // ** NB! A single super-buffer AOSOA[npagC][ndcoup][nx2][neppC] includes data for ndcoup different couplings ** + // ** NB! The ieventAccessRecord and kernelAccess functions refer to the buffer for one individual coupling ** + // ** NB! Use idcoupAccessBuffer to add a fixed offset and locate the buffer for one given individual coupling ** + //-------------------------------------------------------------------------- + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (non-const) ===> fptype* idcoupAccessBuffer( fptype* buffer, const int idcoup ) <===] + // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? + static __host__ __device__ inline fptype* + idcoupAccessBuffer( fptype* buffer, // input "super-buffer" + const int idcoup ) + { + constexpr int ipagC = 0; + constexpr int ieppC = 0; + constexpr int ix2 = 0; + // NB! this effectively adds an offset "idcoup * nx2 * neppC" + return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] + } + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (const) ===> const fptype* idcoupAccessBufferConst( const fptype* buffer, const int idcoup ) <===] + // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? + static __host__ __device__ inline const fptype* + idcoupAccessBufferConst( const fptype* buffer, // input "super-buffer" + const int idcoup ) + { + return idcoupAccessBuffer( const_cast( buffer ), idcoup ); + } + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of couplings that dependent on the running alphas QCD in this specific process + static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagC = ievt / neppC; // #event "C-page" + const int ieppC = ievt %% neppC; // #event in the current event C-page + constexpr int idcoup = 0; + constexpr int ix2 = 0; + return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ix2 ) + { + constexpr int ipagC = 0; + constexpr int ieppC = 0; + // NB! the offset "idcoup * nx2 * neppC" has been added in idcoupAccessBuffer + constexpr int idcoup = 0; + return buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC]; // AOSOA[ipagC][idcoup][ix2][ieppC] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessCouplings : public MemoryAccessCouplingsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessCouplings + { + public: + + // Expose selected functions from MemoryAccessCouplingsBase + static constexpr auto idcoupAccessBuffer = MemoryAccessCouplingsBase::idcoupAccessBuffer; + static constexpr auto idcoupAccessBufferConst = MemoryAccessCouplingsBase::idcoupAccessBufferConst; + + // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. + // For dependent (event-by-event, running-alphas) couplings the value is an AOSOA record [nx2][neppC] + // (real and imaginary SIMD lanes), so consecutive flavor slots are nx2*neppC fptype's apart. + static constexpr int flv_stride = MemoryAccessCouplingsBase::neppC * mgOnGpu::nx2; + + // Expose selected functions from MemoryAccessCouplings + static constexpr auto ieventAccessRecordConst = MemoryAccessCouplings::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2_s = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2Const_s = + KernelAccessHelper::template kernelAccessFieldConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccessIx2( fptype* buffer, + const int ix2 ) + { + fptype& out = kernelAccessIx2_s( buffer, ix2 ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays + constexpr int neppC = MemoryAccessCouplingsBase::neppC; + static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS + static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessIx2Const( const fptype* buffer, + const int ix2 ) + { + return kernelAccessIx2( const_cast( buffer ), ix2 ); + } + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessIx2Const( const fptype* buffer, + const int ix2 ) + { + const fptype& out = kernelAccessIx2Const_s( buffer, ix2 ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays + constexpr int neppC = MemoryAccessCouplingsBase::neppC; + static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS + static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non const, SCALAR OR VECTOR) ===> cxtype_sv_ref kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline cxtype_sv_ref + kernelAccess( fptype* buffer ) + { + /* + fptype_sv& real = kernelAccessIx2( buffer, 0 ); + fptype_sv& imag = kernelAccessIx2( buffer, 1 ); + printf( "C_ACCESS::kernelAccess: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + return cxtype_sv_ref( real, imag ); + */ + return cxtype_sv_ref( kernelAccessIx2( buffer, 0 ), + kernelAccessIx2( buffer, 1 ) ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] + static __host__ __device__ inline cxtype_sv + kernelAccessConst( const fptype* buffer ) + { + /* + const fptype_sv& real = kernelAccessIx2Const( buffer, 0 ); + const fptype_sv& imag = kernelAccessIx2Const( buffer, 1 ); + printf( "C_ACCESS::kernelAccessConst: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + return cxtype_sv( real, imag ); + */ + return cxtype_sv( kernelAccessIx2Const( buffer, 0 ), + kernelAccessIx2Const( buffer, 1 ) ); + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessCouplings HostAccessCouplings; + typedef KernelAccessCouplings DeviceAccessCouplings; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h new file mode 100644 index 0000000000..757de7b6f2 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h @@ -0,0 +1,89 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Apr 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessCouplingsFixed_H +#define MemoryAccessCouplingsFixed_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" +#include "mgOnGpuVectors.h" + +//#include "MemoryAccessHelpers.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for fixed couplings + // This implementation uses a STRUCT[ndcoup][nx2] "super-buffer" layout: in practice, the cIPC global array + // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling + // [If many implementations are used, a suffix _Sv1 should be appended to the class name] + class MemoryAccessCouplingsFixedBase //_Sv1 + { + public: + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (const) ===> const fptype* iicoupAccessBufferConst( const fptype* buffer, const int iicoup ) <===] + static __host__ __device__ inline const fptype* + iicoupAccessBufferConst( const fptype* buffer, // input "super-buffer": in practice, the cIPC global array + const int iicoup ) + { + constexpr int ix2 = 0; + // NB! this effectively adds an offset "iicoup * nx2" + return &( buffer[iicoup * nx2 + ix2] ); // STRUCT[idcoup][ix2] + } + + private: + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessCouplingsFixed + { + public: + + // Expose selected functions from MemoryAccessCouplingsFixedBase + static constexpr auto iicoupAccessBufferConst = MemoryAccessCouplingsFixedBase::iicoupAccessBufferConst; + + // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. + // For fixed (independent) couplings the value is a single scalar complex (real,imag): nx2 fptype's, broadcast across the SIMD vector. + static constexpr int flv_stride = mgOnGpu::nx2; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] + static __host__ __device__ inline const cxtype_sv + kernelAccessConst( const fptype* buffer ) + { + // TRIVIAL ACCESS to fixed-couplings buffers! + //return cxmake( fptype_sv{ buffer[0] }, fptype_sv{ buffer[1] } ); // NO! BUG #339! + const fptype_sv r_sv = fptype_sv{ 0 } + buffer[0]; + const fptype_sv i_sv = fptype_sv{ 0 } + buffer[1]; + return cxmake( r_sv, i_sv ); // ugly but effective + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessCouplingsFixed HostAccessCouplingsFixed; + typedef KernelAccessCouplingsFixed DeviceAccessCouplingsFixed; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h new file mode 100644 index 0000000000..205952e514 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h @@ -0,0 +1,31 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessDenominators_H +#define MemoryAccessDenominators_H 1 + +#include "MemoryAccessGs.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for denominators + // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs + + typedef KernelAccessGs HostAccessDenominators; + typedef KernelAccessGs DeviceAccessDenominators; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h new file mode 100644 index 0000000000..bfcb3e3222 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h @@ -0,0 +1,171 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessGs_H +#define MemoryAccessGs_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for Gs + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessGsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessGs : public MemoryAccessGsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessGs + { + public: + + // Expose selected functions from MemoryAccessGs + static constexpr auto ieventAccessRecord = MemoryAccessGs::ieventAccessRecord; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccess( fptype* buffer ) <===] + static constexpr auto kernelAccess_s = + KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non-const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccess( fptype* buffer ) + { + fptype& out = kernelAccess_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (SCALAR OR VECTOR) ===> fptype_sv* kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv* + kernelAccessP( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + static constexpr auto kernelAccessConst_s = + KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccess( const fptype* buffer ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessConst( const fptype* buffer ) + { + const fptype& out = kernelAccessConst_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessGs HostAccessGs; + typedef KernelAccessGs DeviceAccessGs; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h new file mode 100644 index 0000000000..8b5bd27f4d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h @@ -0,0 +1,158 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessHelpers_H +#define MemoryAccessHelpers_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +//---------------------------------------------------------------------------- + +// A templated helper class that includes the boilerplate code for MemoryAccess classes +template +class MemoryAccessHelper +{ +public: + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = T::ieventAccessRecord; + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline const FT* + ieventAccessRecordConst( const FT* buffer, + const int ievt ) + { + return ieventAccessRecord( const_cast( buffer ), ievt ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + static constexpr auto decodeRecord = T::decodeRecord; + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + decodeRecordConst( const FT* buffer, + Ts... args ) // variadic template + { + return T::decodeRecord( const_cast( buffer ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessField( fptype* buffer, const ievt, Ts... args ) <===] + template + static __host__ __device__ inline FT& + ieventAccessField( FT* buffer, + const int ievt, + Ts... args ) // variadic template + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + return T::decodeRecord( T::ieventAccessRecord( buffer, ievt ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessFieldConst( const fptype* buffer, const ievt, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + ieventAccessFieldConst( const FT* buffer, + const int ievt, + Ts... args ) // variadic template + { + return ieventAccessField( const_cast( buffer ), ievt, args... ); + } +}; + +//---------------------------------------------------------------------------- + +// A templated helper class that includes the boilerplate code for KernelAccess classes +template +class KernelAccessHelper : public MemoryAccessHelper +{ +public: + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non-const) ===> fptype* kernelAccessRecord( fptype* buffer ) <===] + static __host__ __device__ inline FT* + kernelAccessRecord( FT* buffer ) + { + if constexpr( !onDevice ) // requires c++17 also in CUDA (#333) + { + // FIXME #436: clarify that buffer includes all events on device, and only the record for an event subset on host! + // FIXME #436: am I not assuming that the following line is always identical to buffer for all access classes T? + return T::ieventAccessRecord( buffer, 0 ); + } + else + { +#ifdef MGONGPUCPP_GPUIMPL + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid + //printf( "kernelAccessRecord: ievt=%d threadId=%d\n", ievt, threadIdx.x ); + return T::ieventAccessRecord( buffer, ievt ); // NB fptype and fptype_sv coincide for CUDA +#else + throw std::runtime_error( "kernelAccessRecord on device is only implemented in CUDA" ); +#endif + } + } + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const) ===> const fptype* kernelAccessRecordConst( const fptype* buffer ) <===] + static __host__ __device__ inline const FT* + kernelAccessRecordConst( const FT* buffer ) + { + return kernelAccessRecord( const_cast( buffer ) ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessField( fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline FT& + kernelAccessField( FT* buffer, + Ts... args ) // variadic template + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + return T::decodeRecord( kernelAccessRecord( buffer ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessFieldConst( const fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + kernelAccessFieldConst( const FT* buffer, + Ts... args ) // variadic template + { + return kernelAccessField( const_cast( buffer ), args... ); + } + + //-------------------------------------------------------------------------- +}; + +#endif // MemoryAccessHelpers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h new file mode 100644 index 0000000000..ffc74b0f0c --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h @@ -0,0 +1,125 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: D. Massaro (Jan 2026, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessIflavorVec_H +#define MemoryAccessIflavorVec_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for channel ids + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessIflavorVecBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] + static __host__ __device__ inline unsigned int* + ieventAccessRecord( unsigned int* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline unsigned int& + decodeRecord( unsigned int* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessIflavorVec : public MemoryAccessIflavorVecBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] + static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessIflavorVec + { + public: + + // Expose selected functions from MemoryAccessIflavorVec + static constexpr auto ieventAccessRecordConst = MemoryAccessIflavorVec::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] + static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] + static __host__ __device__ inline const uint_sv& + kernelAccessConst( const unsigned int* buffer ) + { + const unsigned int& out = kernelAccessConst_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessIflavorVec HostAccessIflavorVec; + typedef KernelAccessIflavorVec DeviceAccessIflavorVec; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h new file mode 100644 index 0000000000..4236e20602 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h @@ -0,0 +1,147 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessMatrixElements_H +#define MemoryAccessMatrixElements_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for matrix elements + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessMatrixElementsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessMatrixElements : public MemoryAccessMatrixElementsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessMatrixElements + { + public: + + // Expose selected functions from MemoryAccessMatrixElements + static constexpr auto ieventAccessRecord = MemoryAccessMatrixElements::ieventAccessRecord; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccess_s( fptype* buffer ) <===] + static constexpr auto kernelAccess_s = + KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( const fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccess( fptype* buffer ) + { + fptype& out = kernelAccess_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + static constexpr auto kernelAccessConst = + KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessMatrixElements HostAccessMatrixElements; + typedef KernelAccessMatrixElements DeviceAccessMatrixElements; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h new file mode 100644 index 0000000000..3ef4d76fbd --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h @@ -0,0 +1,276 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessMomenta_H +#define MemoryAccessMomenta_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for momenta + // This implementation uses an AOSOA[npagM][npar][np4][neppM] where nevt=npagM*neppM + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessMomentaBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the momenta AOSOA memory buffer layout + // (these are all best kept as a compile-time constants: see issue #23) +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ + // ----------------------------------------------------------------------------------------------- + // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline + // --- This is relevant to ensure coalesced access to momenta in global memory + // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms + // ----------------------------------------------------------------------------------------------- + //static constexpr int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) + static constexpr int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 1.03E9 instead of 1.11E9 in eemumu) +#else + // ----------------------------------------------------------------------------------------------- + // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register + // --- This is relevant to ensure faster access to momenta from C++ memory cache lines + // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded + // --- In practice, neppR, neppM and neppV could now (in principle) all be different + // ----------------------------------------------------------------------------------------------- +#ifdef MGONGPU_CPPSIMD + static constexpr int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance + //static constexpr int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) + //static constexpr int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 4.66E6 instead of 5.09E9 in eemumu) + //static constexpr int neppM = MGONGPU_CPPSIMD*2; // FOR TESTS +#else + static constexpr int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) +#endif +#endif /* clang-format on */ + + // SANITY CHECK: check that neppM is a power of two + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a 4-momentum + static constexpr int np4 = CPPProcess::np4; + + // The number of particles in this physics process + static constexpr int npar = CPPProcess::npar; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagM = ievt / neppM; // #event "M-page" + const int ieppM = ievt % neppM; // #event in the current event M-page + constexpr int ip4 = 0; + constexpr int ipar = 0; + return &( buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM] ); // AOSOA[ipagM][ipar][ip4][ieppM] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ip4, const int ipar" and rename "Field" as "Ip4Ipar"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ip4, + const int ipar ) + { + constexpr int ipagM = 0; + constexpr int ieppM = 0; + return buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessMomenta : public MemoryAccessMomentaBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto decodeRecordIp4Ipar = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto decodeRecordIp4IparConst = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIp4Ipar( fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + static constexpr auto ieventAccessIp4Ipar = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + // DEFAULT VERSION + static constexpr auto ieventAccessIp4IparConst = + MemoryAccessHelper::template ieventAccessFieldConst; + + /* + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + // DEBUG VERSION WITH PRINTOUTS + static __host__ __device__ inline const fptype& + ieventAccessIp4IparConst( const fptype* buffer, + const int ievt, + const int ip4, + const int ipar ) + { + const fptype& out = MemoryAccessHelper::template ieventAccessFieldConst( buffer, ievt, ip4, ipar ); + printf( "ipar=%2d ip4=%2d ievt=%8d out=%8.3f\n", ipar, ip4, ievt, out ); + return out; + } + */ + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessMomenta + { + public: + + // Expose selected functions from MemoryAccessMomenta + static constexpr auto ieventAccessRecordConst = MemoryAccessMomenta::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIp4Ipar( fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto kernelAccessIp4Ipar = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // DEFAULT VERSION + static constexpr auto kernelAccessIp4IparConst_s = + KernelAccessHelper::template kernelAccessFieldConst; + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // DEBUG VERSION WITH PRINTOUTS + static __host__ __device__ inline const fptype& + kernelAccessIp4IparConst_s( const fptype* buffer, + const int ip4, + const int ipar ) + { + const fptype& out = KernelAccessHelper::template kernelAccessFieldConst( buffer, ip4, ipar ); + printf( "ipar=%2d ip4=%2d ievt='kernel' out=%8.3f\n", ipar, ip4, out ); + return out; + } + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> fptype_sv kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // FIXME? Eventually return by const reference and support aligned arrays only? + // FIXME? Currently return by value to support also unaligned and arbitrary arrays + static __host__ __device__ inline fptype_sv + kernelAccessIp4IparConst( const fptype* buffer, + const int ip4, + const int ipar ) + { + const fptype& out = kernelAccessIp4IparConst_s( buffer, ip4, ipar ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + constexpr int neppM = MemoryAccessMomentaBase::neppM; + constexpr bool useContiguousEventsIfPossible = true; // DEFAULT + //constexpr bool useContiguousEventsIfPossible = false; // FOR PERFORMANCE TESTS (treat as arbitrary array even if it is an AOSOA) + // Use c++17 "if constexpr": compile-time branching + if constexpr( useContiguousEventsIfPossible && ( neppM >= neppV ) && ( neppM % neppV == 0 ) ) + { + //constexpr bool skipAlignmentCheck = true; // FASTEST (SEGFAULTS IF MISALIGNED ACCESS, NEEDS A SANITY CHECK ELSEWHERE!) + constexpr bool skipAlignmentCheck = false; // DEFAULT: A BIT SLOWER BUT SAFER [ALLOWS MISALIGNED ACCESS] + if constexpr( skipAlignmentCheck ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! assume aligned AOSOA, skip check" << std::endl; first=false; } // SLOWER (5.06E6) + // FASTEST? (5.09E6 in eemumu 512y) + // This assumes alignment for momenta1d without checking - causes segmentation fault in reinterpret_cast if not aligned! + return mg5amcCpu::fptypevFromAlignedArray( out ); // use reinterpret_cast + } + else if( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! aligned AOSOA, reinterpret cast" << std::endl; first=false; } // SLOWER (5.00E6) + // DEFAULT! A tiny bit (<1%) slower because of the alignment check (5.07E6 in eemumu 512y) + // This explicitly checks buffer alignment to avoid segmentation faults in reinterpret_cast + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWER (4.93E6) + // A bit (1%) slower (5.05E6 in eemumu 512y) + // This does not require buffer alignment, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + return mg5amcCpu::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) + } + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! arbitrary array" << std::endl; first=false; } // SLOWER (5.08E6) + // ?!Used to be much slower, now a tiny bit faster for AOSOA?! (5.11E6 for AOSOA, 4.64E6 for AOS in eemumu 512y) + // This does not even require AOSOA with neppM>=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) + constexpr int ievt0 = 0; // just make it explicit in the code that buffer refers to a given ievt0 and decoderIeppV fetches event ievt0+ieppV + auto decoderIeppv = [buffer, ip4, ipar]( int ieppV ) + -> const fptype& + { return MemoryAccessMomenta::ieventAccessIp4IparConst( buffer, ievt0 + ieppV, ip4, ipar ); }; + return mg5amcCpu::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) + } +#endif + } + + // Is this a HostAccess or DeviceAccess class? + // [this is only needed for a warning printout in rambo.h for nparf==1 #358] + static __host__ __device__ inline constexpr bool + isOnDevice() + { + return onDevice; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessMomenta HostAccessMomenta; + typedef KernelAccessMomenta DeviceAccessMomenta; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h new file mode 100644 index 0000000000..c8d0343b7e --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h @@ -0,0 +1,31 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessNumerators_H +#define MemoryAccessNumerators_H 1 + +#include "MemoryAccessGs.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for numerators + // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs + + typedef KernelAccessGs HostAccessNumerators; + typedef KernelAccessGs DeviceAccessNumerators; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h new file mode 100644 index 0000000000..57f1d02081 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h @@ -0,0 +1,145 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessRandomNumbers_H +#define MemoryAccessRandomNumbers_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "MemoryAccessHelpers.h" + +#ifdef MGONGPUCPP_GPUIMPL +using mg5amcGpu::CPPProcess; +#else +using mg5amcCpu::CPPProcess; +#endif + +//---------------------------------------------------------------------------- + +// A class describing the internal layout of memory buffers for random numbers +// This implementation uses an AOSOA[npagR][nparf][np4][neppR] where nevt=npagR*neppR +// [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] +class MemoryAccessRandomNumbersBase //_AOSOAv1 +{ +public: /* clang-format off */ + + // Number of Events Per Page in the random number AOSOA memory buffer layout + // *** NB Different values of neppR lead to different physics results: the *** + // *** same 1d array is generated, but it is interpreted in different ways *** + static constexpr int neppR = 8; // HARDCODED TO GIVE ALWAYS THE SAME PHYSICS RESULTS! + //static constexpr int neppR = 1; // AOS (tests of sectors/requests) + +private: /* clang-format on */ + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a 4-momentum + static constexpr int np4 = CPPProcess::np4; + + // The number of final state particles in this physics process + static constexpr int nparf = CPPProcess::nparf; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagR = ievt / neppR; // #event "R-page" + const int ieppR = ievt % neppR; // #event in the current event R-page + constexpr int ip4 = 0; + constexpr int iparf = 0; + return &( buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR] ); // AOSOA[ipagR][iparf][ip4][ieppR] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ip4, const int iparf" and rename "Field" as "Ip4Iparf"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ip4, + const int iparf ) + { + constexpr int ipagR = 0; + constexpr int ieppR = 0; + return buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR]; // AOSOA[ipagR][iparf][ip4][ieppR] + } +}; + +//---------------------------------------------------------------------------- + +// A class providing access to memory buffers for a given event, based on explicit event numbers +// Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations +class MemoryAccessRandomNumbers : public MemoryAccessRandomNumbersBase +{ +public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto decodeRecordIp4Iparf = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto decodeRecordIp4IparfConst = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIp4Iparf( fptype* buffer, const ievt, const int ipar, const int iparf ) <===] + static constexpr auto ieventAccessIp4Iparf = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparfConst( const fptype* buffer, const ievt, const int ipar, const int iparf ) <===] + static constexpr auto ieventAccessIp4IparfConst = + MemoryAccessHelper::template ieventAccessFieldConst; +}; + +//---------------------------------------------------------------------------- + +// A class providing access to memory buffers for a given event, based on implicit kernel rules +// Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations +template +class KernelAccessRandomNumbers +{ +public: + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIp4Iparf( fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto kernelAccessIp4Iparf = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIp4IparfConst( const fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto kernelAccessIp4IparfConst = + KernelAccessHelper::template kernelAccessFieldConst; +}; + +//---------------------------------------------------------------------------- + +typedef KernelAccessRandomNumbers HostAccessRandomNumbers; +typedef KernelAccessRandomNumbers DeviceAccessRandomNumbers; + +//---------------------------------------------------------------------------- + +#endif // MemoryAccessRandomNumbers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h new file mode 100644 index 0000000000..8563d8f2f7 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h @@ -0,0 +1,138 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessVectors_H +#define MemoryAccessVectors_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#ifndef MGONGPUCPP_GPUIMPL +namespace mg5amcCpu // this is only needed for CPU SIMD vectorization +{ + +#ifdef MGONGPU_CPPSIMD + //-------------------------------------------------------------------------- + + // Cast one non-const fptype_v reference (one vector of neppV fptype values) from one non-const fptype reference (#435), + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned + inline fptype_v& fptypevFromAlignedArray( fptype& ref ) + { + return *reinterpret_cast( &ref ); + } + + inline uint_v& uintvFromAlignedArray( unsigned int& ref ) + { + return *reinterpret_cast( &ref ); + } + + // Cast one const fptype_v reference (one vector of neppV fptype values) from one const fptype reference, + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned + inline const fptype_v& fptypevFromAlignedArray( const fptype& ref ) + { + return *reinterpret_cast( &ref ); + } + + inline const uint_v& uintvFromAlignedArray( const unsigned int& ref ) + { + return *reinterpret_cast( &ref ); + } + + // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", but that the arrays are not aligned + inline fptype_v fptypevFromUnalignedArray( const fptype& ref ) + { +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (2) + *( &ref + 1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (4) + *( &ref + 1 ), + *( &ref + 2 ), + *( &ref + 3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (8) + *( &ref + 1 ), + *( &ref + 2 ), + *( &ref + 3 ), + *( &ref + 4 ), + *( &ref + 5 ), + *( &ref + 6 ), + *( &ref + 7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (16) + *( &ref + 1 ), + *( &ref + 2 ), + *( &ref + 3 ), + *( &ref + 4 ), + *( &ref + 5 ), + *( &ref + 6 ), + *( &ref + 7 ), + *( &ref + 8 ), + *( &ref + 9 ), + *( &ref + 10 ), + *( &ref + 11 ), + *( &ref + 12 ), + *( &ref + 13 ), + *( &ref + 14 ), + *( &ref + 15 ) }; +#else +#error Internal error! Unknown MGONGPU_CPPSIMD value +#endif + } + + // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, + // with no a priori assumption on how the input fptype array should be decoded + template + inline fptype_v fptypevFromArbitraryArray( Functor decoderIeppv ) + { +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (2) + decoderIeppv( 1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (4) + decoderIeppv( 1 ), + decoderIeppv( 2 ), + decoderIeppv( 3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (8) + decoderIeppv( 1 ), + decoderIeppv( 2 ), + decoderIeppv( 3 ), + decoderIeppv( 4 ), + decoderIeppv( 5 ), + decoderIeppv( 6 ), + decoderIeppv( 7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (16) + decoderIeppv( 1 ), + decoderIeppv( 2 ), + decoderIeppv( 3 ), + decoderIeppv( 4 ), + decoderIeppv( 5 ), + decoderIeppv( 6 ), + decoderIeppv( 7 ), + decoderIeppv( 8 ), + decoderIeppv( 9 ), + decoderIeppv( 10 ), + decoderIeppv( 11 ), + decoderIeppv( 12 ), + decoderIeppv( 13 ), + decoderIeppv( 14 ), + decoderIeppv( 15 ) }; +#else +#error Internal error! Unknown MGONGPU_CPPSIMD value +#endif + } + + //-------------------------------------------------------------------------- +#endif + +} // end namespace +#endif + +#endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h new file mode 100644 index 0000000000..14a37c7fea --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h @@ -0,0 +1,170 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessWavefunctions_H +#define MemoryAccessWavefunctions_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" + +#define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + +#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + // A class describing the internal layout of memory buffers for wavefunctions + // This implementation uses an AOSOA[npagW][nw6][nx2][neppW] where nevt=npagW*neppW + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessWavefunctionsBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the wavefunction AOSOA memory buffer layout + static constexpr int neppW = 1; // AOS (just a test...) + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a (fermion or vector) wavefunction + static constexpr int nw6 = mgOnGpu::nw6; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagW = ievt / neppW; // #event "W-page" + const int ieppW = ievt % neppW; // #event in the current event W-page + constexpr int iw6 = 0; + constexpr int ix2 = 0; + return &( buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW] ); // AOSOA[ipagW][iw6][ix2][ieppW] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int iw6, const int ix2" and rename "Field" as "Iw6Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int iw6, + const int ix2 ) + { + constexpr int ipagW = 0; + constexpr int ieppW = 0; + return buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW]; // AOSOA[ipagW][iw6][ix2][ieppW] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessWavefunctions : public MemoryAccessWavefunctionsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto decodeRecordIw6Ix2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto decodeRecordIw6Ix2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIw6Ix2( fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] + static constexpr auto ieventAccessIw6Ix2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIw6Ix2Const( const fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] + static constexpr auto ieventAccessIw6Ix2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + +#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessWavefunctions + { + public: + +#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIw6Ix2( fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto kernelAccessIw6Ix2 = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIw6Ix2Const( const fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto kernelAccessIw6Ix2Const = + KernelAccessHelper::template kernelAccessFieldConst; + +#else + + static __host__ __device__ inline cxtype_sv* + kernelAccess( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + static __host__ __device__ inline const cxtype_sv* + kernelAccessConst( const fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + +#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessWavefunctions HostAccessWavefunctions; + typedef KernelAccessWavefunctions DeviceAccessWavefunctions; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h new file mode 100644 index 0000000000..c56527f581 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h @@ -0,0 +1,150 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessWeights_H +#define MemoryAccessWeights_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for weights + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessWeightsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessWeights : public MemoryAccessWeightsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessWeights + { + public: + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] + // FINAL IMPLEMENTATION FOR CUDA 11.4 + static constexpr auto kernelAccess = + KernelAccessHelper::template kernelAccessField<>; + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] + // TEMPORARY HACK FOR CUDA 11.1 + static __host__ __device__ inline fptype& + kernelAccess( fptype* buffer ) + { + return KernelAccessHelper::template kernelAccessField<>( buffer ); + } + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + // FINAL IMPLEMENTATION FOR CUDA 11.4 + static constexpr auto kernelAccessConst = + KernelAccessHelper::template kernelAccessFieldConst<>; + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + // TEMPORARY HACK FOR CUDA 11.1 + static __host__ __device__ inline const fptype& + kernelAccessConst( const fptype* buffer ) + { + return KernelAccessHelper::template kernelAccessFieldConst<>( buffer ); + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessWeights HostAccessWeights; + typedef KernelAccessWeights DeviceAccessWeights; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h new file mode 100644 index 0000000000..d259aa7456 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h @@ -0,0 +1,603 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021, based on earlier work by S. Hageboeck) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryBuffers_H +#define MemoryBuffers_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "CPPProcess.h" +#include "GpuRuntime.h" +#include "Parameters.h" +#include "processConfig.h" + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + namespace MemoryBuffers + { + // Process-independent compile-time constants + static constexpr size_t np4 = CPPProcess::np4; + static constexpr size_t nw6 = CPPProcess::nw6; + static constexpr size_t nx2 = mgOnGpu::nx2; + // Process-dependent compile-time constants + static constexpr size_t nparf = CPPProcess::nparf; + static constexpr size_t npar = CPPProcess::npar; + static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; + static constexpr size_t ncolor = CPPProcess::ncolor; + } + + //-------------------------------------------------------------------------- + + // An abstract interface encapsulating a given number of events + class INumberOfEvents + { + public: + virtual ~INumberOfEvents() {} + virtual size_t nevt() const = 0; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating a given number of events + class NumberOfEvents : virtual public INumberOfEvents + { + public: + NumberOfEvents( const size_t nevt ) + : m_nevt( nevt ) {} + virtual ~NumberOfEvents() {} + virtual size_t nevt() const override { return m_nevt; } + private: + const size_t m_nevt; + }; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer (not necessarily an event buffer) + template + class BufferBase : virtual public INumberOfEvents + { + protected: + BufferBase( const size_t size, const bool onDevice ) + : m_size( size ), m_data( nullptr ), m_isOnDevice( onDevice ) {} + public: + virtual ~BufferBase() {} + T* data() { return m_data; } + const T* data() const { return m_data; } + T& operator[]( const size_t index ) { return m_data[index]; } + const T& operator[]( const size_t index ) const { return m_data[index]; } + size_t size() const { return m_size; } + size_t bytes() const { return m_size * sizeof( T ); } + bool isOnDevice() const { return m_isOnDevice; } + virtual size_t nevt() const override { throw std::runtime_error( "This BufferBase is not an event buffer" ); } + protected: + const size_t m_size; + T* m_data; + const bool m_isOnDevice; + }; + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + constexpr bool HostBufferALIGNED = false; // ismisaligned=false + constexpr bool HostBufferMISALIGNED = true; // ismisaligned=true + + // A class encapsulating a C++ host buffer + template + class HostBufferBase : public BufferBase + { + public: + HostBufferBase( const size_t size ) + : BufferBase( size, false ) + { + if constexpr( !ismisaligned ) + this->m_data = new( std::align_val_t( cppAlign ) ) T[size](); + else + this->m_data = new( std::align_val_t( cppAlign ) ) T[size + 1]() + 1; // TEST MISALIGNMENT! + } + virtual ~HostBufferBase() + { + if constexpr( !ismisaligned ) + ::operator delete[]( this->m_data, std::align_val_t( cppAlign ) ); + else + ::operator delete[]( ( this->m_data ) - 1, std::align_val_t( cppAlign ) ); // TEST MISALIGNMENT! + } + static constexpr bool isaligned() { return !ismisaligned; } + public: + static constexpr size_t cppAlign = mgOnGpu::cppAlign; + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA pinned host buffer + template + class PinnedHostBufferBase : public BufferBase + { + public: + PinnedHostBufferBase( const size_t size ) + : BufferBase( size, false ) + { + gpuMallocHost( &( this->m_data ), this->bytes() ); + } + virtual ~PinnedHostBufferBase() + { + gpuFreeHost( this->m_data ); + } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA device buffer + template + class DeviceBufferBase : public BufferBase + { + public: + DeviceBufferBase( const size_t size ) + : BufferBase( size, true ) + { + gpuMalloc( &( this->m_data ), this->bytes() ); + } + virtual ~DeviceBufferBase() + { + gpuFree( this->m_data ); + } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for a given number of events + template + class HostBuffer : public HostBufferBase, virtual private NumberOfEvents + { + public: + HostBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , HostBufferBase( sizePerEvent * nevt ) + { + //std::cout << "HostBuffer::ctor " << this << " " << nevt << std::endl; + } + virtual ~HostBuffer() + { + //std::cout << "HostBuffer::dtor " << this << std::endl; + } + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA pinned host buffer for a given number of events + template + class PinnedHostBuffer : public PinnedHostBufferBase, virtual private NumberOfEvents + { + public: + PinnedHostBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , PinnedHostBufferBase( sizePerEvent * nevt ) {} + virtual ~PinnedHostBuffer() {} + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA device buffer for a given number of events + template + class DeviceBuffer : public DeviceBufferBase, virtual protected NumberOfEvents + { + public: + DeviceBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , DeviceBufferBase( sizePerEvent * nevt ) + { + //std::cout << "DeviceBuffer::ctor " << this << " " << nevt << std::endl; + } + virtual ~DeviceBuffer() + { + //std::cout << "DeviceBuffer::dtor " << this << std::endl; + } + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a simple CUDA device buffer managed on an ad-hoc basis + typedef DeviceBuffer DeviceBufferSimple; + typedef DeviceBuffer DeviceBufferSimple2; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for momenta random numbers + typedef BufferBase BufferRndNumMomenta; + + // The size (number of elements) per event in a memory buffer for momenta random numbers + constexpr size_t sizePerEventRndNumMomenta = MemoryBuffers::np4 * MemoryBuffers::nparf; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for momenta random numbers + typedef HostBuffer HostBufferRndNumMomenta; +#else + // A class encapsulating a CUDA pinned host buffer for momenta random numbers + typedef PinnedHostBuffer PinnedHostBufferRndNumMomenta; + // A class encapsulating a CUDA device buffer for momenta random numbers + typedef DeviceBuffer DeviceBufferRndNumMomenta; +#endif + + //-------------------------------------------------------------------------- + + /* + // A base class encapsulating a memory buffer with ONE fptype per event + typedef BufferBase BufferOneFp; + + // The size (number of elements) per event in a memory buffer with ONE fptype per event + constexpr size_t sizePerEventOneFp = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer with ONE fptype per event + typedef HostBuffer HostBufferOneFp; +#else + // A class encapsulating a CUDA pinned host buffer for gs + typedef PinnedHostBuffer PinnedHostBufferOneFp; + // A class encapsulating a CUDA device buffer for gs + typedef DeviceBuffer DeviceBufferOneFp; +#endif + + // Memory buffers for Gs (related to the event-by-event strength of running coupling constant alphas QCD) + typedef BufferOneFp BufferGs; + typedef HostBufferOneFp HostBufferGs; + typedef PinnedHostBufferOneFp PinnedHostBufferGs; + typedef DeviceBufferOneFp DeviceBufferGs; + */ + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for Gs (related to the event-by-event strength of running coupling constant alphas QCD) + typedef BufferBase BufferGs; + + // The size (number of elements) per event in a memory buffer for Gs + constexpr size_t sizePerEventGs = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for gs + typedef HostBuffer HostBufferGs; +#else + // A class encapsulating a CUDA pinned host buffer for gs + typedef PinnedHostBuffer PinnedHostBufferGs; + // A class encapsulating a CUDA device buffer for gs + typedef DeviceBuffer DeviceBufferGs; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for numerators (of the multichannel single-diagram enhancement factors) + typedef BufferBase BufferNumerators; + + // The size (number of elements) per event in a memory buffer for numerators + // (should be equal to the number of diagrams in the process) + constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for numerators + typedef HostBuffer HostBufferNumerators; +#else + // A class encapsulating a CUDA pinned host buffer for numerators + typedef PinnedHostBuffer PinnedHostBufferNumerators; + // A class encapsulating a CUDA device buffer for numerators + typedef DeviceBuffer DeviceBufferNumerators; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for denominators (of the multichannel single-diagram enhancement factors) + typedef BufferBase BufferDenominators; + + // The size (number of elements) per event in a memory buffer for denominators + constexpr size_t sizePerEventDenominators = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for denominators + typedef HostBuffer HostBufferDenominators; +#else + // A class encapsulating a CUDA pinned host buffer for denominators + typedef PinnedHostBuffer PinnedHostBufferDenominators; + // A class encapsulating a CUDA device buffer for denominators + typedef DeviceBuffer DeviceBufferDenominators; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for couplings that depend on the event-by-event running coupling constant alphas QCD + typedef BufferBase BufferCouplings; + + // The size (number of elements) per event in a memory buffer for random numbers + constexpr size_t sizePerEventCouplings = MemoryBuffers::ndcoup * MemoryBuffers::nx2; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for couplings + typedef HostBuffer HostBufferCouplings; +#else + // A class encapsulating a CUDA pinned host buffer for couplings + typedef PinnedHostBuffer PinnedHostBufferCouplings; + // A class encapsulating a CUDA device buffer for couplings + typedef DeviceBuffer DeviceBufferCouplings; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for momenta + typedef BufferBase BufferMomenta; + + // The size (number of elements) per event in a memory buffer for momenta + constexpr size_t sizePerEventMomenta = MemoryBuffers::np4 * MemoryBuffers::npar; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for momenta + typedef HostBuffer HostBufferMomenta; + //typedef HostBuffer HostBufferMomenta; // TEST MISALIGNMENT! +#else + // A class encapsulating a CUDA pinned host buffer for momenta + typedef PinnedHostBuffer PinnedHostBufferMomenta; + // A class encapsulating a CUDA device buffer for momenta + typedef DeviceBuffer DeviceBufferMomenta; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for sampling weights + typedef BufferBase BufferWeights; + + // The size (number of elements) per event in a memory buffer for sampling weights + constexpr size_t sizePerEventWeights = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for sampling weights + typedef HostBuffer HostBufferWeights; +#else + // A class encapsulating a CUDA pinned host buffer for sampling weights + typedef PinnedHostBuffer PinnedHostBufferWeights; + // A class encapsulating a CUDA device buffer for sampling weights + typedef DeviceBuffer DeviceBufferWeights; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for matrix elements + typedef BufferBase BufferMatrixElements; + + // The size (number of elements) per event in a memory buffer for matrix elements + constexpr size_t sizePerEventMatrixElements = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for matrix elements + typedef HostBuffer HostBufferMatrixElements; +#else + // A class encapsulating a CUDA pinned host buffer for matrix elements + typedef PinnedHostBuffer PinnedHostBufferMatrixElements; + // A class encapsulating a CUDA device buffer for matrix elements + typedef DeviceBuffer DeviceBufferMatrixElements; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for the helicity mask + typedef BufferBase BufferHelicityMask; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for the helicity mask + typedef HostBufferBase HostBufferHelicityMask; +#else + // A class encapsulating a CUDA pinned host buffer for the helicity mask + typedef PinnedHostBufferBase PinnedHostBufferHelicityMask; + // A class encapsulating a CUDA device buffer for the helicity mask + typedef DeviceBufferBase DeviceBufferHelicityMask; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for wavefunctions + typedef BufferBase BufferWavefunctions; + + // The size (number of elements) per event in a memory buffer for wavefunctions + constexpr size_t sizePerEventWavefunctions = MemoryBuffers::nw6 * MemoryBuffers::nx2; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for wavefunctions + typedef HostBuffer HostBufferWavefunctions; +#else + // A class encapsulating a CUDA pinned host buffer for wavefunctions + typedef PinnedHostBuffer PinnedHostBufferWavefunctions; + // A class encapsulating a CUDA device buffer for wavefunctions + typedef DeviceBuffer DeviceBufferWavefunctions; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for helicity random numbers + typedef BufferBase BufferRndNumHelicity; + + // The size (number of elements) per event in a memory buffer for helicity random numbers + constexpr size_t sizePerEventRndNumHelicity = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for helicity random numbers + typedef HostBuffer HostBufferRndNumHelicity; +#else + // A class encapsulating a CUDA pinned host buffer for helicity random numbers + typedef PinnedHostBuffer PinnedHostBufferRndNumHelicity; + // A class encapsulating a CUDA device buffer for helicity random numbers + typedef DeviceBuffer DeviceBufferRndNumHelicity; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for color random numbers + typedef BufferBase BufferRndNumColor; + + // The size (number of elements) per event in a memory buffer for color random numbers + constexpr size_t sizePerEventRndNumColor = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for color random numbers + typedef HostBuffer HostBufferRndNumColor; +#else + // A class encapsulating a CUDA pinned host buffer for color random numbers + typedef PinnedHostBuffer PinnedHostBufferRndNumColor; + // A class encapsulating a CUDA device buffer for color random numbers + typedef DeviceBuffer DeviceBufferRndNumColor; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for channel ids + typedef BufferBase BufferChannelIds; + + // The size (number of elements) per event in a memory buffer for channel ids + constexpr size_t sizePerEventChannelId = 1; + +#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) + // A class encapsulating a C++ host buffer for channel ids + typedef HostBuffer HostBufferChannelIds; +#else + // A class encapsulating a CUDA pinned host buffer for channel ids + typedef PinnedHostBuffer PinnedHostBufferChannelIds; + // A class encapsulating a CUDA device buffer for channel ids + typedef DeviceBuffer DeviceBufferChannelIds; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for channel ids + typedef BufferBase BufferIflavorVec; + + // The size (number of elements) per event in a memory buffer for channel ids + constexpr size_t sizePerEventIflavorVec = 1; + +#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) + // A class encapsulating a C++ host buffer for channel ids + typedef HostBuffer HostBufferIflavorVec; +#else + // A class encapsulating a CUDA pinned host buffer for channel ids + typedef PinnedHostBuffer PinnedHostBufferIflavorVec; + // A class encapsulating a CUDA device buffer for channel ids + typedef DeviceBuffer DeviceBufferIflavorVec; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for helicity selection + typedef BufferBase BufferSelectedHelicity; + + // The size (number of elements) per event in a memory buffer for helicity selection + constexpr size_t sizePerEventSelectedHelicity = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for helicity selection + typedef HostBuffer HostBufferSelectedHelicity; +#else + // A class encapsulating a CUDA pinned host buffer for helicity selection + typedef PinnedHostBuffer PinnedHostBufferSelectedHelicity; + // A class encapsulating a CUDA device buffer for helicity selection + typedef DeviceBuffer DeviceBufferSelectedHelicity; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for color selection + typedef BufferBase BufferSelectedColor; + + // The size (number of elements) per event in a memory buffer for color selection + constexpr size_t sizePerEventSelectedColor = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for color selection + typedef HostBuffer HostBufferSelectedColor; +#else + // A class encapsulating a CUDA pinned host buffer for color selection + typedef PinnedHostBuffer PinnedHostBufferSelectedColor; + // A class encapsulating a CUDA device buffer for color selection + typedef DeviceBuffer DeviceBufferSelectedColor; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // The size (number of elements) per event in a memory buffer for jamps + constexpr size_t sizePerEventJamps = MemoryBuffers::ncolor * MemoryBuffers::nx2; + + // A class encapsulating a CUDA device buffer for color selection + typedef DeviceBuffer DeviceBufferJamps; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + template + void copyDeviceFromHost( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy + { + if( dst.size() != src.size() ) + { + std::ostringstream sstr; + sstr << "Size (#elements) mismatch in copyDeviceFromHost: dst=" << dst.size() << ", src=" << src.size(); + throw std::runtime_error( sstr.str() ); + } + if( dst.bytes() != src.bytes() ) + { + std::ostringstream sstr; + sstr << "Size (#bytes) mismatch in copyDeviceFromHost: dst=" << dst.bytes() << ", src=" << src.bytes(); + throw std::runtime_error( sstr.str() ); + } + // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array + gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyHostToDevice ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + template + void copyHostFromDevice( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy + { + if( dst.size() != src.size() ) + { + std::ostringstream sstr; + sstr << "Size (#elements) mismatch in copyHostFromDevice: dst=" << dst.size() << ", src=" << src.size(); + throw std::runtime_error( sstr.str() ); + } + if( dst.bytes() != src.bytes() ) + { + std::ostringstream sstr; + sstr << "Size (#bytes) mismatch in copyHostFromDevice: dst=" << dst.bytes() << ", src=" << src.bytes(); + throw std::runtime_error( sstr.str() ); + } + // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array + gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyDeviceToHost ); + } +#endif + + //-------------------------------------------------------------------------- +} + +#endif // MemoryBuffers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc new file mode 100644 index 0000000000..d40b1b4174 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc @@ -0,0 +1,246 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "RamboSamplingKernels.h" + +#include "GpuRuntime.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessRandomNumbers.h" +#include "MemoryAccessWeights.h" +#include "MemoryBuffers.h" +#include "rambo.h" // inline classic (massive) RAMBO, ported from standalone_cpp +#include "massless_rambo.h" // inline implementation of massless RAMBO algorithms and kernels + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + MasslessRamboSamplingKernelHost::MasslessRamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t nevt ) + : SamplingKernelBase( energy, rndmom, momenta, weights ) + , NumberOfEvents( nevt ) + { + if( m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: rndmom must be a host array" ); + if( m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: momenta must be a host array" ); + if( m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: weights must be a host array" ); + if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with rndmom" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with weights" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Sanity checks for memory access (random number buffer) + constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout + static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); + if( nevt % neppR != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppR=" << neppR; + throw std::runtime_error( sstr.str() ); + } + } + + //-------------------------------------------------------------------------- + + void + MasslessRamboSamplingKernelHost::getMomentaInitial() + { + constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; + // ** START LOOP ON IEVT ** + for( size_t ievt = 0; ievt < nevt(); ++ievt ) + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); + getMomentaInitial( m_energy, ievtMomenta ); + } + // ** END LOOP ON IEVT ** + } + + //-------------------------------------------------------------------------- + + void + MasslessRamboSamplingKernelHost::getMomentaFinal() + { + constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; + // ** START LOOP ON IEVT ** + for( size_t ievt = 0; ievt < nevt(); ++ievt ) + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + const fptype* ievtRndmom = MemoryAccessRandomNumbers::ieventAccessRecordConst( m_rndmom.data(), ievt ); + fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); + fptype* ievtWeights = MemoryAccessWeights::ieventAccessRecord( m_weights.data(), ievt ); + getMomentaFinal( m_energy, ievtRndmom, ievtMomenta, ievtWeights ); + } + // ** END LOOP ON IEVT ** + } + + //-------------------------------------------------------------------------- + + RamboSamplingKernelHost::RamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED + const std::vector& masses, // input: external-leg masses + const int ninitial, // input: #initial-state particles + const size_t nevt, // input: #events + BufferMomenta& momenta, // output: momenta + BufferWeights& weights ) // output: weights + : SamplingKernelBase( energy, rndmom, momenta, weights ) + , NumberOfEvents( nevt ) + , m_masses( masses.begin(), masses.end() ) + , m_ninitial( ninitial ) + { + if( m_momenta.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: momenta must be a host array" ); + if( m_weights.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: weights must be a host array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with weights" ); + + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "RamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + } + + //-------------------------------------------------------------------------- + + void + RamboSamplingKernelHost::getMomentaInitial() + { + // NOOP + } + + //-------------------------------------------------------------------------- + + void + RamboSamplingKernelHost::getMomentaFinal() + { + const int npar = (int)m_masses.size(); + // ** START LOOP ON IEVT ** + for( size_t ievt = 0; ievt < nevt(); ++ievt ) + { + // Clas. RAMBO returns [E,px,py,pz] vector per ex. particle + // own RNG, intial final once + // For reproducibility betwn fptype = FP32/FP64 generation in FP64 + double wgt = 0.; + const std::vector> point = + rambo::get_momenta( m_ninitial, (double)m_energy, m_masses, wgt ); + for( int ipar = 0; ipar < npar; ++ipar ) + for( int ip4 = 0; ip4 < 4; ++ip4 ) + MemoryAccessMomenta::ieventAccessIp4Ipar( m_momenta.data(), ievt, ip4, ipar ) = (fptype)point[ipar][ip4]; + MemoryAccessWeights::ieventAccess( m_weights.data(), ievt ) = (fptype)wgt; + } + // ** END LOOP ON IEVT ** + } + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + MasslessRamboSamplingKernelDevice::MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t gpublocks, + const size_t gputhreads ) + : SamplingKernelBase( energy, rndmom, momenta, weights ) + , NumberOfEvents( gpublocks * gputhreads ) + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + if( !m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: rndmom must be a device array" ); + if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: momenta must be a device array" ); + if( !m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: weights must be a device array" ); + if( m_gpublocks == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gpublocks must be > 0" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gputhreads must be > 0" ); + if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with rndmom" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with momenta" ); + if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with weights" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( m_gputhreads % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelHost: gputhreads should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Sanity checks for memory access (random number buffer) + constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout + static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); + if( m_gputhreads % neppR != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelDevice: gputhreads should be a multiple of neppR=" << neppR; + throw std::runtime_error( sstr.str() ); + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + getMomentaInitialDevice( const fptype energy, + fptype* momenta ) + { + constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; + return getMomentaInitial( energy, momenta ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + MasslessRamboSamplingKernelDevice::getMomentaInitial() + { + gpuLaunchKernel( getMomentaInitialDevice, m_gpublocks, m_gputhreads, m_energy, m_momenta.data() ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + getMomentaFinalDevice( const fptype energy, + const fptype* rndmom, + fptype* momenta, + fptype* wgts ) + { + constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; + return getMomentaFinal( energy, rndmom, momenta, wgts ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + MasslessRamboSamplingKernelDevice::getMomentaFinal() + { + gpuLaunchKernel( getMomentaFinalDevice, m_gpublocks, m_gputhreads, m_energy, m_rndmom.data(), m_momenta.data(), m_weights.data() ); + } +#endif + + //-------------------------------------------------------------------------- +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h new file mode 100644 index 0000000000..6244d55618 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h @@ -0,0 +1,173 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef RAMBOSAMPLINGKERNELS_H +#define RAMBOSAMPLINGKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryBuffers.h" + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // A base class encapsulating phase space sampling on a CPU host or on a GPU device + class SamplingKernelBase //: virtual public ISamplingKernel + { + protected: + + // Constructor from existing input and output buffers + SamplingKernelBase( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights ) // output: weights + : m_energy( energy ) + , m_rndmom( rndmom ) + , m_momenta( momenta ) + , m_weights( weights ) + { + } + + public: + + // Destructor + virtual ~SamplingKernelBase() {} + + // Get momenta of initial state particles + virtual void getMomentaInitial() = 0; + + // Get momenta of final state particles and weights + virtual void getMomentaFinal() = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The energy + const fptype m_energy; + + // The buffer for the input random numbers + const BufferRndNumMomenta& m_rndmom; + + // The buffer for the output momenta + BufferMomenta& m_momenta; + + // The buffer for the output weights + BufferWeights& m_weights; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating RAMBO phase space sampling on a CPU host + class MasslessRamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MasslessRamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t nevt ); + + // Destructor + virtual ~MasslessRamboSamplingKernelHost() {} + + // Get momenta of initial state particles + void getMomentaInitial() override final; + + // Get momenta of final state particles and weights + void getMomentaFinal() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + }; + + //-------------------------------------------------------------------------- + + // Compability port mirroring the massless momenta implementation + // For now own RNG internally (or keep to match Fortran) TODO + // rndmom just for interface (maybe delete later) TODO + class RamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents + { + public: + + RamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED + const std::vector& masses, // input: external-leg masses + const int ninitial, // input: #n initial-state particles + const size_t nevt, // input: #n events + BufferMomenta& momenta, // output: momenta + BufferWeights& weights); // output: weights + + virtual ~RamboSamplingKernelHost() {} + + // No-op, kept to match the massless + void getMomentaInitial() override final; + + // All the magic here + void getMomentaFinal() override final; + + bool isOnDevice() const override final { return false; } + + private: + + // The EXTERNAL masses + std::vector m_masses; + + // The number of inital particles + const int m_ninitial; + }; + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating RAMBO phase space sampling on a GPU device + class MasslessRamboSamplingKernelDevice final : public SamplingKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t gpublocks, + const size_t gputhreads ); + + // Destructor + virtual ~MasslessRamboSamplingKernelDevice() {} + + // Get momenta of initial state particles + void getMomentaInitial() override final; + + // Get momenta of final state particles and weights + void getMomentaFinal() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + }; +#endif + + //-------------------------------------------------------------------------- +} +#endif // RAMBOSAMPLINGKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h new file mode 100644 index 0000000000..0fb880bb36 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h @@ -0,0 +1,192 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef RANDOMNUMBERKERNELS_H +#define RANDOMNUMBERKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryBuffers.h" + +// Forward definition from curand.h (the full header is only needed in CurandRandomKernel.cc) +struct curandGenerator_st; + +// Forward definition from hiprand.h (the full header is only needed in HiprandRandomKernel.cc) +struct rocrand_generator_base_type; +typedef rocrand_generator_base_type hiprandGenerator_st; + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + /* + // An interface encapsulating random number generation on a CPU host or on a GPU device + class IRandomNumberKernel + { + public: + + // Destructor + virtual ~IRandomNumberKernel(){} + + // Seed the random number generator + virtual void seedGenerator( const unsigned int seed ) = 0; + + // Generate the random number array + virtual void generateRnarray() = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + }; + */ + + //-------------------------------------------------------------------------- + + // A base class encapsulating random number generation on a CPU host or on a GPU device + class RandomNumberKernelBase //: virtual public IRandomNumberKernel + { + + protected: + + // Constructor from an existing output buffer + RandomNumberKernelBase( BufferRndNumMomenta& rnarray ) + : m_rnarray( rnarray ) {} + + public: + + // Destructor + virtual ~RandomNumberKernelBase() {} + + // Seed the random number generator + virtual void seedGenerator( const unsigned int seed ) = 0; + + // Generate the random number array + virtual void generateRnarray() = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The buffer for the output random numbers + BufferRndNumMomenta& m_rnarray; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating common random number generation on a CPU host + class CommonRandomNumberKernel final : public RandomNumberKernelBase + { + public: + + // Constructor from an existing output buffer + CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ); + + // Destructor + ~CommonRandomNumberKernel() {} + + // Seed the random number generator + void seedGenerator( const unsigned int seed ) override final { m_seed = seed; }; + + // Generate the random number array + void generateRnarray() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // The generator seed + unsigned int m_seed; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating CURAND random number generation on a CPU host or on a GPU device + class CurandRandomNumberKernel final : public RandomNumberKernelBase + { + public: + + // Constructor from an existing output buffer + CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); + + // Destructor + ~CurandRandomNumberKernel(); + + // Seed the random number generator + void seedGenerator( const unsigned int seed ) override final; + + // Generate the random number array + void generateRnarray() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return m_isOnDevice; } + + private: + + // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void createGenerator(); + + // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void destroyGenerator(); + + private: + + // Is this a host or device kernel? + const bool m_isOnDevice; + + // The curand generator + // (NB: curand.h defines typedef curandGenerator_t as a pointer to forward-defined 'struct curandGenerator_st') + curandGenerator_st* m_rnGen; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating HIPRAND random number generation on a CPU host or on a GPU device + class HiprandRandomNumberKernel final : public RandomNumberKernelBase + { + public: + + // Constructor from an existing output buffer + HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); + + // Destructor + ~HiprandRandomNumberKernel(); + + // Seed the random number generator + void seedGenerator( const unsigned int seed ) override final; + + // Generate the random number array + void generateRnarray() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return m_isOnDevice; } + + private: + + // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void createGenerator(); + + // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void destroyGenerator(); + + private: + + // Is this a host or device kernel? + const bool m_isOnDevice; + + // The hiprand generator + // (NB: hiprand.h defines typedef hiprandGenerator_t as a pointer to forward-defined 'struct hiprandGenerator_st') + hiprandGenerator_st* m_rnGen; + }; + + //-------------------------------------------------------------------------- +} +#endif // RANDOMNUMBERKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc new file mode 100644 index 0000000000..68e93edb50 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc @@ -0,0 +1,1159 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: O. Mattelaer (Nov 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, D. Massaro, O. Mattelaer, S. Roiser, J. Teig, A. Thete, A. Valassi (2020-2026). +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== +// +// Standalone script for MadGraph7 standalone mode. +// Generates phase-space points with RAMBO and evaluates the matrix element +// through the UMAMI interface (umami.h). +// +// Two run modes: +// * matrix (default): evaluates one phase-space point (generated with the +// classic standalone RAMBO, so identical to the one of +// the Fortran/C++ standalone 'check' drivers at the +// same energy) and prints it together with the matrix +// element of every flavor combination. +// * perf : runs nblocks*nthreads*niter events on a single flavor +// and prints performance counters. +// +//========================================================================== + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "GpuAbstraction.h" +#include "GpuRuntime.h" +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" +#include "RamboSamplingKernels.h" +#include "RandomNumberKernels.h" +#include "epoch_process_id.h" +#include "read_slha.h" +#include "timermap.h" +#include "umami.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STRINGIFY( s ) #s +#define XSTRINGIFY( s ) STRINGIFY( s ) +#define SEP79 79 + +namespace +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + // Fixed physics inputs + fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV + constexpr unsigned long long kSeed = 20200805ULL; // reproducible RAMBO seed + + // Matrix-mode always runs 8 events on a single flavor index. + constexpr unsigned int kMatrixBlocks = 1; + constexpr unsigned int kMatrixThreads = 8; + + // Power of GeV of the matrix-element output; depends only on the number of external legs. + constexpr int kMEGeVExponent = -( 2 * CPPProcess::npar - 8 ); + + bool is_number( const char* s ) + { + const char* t = s; + while( *t != '\0' && isdigit( *t ) ) ++t; + return (int)strlen( s ) == t - s; + } + + // Accepts plain decimal numbers such as "1000" or "1000.0" (used for the + // optional energy argument of matrix mode). + bool is_float( const char* s ) + { + int ndots = 0; + const char* t = s; + while( *t != '\0' && ( isdigit( *t ) || ( *t == '.' && ndots++ == 0 ) ) ) ++t; + return t != s && (int)strlen( s ) == t - s; + } + + enum Mode { MODE_MATRIX, MODE_PERF }; + + enum RamboType { RAMBO_MASSIVE, RAMBO_MASSLESS }; + + // One external-particle list per LHE event, each particle stored as (E, px, py, pz). + using LheEvent = std::array, CPPProcess::npar>; + + bool read_lhe_events( const std::string& path, std::vector& events ) + { + constexpr int npar = CPPProcess::npar; + std::ifstream in( path ); + if( !in ) + { + std::cerr << "ERROR! cannot open LHE file '" << path << "'" << std::endl; + return false; + } + std::string line; + while( std::getline( in, line ) ) + { + if( line.find( "" ) == std::string::npos ) continue; + if( !std::getline( in, line ) ) break; + std::istringstream hdr( line ); + int nptcl = 0; + hdr >> nptcl; + if( nptcl != npar ) + { + std::cerr << "ERROR! LHE event has " << nptcl << " particles, expected " << npar << std::endl; + return false; + } + // particle lines: pdg status mother1 mother2 color1 color2 px py pz E m lifetime spin + LheEvent ev; + int ipar = 0; + while( ipar < npar && std::getline( in, line ) ) + { + if( line.empty() ) continue; + std::istringstream ls( line ); + long pdg; + int status, m1, m2, c1, c2; + double px, py, pz, E; + if( !( ls >> pdg >> status >> m1 >> m2 >> c1 >> c2 >> px >> py >> pz >> E ) ) + { + std::cerr << "ERROR! malformed LHE particle line: " << line << std::endl; + return false; + } + ev[ipar] = { E, px, py, pz }; + ++ipar; + } + if( ipar != npar ) + { + std::cerr << "ERROR! truncated LHE event (got " << ipar << " of " << npar << " particles)" << std::endl; + return false; + } + events.push_back( ev ); + } + if( events.empty() ) + { + std::cerr << "ERROR! no events found in '" << path << "'" << std::endl; + return false; + } + return true; + } + + int usage( const char* argv0, int ret = 1 ) + { + std::cout + << "Usage:\n" + << " " << argv0 << " [matrix] [-v|--verbose] []\n" + << " " << argv0 << " perf [-v|--verbose] [-f|--flavor ] [--rambo-massless]" + << " [-e|--events ] [<#blocksPerGrid> <#threadsPerBlock>] <#iterations>\n" + << " " << argv0 << " -p [opts] (legacy alias for `perf`)\n" + << "\n" + << "Subcommands:\n" + << " matrix (default) Evaluate one phase-space point (classic standalone\n" + << " RAMBO, identical to the Fortran/C++ 'check' drivers,\n" + << " Ecms = , default 1000 GeV) and print it with\n" + << " the matrix element for each flavor combination.\n" + << " With -v also prints backend/fptype/hardcodePARAM header.\n" + << " perf Run #blocks*#threads events over #iterations iterations\n" + << " on a single flavor index, then print performance counters.\n" + << " Always prints inputs + backend/fptype header.\n" + << " With -v also dumps every event's phase-space point and ME.\n" + << "\n" + << "Options:\n" + << " -e|--events (perf only) Read the external momenta from an LHE\n" + << " file instead of generating them with RAMBO. The events are\n" + << " processed in batches of #blocks*#threads; #iterations is\n" + << " ignored (derived from the number of events in the file).\n" + << "\n" + << "perf-mode defaults if positional args are omitted:\n" + << " #blocksPerGrid = 64, #threadsPerBlock = 256, #iterations = 1.\n"; + return ret; + } + + // AOSOA -> UMAMI SoA single-event helper. Layout reminder: + // AOSOA: aosoa[i_page * npar*4*neppM + ipar*4*neppM + ip4*neppM + i_vector] + // UMAMI: soa[ip4 * npar*nevt + ipar*nevt + ievt] + __host__ __device__ inline void + aosoa_to_umami_one( const fptype* aosoa, + double* soa, + std::size_t ievt, + std::size_t nevt ) + { + constexpr int npar = CPPProcess::npar; + for( int ipar = 0; ipar < npar; ++ipar ) + { + for( int ip4 = 0; ip4 < 4; ++ip4 ) + { + soa[(std::size_t)ip4 * npar * nevt + (std::size_t)ipar * nevt + ievt] = + (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, ip4, ipar ); + } + } + } + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + aosoa_to_umami_kernel( const fptype* aosoa, + double* soa, + std::size_t nevt ) + { + std::size_t ievt = blockDim.x * blockIdx.x + threadIdx.x; + if( ievt >= nevt ) return; + aosoa_to_umami_one( aosoa, soa, ievt, nevt ); + } +#endif + + const char* backend_label() + { +#ifdef __CUDACC__ + return "CUDA"; +#elif defined( __HIPCC__ ) + return "HIP"; +#else + return "CPP"; +#endif + } + + const char* fp_label() + { +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + return "MIXED"; +#elif defined MGONGPU_FPTYPE_DOUBLE + return "DOUBLE"; +#elif defined MGONGPU_FPTYPE_FLOAT + return "FLOAT"; +#else + return "UNKNOWN"; +#endif + } + + void print_run_header( std::ostream& os ) + { + os << "Process = " << XSTRINGIFY( MG_EPOCH_PROCESS_ID ) << "_" << backend_label() +#ifdef MGONGPU_HARDCODE_PARAM + << " [hardcodePARAM=1]" << std::endl +#else + << " [hardcodePARAM=0]" << std::endl +#endif + << "FP precision = " << fp_label() << std::endl + << "Random number generation = COMMON RANDOM HOST" << std::endl; + } + + void print_momenta_table( std::ostream& os, const fptype* aosoa, unsigned int ievt ) + { + auto constexpr prec = std::numeric_limits::digits10; + constexpr int npar = CPPProcess::npar; + os << std::string( SEP79, '-' ) << std::endl + << " n E px py pz" << std::endl; + for( int ipar = 0; ipar < npar; ++ipar ) + { + double E = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 0, ipar ); + double px = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 1, ipar ); + double py = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 2, ipar ); + double pz = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 3, ipar ); + os << std::scientific << std::setprecision( prec ) + << std::setw( 4 ) << ipar + 1 + << std::setw( prec + 10 ) << E + << std::setw( prec + 10 ) << px + << std::setw( prec + 10 ) << py + << std::setw( prec + 10 ) << pz + << std::endl + << std::defaultfloat; + } + os << std::string( SEP79, '-' ) << std::endl; + } + + // Run sigmaKin via UMAMI for `nevt` events and copy back the MEs. + // Both the momenta (UMAMI SoA layout) and the per-event flavor buffer must be set + // by the caller. On GPU the buffers are device pointers and `hstMEs` receives the + // host-side copy; on CPU `umamiMEs` is the output buffer. + bool run_umami( + UmamiHandle handle, + unsigned int nevt, + mgOnGpu::TimerMap& timermap, + double& wavetime, +#ifdef MGONGPUCPP_GPUIMPL + const DeviceBufferBase& devUmamiMomenta, + const DeviceBufferBase& devFlv, + DeviceBufferBase& devUmamiMEs, + std::vector& hstMEs +#else + const std::vector& umamiMomenta, + const std::vector& flvVec, + std::vector& umamiMEs +#endif + ) + { + constexpr unsigned int UmamiInKeyNum = 2; + timermap.start( "3a SigmaKin" ); + UmamiInputKey in_keys[UmamiInKeyNum] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX }; + UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; +#ifdef MGONGPUCPP_GPUIMPL + const void* inputs[UmamiInKeyNum] = { devUmamiMomenta.data(), devFlv.data() }; + void* outputs[1] = { devUmamiMEs.data() }; +#else + const void* inputs[UmamiInKeyNum] = { umamiMomenta.data(), flvVec.data() }; + void* outputs[1] = { umamiMEs.data() }; +#endif + UmamiStatus st = umami_matrix_element( + handle, nevt, nevt, 0, UmamiInKeyNum, in_keys, inputs, 1, out_keys, outputs ); + wavetime += timermap.stop(); + if( st != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; + return false; + } + +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "3b CpDTHmes" ); + gpuMemcpy( hstMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); + wavetime += timermap.stop(); +#endif + return true; + } + + // -------------------------------------------------------------------------- + // Classic MadGraph standalone RAMBO (RANMAR generator seeded with the fixed + // values 1802/9373), reproducing the exact phase-space point used by the + // Fortran and C++ standalone 'check' drivers at the same energy, so that + // matrix-mode output can be compared line by line across backends. + // Host-side only; ported from madgraph/iolibs/template_files/rambo.cc. + // -------------------------------------------------------------------------- + namespace classic_rambo + { + struct Random + { + double ranu[98]; + double ranc, rancd, rancm; + int iranmr, jranmr; + + // universal random number generator proposed by Marsaglia and Zaman + double ranmar() + { + double uni = ranu[iranmr] - ranu[jranmr]; + if( uni < 0 ) uni = uni + 1; + ranu[iranmr] = uni; + iranmr = iranmr - 1; + jranmr = jranmr - 1; + if( iranmr == 0 ) iranmr = 97; + if( jranmr == 0 ) jranmr = 97; + ranc = ranc - rancd; + if( ranc < 0 ) ranc = ranc + rancm; + uni = uni - ranc; + if( uni < 0 ) uni = uni + 1; + return uni; + } + + void rmarin( int ij, int kl ) + { + int i = ij / 177 % 177 + 2; + int j = ij % 177 + 2; + int k = ( kl / 169 ) % 178 + 1; + int l = kl % 169; + for( int ii = 1; ii < 98; ii++ ) + { + double s = 0; + double t = .5; + for( int jj = 1; jj < 25; jj++ ) + { + int m = ( ( i * j % 179 ) * k ) % 179; + i = j; + j = k; + k = m; + l = ( 53 * l + 1 ) % 169; + if( ( l * m ) % 64 >= 32 ) s = s + t; + t = .5 * t; + } + ranu[ii] = s; + } + ranc = 362436. / 16777216.; + rancd = 7654321. / 16777216.; + rancm = 16777213. / 16777216.; + iranmr = 97; + jranmr = 33; + } + }; + + inline double rn() + { + static Random rand; + static bool init = true; + if( init ) + { + init = false; + rand.rmarin( 1802, 9373 ); + } + double ran; + while( true ) + { + ran = rand.ranmar(); + if( ran > 1e-16 ) break; + } + return ran; + } + + // RAMBO: democratic multi-particle phase space generator (S.D. Ellis, + // R. Kleiss, W.J. Stirling); weights are logarithmic. + inline std::vector> + rambo( double et, const std::vector& xm, double& wt ) + { + const int n = (int)xm.size(); + std::vector> q( n, std::vector( 4 ) ); + std::vector> p( n, std::vector( 4 ) ); + std::vector z( n ), r( 4 ), b( 3 ), p2( n ), xm2( n ), e( n ), v( n ); + const double acc = 1e-14; + const int itmax = 6; + const double twopi = 8. * atan( 1. ); + const double po2log = log( twopi / 4. ); + + // factorials for the phase-space weight + z[1] = po2log; + for( int k = 2; k < n; k++ ) z[k] = z[k - 1] + po2log - 2. * log( double( k - 1 ) ); + for( int k = 2; k < n; k++ ) z[k] = z[k] - log( double( k ) ); + + if( n < 1 || n > 101 ) + { + std::cout << "Too few or many particles: " << n << std::endl; + exit( -1 ); + } + double xmt = 0.; + int nm = 0; + for( int i = 0; i < n; i++ ) + { + if( xm[i] != 0. ) nm = nm + 1; + xmt = xmt + std::abs( xm[i] ); + } + if( xmt > et ) + { + std::cout << "Too low energy: " << et << " needed " << xmt << std::endl; + exit( -1 ); + } + + // generate n massless momenta in infinite phase space + for( int i = 0; i < n; i++ ) + { + double r1 = rn(); + double c = 2. * r1 - 1.; + double s = sqrt( 1. - c * c ); + double f = twopi * rn(); + r1 = rn(); + double r2 = rn(); + q[i][0] = -log( r1 * r2 ); + q[i][3] = q[i][0] * c; + q[i][2] = q[i][0] * s * cos( f ); + q[i][1] = q[i][0] * s * sin( f ); + } + + // parameters of the conformal transformation + for( int k = 0; k < 4; k++ ) r[k] = 0.; + for( int i = 0; i < n; i++ ) + for( int k = 0; k < 4; k++ ) r[k] = r[k] + q[i][k]; + double rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); + for( int k = 1; k < 4; k++ ) b[k - 1] = -r[k] / rmas; + double g = r[0] / rmas; + double a = 1. / ( 1. + g ); + double x = et / rmas; + + // transform the q's conformally into the p's + for( int i = 0; i < n; i++ ) + { + double bq = b[0] * q[i][1] + b[1] * q[i][2] + b[2] * q[i][3]; + for( int k = 1; k < 4; k++ ) p[i][k] = x * ( q[i][k] + b[k - 1] * ( q[i][0] + a * bq ) ); + p[i][0] = x * ( g * q[i][0] + bq ); + } + + wt = po2log; + if( n != 2 ) wt = ( 2. * n - 4. ) * log( et ) + z[n - 1]; + + // massless case is done + if( nm == 0 ) return p; + + // massive particles: rescale the momenta by a factor x + double xmax = sqrt( 1. - pow( xmt / et, 2 ) ); + for( int i = 0; i < n; i++ ) + { + xm2[i] = pow( xm[i], 2 ); + p2[i] = pow( p[i][0], 2 ); + } + int iter = 0; + x = xmax; + double accu = et * acc; + while( true ) + { + double f0 = -et; + double g0 = 0.; + double x2 = x * x; + for( int i = 0; i < n; i++ ) + { + e[i] = sqrt( xm2[i] + x2 * p2[i] ); + f0 = f0 + e[i]; + g0 = g0 + p2[i] / e[i]; + } + if( std::abs( f0 ) <= accu ) break; + iter = iter + 1; + if( iter > itmax ) + { + std::cout << "Too many iterations without desired accuracy: " << itmax << std::endl; + break; + } + x = x - f0 / ( x * g0 ); + } + for( int i = 0; i < n; i++ ) + { + v[i] = x * p[i][0]; + for( int k = 1; k < 4; k++ ) p[i][k] = x * p[i][k]; + p[i][0] = e[i]; + } + + double wt2 = 1.; + double wt3 = 0.; + for( int i = 0; i < n; i++ ) + { + wt2 = wt2 * v[i] / e[i]; + wt3 = wt3 + pow( v[i], 2 ) / e[i]; + } + double wtm = ( 2. * n - 3. ) * log( x ) + log( wt2 / wt3 * et ); + wt = wt + wtm; + return p; + } + + // Auxiliary function changing convention between MadGraph5_aMC@NLO and + // RAMBO four-momenta (same as get_momenta in the standalone_cpp driver). + inline std::vector> + get_momenta( int ninitial, double energy, const std::vector& masses, double& wgt ) + { + const int nexternal = (int)masses.size(); + const int nfinal = nexternal - ninitial; + const double e2 = pow( energy, 2 ); + const double m1 = masses[0]; + + if( ninitial == 1 ) + { + std::vector> p( 1, std::vector( 4, 0. ) ); + p[0][0] = m1; + std::vector finalmasses( masses.begin() + 1, masses.end() ); + std::vector> p_rambo = rambo( m1, finalmasses, wgt ); + p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); + return p; + } + + if( ninitial != 2 ) + { + std::cout << "Rambo needs 1 or 2 incoming particles" << std::endl; + exit( -1 ); + } + + double etot = energy; + if( nfinal == 1 ) etot = m1; + const double m2 = masses[1]; + const double mom = sqrt( ( pow( e2, 2 ) - 2 * e2 * pow( m1, 2 ) + pow( m1, 4 ) - 2 * e2 * pow( m2, 2 ) - 2 * pow( m1, 2 ) * pow( m2, 2 ) + pow( m2, 4 ) ) / ( 4 * e2 ) ); + const double energy1 = sqrt( pow( mom, 2 ) + pow( m1, 2 ) ); + const double energy2 = sqrt( pow( mom, 2 ) + pow( m2, 2 ) ); + std::vector> p( 2, std::vector( 4, 0. ) ); + p[0][0] = energy1; + p[0][3] = mom; + p[1][0] = energy2; + p[1][3] = -mom; + + if( nfinal == 1 ) + { + p.push_back( std::vector( 4, 0. ) ); + p[2][0] = etot; + wgt = 1; + return p; + } + std::vector finalmasses( masses.begin() + 2, masses.end() ); + std::vector> p_rambo = rambo( etot, finalmasses, wgt ); + p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); + return p; + } + } + + // -------------------------------------------------------------------------- + // matrix mode: same PS point fed to every flavor combination, print event 0. + // The point is generated with the classic standalone RAMBO so it is + // identical to the one of the Fortran/C++ standalone 'check' drivers. + // -------------------------------------------------------------------------- + int run_matrix_mode( bool verbose ) + { + constexpr unsigned int nevt = kMatrixBlocks * kMatrixThreads; + const unsigned int nFlavors = CPPProcess::nmaxflavor; + + mgOnGpu::TimerMap timermap; + +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "00 GpuInit" ); + GpuRuntime gpuRuntime( false ); + + PinnedHostBufferRndNumMomenta hstRndmom( nevt ); + PinnedHostBufferMomenta hstMomenta( nevt ); + PinnedHostBufferWeights hstWeights( nevt ); + DeviceBufferRndNumMomenta devRndmom( nevt ); + DeviceBufferMomenta devMomenta( nevt ); + DeviceBufferWeights devWeights( nevt ); + DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + DeviceBufferBase devUmamiMEs( nevt ); + DeviceBufferBase devFlv( nevt ); + std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + std::vector flvVec( nevt ); + std::vector hstUmamiMEs( nevt ); +#else + HostBufferRndNumMomenta hstRndmom( nevt ); + HostBufferMomenta hstMomenta( nevt ); + HostBufferWeights hstWeights( nevt ); + std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + std::vector umamiMEs( nevt ); + std::vector flvVec( nevt ); +#endif + + UmamiHandle umami_handle = nullptr; + if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_initialize failed" << std::endl; + return 2; + } + + // Generate one shared phase-space point used by every flavor, with the + // classic standalone RAMBO so it matches the Fortran/C++ 'check' drivers. + CPPProcess process; + process.initProc( "../../Cards/param_card.dat" ); + double rambowgt = 0.; + + // Retrieve masses + int npar_meta = 0; + if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + std::vector massesD( npar_meta ); + if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + const std::vector masses( massesD.begin(), massesD.end() ); + + std::vector> point = + classic_rambo::get_momenta( CPPProcess::npari, (double)kEnergy, masses, rambowgt ); + + // alpha_s from the param card so the couplings match the Fortran/C++ + // 'check' drivers (UMAMI otherwise falls back to a hardcoded g_s). + SLHAReader slha( "../../Cards/param_card.dat", false ); + const double alphaS = slha.get_block_entry( "sminputs", 3, 1.180000e-01 ); + std::vector alphasVec( nevt, alphaS ); +#ifdef MGONGPUCPP_GPUIMPL + DeviceBufferBase devAlphaS( nevt ); + gpuMemcpy( devAlphaS.data(), alphasVec.data(), nevt * sizeof( double ), gpuMemcpyHostToDevice ); +#endif + + // Always massive RAMBO + std::unique_ptr prsk( + new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); + prsk->getMomentaInitial(); + prsk->getMomentaFinal(); + + // Fill the UMAMI SoA buffer with nevt copies of the same event: + // soa[ip4 * npar*nevt + ipar*nevt + ievt] + for( int ip4 = 0; ip4 < 4; ++ip4 ) + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + for( unsigned int ievt = 0; ievt < nevt; ++ievt ) + umamiMomenta[(std::size_t)ip4 * CPPProcess::npar * nevt + (std::size_t)ipar * nevt + ievt] = point[ipar][ip4]; +#ifdef MGONGPUCPP_GPUIMPL + gpuMemcpy( devUmamiMomenta.data(), umamiMomenta.data(), umamiMomenta.size() * sizeof( double ), gpuMemcpyHostToDevice ); + // Host only implementation now (copy) + copyDeviceFromHost( devMomenta, hstMomenta ); + gpuLaunchKernel( aosoa_to_umami_kernel, kMatrixBlocks, kMatrixThreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); + checkGpu( gpuPeekAtLastError() ); +#else + for( std::size_t ievt = 0; ievt < nevt; ++ievt ) + aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); +#endif + + if( verbose ) + { + std::cout << std::string( SEP79, '*' ) << std::endl; + print_run_header( std::cout ); + std::cout << std::string( SEP79, '*' ) << std::endl; + } + + std::cout << "Phase space point:" << std::endl + << std::string( SEP79, '-' ) << std::endl + << " n E px py pz" << std::endl; + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + { + std::cout << std::scientific << std::setprecision( 7 ) + << std::setw( 2 ) << ipar + 1 + << std::setw( 16 ) << point[ipar][0] + << std::setw( 16 ) << point[ipar][1] + << std::setw( 16 ) << point[ipar][2] + << std::setw( 16 ) << point[ipar][3] + << std::endl + << std::defaultfloat; + } + std::cout << std::string( SEP79, '-' ) << std::endl; + + for( unsigned int iflav = 0; iflav < nFlavors; ++iflav ) + { + std::fill( flvVec.begin(), flvVec.end(), iflav ); +#ifdef MGONGPUCPP_GPUIMPL + gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); +#endif + timermap.start( "3a SigmaKin" ); + UmamiInputKey in_keys[3] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S }; + UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; +#ifdef MGONGPUCPP_GPUIMPL + const void* inputs[3] = { devUmamiMomenta.data(), devFlv.data(), devAlphaS.data() }; + void* outputs[1] = { devUmamiMEs.data() }; +#else + const void* inputs[3] = { umamiMomenta.data(), flvVec.data(), alphasVec.data() }; + void* outputs[1] = { umamiMEs.data() }; +#endif + UmamiStatus st = umami_matrix_element( + umami_handle, nevt, nevt, 0, 3, in_keys, inputs, 1, out_keys, outputs ); + timermap.stop(); + if( st != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; + umami_free( umami_handle ); + return 3; + } +#ifdef MGONGPUCPP_GPUIMPL + gpuMemcpy( hstUmamiMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); + const double* mes = hstUmamiMEs.data(); +#else + const double* mes = umamiMEs.data(); +#endif + + std::cout << " PDG"; + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + std::cout << std::setw( 12 ) << CPPProcess::flavorPDG( iflav, ipar ); + std::cout << std::endl + << " Matrix element = " << std::scientific << std::setprecision( 16 ) + << mes[0] << " GeV^" << kMEGeVExponent << std::endl + << std::defaultfloat + << std::string( SEP79, '-' ) << std::endl; + } + + umami_free( umami_handle ); + return 0; + } + + // -------------------------------------------------------------------------- + // perf mode: nblocks*nthreads events per iteration on a single flavor. + // -------------------------------------------------------------------------- + int run_perf_mode( bool verbose, + unsigned int gpublocks, + unsigned int gputhreads, + unsigned int niter, + unsigned int flavorID, + RamboType ramboType, + const std::string& lheFile = "" ) + { + const unsigned int nevt = gpublocks * gputhreads; + + // LHE instead of generating. Processed in batches of nevt and + // niter is derived from the number of events read. + std::vector lheEvents; + if( !lheFile.empty() ) + { + if( !read_lhe_events( lheFile, lheEvents ) ) return 2; + niter = (unsigned int)( ( lheEvents.size() + nevt - 1 ) / nevt ); + std::cout << "Reading events from LHE file = " << lheFile + << " (" << lheEvents.size() << " events, " << niter + << " batches of " << nevt << ")" << std::endl; + } + + mgOnGpu::TimerMap timermap; + +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "00 GpuInit" ); + GpuRuntime gpuRuntime( false ); + + PinnedHostBufferRndNumMomenta hstRndmom( nevt ); + PinnedHostBufferMomenta hstMomenta( nevt ); + PinnedHostBufferWeights hstWeights( nevt ); + DeviceBufferRndNumMomenta devRndmom( nevt ); + DeviceBufferMomenta devMomenta( nevt ); + DeviceBufferWeights devWeights( nevt ); + DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + DeviceBufferBase devUmamiMEs( nevt ); + DeviceBufferBase devFlv( nevt ); + std::vector flvVec( nevt, flavorID ); + std::vector hstUmamiMEs( nevt ); + // perf-mode runs a single flavor, so the device-side flavor buffer is filled once. + gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); +#else + HostBufferRndNumMomenta hstRndmom( nevt ); + HostBufferMomenta hstMomenta( nevt ); + HostBufferWeights hstWeights( nevt ); + std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + std::vector umamiMEs( nevt ); + std::vector flvVec( nevt, flavorID ); +#endif + + std::unique_ptr prnk( + new CommonRandomNumberKernel( hstRndmom ) ); + + UmamiHandle umami_handle = nullptr; + if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_initialize failed" << std::endl; + return 2; + } + + // Retrieve masses + std::vector masses; + if( ramboType != RAMBO_MASSLESS) + { + int npar_meta = 0; + if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + std::vector massesD( npar_meta ); + if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + masses.assign( massesD.begin(), massesD.end() ); + } + + std::unique_ptr prsk; + if( ramboType != RAMBO_MASSLESS ) + { + // Massive host only (copy) + prsk.reset( new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); + } + else + { +#ifdef MGONGPUCPP_GPUIMPL + prsk.reset( new MasslessRamboSamplingKernelDevice( kEnergy, devRndmom, devMomenta, devWeights, gpublocks, gputhreads ) ); +#else + prsk.reset( new MasslessRamboSamplingKernelHost( kEnergy, hstRndmom, hstMomenta, hstWeights, nevt ) ); +#endif + } + + std::unique_ptr genrtimes( new double[niter] ); + std::unique_ptr rambtimes( new double[niter] ); + std::unique_ptr wavetimes( new double[niter] ); + + unsigned int nevtABN = 0; + unsigned int nevtZERO = 0; + double sumME = 0.; + double sumMEsq = 0.; + double minME = std::numeric_limits::infinity(); + double maxME = -std::numeric_limits::infinity(); + unsigned int nevtALL = 0; + + for( unsigned int iiter = 0; iiter < niter; ++iiter ) + { + double genrtime = 0; + double rambtime = 0; + unsigned int nreal = nevt; // number of real (non-padding) events in this batch + if( lheFile.empty() ) + { + timermap.start( "1a GenSeed " ); + prnk->seedGenerator( kSeed + iiter ); + genrtime += timermap.stop(); + timermap.start( "1b GenRnGen" ); + prnk->generateRnarray(); + genrtime += timermap.stop(); +#ifdef MGONGPUCPP_GPUIMPL + if( ramboType == RAMBO_MASSLESS ) + { + timermap.start( "1c CpHTDrnd" ); + copyDeviceFromHost( devRndmom, hstRndmom ); + genrtime += timermap.stop(); + } +#endif + + timermap.start( "2a RamboIni" ); + prsk->getMomentaInitial(); + rambtime += timermap.stop(); + timermap.start( "2b RamboFin" ); + prsk->getMomentaFinal(); + rambtime += timermap.stop(); +#ifdef MGONGPUCPP_GPUIMPL + // Massive host only (copy) + if( ramboType != RAMBO_MASSLESS ) + { + timermap.start( "2c CpHTDmom" ); + copyDeviceFromHost( devMomenta, hstMomenta ); + rambtime += timermap.stop(); + } +#endif + } + else + { + // Fill this batch from the LHE events (AOSOA layout, (E,px,py,pz) per leg). + // padded by repeating its last real event so the SIMD page is valid + // only the nreal real events are counted below. + timermap.start( "2e ReadLHE " ); + const std::size_t base = (std::size_t)iiter * nevt; + nreal = (unsigned int)std::min( nevt, lheEvents.size() - base ); + for( unsigned int ievt = 0; ievt < nevt; ++ievt ) + { + const std::size_t src = base + std::min( ievt, (std::size_t)nreal - 1 ); + const LheEvent& ev = lheEvents[src]; + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + for( int ip4 = 0; ip4 < 4; ++ip4 ) + MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar ) = (fptype)ev[ipar][ip4]; + } + rambtime += timermap.stop(); +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "2c CpHTDmom" ); + copyDeviceFromHost( devMomenta, hstMomenta ); + rambtime += timermap.stop(); +#endif + } + + timermap.start( "2d Aosoa2U " ); +#ifdef MGONGPUCPP_GPUIMPL + gpuLaunchKernel( aosoa_to_umami_kernel, gpublocks, gputhreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); + checkGpu( gpuPeekAtLastError() ); +#else + for( std::size_t ievt = 0; ievt < nevt; ++ievt ) + aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); +#endif + rambtime += timermap.stop(); + + double wavetime = 0; + if( !run_umami( umami_handle, nevt, timermap, wavetime, +#ifdef MGONGPUCPP_GPUIMPL + devUmamiMomenta, devFlv, devUmamiMEs, hstUmamiMEs +#else + umamiMomenta, flvVec, umamiMEs +#endif + ) ) + { + umami_free( umami_handle ); + return 3; + } + +#ifdef MGONGPUCPP_GPUIMPL + if( verbose ) + { + timermap.start( "3c CpDTHmom" ); + copyHostFromDevice( hstMomenta, devMomenta ); + wavetime += timermap.stop(); + } + const double* mes = hstUmamiMEs.data(); +#else + const double* mes = umamiMEs.data(); +#endif + + timermap.start( "4@ UpdtStat" ); + for( unsigned int ievt = 0; ievt < nreal; ++ievt ) + { + double me = mes[ievt]; + ++nevtALL; + if( !std::isfinite( me ) ) + ++nevtABN; + else if( me == 0. ) + ++nevtZERO; + sumME += me; + sumMEsq += me * me; + if( me < minME ) minME = me; + if( me > maxME ) maxME = me; + } + + genrtimes[iiter] = genrtime; + rambtimes[iiter] = rambtime; + wavetimes[iiter] = wavetime; + + if( verbose ) + { + std::cout << std::string( SEP79, '*' ) << std::endl + << "Iteration #" << iiter + 1 << " of " << niter << std::endl; + for( unsigned int ievt = 0; ievt < nreal; ++ievt ) + { + std::cout << "Event #" << ievt + 1 << std::endl; + print_momenta_table( std::cout, hstMomenta.data(), ievt ); + std::cout << " Matrix element = " << std::scientific << std::setprecision( 16 ) + << mes[ievt] << " GeV^" << kMEGeVExponent << std::endl + << std::defaultfloat + << std::string( SEP79, '-' ) << std::endl; + } + } + } + + double sumgtim = 0, sumrtim = 0, sumwtim = 0; + double minwtim = wavetimes[0], maxwtim = wavetimes[0]; + for( unsigned int i = 0; i < niter; ++i ) + { + sumgtim += genrtimes[i]; + sumrtim += rambtimes[i]; + sumwtim += wavetimes[i]; + minwtim = std::min( minwtim, wavetimes[i] ); + maxwtim = std::max( maxwtim, wavetimes[i] ); + } + double meanwtim = sumwtim / niter; + + unsigned int nevtGood = nevtALL - nevtABN; + double meanME = ( nevtGood > 0 ) ? sumME / nevtGood : 0.; + double varME = ( nevtGood > 0 ) ? sumMEsq / nevtGood - meanME * meanME : 0.; + double stdME = ( varME > 0 ) ? std::sqrt( varME ) : 0.; + + std::cout << std::string( SEP79, '*' ) << std::endl; + print_run_header( std::cout ); + std::cout << "NumBlocksPerGrid = " << gpublocks << std::endl + << "NumThreadsPerBlock = " << gputhreads << std::endl + << "NumIterations = " << niter << std::endl + << "FlavorIndex = " << flavorID << " / " << CPPProcess::nmaxflavor << std::endl + << std::string( SEP79, '-' ) << std::endl + << "NaN/abnormal MEs = " << nevtABN << std::endl + << "Zero MEs = " << nevtZERO << std::endl + << std::string( SEP79, '-' ) << std::endl + << "NumberOfEntries = " << niter << std::endl + << std::scientific + << "TotalTime[Rnd+Rmb+ME] (123) = ( " << sumgtim + sumrtim + sumwtim << " ) sec" << std::endl + << "TotalTime[Rambo+ME] (23) = ( " << sumrtim + sumwtim << " ) sec" << std::endl + << "TotalTime[RndNumGen] (1) = ( " << sumgtim << " ) sec" << std::endl + << "TotalTime[Rambo] (2) = ( " << sumrtim << " ) sec" << std::endl + << "TotalTime[MatrixElems] (3) = ( " << sumwtim << " ) sec" << std::endl + << "MeanTimeInMatrixElems = ( " << meanwtim << " ) sec" << std::endl + << "[Min,Max]TimeInMatrixElems = [ " << minwtim << " , " << maxwtim << " ] sec" << std::endl + << std::string( SEP79, '-' ) << std::endl + << "TotalEventsComputed = " << nevtALL << std::endl + << "EvtsPerSec[Rnd+Rmb+ME](123) = ( " << nevtALL / ( sumgtim + sumrtim + sumwtim ) << " ) sec^-1" << std::endl + << "EvtsPerSec[Rmb+ME] (23) = ( " << nevtALL / ( sumrtim + sumwtim ) << " ) sec^-1" << std::endl + << "EvtsPerSec[MatrixElems] (3) = ( " << nevtALL / sumwtim << " ) sec^-1" << std::endl + << std::defaultfloat + << std::string( SEP79, '*' ) << std::endl + << "MeanMatrixElemValue = ( " << meanME << " +- " << stdME / std::sqrt( (double)std::max( 1u, nevtGood ) ) + << " ) GeV^" << kMEGeVExponent << std::endl + << "[Min,Max]MatrixElemValue = [ " << minME << " , " << maxME << " ] GeV^" << kMEGeVExponent << std::endl + << std::string( SEP79, '*' ) << std::endl; + timermap.dump(); + std::cout << std::string( SEP79, '*' ) << std::endl; + + umami_free( umami_handle ); + return 0; + } +} + +int main( int argc, char** argv ) +{ + + Mode mode = MODE_MATRIX; + RamboType ramboType = RAMBO_MASSIVE; // default + bool ramboTypeSet = false; + bool verbose = false; + unsigned int flavorID = 0; + unsigned int gpublocks = 64; + unsigned int gputhreads = 256; + unsigned int niter = 1; + unsigned int numvec[3] = { 0, 0, 0 }; + int nnum = 0; + std::string lheFile; // -e/--events: read momenta from this LHE file (perf mode only) + + // Optional leading subcommand (no leading dash). + int firstArg = 1; + if( firstArg < argc ) + { + std::string a = argv[firstArg]; + if( a == "matrix" ) { mode = MODE_MATRIX; ++firstArg; } + else if( a == "perf" ) { mode = MODE_PERF; ++firstArg; } + } + + double energyArg = -1.; + + for( int argn = firstArg; argn < argc; ++argn ) + { + std::string arg = argv[argn]; + if( arg == "--verbose" || arg == "-v" ) + verbose = true; + else if( arg == "--performance" || arg == "-p" ) + mode = MODE_PERF; // legacy alias + else if( ( arg == "--flavor" || arg == "-f" ) && argn + 1 < argc && is_number( argv[argn + 1] ) ) + flavorID = strtoul( argv[++argn], nullptr, 0 ); + else if( arg == "--rambo-massless" ) + { + std::string r = argv[++argn]; + ramboType = RAMBO_MASSLESS; + ramboTypeSet = true; + } + else if( ( arg == "--events" || arg == "-e" ) && argn + 1 < argc ) + { + lheFile = argv[++argn]; + mode = MODE_PERF; // reading events from file only makes sense in perf mode + } + else if( is_number( argv[argn] ) && nnum < 3 ) + { + numvec[nnum++] = strtoul( argv[argn], nullptr, 0 ); + if( energyArg < 0 ) energyArg = atof( argv[argn] ); + } + else if( is_float( argv[argn] ) && energyArg < 0 ) + { + // decimal number: only meaningful as the matrix-mode energy + energyArg = atof( argv[argn] ); + } + else + return usage( argv[0] ); + } +//ENERGY CHANGE FOR THE MATRIX MODE +// (default 1000 GeV as for the Fortran/C++ standalone 'check' drivers; +// can be overridden with a single positional argument) + if( mode == MODE_MATRIX ) kEnergy = ( energyArg > 0 ) ? energyArg : 1000.; + + if( mode == MODE_MATRIX ) + { + if( ramboType == RAMBO_MASSLESS ) + { + std::cerr << "ERROR: matrix mode only supports the classic RAMBO (-r c)." << std::endl; + return usage( argv[0] ); + } + if( nnum > 1 ) + { + std::cerr << "WARNING: extra positional args are ignored in matrix mode " + << "(dimensions are fixed at " << kMatrixBlocks << " " << kMatrixThreads << " 1)." + << std::endl; + } + return run_matrix_mode( verbose ); + } + + // perf mode + if( nnum == 3 ) + { + gpublocks = numvec[0]; + gputhreads = numvec[1]; + niter = numvec[2]; + } + else if( nnum == 1 ) + { + niter = numvec[0]; + } + else if( nnum != 0 ) + { + return usage( argv[0] ); + } + if( niter == 0 && lheFile.empty() ) return usage( argv[0] ); // niter is derived from the file in LHE mode + + if( flavorID >= CPPProcess::nmaxflavor ) + { + std::cerr << "ERROR: flavor index " << flavorID + << " is out of range [0, " << CPPProcess::nmaxflavor << ")." << std::endl; + return 1; + } + + return run_perf_mode( verbose, gpublocks, gputhreads, niter, flavorID, ramboType, lheFile ); +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc new file mode 100644 index 0000000000..30c6799932 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc @@ -0,0 +1,434 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "color_sum.h" + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessMatrixElements.h" + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors + + //-------------------------------------------------------------------------- + + // *** COLOR MATRIX BELOW *** +%(color_matrix_lines)s + +#ifdef MGONGPUCPP_GPUIMPL + // The normalized color matrix (divide each column by denom) + template + struct NormalizedColorMatrix + { + constexpr __host__ __device__ NormalizedColorMatrix() + : value() + { + for( int icol = 0; icol < ncolor; icol++ ) + for( int jcol = 0; jcol < ncolor; jcol++ ) + value[icol * ncolor + jcol] = colorMatrix[icol][jcol] / colorDenom[icol]; + } + T value[ncolor * ncolor]; + }; + // The fptype2 version is the default used by kernels (supporting mixed floating point mode also in blas) + static __device__ fptype2 s_pNormalizedColorMatrix2[ncolor * ncolor]; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void createNormalizedColorMatrix() + { + static bool first = true; + if( first ) + { + first = false; + constexpr NormalizedColorMatrix normalizedColorMatrix2; + gpuMemcpyToSymbol( s_pNormalizedColorMatrix2, normalizedColorMatrix2.value, ncolor * ncolor * sizeof( fptype2 ) ); + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + void + color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity + const int ievt0 ) // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) + { + // Pre-compute a constexpr triangular color matrix properly normalized #475 + struct TriangularNormalizedColorMatrix + { + // See https://stackoverflow.com/a/34465458 + __host__ __device__ constexpr TriangularNormalizedColorMatrix() + : value() + { + for( int icol = 0; icol < ncolor; icol++ ) + { + // Diagonal terms + value[icol][icol] = colorMatrix[icol][icol] / colorDenom[icol]; + // Off-diagonal terms + for( int jcol = icol + 1; jcol < ncolor; jcol++ ) + value[icol][jcol] = 2 * colorMatrix[icol][jcol] / colorDenom[icol]; + } + } + fptype2 value[ncolor][ncolor]; + }; + static constexpr auto cf2 = TriangularNormalizedColorMatrix(); + // Use the property that M is a real matrix (see #475): + // we can rewrite the quadratic form (A-iB)(M)(A+iB) as AMA - iBMA + iBMA + BMB = AMA + BMB + // In addition, on C++ use the property that M is symmetric (see #475), + // and also use constexpr to compute "2*" and "/colorDenom[icol]" once and for all at compile time: + // we gain (not a factor 2...) in speed here as we only loop over the up diagonal part of the matrix. + // Strangely, CUDA is slower instead, so keep the old implementation for the moment. + fptype_sv deltaMEs = { 0 }; +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype_sv deltaMEs_next = { 0 }; + // Mixed mode: merge two neppV vectors into one neppV2 vector + fptype2_sv jampR_sv[ncolor]; + fptype2_sv jampI_sv[ncolor]; + for( int icol = 0; icol < ncolor; icol++ ) + { + jampR_sv[icol] = fpvmerge( cxreal( allJamp_sv[icol] ), cxreal( allJamp_sv[ncolor + icol] ) ); + jampI_sv[icol] = fpvmerge( cximag( allJamp_sv[icol] ), cximag( allJamp_sv[ncolor + icol] ) ); + } +#else + const cxtype_sv* jamp_sv = allJamp_sv; +#endif + // Loop over icol + for( int icol = 0; icol < ncolor; icol++ ) + { + // Diagonal terms +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype2_sv& jampRi_sv = jampR_sv[icol]; + fptype2_sv& jampIi_sv = jampI_sv[icol]; +#else + fptype2_sv jampRi_sv = (fptype2_sv)( cxreal( jamp_sv[icol] ) ); + fptype2_sv jampIi_sv = (fptype2_sv)( cximag( jamp_sv[icol] ) ); +#endif + fptype2_sv ztempR_sv = cf2.value[icol][icol] * jampRi_sv; + fptype2_sv ztempI_sv = cf2.value[icol][icol] * jampIi_sv; + // Loop over jcol + for( int jcol = icol + 1; jcol < ncolor; jcol++ ) + { + // Off-diagonal terms +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype2_sv& jampRj_sv = jampR_sv[jcol]; + fptype2_sv& jampIj_sv = jampI_sv[jcol]; +#else + fptype2_sv jampRj_sv = (fptype2_sv)( cxreal( jamp_sv[jcol] ) ); + fptype2_sv jampIj_sv = (fptype2_sv)( cximag( jamp_sv[jcol] ) ); +#endif + ztempR_sv += cf2.value[icol][jcol] * jampRj_sv; + ztempI_sv += cf2.value[icol][jcol] * jampIj_sv; + } + fptype2_sv deltaMEs2 = ( jampRi_sv * ztempR_sv + jampIi_sv * ztempI_sv ); // may underflow #831 +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + deltaMEs += fpvsplit0( deltaMEs2 ); + deltaMEs_next += fpvsplit1( deltaMEs2 ); +#else + deltaMEs += deltaMEs2; +#endif + } + // *** STORE THE RESULTS *** + using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv += deltaMEs; // fix #435 +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype* MEs_next = E_ACCESS::ieventAccessRecord( allMEs, ievt0 + neppV ); + fptype_sv& MEs_sv_next = E_ACCESS::kernelAccess( MEs_next ); + MEs_sv_next += deltaMEs_next; +#endif + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity + const int nGoodHel, // input: number of good helicities + const int nevtIfAllHelicities ) // input: zero in single-helicity mode, number of events in multi-helicity mode + { + if (nevtIfAllHelicities) { + int ighel = blockIdx.y; + allMEs = allMEs + ighel * nevtIfAllHelicities; // MEs for one specific helicity ighel + allJamps = allJamps + ighel * nevtIfAllHelicities; // Jamps for one specific helicity ighel + } + using J_ACCESS = DeviceAccessJamp; + fptype jampR[ncolor]; + fptype jampI[ncolor]; + for( int icol = 0; icol < ncolor; icol++ ) + { + constexpr int ihel0 = 0; // the input buffer allJamps already points to a specific helicity + cxtype jamp = J_ACCESS::kernelAccessIcolIhelNhelConst( allJamps, icol, ihel0, nGoodHel ); + jampR[icol] = jamp.real(); + jampI[icol] = jamp.imag(); + } + // Loop over icol + fptype deltaMEs = { 0 }; + for( int icol = 0; icol < ncolor; icol++ ) + { + fptype2 ztempR = { 0 }; + fptype2 ztempI = { 0 }; + fptype2 jampRi = jampR[icol]; + fptype2 jampIi = jampI[icol]; + // OLD IMPLEMENTATION (ihel3: symmetric square matrix) - Loop over all jcol + //for( int jcol = 0; jcol < ncolor; jcol++ ) + //{ + // fptype2 jampRj = jampR[jcol]; + // fptype2 jampIj = jampI[jcol]; + // ztempR += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix + // ztempI += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix + //} + // NEW IMPLEMENTATION #475 (ihel3p1: triangular lower diagonal matrix) - Loop over jcol < icol + ztempR += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampRi; // use fptype2 version of color matrix + ztempI += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampIi; // use fptype2 version of color matrix + for( int jcol = 0; jcol < icol; jcol++ ) + { + fptype2 jampRj = jampR[jcol]; + fptype2 jampIj = jampI[jcol]; + ztempR += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix + ztempI += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix + } + deltaMEs += ztempR * jampRi; + deltaMEs += ztempI * jampIi; + } + // *** STORE THE RESULTS *** + using E_ACCESS = DeviceAccessMatrixElements; // non-trivial access: buffer includes all events + // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) + E_ACCESS::kernelAccess( allMEs ) += deltaMEs; // fix #435 + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +#ifndef MGONGPU_HAS_NO_BLAS +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + __global__ void + convertD2F_Jamps( fptype2* allJampsFpt2, // output: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel + const fptype* allJamps, // input: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel + const int nhel ) // input: number of good helicities nGoodHel + { + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + constexpr int ihel = 0; // the input buffer allJamps already points to a specific helicity + // NB! From a functional point of view, any striding will be ok here as long as ncolor*2*nevt elements are all correctly copied! + // NB! Just in case this may be better for performance reasons, however, the same striding as in compute_jamps and cuBLAS is used here + for( int ix2 = 0; ix2 < mgOnGpu::nx2; ix2++ ) + for( int icol = 0; icol < ncolor; icol++ ) + allJampsFpt2[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] = + allJamps[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt]; + } +#endif +#endif +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +#ifndef MGONGPU_HAS_NO_BLAS +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + __global__ void + convertF2D_MEs( fptype* allMEs, // output: allMEs[nevt] for one specific helicity + const fptype2* allMEsFpt2 ) // input: allMEs[nevt] for one specific helicity + { + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + allMEs[ievt] = allMEsFpt2[ievt]; + } +#endif +#endif +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ +#ifndef MGONGPU_HAS_NO_BLAS + void + color_sum_blas( fptype* ghelAllMEs, // output: allMEs super-buffer[nhel][nevt], add |M|^2 separately for each helicity + const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nhel][nevt] for nhel good helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nhel good helicities + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) +#else + gpuStream_t* /*ghelStreams*/, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) +#endif + const int nhel, // input: number of good helicities (nhel == nGoodHel) + const int gpublocks, // input: cuda gpublocks + const int gputhreads ) // input: cuda gputhreads + { + const int nevt = gpublocks * gputhreads; + + // Get the address associated with the normalized color matrix in device memory + static fptype2* devNormColMat = nullptr; + if( !devNormColMat ) gpuGetSymbolAddress( (void**)&devNormColMat, s_pNormalizedColorMatrix2 ); + +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Mixed precision mode: need two fptype2[2*ncolor*nhel*nevt] buffers and one fptype2[nhel*nevt] buffers for the nhel helicities + fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of first fptype2[ncolor*2*nhel*nevt] buffer + fptype2* ghelAllJampsFpt2 = ghelAllBlasTmp + ncolor * mgOnGpu::nx2 * nhel * nevt; // start of second fptype2[ncolor*2*nhel*nevt] buffer + fptype2* ghelAllMEsFpt2 = ghelAllBlasTmp + 2 * ncolor * mgOnGpu::nx2 * nhel * nevt; // start of fptype2[nhel*nevt] buffer + // Convert jamps from double to float + for( int ighel = 0; ighel < nhel; ighel++ ) + { + const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // jamps for a single helicity ihel + fptype2* hAllJampsFpt2 = ghelAllJampsFpt2 + ighel * nevt; // jamps for a single helicity ihel + gpuLaunchKernelStream( convertD2F_Jamps, gpublocks, gputhreads, ghelStreams[ighel], hAllJampsFpt2, hAllJamps, nhel ); + } + // Real and imaginary components + const fptype2* ghelAllJampsReal = ghelAllJampsFpt2; + const fptype2* ghelAllJampsImag = ghelAllJampsFpt2 + ncolor * nhel * nevt; +#else + // Standard single or double precision mode: need one fptype2[ncolor*2*nhel*nevt] buffer + static_assert( std::is_same::value ); + fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of fptype2[ncolor*2*nhel*nevt] buffer + fptype2* ghelAllMEsFpt2 = ghelAllMEs; + // Real and imaginary components + const fptype2* ghelAllJampsReal = ghelAllJamps; // this is not a cast (the two types are identical) + const fptype2* ghelAllJampsImag = ghelAllJamps + ncolor * nhel * nevt; // this is not a cast (the two types are identical) +#endif + // Real and imaginary components + fptype2* ghelAllZtempReal = ghelAllZtempBoth; + fptype2* ghelAllZtempImag = ghelAllZtempBoth + ncolor * nhel * nevt; + + // Note: striding for cuBLAS from DeviceAccessJamp: + // - ghelAllJamps(icol,ihel,ievt).real is ghelAllJamps[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] + // - ghelAllJamps(icol,ihel,ievt).imag is ghelAllJamps[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] + + // Step 1: Compute Ztemp[ncolor][nhel*nevt] = ColorMatrix[ncolor][ncolor] * JampsVector[ncolor][nhel*nevt] for both real and imag + // In this case alpha=1 and beta=0: the operation is Ztemp = alpha * ColorMatrix * JampsVector + beta * Ztemp + fptype2 alpha1 = 1; + fptype2 beta1 = 0; + const int ncolorM = ncolor; + const int nevtN = nhel*nevt; + const int ncolorK = ncolor; + checkGpuBlas( gpuBlasTgemm( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose ColMat + GPUBLAS_OP_T, // transpose JampsV (new1) + ncolorM, nevtN, ncolorK, + &alpha1, + devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK + ghelAllJampsReal, nevtN, // JampsV is nevtN x ncolorK + &beta1, + ghelAllZtempReal, ncolorM ) ); // Ztemp is ncolorM x nevtN + checkGpuBlas( gpuBlasTgemm( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose ColMat + GPUBLAS_OP_T, // transpose JampsV (new1) + ncolorM, nevtN, ncolorK, + &alpha1, + devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK + ghelAllJampsImag, nevtN, // JampsV is nevtN x ncolorK (new1) + &beta1, + ghelAllZtempImag, ncolorM ) ); // Ztemp is ncolorM x nevtN + + // Step 2: For each ievt, compute the dot product of JampsVector[ncolor][ievt] dot tmp[ncolor][ievt] + // In this case alpha=1 and beta=1: the operation is ME = alpha * ( Tmp dot JampsVector ) + beta * ME + // Use cublasSgemmStridedBatched to perform these batched dot products in one call + fptype2 alpha2 = 1; + fptype2 beta2 = 1; + checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose JampsV (new1) + GPUBLAS_OP_N, // do not transpose Tmp + 1, 1, ncolor, // result is 1x1 (dot product) + &alpha2, + ghelAllJampsReal, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column + ghelAllZtempReal, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column + &beta2, + ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) + nevtN ) ); // there are nevtN (nhel*nevt) "batches" + checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose JampsV (new1) + GPUBLAS_OP_N, // do not transpose Tmp + 1, 1, ncolor, // result is 1x1 (dot product) + &alpha2, + ghelAllJampsImag, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column (new1) + ghelAllZtempImag, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column + &beta2, + ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) + nevtN ) ); // there are nevt (nhel*nevt) "batches" + +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Convert MEs from float to double + for( int ighel = 0; ighel < nhel; ighel++ ) + { + fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for a single helicity ihel + fptype2* hAllMEsFpt2 = ghelAllMEsFpt2 + ighel * nevt; // MEs for a single helicity ihel + gpuLaunchKernelStream( convertF2D_MEs, gpublocks, gputhreads, ghelStreams[ighel], hAllMEs, hAllMEsFpt2 ); + } +#endif + } +#endif /* clang-format on */ +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) + const int nGoodHel, // input: number of good helicities + const int gpublocks, // input: cuda gpublocks + const int gputhreads, // input: cuda gputhreads + const bool processAllHelicities ) // input: if true, use blockIdx.y to index helicities + { + const int nevt = gpublocks * gputhreads; + // CASE 1: KERNEL + if( !pBlasHandle ) + { + assert( ghelAllBlasTmp == nullptr ); // sanity check for HASBLAS=hasNoBlas or CUDACPP_RUNTIME_BLASCOLORSUM not set + if (processAllHelicities) { + gpuLaunchKernel2D( color_sum_kernel, gpublocks, nGoodHel, gputhreads, ghelStreams[0], ghelAllMEs, ghelAllJamps, nGoodHel, nevt ); + } else { + // Loop over helicities + for( int ighel = 0; ighel < nGoodHel; ighel++ ) + { + fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for one specific helicity ighel + const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // Jamps for one specific helicity ighel + gpuStream_t hStream = ghelStreams[ighel]; + gpuLaunchKernelStream( color_sum_kernel, gpublocks, gputhreads, hStream, hAllMEs, hAllJamps, nGoodHel, 0 ); + } + } + } + // CASE 2: BLAS + else + { +#ifdef MGONGPU_HAS_NO_BLAS + assert( false ); // sanity check: no path to this statement for HASBLAS=hasNoBlas +#else + if (processAllHelicities) { + assert( false ); // BLAS in async mode not supported for now + } else { + checkGpu( gpuDeviceSynchronize() ); // do not start the BLAS color sum for all helicities until the loop over helicities has completed + // Reset the tmp buffer +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( 2 * ncolor * mgOnGpu::nx2 + 1 ) * sizeof( fptype2 ) ); +#else + gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( ncolor * mgOnGpu::nx2 ) * sizeof( fptype2 ) ); +#endif + // Delegate the color sum to BLAS for + color_sum_blas( ghelAllMEs, ghelAllJamps, ghelAllBlasTmp, pBlasHandle, ghelStreams, nGoodHel, gpublocks, gputhreads ); + } +#endif + } + } +#endif + + //-------------------------------------------------------------------------- + +} // end namespace diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h new file mode 100644 index 0000000000..347184c4e1 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h @@ -0,0 +1,105 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef COLOR_SUM_H +#define COLOR_SUM_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#include "CPPProcess.h" +#include "GpuAbstraction.h" + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + class DeviceAccessJamp + { + public: + static __device__ inline cxtype_ref + kernelAccessIcolIhelNhel( fptype* buffer, const int icol, const int ihel, const int nhel ) + { + const int ncolor = CPPProcess::ncolor; // the number of leading colors + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) + //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" + // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) + // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS + //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" + // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) + return cxtype_ref( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], + buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); + } + static __device__ inline const cxtype + kernelAccessIcolIhelNhelConst( const fptype* buffer, const int icol, const int ihel, const int nhel ) + { + const int ncolor = CPPProcess::ncolor; // the number of leading colors + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) + //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" + // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) + // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS + //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" + // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) + return cxtype( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], + buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); + } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void createNormalizedColorMatrix(); +#endif + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + void + color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity + const int ievt0 ); // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) + const int nGoodHel, // input: number of good helicities + const int gpublocks, // input: cuda gpublocks + const int gputhreads, // input: cuda gputhreads + const bool processAllHelicities); // input: if true, use blockIdx.y to index helicities +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity + const int nGoodHel, // input: number of good helicities + const int nevtIfAllHelicities); // input: zero in single-helicity mode, number of events in multi-helicity mode +#endif + + //-------------------------------------------------------------------------- +} + +#endif // COLOR_SUM_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h new file mode 100644 index 0000000000..027f1aa44e --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h @@ -0,0 +1,68 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: O. Mattelaer, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef COLORAMPS_H +#define COLORAMPS_H 1 + +#include "CPPProcess.h" + + +namespace mgOnGpu +{ + // Summary of numbering and indexing conventions for the relevant concepts (see issue #826 and PR #852) + // - Diagram number (no variable) in [0, N_diagrams-1]: all values are allowed (N_diagrams distinct values) + // It follows the same C-style indexing of MadSpace + // => this number is displayed for information before each block of code in CPPProcess.cc + // - Channel number ("channelId" in C, CHANNEL_ID in F) in [1, N_channels]: not all values are allowed (N_config <= N_channels <= N_diagrams distinct values) + // *** NB channelId is a diagram number: but ALL diagrams > N_channels, and also some < N_channels, do not have an associated SDE config number (#919) *** + // => this number (with F indexing as in ps/pdf output) is passed around as an API argument between cudacpp functions + // Note: the old API passes around a single CHANNEL_ID (and uses CHANNEL_ID=0 to indicate no-multichannel mode, but this is not used in coloramps.h), + // while the new API passes around an array of CHANNEL_ID's (and uses a NULL array pointer to indicate no-multichannel mode) + // - Channel number in C indexing: "channelID - 1" + // => this number (with C indexing) is used as the index of the channel2iconfig array below + // - Config number ("iconfig" in C, ICONFIG in F) in [1, N_config]: all values are allowed (N_config <= N_channels <= N_diagrams distinct values) + // - Config number in C indexing: "iconfig - 1" + // => this number (with C indexing) is used as the index of the icolamp array below + + // The number of channels in the channel2iconfig array below + // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** + constexpr unsigned int nchannels = %(nb_diag)i; +#ifdef MGONGPUCPP_GPUIMPL + static_assert( nchannels <= mg5amcGpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 +#else + static_assert( nchannels <= mg5amcCpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 +#endif + + // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) + // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) + // This array has N_diagrams elements, but only N_config <= N_diagrams valid values (iconfig>0) + // (NB: this array is created on the host in C++ code and on the device in GPU code, but a host copy is also needed in runTest #917) + __device__ constexpr int channel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed +%(channelc2iconfig_lines)s + }; + + // Host copy of the channel2iconfig array (this is needed in runTest #917) +#ifndef MGONGPUCPP_GPUIMPL + constexpr const int* hostChannel2iconfig = channel2iconfig; +#else + constexpr int hostChannel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed +%(channelc2iconfig_lines)s + }; +#endif + + // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) + constexpr unsigned int nconfigSDE = %(nb_channel)s; + + // Map iconfig to the mask of allowed colors (e.g. "colormask = icolamp[iconfig - 1]": input index uses C indexing) + // This array has N_config <= N_diagrams elements + // (NB: this array is created on the host in C++ code and on the device in GPU code) + __device__ constexpr bool icolamp[%(nb_channel)s][%(nb_color)s] = { // note: a trailing comma in the initializer list is allowed +%(is_LC)s + }; + +} + +#endif // COLORAMPS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h new file mode 100644 index 0000000000..32c1bbb963 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h @@ -0,0 +1,335 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Feb 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef constexpr_math_h +#define constexpr_math_h 1 + +#include "mgOnGpuConfig.h" + +#include +#include +#include +#include + +// SWITCH ON/OFF DEBUGGING +#undef CONSTEXPR_MATH_DEBUG // no-debug +//#define CONSTEXPR_MATH_DEBUG 1 // debug + +// FOR DEBUGGING +#ifdef CONSTEXPR_MATH_DEBUG +#define CONSTEXPRMATHFUN inline +#define CONSTEXPRMATHVAR const +#else +#define CONSTEXPRMATHFUN constexpr +#define CONSTEXPRMATHVAR constexpr +#endif + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) + constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) + { + return curr == prev ? curr : sqrtNewtonRaphson( xx, 0.5 * ( curr + xx / curr ), curr ); + } + constexpr long double constexpr_sqrt( const long double xx ) + { + return xx >= 0 // && x < std::numeric_limits::infinity() // avoid -Wtautological-constant-compare warning in fast math + ? sqrtNewtonRaphson( xx, xx, 0 ) + : std::numeric_limits::quiet_NaN(); + } + + // SQRT constants + constexpr long double constexpr_sqrt2 = constexpr_sqrt( 2 ); + + // Constexpr implementation of floor (see https://stackoverflow.com/a/66146159) + constexpr int constexpr_floor( const long double xx ) + { + const int i = static_cast( xx ); + return xx < i ? i - 1 : i; + } + + // Constexpr implementation of pow + constexpr long double constexpr_pow( const long double base, const long double exp, const bool requireExpGe0 = false ) + { + // NB(1): this iterative integer implementation of constexpr_pow requires exponent >= 0 + if( requireExpGe0 ) assert( exp >= 0 ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + if( exp < 0 ) return 1. / constexpr_pow( base, -exp, true ); + // NB(2): this iterative integer implementation of constexpr_pow requires an integer exponent, excexpt for special cases (1/2, 1/4) + if( exp == 0.5 ) return constexpr_sqrt( base ); + if( exp == 0.25 ) return constexpr_sqrt( constexpr_sqrt( base ) ); + const int iexp = constexpr_floor( exp ); + assert( static_cast( iexp ) == exp ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + // Iterative implementation of pow if exp is a non negative integer + return iexp == 0 ? 1 : base * constexpr_pow( base, iexp - 1 ); + } + + // PI constants + // NB1: M_PIl from from cmath is not defined on Mac + // NB2: std::numbers::pi needs c++20 but we are still using c++17 + // NB3: I could use my constexpr_atan(1)*4... but a literal is better? + //constexpr long double constexpr_pi = M_PIl; // pi + //constexpr long double constexpr_pi_by_2 = M_PI_2l; // pi/2 + //constexpr long double constexpr_pi_by_4 = M_PI_4l; // pi/4 + constexpr long double constexpr_pi = 3.141592653589793238462643383279502884L; // same as M_PIl in gcc + constexpr long double constexpr_pi_by_2 = 1.570796326794896619231321691639751442L; // same as M_PI_2l in gcc + constexpr long double constexpr_pi_by_4 = 0.785398163397448309615660845819875721L; // same as M_PI_4l in gcc + static_assert( constexpr_pi_by_4 * 4 == constexpr_pi ); + static_assert( constexpr_pi_by_4 * 2 == constexpr_pi_by_2 ); + static_assert( constexpr_pi_by_2 * 2 == constexpr_pi ); + + // Constexpr implementation of sin for 0= constexpr_pi_by_4 ) std::cout << "sinTaylor xx=" << xx << std::endl; +#endif + assert( xx >= 0 && "The argument of sinTaylor is lower than the expected range [0,pi/4)" ); + assert( xx < constexpr_pi_by_4 && "The argument of sinTaylor is higher than the expected range [0,pi/4)" ); + long double sinx = 0; + int ipow = 1; + long double delta = xx; + while( true ) + { + long double sinxlast = sinx; + sinx += delta; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + if( sinx == sinxlast ) break; + // Next iteration + ipow += 2; + delta *= -xx * xx / ( ipow - 1 ) / ipow; + } +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + return sinx; + } + +#ifdef CONSTEXPR_MATH_DEBUG + // Debug printouts for trig functions + inline void debug_constexpr_trig( const long double xx, size_t call ) + { + CONSTEXPRMATHVAR long double xxminuspi = xx - constexpr_pi; + CONSTEXPRMATHVAR long double twopiminusxx = 2 * constexpr_pi - xx; + std::cout << std::setprecision( 40 ); + std::cout << "constexpr_sin_quad call=" << call << std::endl + << " xx=" << xx << std::endl + << " (xx-pi)=" << xxminuspi << std::endl + << " (2pi-xx)=" << twopiminusxx << std::endl; + std::cout << std::setprecision( 6 ); + if( xx < 0 ) // (-inf, 0) + std::cout << " -- case 1 (xx < 0)" << std::endl; + else if( xx == 0 ) // [0] *** NEW + std::cout << " -- case 2 (xx == 0)" << std::endl; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + std::cout << " -- case 3 (xx < pi/4)" << std::endl; + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + std::cout << " -- case 4 (xx == pi/4)" << std::endl; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + std::cout << " -- case 5 (xx < pi/2)" << std::endl; + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + std::cout << " -- case 6 (xx == pi/2)" << std::endl; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + std::cout << " -- case 7 (xx < 3*pi/4)" << std::endl; + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + std::cout << " -- case 8 (xx == 3*pi/4)" << std::endl; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + std::cout << " -- case 9 (xx < pi)" << std::endl; + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + std::cout << " -- case 10 (xx == pi)" << std::endl; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + std::cout << " -- case 11 (xx < 2*pi)" << std::endl; + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + std::cout << " -- case 12 (xx == 2*pi)" << std::endl; + else // (2*pi, +inf) + std::cout << " -- case 13 (xx > 2*pi)" << std::endl; + } +#endif + + // Mapping to [0,2*pi) range (long double signature) + constexpr long double mapIn0to2Pi( const long double xx ) + { + return xx - constexpr_floor( xx / 2 / constexpr_pi ) * 2 * constexpr_pi; + } + + // Constexpr implementation of cos (long double signature) + CONSTEXPRMATHFUN long double constexpr_cos_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 1; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx ), 2 ) ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return sinTaylor( constexpr_pi_by_2 - xx ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 0; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return -sinTaylor( xx - constexpr_pi_by_2 ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return -1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return -constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi - xx ), 2 ) ); + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + return -1; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return constexpr_cos_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 1; + else // (2*pi, +inf) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of cos (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_cos( const double x ) + { + return constexpr_cos_quad( x ); + } + + // Constexpr implementation of sin (long double signature) + CONSTEXPRMATHFUN long double constexpr_sin_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 0; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return sinTaylor( xx ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi_by_2 - xx ), 2 ) ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 1; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx - constexpr_pi_by_2 ), 2 ) ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return sinTaylor( constexpr_pi - xx ); + else if( xx == constexpr_pi ) // [pi] *** NEW (1st fix #903 infinite recursion) + return 0; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return -constexpr_sin_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 0; + else // (2*pi, +inf) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of sin (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_sin( const double x ) + { + return constexpr_sin_quad( x ); + } + + // Constexpr implementation of tan (long double signature) + CONSTEXPRMATHFUN long double constexpr_tan_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } + if( xx < 0 ) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + else if( xx < 2 * constexpr_pi ) // [0, 2*pi) + return constexpr_sin_quad( xx, assume0to2Pi ) / constexpr_cos_quad( xx, assume0to2Pi ); + else // [8/4*pi, +inf) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of tan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_tan( const double x ) + { + return constexpr_tan_quad( x ); + } + + // Constexpr implementation of atan for -1= -1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + assert( xx < 1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + long double atanx = 0; + int ipow = 1; + long double xpow = xx; + while( true ) + { + long double atanxlast = atanx; + atanx += xpow / ipow; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", xpow=" << xpow << ", atanx=" << atanx << std::endl; // for debugging (not constexpr) +#endif + if( atanx == atanxlast ) break; + // Next iteration + ipow += 2; + xpow *= -xx * xx; + } + return atanx; + } + + // Constexpr implementation of atan (long double signature) + CONSTEXPRMATHFUN long double constexpr_atan_quad( const long double xx ) + { + if( xx > 1 ) + return constexpr_pi_by_2 - atanTaylor( 1 / xx ); + else if( xx == 1 ) + return constexpr_pi_by_4; + else if( xx > -1 ) + return atanTaylor( xx ); + else if( xx == -1 ) + return -constexpr_pi_by_4; + else // if( xx < -1 ) + return -constexpr_pi_by_2 - atanTaylor( 1 / xx ); + } + + // Constexpr implementation of atan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_atan( const double x ) + { + return constexpr_atan_quad( x ); + } +} + +#endif // constexpr_math_h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_hel_amps_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_hel_amps_h.inc new file mode 100644 index 0000000000..8dde0fff4b --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_hel_amps_h.inc @@ -0,0 +1,90 @@ +// Copyright (C) 2010 The ALOHA Development team and Contributors. +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Sep 2010) for the MG5aMC backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== +// This file has been automatically generated for %(output_name)s by +%(info_lines)s +//========================================================================== + +#ifndef HelAmps_%(model_name)s_H +#define HelAmps_%(model_name)s_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#include "Parameters.h" + +#include +//#include +//#include +//#include +//#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + + // ALOHA-style object for easy flavor consolidation and non-template API + struct ALOHAOBJ { + + static constexpr int np4 = 4; // dimensions of 4-momenta (E,px,py,pz) + static constexpr int nw6 = %(nwave)d; // dimensions of each wavefunction (notice, this is +1 in case of FD gauge) + fptype_sv * pvec; + fptype * w; + int flv_index; + + __host__ __device__ ALOHAOBJ() {} + __host__ __device__ ALOHAOBJ(fptype_sv * pvec_sv, cxtype_sv * w_sv, int flv = -1) + : pvec(pvec_sv), w(reinterpret_cast(w_sv)), flv_index(flv) {} + }; + + struct FLV_COUPLING_VIEW { + + const int* const partner1; + const int* const partner2; + const fptype* const value; + + __host__ __device__ + FLV_COUPLING_VIEW(const int* p1, const int* p2, const fptype* v) + : partner1(p1), partner2(p2), value(v) {} + }; + + // FSTRIDE is the number of fptype's used to store one flavor slot of the value buffer: + // - independent (fixed) flavored couplings: FSTRIDE = nx2 = 2 (a single scalar complex, broadcast across the SIMD vector) + // - dependent (event-by-event, running-alphas) flavored couplings: FSTRIDE = nx2*neppC (an AOSOA SIMD record) + // It must match C_ACCESS::flv_stride of the access type the consuming vertex routine is instantiated with. + template + class FLV_COUPLING_ARRAY { + + static_assert(SIZE >= 0, "flvCOUPs SIZE must be non-negative"); + static_assert(STRIDE > 0, "flvCOUPs STRIDE must be positive"); + static_assert(FSTRIDE > 0, "flvCOUPs FSTRIDE must be positive"); + const int* const partner1; + const int* const partner2; + const fptype* const value; + + public: + __host__ __device__ + FLV_COUPLING_ARRAY(const int* p1, const int* p2, const fptype* v) + : partner1(p1), partner2(p2), value(v) {} + + __host__ __device__ + FLV_COUPLING_VIEW operator[](const int i) const { + return FLV_COUPLING_VIEW{ + partner1 + i*STRIDE, + partner2 + i*STRIDE, + value + i*FSTRIDE*STRIDE + }; + } + }; +%(function_declarations)s diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_cc.inc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_cc.inc new file mode 100644 index 0000000000..4650fc76c8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_cc.inc @@ -0,0 +1,101 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== +// This file has been automatically generated for CUDA/C++ standalone by +%(info_lines)s +//========================================================================== + +#include "Parameters.h" + +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +using namespace mg5amcGpu; +#else +using namespace mg5amcCpu; +#endif + +#ifndef MGONGPU_HARDCODE_PARAM + +// Initialize static instance +Parameters* Parameters::instance = 0; + +// Function to get static instance - only one instance per program +Parameters* +Parameters::getInstance() +{ + if( instance == 0 ) + instance = new Parameters(); + return instance; +} + +void +Parameters::setIndependentParameters( SLHAReader& slha ) +{ + zero = 0; // define "zero" + ZERO = 0; // define "zero" + std::vector indices( 2, 0 ); // prepare a vector for indices + %(set_independent_parameters)s +} + +void +Parameters::setIndependentCouplings() +{ + %(set_independent_couplings)s + %(set_flv_couplings)s +} + +/* +void +Parameters::setDependentParameters() // now computed event-by-event (running alphas #373) +{ + %(set_dependent_parameters)s +} + +void +Parameters::setDependentCouplings() // now computed event-by-event (running alphas #373) +{ + %(set_dependent_couplings)s +} +*/ + +#endif + +// Routines for printing out parameters +void +Parameters::printIndependentParameters() +{ + std::cout << "model parameters independent of event kinematics:" << std::endl; + std::cout << "(Warning: aS in the runcard is ignored because event-by-event Gs are hardcoded or retrieved from Fortran)" << std::endl; + %(print_independent_parameters)s +} + +void +Parameters::printIndependentCouplings() +{ + std::cout << "model couplings independent of event kinematics:" << std::endl; + %(print_independent_couplings)s +} + +/* +void +Parameters::printDependentParameters() // now computed event-by-event (running alphas #373) +{ + std::cout << "model parameters dependent on event kinematics:" << std::endl; + %(print_dependent_parameters)s +} + +void +Parameters::printDependentCouplings() // now computed event-by-event (running alphas #373) +{ + std::cout << "model couplings dependent on event kinematics:" << std::endl; + %(print_dependent_couplings)s +} +*/ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_h.inc new file mode 100644 index 0000000000..c13f8ebe2f --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_h.inc @@ -0,0 +1,264 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. +// ========================================================================== +// This file has been automatically generated for CUDA/C++ standalone by +%(info_lines)s +//========================================================================== + +#ifndef Parameters_H +#define Parameters_H + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" +#include "mgOnGpuVectors.h" + +#include "constexpr_math.h" + +//========================================================================== + +// AV Jan 2024 (PR #625): this ugly #define was the only way I found to avoid creating arrays[nBsm] in CPPProcess.cc if nBsm is 0 +// The problem is that nBsm is determined when generating Parameters.h, which happens after CPPProcess.cc has already been generated +// For simplicity, keep this code hardcoded also for SM processes (a nullptr is needed as in the case nBsm == 0) +%(bsmdefine)s + +#ifndef MGONGPU_HARDCODE_PARAM%(eftwarn0)s + +#include "read_slha.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // Define FLV_COUPLING struct + struct FLV_COUPLING { + static constexpr int max_flavor = %(max_flavor)d; + + int partner1[max_flavor]; + int partner2[max_flavor]; + cxtype* value[max_flavor]; // because it will refer already calculated couplings + + FLV_COUPLING() { + for (int i = 0; i < max_flavor; ++i) { + partner1[i] = -1; + partner2[i] = -1; + value[i] = nullptr; // flavor slots with no coupling stay null so the + // cIPF_value setup (value[j] ? *value[j] : 0) does + // not dereference an uninitialised pointer + } + } + }; + + class Parameters + { + public: + + static Parameters* getInstance(); + + // Define "zero" + double zero, ZERO; + + %(independent_parameters)s + + %(independent_couplings)s + + %(dependent_parameters)s + + %(dependent_couplings)s + + %(flavor_independent_couplings)s + + %(flavor_dependent_couplings)s + + // Set parameters that are unchanged during the run + void setIndependentParameters( SLHAReader& slha ); + + // Set couplings that are unchanged during the run + void setIndependentCouplings(); + + // Set parameters that are changed event by event + //void setDependentParameters(); // now computed event-by-event (running alphas #373) + + // Set couplings that are changed event by event + //void setDependentCouplings(); // now computed event-by-event (running alphas #373) + + // Print parameters that are unchanged during the run + void printIndependentParameters(); + + // Print couplings that are unchanged during the run + void printIndependentCouplings(); + + // Print parameters that are changed event by event + //void printDependentParameters(); // now computed event-by-event (running alphas #373) + + // Print couplings that are changed event by event + //void printDependentCouplings(); // now computed event-by-event (running alphas #373) + + // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; + static constexpr int nBsmIndepParam = %(nbsmip)i; + %(hasbsmip)sdouble mdl_bsmIndepParam[nBsmIndepParam]; + + private: + + static Parameters* instance; + }; + +} // end namespace mg5amcGpu/mg5amcCpu + +#else%(eftwarn1)s + +#include +#include + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // Hardcoded constexpr physics parameters + namespace Parameters // keep the same name rather than HardcodedParameters for simplicity + { + // Model parameters independent of aS + constexpr double zero = 0; + constexpr double ZERO = 0; + %(hardcoded_independent_parameters)s + // Model couplings independent of aS + %(hardcoded_independent_couplings)s + // Model parameters dependent on aS + %(hardcoded_dependent_parameters)s + // Model couplings dependent on aS + %(hardcoded_dependent_couplings)s + // Print parameters that are unchanged during the run + void printIndependentParameters(); + + // Print couplings that are unchanged during the run + void printIndependentCouplings(); + + // Print parameters that are changed event by event + //void printDependentParameters(); // now computed event-by-event (running alphas #373) + + // Print couplings that are changed event by event + //void printDependentCouplings(); // now computed event-by-event (running alphas #373) + + // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; + constexpr int nBsmIndepParam = %(nbsmip)i; + %(hasbsmip)s__device__ constexpr double mdl_bsmIndepParam[nBsmIndepParam] = { %(bsmip)s }; + } + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif + +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + namespace Parameters_dependentCouplings + { + constexpr size_t ndcoup = %(ndcoup)d; // #couplings that vary event by event because they depend on the running alphas QCD +%(idcoup)s + struct DependentCouplings_sv + { +%(dcoupdecl)s + }; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" // e.g. <> +#pragma GCC diagnostic ignored "-Wunused-variable" // e.g. <> +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> +#ifdef MGONGPUCPP_GPUIMPL +#pragma nv_diagnostic push +#pragma nv_diag_suppress 177 // e.g. <> +#endif + __host__ __device__ inline const DependentCouplings_sv computeDependentCouplings_fromG( const fptype_sv& G_sv, const double* bsmIndepParamPtr ) + { +#ifdef MGONGPU_HARDCODE_PARAM + using namespace Parameters; +#else%(eftspecial0)s +#endif + // NB: hardcode cxtype cI(0,1) instead of cxtype (or hardcoded cxsmpl) mdl_complexi (which exists in Parameters) because: + // (1) mdl_complexi is always (0,1); (2) mdl_complexi is undefined in device code; (3) need cxsmpl conversion to cxtype in code below + const cxtype cI( 0., 1. ); + DependentCouplings_sv out; +#if not( defined MGONGPU_CPPSIMD && defined MGONGPU_FPTYPE_FLOAT ) + // Couplings are (scalar, or vector of) doubles, or scalar floats - default implementation + { + const fptype_sv& G = G_sv; + // Model parameters dependent on aS +%(dcoupsetdpar)s + // Model couplings dependent on aS +%(dcoupsetdcoup)s + } +#else + // Couplings are VECTORS OF FLOATS: #439 special handling is needed (variable Gs are vector floats, fixed parameters are scalar doubles) + // Use an explicit loop to avoid <> + // Problems may come e.g. in EFTs from multiplying a vector float (related to aS-dependent G) by a scalar double (aS-independent parameters) + // (NB in pure SM processes this special handling is not needed, but we keep it here for simplicity, see PR #824)%(dcoupoutfptypev2)s + for( int i = 0; i < neppV; i++ ) + { + const fptype& G = G_sv[i]; + // Model parameters dependent on aS +%(dcoupsetdpar2)s + // Model couplings dependent on aS + %(dcoupsetdcoup2)s + }%(dcoupoutdcoup2)s +#endif + return out; + } +#ifdef MGONGPUCPP_GPUIMPL +#pragma GCC diagnostic pop +#pragma nv_diagnostic pop +#endif + } + + //========================================================================== + + namespace Parameters_independentCouplings + { + constexpr size_t nicoup = %(nicoup)d; // #couplings that are fixed for all events because they do not depend on the running alphas QCD +%(iicoup)s + } + + //========================================================================== + +#pragma GCC diagnostic push +#ifndef __clang__ +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> +#endif + // Compute the output couplings (e.g. gc10 and gc11) from the input gs + template + __device__ inline void + G2COUP( const fptype gs[], + fptype couplings[], + const double* bsmIndepParamPtr ) + { + mgDebug( 0, __FUNCTION__ ); + using namespace Parameters_dependentCouplings; + const fptype_sv& gs_sv = G_ACCESS::kernelAccessConst( gs ); + DependentCouplings_sv couplings_sv = computeDependentCouplings_fromG( gs_sv, bsmIndepParamPtr ); +%(dcoupaccessbuffer)s%(dcoupkernelaccess)s%(dcoupcompute)s + mgDebug( 1, __FUNCTION__ ); + return; + } +#pragma GCC diagnostic pop + +} // end namespace mg5amcGpu/mg5amcCpu + +//========================================================================== + +#endif // Parameters_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc new file mode 100644 index 0000000000..05c3b3e0c4 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc @@ -0,0 +1,161 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Oct 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "Bridge.h" +#include "CPPProcess.h" +#include "GpuRuntime.h" + +extern "C" +{ + /** + * The namespace where the Bridge class is taken from. + * + * In the current implementation, two separate shared libraries are created for the GPU/CUDA and CPU/C++ implementations. + * Actually, two shared libraries for GPU and CPU are created for each of the five SIMD implementations on CPUs (none, sse4, avx2, 512y, 512z). + * A single fcreatebridge_ symbol is created in each library with the same name, connected to the appropriate Bridge on CPU or GPU. + * The Fortran MadEvent code is always the same: the choice whether to use a CPU or GPU implementation is done by linking the appropriate library. + * As the names of the two CPU/GPU libraries are the same in the five SIMD implementations, the choice of SIMD is done by setting LD_LIBRARY_PATH. + * + * In a future implementation, a single heterogeneous shared library may be created, with the same interface. + * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. + * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. + */ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + /** + * The floating point precision used in Fortran arrays. + * This is presently hardcoded to double precision (REAL*8). + */ + using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays + //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays + + /** + * Create a Bridge and return its pointer. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param nevtF the pointer to the number of events in the Fortran arrays + * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + */ + void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) + { +#ifdef MGONGPUCPP_GPUIMPL + GpuRuntime::setUp(); +#endif + // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) + // FIXME: disable OMP in Bridge when called from Fortran + *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); + } + + /** + * Delete a Bridge. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + */ + void fbridgedelete_( CppObjectInFortran** ppbridge ) + { + Bridge* pbridge = dynamic_cast*>( *ppbridge ); + if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); + delete pbridge; +#ifdef MGONGPUCPP_GPUIMPL + GpuRuntime::tearDown(); +#endif + } + + /** + * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant alphas) + * @param rndhel the pointer to the input random numbers for helicity selection + * @param rndcol the pointer to the input random numbers for color selection + * @param channelIds the pointer to the input channels i.e. Feynman diagrams to enhance (1 to n: 0 is an invalid value!) + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void fbridgesequence_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ) + { + Bridge* pbridge = dynamic_cast*>( *ppbridge ); + //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); + if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); +#ifdef MGONGPUCPP_GPUIMPL + // Use the device/GPU implementation in the CUDA library + // (there is also a host implementation in this library) + pbridge->gpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); +#else + // Use the host/CPU implementation in the C++ library + // (there is no device implementation in this library) + pbridge->cpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); +#endif + } + + /** + * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++, without multi-channel mode. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant alphas) + * @param rndhel the pointer to the input random numbers for helicity selection + * @param rndcol the pointer to the input random numbers for color selection + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ) + { + //printf("fbridgesequence_nomultichannel_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); + fbridgesequence_( ppbridge, momenta, gs, iflavorVec, rndhel, rndcol, nullptr, mes, selhel, selcol, pgoodHelOnly ); + } + + /** + * Retrieve the number of good helicities for helicity filtering in the Bridge. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param pngoodhel the pointer to the output number of good helicities + * @param pntothel the pointer to the output total number of helicities + */ + void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, + unsigned int* pngoodhel, + unsigned int* pntothel ) + { + Bridge* pbridge = dynamic_cast*>( *ppbridge ); + if( pbridge == 0 ) throw std::runtime_error( "fbridgegetngoodhel_: invalid Bridge address" ); + *pngoodhel = pbridge->nGoodHel(); + *pntothel = pbridge->nTotHel(); + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h new file mode 100644 index 0000000000..e676df0a33 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h @@ -0,0 +1,52 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: Z. Wettersten (Oct 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, A. Thete, A. Valassi (2025). + +#include "Bridge.h" +#include "CPPProcess.h" +#include "GpuRuntime.h" + +#ifndef _FBRIDGE_H_ +#define _FBRIDGE_H_ + +extern "C" +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + using FORTRANFPTYPE = double; + + void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ); + + void fbridgedelete_( CppObjectInFortran** ppbridge ); + + void fbridgesequence_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ); + + void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ); + + void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, unsigned int* pngoodhel, unsigned int* pntothel ); +} +#endif // _FBRIDGE_H_ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc new file mode 100644 index 0000000000..8a4468fb3c --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc @@ -0,0 +1,166 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Feb 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "mgOnGpuConfig.h" + +#include "Bridge.h" +#include "CPPProcess.h" +#include "MemoryBuffers.h" +#include "RamboSamplingKernels.h" +#include "RandomNumberKernels.h" + +//-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + template + class Sampler final : public CppObjectInFortran + { + public: + // Constructor + // @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran arrays + // @param nparF (NEXTERNAL, nexternal.inc) number of external particles in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) + // @param np4F number of momenta components, usually 4, in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) + Sampler( int nevtF, int nparF, int np4F ); + // Destructor + virtual ~Sampler() {} + // Delete copy/move constructors and assignment operators + Sampler( const Sampler& ) = delete; + Sampler( Sampler&& ) = delete; + Sampler& operator=( const Sampler& ) = delete; + Sampler& operator=( Sampler&& ) = delete; + // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta + void samplerHostSequence( FORTRANFPTYPE* fortranMomenta ); + private: + const int m_nevt; // The number of events in each iteration + int m_iiter; // The iteration counter (for random number seeding) +#ifndef MGONGPUCPP_GPUIMPL + HostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers + HostBufferMomenta m_hstMomenta; // Memory buffers for momenta + HostBufferWeights m_hstWeights; // Memory buffers for sampling weights +#else + PinnedHostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers + PinnedHostBufferMomenta m_hstMomenta; // Memory buffers for momenta + PinnedHostBufferWeights m_hstWeights; // Memory buffers for sampling weights +#endif + std::unique_ptr m_prnk; // The appropriate RandomNumberKernel + std::unique_ptr m_prsk; // The appropriate SamplingKernel + // HARDCODED DEFAULTS + static constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) + }; + + template + Sampler::Sampler( int nevtF, int nparF, int np4F ) + : m_nevt( nevtF ) + , m_iiter( 0 ) + , m_hstRndmom( nevtF ) + , m_hstMomenta( nevtF ) + , m_hstWeights( nevtF ) + , m_prnk( new CommonRandomNumberKernel( m_hstRndmom ) ) + , m_prsk( new RamboSamplingKernelHost( energy, m_hstRndmom, m_hstMomenta, m_hstWeights, nevtF ) ) + { + if( nparF != CPPProcess::npar ) throw std::runtime_error( "Sampler constructor: npar mismatch" ); + if( np4F != CPPProcess::np4 ) throw std::runtime_error( "Sampler constructor: np4 mismatch" ); + std::cout << "WARNING! Instantiate host Sampler (nevt=" << m_nevt << ")" << std::endl; + } + + // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta + template + void Sampler::samplerHostSequence( FORTRANFPTYPE* fortranMomenta ) + { + std::cout << "Iteration #" << m_iiter + 1 << std::endl; + // === STEP 1 OF 3 + // --- 1a. Seed rnd generator (to get same results on host and device in curand) + // [NB This should not be necessary using the host API: "Generation functions + // can be called multiple times on the same generator to generate successive + // blocks of results. For pseudorandom generators, multiple calls to generation + // functions will yield the same result as a single call with a large size."] + // *** NB! REMEMBER THAT THE FORTRAN SAMPLER ALWAYS USES COMMON RANDOM NUMBERS! *** + constexpr unsigned long long seed = 20200805; + m_prnk->seedGenerator( seed + m_iiter ); + m_iiter++; + // --- 1b. Generate all relevant numbers to build nevt events (i.e. nevt phase space points) on the host + m_prnk->generateRnarray(); + //std::cout << "Got random numbers" << std::endl; + // === STEP 2 OF 3 + // --- 2a. Fill in momenta of initial state particles on the device + m_prsk->getMomentaInitial(); + //std::cout << "Got initial momenta" << std::endl; + // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device + // (i.e. map random numbers to final-state particle momenta for each of nevt events) + m_prsk->getMomentaFinal(); + //std::cout << "Got final momenta" << std::endl; + // --- 2c. TransposeC2F + hst_transposeMomentaC2F( m_hstMomenta.data(), fortranMomenta, m_nevt ); + } +} + +//-------------------------------------------------------------------------- + +extern "C" +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + /** + * The floating point precision used in Fortran arrays. + * This is presently hardcoded to double precision (REAL*8). + */ + using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays + //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays + + /** + * Create a Sampler and return its pointer. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) + * @param nevtF the pointer to the number of events in the Fortran arrays + * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + */ + void fsamplercreate_( CppObjectInFortran** ppsampler, const int* pnevtF, const int* pnparF, const int* pnp4F ) + { + *ppsampler = new Sampler( *pnevtF, *pnparF, *pnp4F ); + } + + /** + * Delete a Sampler. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) + */ + void fsamplerdelete_( CppObjectInFortran** ppsampler ) + { + Sampler* psampler = dynamic_cast*>( *ppsampler ); + if( psampler == 0 ) throw std::runtime_error( "fsamplerdelete_: invalid Sampler address" ); + delete psampler; + } + + /** + * Execute the matrix-element calculation "sequence" via a Sampler on GPU/CUDA or CUDA/C++. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) + * @param momenta the pointer to the input 4-momenta + * @param mes the pointer to the output matrix elements + */ + void fsamplersequence_( CppObjectInFortran** ppsampler, FORTRANFPTYPE* momenta ) + { + Sampler* psampler = dynamic_cast*>( *ppsampler ); + if( psampler == 0 ) throw std::runtime_error( "fsamplersequence_: invalid Sampler address" ); + // Use the host/CPU implementation (there is no device implementation) + psampler->samplerHostSequence( momenta ); + } +} + +//-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h new file mode 100644 index 0000000000..bcd3990bed --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h @@ -0,0 +1,200 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: S. Roiser (Feb 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== + +#ifndef MASSLESS_RAMBO_H +#define MASSLESS_RAMBO_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +#include "CPPProcess.h" + +#include +#include +#include + +// Simplified rambo version for 2 to N (with N>=2) processes with massless particles +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +namespace massless_rambo { + + constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) + constexpr int npari = CPPProcess::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- + constexpr int nparf = CPPProcess::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- + constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- + + //-------------------------------------------------------------------------- + + // Fill in the momenta of the initial particles + // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] + template + __host__ __device__ void + ramboGetMomentaInitial( const fptype energy, // input: energy + fptype* momenta ) // output: momenta for one event or for a set of events + { + const fptype energy1 = energy / 2; + const fptype energy2 = energy / 2; + const fptype mom = energy / 2; + M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 0 ) = energy1; + M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 0 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 0 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 0 ) = mom; + M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 1 ) = energy2; + M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 1 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 1 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 1 ) = -mom; + } + + //-------------------------------------------------------------------------- + + // Fill in the momenta of the final particles using the RAMBO algorithm + // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] + template + __host__ __device__ void + ramboGetMomentaFinal( const fptype energy, // input: energy + const fptype* rndmom, // input: random numbers in [0,1] for one event or for a set of events + fptype* momenta, // output: momenta for one event or for a set of events + fptype* wgts ) // output: weights for one event or for a set of events + { + /**************************************************************************** + * rambo * + * ra(ndom) m(omenta) b(eautifully) o(rganized) * + * * + * a democratic multi-particle phase space generator * + * authors: s.d. ellis, r. kleiss, w.j. stirling * + * this is version 1.0 - written by r. kleiss * + * -- adjusted by hans kuijf, weights are logarithmic (1990-08-20) * + * -- adjusted by madgraph@sheffield_gpu_hackathon team (2020-07-29) * + * * + ****************************************************************************/ + + // output weight + fptype& wt = W_ACCESS::kernelAccess( wgts ); + + // AV special case nparf==1 (issue #358) + if constexpr( nparf == 1 ) + { + static bool first = true; + if( first ) + { +#ifdef MGONGPUCPP_GPUIMPL + if constexpr( M_ACCESS::isOnDevice() ) // avoid + { + const int ievt0 = 0; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid + if( ievt == ievt0 ) + printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); + } + else +#endif + { + printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); + } + first = false; + } + const int iparf = 0; + for( int i4 = 0; i4 < np4; i4++ ) + { + M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = 0; + for( int ipari = 0; ipari < npari; ipari++ ) + { + M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) += M_ACCESS::kernelAccessIp4Ipar( momenta, i4, ipari ); + } + } + wt = 1; + return; + } + + // initialization step: factorials for the phase space weight + const fptype twopi = 8. * atan( 1. ); + const fptype po2log = log( twopi / 4. ); + fptype z[nparf]; + if constexpr( nparf > 1 ) // avoid build warning on clang (related to #358) + z[1] = po2log; + for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = z[kpar - 1] + po2log - 2. * log( fptype( kpar - 1 ) ); + for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = ( z[kpar] - log( fptype( kpar ) ) ); + + // generate n massless momenta in infinite phase space + fptype q[nparf][np4]; + for( int iparf = 0; iparf < nparf; iparf++ ) + { + const fptype r1 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 0, iparf ); + const fptype r2 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 1, iparf ); + const fptype r3 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 2, iparf ); + const fptype r4 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 3, iparf ); + const fptype c = 2. * r1 - 1.; + const fptype s = sqrt( 1. - c * c ); + const fptype f = twopi * r2; + q[iparf][0] = -log( r3 * r4 ); + q[iparf][3] = q[iparf][0] * c; + q[iparf][2] = q[iparf][0] * s * cos( f ); + q[iparf][1] = q[iparf][0] * s * sin( f ); + } + + // calculate the parameters of the conformal transformation + fptype r[np4]; + fptype b[np4 - 1]; + for( int i4 = 0; i4 < np4; i4++ ) r[i4] = 0.; + for( int iparf = 0; iparf < nparf; iparf++ ) + { + for( int i4 = 0; i4 < np4; i4++ ) r[i4] = r[i4] + q[iparf][i4]; + } + const fptype rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); + for( int i4 = 1; i4 < np4; i4++ ) b[i4 - 1] = -r[i4] / rmas; + const fptype g = r[0] / rmas; + const fptype a = 1. / ( 1. + g ); + const fptype x0 = energy / rmas; + + // transform the q's conformally into the p's (i.e. the 'momenta') + for( int iparf = 0; iparf < nparf; iparf++ ) + { + fptype bq = b[0] * q[iparf][1] + b[1] * q[iparf][2] + b[2] * q[iparf][3]; + for( int i4 = 1; i4 < np4; i4++ ) + { + M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = x0 * ( q[iparf][i4] + b[i4 - 1] * ( q[iparf][0] + a * bq ) ); + } + M_ACCESS::kernelAccessIp4Ipar( momenta, 0, iparf + npari ) = x0 * ( g * q[iparf][0] + bq ); + } + + // calculate weight (NB return log of weight) + wt = po2log; + if( nparf != 2 ) wt = ( 2. * nparf - 4. ) * log( energy ) + z[nparf - 1]; + +#ifndef MGONGPUCPP_GPUIMPL + // issue warnings if weight is too small or too large + static int iwarn[5] = { 0, 0, 0, 0, 0 }; + if( wt < -180. ) + { + if( iwarn[0] <= 5 ) std::cout << "Too small wt, risk for underflow: " << wt << std::endl; + iwarn[0] = iwarn[0] + 1; + } + if( wt > 174. ) + { + if( iwarn[1] <= 5 ) std::cout << "Too large wt, risk for overflow: " << wt << std::endl; + iwarn[1] = iwarn[1] + 1; + } +#endif + + // return for weighted massless momenta + // nothing else to do in this event if all particles are massless (nm==0) + + return; + } + + //-------------------------------------------------------------------------- +} +} + +#endif // MASSLESS_RAMBO_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h new file mode 100644 index 0000000000..09c76a6bd5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h @@ -0,0 +1,296 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jul 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCONFIG_H +#define MGONGPUCONFIG_H 1 + +// Is this a GPU (CUDA, HIP) or CPU implementation? +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#define MGONGPUCPP_GPUIMPL cuda +#elif defined __HIPCC__ +#define MGONGPUCPP_GPUIMPL hip +#include "hip/hip_runtime.h" // needed for blockDim, blockIdx, threadIdx: better in mgOnGpuConfig.h than in GpuAbstraction.h +#else +#undef MGONGPUCPP_GPUIMPL +#endif + +// Make sure that __HIP_PLATFORM_NVIDIA__ is undefined +// (__HIP_PLATFORM_AMD__ is defined by hipcc or in HiprandRandomNumberKernel.cc) +#undef __HIP_PLATFORM_NVIDIA__ // disable hiprand for NVidia (curand) + +// ** NB1 Throughputs (e.g. 6.8E8) are events/sec for "./gcheck.exe -p 65536 128 12" +// ** NB2 Baseline on b7g47n0004 fluctuates (probably depends on load on other VMs) + +// Choose if curand is supported for generating random numbers +// For HIP, by default, do not allow curand to be used (hiprand or common random numbers will be used instead) +// For both CUDA and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_CURAND +// (there exist CUDA installations, e.g. using the HPC package, which do not include curand - see PR #784 and #785) +#if defined __HIPCC__ +#define MGONGPU_HAS_NO_CURAND 1 +#else +//#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#undef MGONGPU_HAS_NO_CURAND // default +////#define MGONGPU_HAS_NO_CURAND 1 +//#else +//#undef MGONGPU_HAS_NO_CURAND // default +////#define MGONGPU_HAS_NO_CURAND 1 +//#endif +#endif + +// Choose if hiprand is supported for generating random numbers +// For CUDA, by default, do not allow hiprand to be used (curand or common random numbers will be used instead) +// For both HIP and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_HIPRAND +// (there may exist HIP installations which do not include hiprand?) +#if defined __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#define MGONGPU_HAS_NO_HIPRAND 1 +#else +//#ifdef __HIPCC__ +//#undef MGONGPU_HAS_NO_HIPRAND // default +////#define MGONGPU_HAS_NO_HIPRAND 1 +//#else +//#undef MGONGPU_HAS_NO_HIPRAND // default +////#define MGONGPU_HAS_NO_HIPRAND 1 +//#endif +#endif + +// Choose floating point precision (for everything but color algebra #537) +// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE_FLOAT, nothing happens (issue #167) +#if not defined MGONGPU_FPTYPE_DOUBLE and not defined MGONGPU_FPTYPE_FLOAT +// Floating point precision (CHOOSE ONLY ONE) +#define MGONGPU_FPTYPE_DOUBLE 1 // default +//#define MGONGPU_FPTYPE_FLOAT 1 // 2x faster +#endif + +// Choose floating point precision (for color algebra alone #537) +// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE2_FLOAT, nothing happens (issue #167) +#if not defined MGONGPU_FPTYPE2_DOUBLE and not defined MGONGPU_FPTYPE2_FLOAT +// Floating point precision (CHOOSE ONLY ONE) +#define MGONGPU_FPTYPE2_DOUBLE 1 // default +//#define MGONGPU_FPTYPE2_FLOAT 1 // 2x faster +#endif + +// Choose whether to inline all HelAmps functions +// This optimization can gain almost a factor 4 in C++, similar to -flto (issue #229) +// By default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_INLINE_HELAMPS +//#undef MGONGPU_INLINE_HELAMPS // default +////#define MGONGPU_INLINE_HELAMPS 1 + +// Choose whether to hardcode the cIPD physics parameters rather than reading them from user cards +// This optimization can gain 20%% in CUDA in eemumu (issue #39) +// By default, do not hardcode, but allow this macro to be set from outside with e.g. -DMGONGPU_HARDCODE_PARAM +// ** NB: The option to use hardcoded cIPD physics parameters is supported again even now when alphas is running (#373) +// ** NB: Note however that it now only refers to cIPD parameters (cIPC parameters are always accessed through global memory) +//#undef MGONGPU_HARDCODE_PARAM // default +////#define MGONGPU_HARDCODE_PARAM 1 + +/* clang-format off */ +// Complex type in CUDA: thrust or cucomplex or cxsmpl (CHOOSE ONLY ONE) +// (NB THIS IS MGONGPU_*CU*CXTYPE_xxx) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#define MGONGPU_CUCXTYPE_THRUST 1 // default (~1.15E9/double, ~3.2E9/float) +//#define MGONGPU_CUCXTYPE_CUCOMPLEX 1 // ~10 percent slower (1.03E9/double, ~2.8E9/float) +//#define MGONGPU_CUCXTYPE_CXSMPL 1 // ~10 percent slower (1.00E9/double, ~2.9E9/float) + +// Complex type in HIP: cxsmpl (ONLY ONE OPTION POSSIBLE? #810) +// (NB THIS IS MGONGPU_*HIP*CXTYPE_xxx) +#elif defined __HIPCC__ +#define MGONGPU_HIPCXTYPE_CXSMPL 1 // default for HIP + +// Complex type in C++: std::complex or cxsmpl (CHOOSE ONLY ONE) +// (NB THIS IS MGONGPU_*CPP*CXTYPE_xxx) +#else +//#define MGONGPU_CPPCXTYPE_STDCOMPLEX 1 // ~8 percent slower on float, same on double (5.1E6/double, 9.4E6/float) +#define MGONGPU_CPPCXTYPE_CXSMPL 1 // new default (5.1E6/double, 10.2E6/float) +#endif + +// Choose if cuBLAS and hipBLAS are supported for generating random numbers +// For both CUDA and HIP, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_BLAS +// (there may exist CUDA/HIP installations, e.g. using the HPC package, which do not include cuBLAS/hipBLAS?) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#undef MGONGPU_HAS_NO_BLAS // default +////#define MGONGPU_HAS_NO_BLAS 1 +#elif defined __HIPCC__ +//#undef MGONGPU_HAS_NO_BLAS // default +////#define MGONGPU_HAS_NO_BLAS 1 +#else +#define MGONGPU_HAS_NO_BLAS 1 +#endif + +// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#undef MGONGPU_NSIGHT_DEBUG // default in CUDA +//#define MGONGPU_NSIGHT_DEBUG 1 // CURRENTLY NO LONGER SUPPORTED! +#else +#undef MGONGPU_NSIGHT_DEBUG // only option in HIP or C++ +#endif /* clang-format on */ + +// SANITY CHECKS (floating point precision for everything but color algebra #537) +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE_DOUBLE or defined MGONGPU_FPTYPE_FLOAT +#endif + +// SANITY CHECKS (floating point precision for color algebra alone #537) +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE2_DOUBLE or defined MGONGPU_FPTYPE2_FLOAT +#endif +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You cannot use double precision for color algebra and single precision elsewhere +#endif + +// SANITY CHECKS (CUDA complex number implementation) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CUCOMPLEX +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX for CUDA +#elif defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CXSMPL for CUDA +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX and defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE OF MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL for CUDA +#endif +#endif + +// SANITY CHECKS (C++ complex number implementation) +#ifndef MGONGPUCPP_GPUIMPL +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX and defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL for C++ +#endif +#endif + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu +{ + + // --- Type definitions + + // Floating point type (for everything but color algebra #537): fptype +#if defined MGONGPU_FPTYPE_DOUBLE + typedef double fptype; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE_FLOAT + typedef float fptype; // single precision (4 bytes, fp32) +#endif + + // Floating point type (for color algebra alone #537): fptype2 +#if defined MGONGPU_FPTYPE2_DOUBLE + typedef double fptype2; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE2_FLOAT + typedef float fptype2; // single precision (4 bytes, fp32) +#endif + + // --- Platform-specific software implementation details + + // Maximum number of blocks per grid + // ** NB Some arrays of pointers will be allocated statically to fit all these blocks + // ** (the actual memory for each block will then be allocated dynamically only for existing blocks) + //const int nbpgMAX = 2048; + + // Maximum number of threads per block + //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? + const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" + + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) + // Only needed for C++ code but can be enforced also in NVCC builds of C++ code using CUDA>=11.2 and C++17 (#318, #319, #333) +#ifndef MGONGPUCPP_GPUIMPL + constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) +#endif + +} + +// Expose typedefs and operators outside the namespace +using mgOnGpu::fptype; +using mgOnGpu::fptype2; + +// Undefine ARM_NEON (hack for cppnone on Apple silicon ARM) +#ifdef MGONGPU_NOARMNEON +#undef __ARM_NEON +#endif + +// C++ SIMD vectorization width (this will be used to set neppV) +#ifdef MGONGPUCPP_GPUIMPL // CUDA and HIP implementations have no SIMD +#undef MGONGPU_CPPSIMD +#elif defined __AVX512VL__ && defined MGONGPU_PVW512 // C++ "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 8 +#else +#define MGONGPU_CPPSIMD 16 +#endif +#elif defined __AVX512VL__ // C++ "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif +#elif defined __AVX2__ // C++ "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif +#elif defined __SSE4_2__ // C++ "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [Power9 default] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 +#else +#define MGONGPU_CPPSIMD 4 +#endif +#elif defined __ARM_NEON // C++ "sse4" ARM NEON (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [ARM default] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 +#else +#define MGONGPU_CPPSIMD 4 +#endif +#else // C++ "none" i.e. no SIMD +#undef MGONGPU_CPPSIMD +#endif + +/* clang-format off */ +// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation [NB: CURRENTLY NO LONGER SUPPORTED!] +// Arguments (not used so far): text is __FUNCTION__, code is 0 (start) or 1 (end) +//#if defined __CUDACC__ && defined MGONGPU_NSIGHT_DEBUG // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#define mgDebugDeclare() __shared__ float mgDebugCounter[mgOnGpu::ntpbMAX]; +//#define mgDebugInitialise() { mgDebugCounter[threadIdx.x] = 0; } +//#define mgDebug( code, text ) { mgDebugCounter[threadIdx.x] += 1; } +//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%%f\n", mgDebugCounter[threadIdx.x] ); } +//#else +#define mgDebugDeclare() /*noop*/ +#define mgDebugInitialise() /*noop*/ +#define mgDebug( code, text ) /*noop*/ +#define mgDebugFinalise() /*noop*/ +//#endif /* clang-format on */ + +// Define empty CUDA/HIP declaration specifiers for C++ +#ifndef MGONGPUCPP_GPUIMPL +#define __global__ +#define __host__ +#define __device__ +#endif + +// For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two (https://stackoverflow.com/a/108360) +inline constexpr bool +ispoweroftwo( int n ) +{ + return ( n > 0 ) && !( n & ( n - 1 ) ); +} + +// Compiler version support (#96): require nvcc from CUDA >= 11.2, e.g. to use C++17 (see #333) +#ifdef __NVCC__ +#if( __CUDACC_VER_MAJOR__ < 11 ) || ( __CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ < 2 ) +#error Unsupported CUDA version: please use CUDA >= 11.2 +#endif +#endif + +// Compiler version support (#96): require clang >= 11 +#if defined __clang__ +#if( __clang_major__ < 11 ) +#error Unsupported clang version: please use clang >= 11 +#endif +// Compiler version support (#96): require gcc >= 9.3, e.g. for some OMP issues (see #269) +// [NB skip this check for the gcc toolchain below clang or icx (TEMPORARY? #355)] +#elif defined __GNUC__ +#if( __GNUC__ < 9 ) || ( __GNUC__ == 9 && __GNUC_MINOR__ < 3 ) +#error Unsupported gcc version: please gcc >= 9.3 +#endif +#endif + +#endif // MGONGPUCONFIG_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h new file mode 100644 index 0000000000..d6e1d02e05 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h @@ -0,0 +1,745 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022, based on earlier work by D. Smith) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCXTYPES_H +#define MGONGPUCXTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +#include + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) HEADERS +//========================================================================== + +#include + +// Complex type in cuda: thrust or cucomplex or cxsmpl +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_THRUST +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) +#include +#pragma clang diagnostic pop +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX +#include +#elif not defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL +#endif +// Complex type in HIP: cxsmpl +#elif defined __HIPCC__ +#if not defined MGONGPU_HIPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_HIPCXTYPE_CXSMPL +#endif +#else +// Complex type in c++ or HIP: std::complex or cxsmpl +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX +#include +#elif not defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL +#endif +#endif + +//========================================================================== +// COMPLEX TYPES: INSTRUMENTED CUCOMPLEX CLASS (cucomplex) +//========================================================================== + +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_CUCOMPLEX +namespace mg5amcGpu +{ +#if defined MGONGPU_FPTYPE_DOUBLE + class cucomplex + { + public: + __host__ __device__ cucomplex( const double& r = 0, const double& i = 0 ) + : m_ri( make_cuDoubleComplex( r, i ) ) {} + __host__ __device__ constexpr cucomplex( const cuDoubleComplex& ri ) + : m_ri( ri ) {} + //__host__ __device__ operator cuDoubleComplex&() { return m_ri; } + __host__ __device__ constexpr operator cuDoubleComplex() const { return m_ri; } + __host__ __device__ double real() const { return cuCreal( m_ri ); } + __host__ __device__ double imag() const { return cuCimag( m_ri ); } + inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) + { + m_ri = cuCadd( m_ri, c ); + return *this; + } + inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) + { + m_ri = cuCsub( m_ri, c ); + return *this; + } + private: + cuDoubleComplex m_ri; + }; +#elif defined MGONGPU_FPTYPE_FLOAT + class cucomplex + { + public: + __host__ __device__ cucomplex( const float& r = 0, const float& i = 0 ) + : m_ri( make_cuFloatComplex( r, i ) ) {} + __host__ __device__ constexpr cucomplex( const cuFloatComplex& ri ) + : m_ri( ri ) {} + //__host__ __device__ operator cuFloatComplex&() { return m_ri; } + __host__ __device__ constexpr operator cuFloatComplex() const { return m_ri; } + __host__ __device__ float real() const { return cuCrealf( m_ri ); } + __host__ __device__ float imag() const { return cuCimagf( m_ri ); } + inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) + { + m_ri = cuCaddf( m_ri, c ); + return *this; + } + inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) + { + m_ri = cuCsubf( m_ri, c ); + return *this; + } + private: + cuFloatComplex m_ri; + }; +#endif +} +#endif +#endif + +//========================================================================== +// COMPLEX TYPES: SIMPLE COMPLEX CLASS (cxsmpl) +//========================================================================== + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu /* clang-format off */ +{ + // The number of floating point types in a complex type (real, imaginary) + constexpr int nx2 = 2; + + // --- Type definition (simple complex type derived from cxtype_v) + template + class cxsmpl + { + public: + __host__ __device__ constexpr cxsmpl() : m_real( 0 ), m_imag( 0 ) {} + cxsmpl( const cxsmpl& ) = default; + cxsmpl( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl( const FP& r, const FP& i = 0 ) : m_real( r ), m_imag( i ) {} + __host__ __device__ constexpr cxsmpl( const std::complex& c ) : m_real( c.real() ), m_imag( c.imag() ) {} + cxsmpl& operator=( const cxsmpl& ) = default; + cxsmpl& operator=( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl& operator+=( const cxsmpl& c ) { m_real += c.real(); m_imag += c.imag(); return *this; } + __host__ __device__ constexpr cxsmpl& operator-=( const cxsmpl& c ) { m_real -= c.real(); m_imag -= c.imag(); return *this; } + __host__ __device__ constexpr const FP& real() const { return m_real; } + __host__ __device__ constexpr const FP& imag() const { return m_imag; } + template __host__ __device__ constexpr operator cxsmpl() const { return cxsmpl( m_real, m_imag ); } +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef MGONGPU_CUCXTYPE_THRUST + template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX + __host__ __device__ constexpr operator mg5amcGpu::cucomplex() const { return mg5amcGpu::cucomplex( m_real, m_imag ); } +#endif +#else +#ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX + template __host__ __device__ constexpr operator std::complex() const { return std::complex( m_real, m_imag ); } +#endif +#endif + private: + FP m_real, m_imag; // RI + }; + + template + constexpr // (NB: now valid code? in the past this failed as "a constexpr function cannot have a nonliteral return type mgOnGpu::cxsmpl") + inline __host__ __device__ cxsmpl + conj( const cxsmpl& c ) + { + return cxsmpl( c.real(), -c.imag() ); + } +} /* clang-format on */ + +// Expose the cxsmpl class outside the namespace +using mgOnGpu::cxsmpl; + +// Printout to stream for user defined types +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + template + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxsmpl& c ) + { + //out << std::complex( c.real(), c.imag() ); + out << "(" << c.real() << ", " << c.imag() << ")"; // add a space after the comma + return out; + } + + // Operators for cxsmpl + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl a ) + { + return a; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a ) + { + return cxsmpl( -a.real(), -a.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() + b.real(), a.imag() + b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) + b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() - b.real(), a.imag() - b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) - b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const double& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const double& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const cxsmpl& b ) + { + FP bnorm = b.real() * b.real() + b.imag() * b.imag(); + return cxsmpl( ( a.real() * b.real() + a.imag() * b.imag() ) / bnorm, + ( a.imag() * b.real() - a.real() * b.imag() ) / bnorm ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) / b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const FP& b ) + { + return a + cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const FP& b ) + { + return a - cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const FP& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const FP& b ) + { + return a / cxsmpl( b, 0 ); + } +} + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // --- Type definitions (complex type: cxtype) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_THRUST + typedef thrust::complex cxtype; +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX + typedef cucomplex cxtype; +#else + typedef cxsmpl cxtype; +#endif +#else // c++ +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX + typedef std::complex cxtype; +#else + typedef cxsmpl cxtype; +#endif +#endif + + // SANITY CHECK: memory access may be based on casts of fptype[2] to cxtype (e.g. for wavefunctions) + static_assert( sizeof( cxtype ) == mgOnGpu::nx2 * sizeof( fptype ), "sizeof(cxtype) is not 2*sizeof(fptype)" ); +} + +// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) +// DO NOT expose typedefs and operators outside the namespace +//using mgOnGpu::cxtype; + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL + + //------------------------------ + // CUDA or C++ - using cxsmpl + //------------------------------ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // cxsmpl constructor + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return c.real(); // cxsmpl::real() + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return c.imag(); // cxsmpl::imag() + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( cxsmpl ) + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +#endif // #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL + + //========================================================================== + +#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST // cuda + thrust (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) + + //------------------------------ + // CUDA - using thrust::complex + //------------------------------ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // thrust::complex constructor + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return c.real(); // thrust::complex::real() + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return c.imag(); // thrust::complex::imag() + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( thrust::complex ) + } + + inline __host__ __device__ const cxtype& + cxmake( const cxtype& c ) + { + return c; + } + +#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST + + //========================================================================== + +#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX // cuda + cucomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) + + //------------------------------ + // CUDA - using cuComplex + //------------------------------ + +#if defined MGONGPU_FPTYPE_DOUBLE // cuda + cucomplex + double + + //+++++++++++++++++++++++++ + // cuDoubleComplex ONLY + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return make_cuDoubleComplex( r, i ); + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return cuCreal( c ); // returns by value + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return cuCimag( c ); // returns by value + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const cxtype& b ) + { + return cuCadd( a, b ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const cxtype& b ) + { + return cuCsub( a, b ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const cxtype& b ) + { + return cuCmul( a, b ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const cxtype& b ) + { + return cuCdiv( a, b ); + } + + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + //out << std::complex( cxreal( c ), cximag( c ) ); + out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma + return out; + } + +#elif defined MGONGPU_FPTYPE_FLOAT // cuda + cucomplex + float + + //+++++++++++++++++++++++++ + // cuFloatComplex ONLY + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return make_cuFloatComplex( r, i ); + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return cuCrealf( c ); // returns by value + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return cuCimagf( c ); // returns by value + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const cxtype& b ) + { + return cuCaddf( a, b ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const cxtype& b ) + { + return cuCsubf( a, b ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const cxtype& b ) + { + return cuCmulf( a, b ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const cxtype& b ) + { + return cuCdivf( a, b ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cucomplex (cast double-to-float) + { + return cxmake( (fptype)c.real(), (fptype)c.imag() ); + } + + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + //out << std::complex( cxreal( c ), cximag( c ) ); + out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma + return out; + } + +#endif + + //+++++++++++++++++++++++++ + // cuDoubleComplex OR + // cuFloatComplex + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + operator+( const cxtype a ) + { + return a; + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a ) + { + return cxmake( -cxreal( a ), -cximag( a ) ); + } + + inline __host__ __device__ cxtype + operator+( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) + b; + } + + inline __host__ __device__ cxtype + operator-( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) - b; + } + + inline __host__ __device__ cxtype + operator*( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) * b; + } + + inline __host__ __device__ cxtype + operator/( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) / b; + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const fptype& b ) + { + return a + cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const fptype& b ) + { + return a - cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const fptype& b ) + { + return a * cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const fptype& b ) + { + return a / cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return cxmake( cxreal( c ), -cximag( c ) ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cucomplex (float-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX + + //========================================================================== + +#if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX // c++/hip + stdcomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) + + //------------------------------ + // C++ - using std::complex + //------------------------------ + + inline cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // std::complex constructor + } + + inline fptype + cxreal( const cxtype& c ) + { + return c.real(); // std::complex::real() + } + + inline fptype + cximag( const cxtype& c ) + { + return c.imag(); // std::complex::imag() + } + + inline cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( std::complex ) + } + + inline const cxtype& + cxmake( const cxtype& c ) // std::complex to std::complex (float-to-float or double-to-double) + { + return c; + } + +#if defined MGONGPU_FPTYPE_FLOAT + inline cxtype + cxmake( const std::complex& c ) // std::complex to std::complex (cast double-to-float) + { + return cxmake( (fptype)c.real(), (fptype)c.imag() ); + } +#endif + +#endif // #if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX + + //========================================================================== + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +} // end namespace mg5amcGpu/mg5amcCpu + +//========================================================================== +// COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] + // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined + // It is now always included in the code because it is needed also to access an fptype wavefunction buffer as a cxtype + class cxtype_ref + { + public: + cxtype_ref() = delete; + cxtype_ref( const cxtype_ref& ) = delete; + cxtype_ref( cxtype_ref&& ) = default; // copy const refs + __host__ __device__ cxtype_ref( fptype& r, fptype& i ) + : m_preal( &r ), m_pimag( &i ) {} // copy (create from) const refs + cxtype_ref& operator=( const cxtype_ref& ) = delete; + //__host__ __device__ cxtype_ref& operator=( cxtype_ref&& c ) {...} // REMOVED! Should copy refs or copy values? No longer needed in cxternary + __host__ __device__ cxtype_ref& operator=( const cxtype& c ) + { + *m_preal = cxreal( c ); + *m_pimag = cximag( c ); + return *this; + } // copy (assign) non-const values + __host__ __device__ operator cxtype() const { return cxmake( *m_preal, *m_pimag ); } + private: + fptype* const m_preal; // const pointer to non-const fptype R + fptype* const m_pimag; // const pointer to non-const fptype I + }; + + // Printout to stream for user defined types + inline __host__ __device__ std::ostream& + operator<<( std::ostream& out, const cxtype_ref& c ) + { + out << (cxtype)c; + return out; + } + +} // end namespace mg5amcGpu/mg5amcCpu + +//========================================================================== + +#endif // MGONGPUCXTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h new file mode 100644 index 0000000000..a2489d05d5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h @@ -0,0 +1,102 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUFPTYPES_H +#define MGONGPUFPTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include +#include + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL // cuda +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //========================================================================== + +#ifdef MGONGPUCPP_GPUIMPL // cuda + + //------------------------------ + // Floating point types - Cuda + //------------------------------ + + /* + inline __host__ __device__ fptype + fpmax( const fptype& a, const fptype& b ) + { + return max( a, b ); + } + + inline __host__ __device__ fptype + fpmin( const fptype& a, const fptype& b ) + { + return min( a, b ); + } + */ + + inline __host__ __device__ const fptype& + fpmax( const fptype& a, const fptype& b ) + { + return ( ( b < a ) ? a : b ); + } + + inline __host__ __device__ const fptype& + fpmin( const fptype& a, const fptype& b ) + { + return ( ( a < b ) ? a : b ); + } + + inline __host__ __device__ fptype + fpsqrt( const fptype& f ) + { +#if defined MGONGPU_FPTYPE_FLOAT + // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__SINGLE.html + return sqrtf( f ); +#else + // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__DOUBLE.html + return sqrt( f ); +#endif + } + +#endif // #ifdef MGONGPUCPP_GPUIMPL + + //========================================================================== + +#ifndef MGONGPUCPP_GPUIMPL + + //------------------------------ + // Floating point types - C++ + //------------------------------ + + inline const fptype& + fpmax( const fptype& a, const fptype& b ) + { + return std::max( a, b ); + } + + inline const fptype& + fpmin( const fptype& a, const fptype& b ) + { + return std::min( a, b ); + } + + inline fptype + fpsqrt( const fptype& f ) + { + return std::sqrt( f ); + } + +#endif // #ifndef MGONGPUCPP_GPUIMPL + + //========================================================================== + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h new file mode 100644 index 0000000000..74d93f05a3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h @@ -0,0 +1,932 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Nov 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi, Z. Wettersten (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUVECTORS_H +#define MGONGPUVECTORS_H 1 + +#include "mgOnGpuCxtypes.h" +#include "mgOnGpuFptypes.h" + +#include + +//========================================================================== + +//------------------------------ +// Vector types - C++ +//------------------------------ + +#ifdef __clang__ +// If set: return a pair of (fptype&, fptype&) by non-const reference in cxtype_v::operator[] +// This is forbidden in clang ("non-const reference cannot bind to vector element") +// See also https://stackoverflow.com/questions/26554829 +//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // clang test (compilation fails also on clang 12.0, issue #182) +#undef MGONGPU_HAS_CPPCXTYPEV_BRK // clang default +#elif defined __INTEL_COMPILER +//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // icc default? +#undef MGONGPU_HAS_CPPCXTYPEV_BRK // icc test +#else +#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // gcc default +//#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) +#endif + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#ifdef MGONGPU_CPPSIMD + + const int neppV = MGONGPU_CPPSIMD; + + // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) + static_assert( mgOnGpu::cppAlign % ( neppV * sizeof( fptype ) ) == 0 ); + + // SANITY CHECK: check that neppV is a power of two + static_assert( ispoweroftwo( neppV ), "neppV is not a power of 2" ); + + // --- Type definition (using vector compiler extensions: need -march=...) + // For gcc: https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html + // For clang: https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors +#ifdef __clang__ + typedef fptype fptype_v __attribute__( ( ext_vector_type( neppV ) ) ); // RRRR +#else + typedef fptype fptype_v __attribute__( ( vector_size( neppV * sizeof(fptype) ), aligned( neppV * sizeof(fptype) ) ) ); // RRRR +#endif + + // Mixed fptypes #537: float for color algebra and double elsewhere +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int neppV2 = MGONGPU_CPPSIMD * 2; + static_assert( mgOnGpu::cppAlign % ( neppV2 * sizeof( fptype2 ) ) == 0 ); + static_assert( ispoweroftwo( neppV2 ), "neppV2 is not a power of 2" ); +#ifdef __clang__ + typedef fptype2 fptype2_v __attribute__( ( ext_vector_type( neppV2 ) ) ); // RRRRRRRR +#else + typedef fptype2 fptype2_v __attribute__( ( vector_size( neppV2 * sizeof( fptype2 ) ), aligned( neppV2 * sizeof( fptype2 ) ) ) ); // RRRRRRRR +#endif +#else + typedef fptype_v fptype2_v; +#endif + + // --- Type definition (using vector compiler extensions: need -march=...) + class cxtype_v // no need for "class alignas(2*sizeof(fptype_v)) cxtype_v" + { + public: + // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) + // See https://en.cppreference.com/w/c/language/array_initialization#Notes + cxtype_v() + : m_real{ 0 }, m_imag{ 0 } {} // RRRR=0000 IIII=0000 + cxtype_v( const cxtype_v& ) = default; + cxtype_v( cxtype_v&& ) = default; + cxtype_v( const fptype_v& r, const fptype_v& i ) + : m_real( r ), m_imag( i ) {} + cxtype_v( const fptype_v& r ) + : m_real( r ), m_imag{ 0 } {} // IIII=0000 + cxtype_v( const fptype& r ) + : m_real( fptype_v{} + r ), m_imag{ 0 } {} // IIII=0000 + cxtype_v& operator=( const cxtype_v& ) = default; + cxtype_v& operator=( cxtype_v&& ) = default; + cxtype_v& operator+=( const cxtype_v& c ) + { + m_real += c.real(); + m_imag += c.imag(); + return *this; + } + cxtype_v& operator-=( const cxtype_v& c ) + { + m_real -= c.real(); + m_imag -= c.imag(); + return *this; + } +#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK + // NB: THIS IS THE FUNDAMENTAL DIFFERENCE BETWEEN MGONGPU_HAS_CPPCXTYPEV_BRK DEFINED AND NOT DEFINED + // NB: the alternative "clang" implementation is simpler: it simply does not have any bracket operator[] + //cxtype_ref operator[]( size_t i ) const { return cxtype_ref( m_real[i], m_imag[i] ); } // gcc14.2 build fails #1004 + cxtype_ref operator[]( size_t i ) { return cxtype_ref( m_real[i], m_imag[i] ); } + cxtype operator[]( size_t i ) const { return cxtype( m_real[i], m_imag[i] ); } +#endif + const fptype_v& real() const + { + return m_real; + } + const fptype_v& imag() const { return m_imag; } + private: + fptype_v m_real, m_imag; // RRRRIIII + }; + + // --- Type definition (using vector compiler extensions: need -march=...) +#ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors + typedef unsigned int uint_v __attribute__( ( ext_vector_type( neppV ) ) ); +#if defined MGONGPU_FPTYPE_DOUBLE + typedef long int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb +#elif defined MGONGPU_FPTYPE_FLOAT + typedef int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb +#endif +#else // gcc + typedef unsigned int uint_v __attribute__( ( vector_size( neppV * sizeof( unsigned int ) ), aligned( neppV * sizeof( unsigned int ) ) ) ); +#if defined MGONGPU_FPTYPE_DOUBLE + typedef long int bool_v __attribute__( ( vector_size( neppV * sizeof( long int ) ), aligned( neppV * sizeof( long int ) ) ) ); // bbbb +#elif defined MGONGPU_FPTYPE_FLOAT + typedef int bool_v __attribute__( ( vector_size( neppV * sizeof( int ) ), aligned( neppV * sizeof( int ) ) ) ); // bbbb +#endif +#endif + +#else // i.e #ifndef MGONGPU_CPPSIMD (this includes #ifdef MGONGPUCPP_GPUIMPL) + + const int neppV = 1; + +#endif // #ifdef MGONGPU_CPPSIMD +} + +//-------------------------------------------------------------------------- + +// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) +// DO NOT expose typedefs outside the namespace +//using mgOnGpu::neppV; +//#ifdef MGONGPU_CPPSIMD +//using mgOnGpu::fptype_v; +//using mgOnGpu::fptype2_v; +//using mgOnGpu::cxtype_v; +//using mgOnGpu::bool_v; +//#endif + +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#ifndef MGONGPUCPP_GPUIMPL + + // Printout to stream for user defined types + +#ifndef MGONGPU_CPPCXTYPE_CXSMPL // operator<< for cxsmpl has already been defined! + inline std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + out << "[" << cxreal( c ) << "," << cximag( c ) << "]"; + //out << cxreal(c) << "+i" << cximag(c); + return out; + } +#endif + + /* +#ifdef MGONGPU_CPPSIMD + inline std::ostream& + operator<<( std::ostream& out, const bool_v& v ) + { + out << "{ " << v[0]; + for ( int i=1; i 0 ) outi = fpsqrt( (fptype)v[i] ); + out[i] = outi; + } + return out; + } + + inline fptype_v + fpsqrt( const fptype_v& v ) + { + // See https://stackoverflow.com/questions/18921049/gcc-vector-extensions-sqrt + fptype_v out = {}; // avoid warning 'out' may be used uninitialized: see #594 + for( int i = 0; i < neppV; i++ ) out[i] = fpsqrt( v[i] ); + return out; + } +#endif + + /* +#ifdef MGONGPU_CPPSIMD + inline fptype_v + fpvmake( const fptype v[neppV] ) + { + fptype_v out = {}; // see #594 + for ( int i=0; i + +#ifdef MGONGPUCPP_GPUIMPL +using namespace mg5amcGpu; +#else +using namespace mg5amcCpu; +#endif + +struct CUDA_CPU_TestBase : public TestDriverBase +{ + static constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static constexpr int np4 = CPPProcess::np4; + static constexpr int npar = CPPProcess::npar; + static_assert( gputhreads % neppM == 0, "ERROR! #threads/block should be a multiple of neppM" ); + static_assert( gputhreads <= mgOnGpu::ntpbMAX, "ERROR! #threads/block should be <= ntpbMAX" ); + CUDA_CPU_TestBase( const std::string& refFileName ) + : TestDriverBase( npar, refFileName ) {} + // Does this test use channelIds? + virtual bool useChannelIds() const = 0; + // Set channelId array (in the same way for CUDA and CPU tests) + static constexpr unsigned int warpSize = 32; // FIXME: add a sanity check in madevent that this is the minimum? (would need to expose this from cudacpp to madevent) + static void setChannelIds( BufferChannelIds& hstChannelIds, std::size_t iiter ) + { + static const char* debugC = getenv( "CUDACPP_RUNTEST_DEBUG" ); + static const bool debug = ( debugC != 0 ) && ( std::string( debugC ) != "" ); + // Fill channelIds for multi-channel tests #896 + // (NB: these are only used if useChannelIds == true) + // TEMPORARY(0): debug multichannel tests with channelId=1 for all events + //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1; + // TEMPORARY(1): debug multichannel tests with channelId=1,2,..,ndiag,1,2,..ndiag,... (every event gets a different channel, no warps) + //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1 + i % CPPProcess::ndiagrams; + // ALMOST FINAL test implementation: 1111222233331111... (every 32-event warp gets a different channel) + // FINAL(?) test implementation: 2222333344442222... (every 32-event warp gets a different channel, skip those without associated iconfig #917) + static_assert( nevt % warpSize == 0, "ERROR! nevt should be a multiple of warpSize" ); + constexpr unsigned int nWarp = nevt / warpSize; + for( unsigned int iWarp = 0; iWarp < nWarp; ++iWarp ) + { + //const unsigned int channelId = 1 + ( iWarp + iiter * nWarp ) % CPPProcess::ndiagrams; // bug #917 + const int iconfig = 1 + ( iWarp + iiter * nWarp ) % mgOnGpu::nconfigSDE; + unsigned int channelId = 0; + //for( unsigned int idiagram = 1; idiagram < CPPProcess::ndiagrams; idiagram++ ) // two bugs #920 and #919 + for( unsigned int idiagram = 0; idiagram < mgOnGpu::nchannels; idiagram++ ) // fix #920 and work around #919 + { + if( mgOnGpu::hostChannel2iconfig[idiagram] == iconfig ) + { + channelId = idiagram + 1; // fix #917 (NB add +1 because channelId uses F indexing) + break; + } + } + assert( channelId > 0 ); // sanity check that the channelId for the given iconfig was found + if( debug ) std::cout << "CUDA_CPU_TestBase::setChannelIds: iWarp=" << iWarp << ", iconfig=" << iconfig << ", channelId=" << channelId << std::endl; + for( unsigned int i = 0; i < warpSize; ++i ) + hstChannelIds[iWarp * warpSize + i] = channelId; + } + } +}; + +#ifndef MGONGPUCPP_GPUIMPL +struct CPUTest : public CUDA_CPU_TestBase +{ + // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) + // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] + CPPProcess process; + HostBufferRndNumMomenta hstRndMom; + HostBufferChannelIds hstChannelIds; + HostBufferMomenta hstMomenta; + HostBufferGs hstGs; + HostBufferRndNumHelicity hstRndHel; + HostBufferRndNumColor hstRndCol; + HostBufferWeights hstWeights; + HostBufferMatrixElements hstMatrixElements; + HostBufferSelectedHelicity hstSelHel; + HostBufferSelectedColor hstSelCol; + HostBufferHelicityMask hstIsGoodHel; + std::unique_ptr pmek; + + // Create a process object + // Read param_card and set parameters + // ** WARNING EVIL EVIL ** + // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. + // Don't remove! + CPUTest( const std::string& refFileName ) + : CUDA_CPU_TestBase( refFileName ) + , process( /*verbose=*/false ) + , hstRndMom( nevt ) + , hstChannelIds( nevt ) + , hstMomenta( nevt ) + , hstGs( nevt ) + , hstRndHel( nevt ) + , hstRndCol( nevt ) + , hstWeights( nevt ) + , hstMatrixElements( nevt ) + , hstSelHel( nevt ) + , hstSelCol( nevt ) + , hstIsGoodHel( CPPProcess::ncomb ) + , pmek( new MatrixElementKernelHost( hstMomenta, hstGs, hstRndHel, hstRndCol, hstChannelIds, hstMatrixElements, hstSelHel, hstSelCol, nevt ) ) + { + // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? + // FIXME: the CPPProcess should really be a singleton? + process.initProc( "../../Cards/param_card.dat" ); + } + + virtual ~CPUTest() {} + + void prepareRandomNumbers( unsigned int iiter ) override + { + // Random numbers for momenta + CommonRandomNumberKernel rnk( hstRndMom ); + rnk.seedGenerator( 1337 + iiter ); + rnk.generateRnarray(); + // Random numbers for helicity and color selection (fix #931) + CommonRandomNumberKernel rnk2( hstRndHel ); + rnk2.seedGenerator( 1338 + iiter ); + rnk2.generateRnarray(); + CommonRandomNumberKernel rnk3( hstRndCol ); + rnk3.seedGenerator( 1339 + iiter ); + rnk3.generateRnarray(); + } + + void prepareMomenta( fptype energy ) override + { + RamboSamplingKernelHost rsk( energy, hstRndMom, hstMomenta, hstWeights, nevt ); + // --- 2a. Fill in momenta of initial state particles on the device + rsk.getMomentaInitial(); + // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device + // (i.e. map random numbers to final-state particle momenta for each of nevt events) + rsk.getMomentaFinal(); + } + + void runSigmaKin( std::size_t iiter ) override + { + constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) + for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; + setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 + if( iiter == 0 ) pmek->computeGoodHelicities(); + pmek->computeMatrixElements( useChannelIds() ); + } + + fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override + { + assert( ipar < npar ); + assert( ip4 < np4 ); + return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); + } + + fptype getMatrixElement( std::size_t ievt ) const override + { + return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); + } + + int getChannelId( std::size_t ievt ) const override + { + return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); + } + + int getSelectedHelicity( std::size_t ievt ) const override + { + //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... + return hstSelHel.data()[ievt]; + } + + int getSelectedColor( std::size_t ievt ) const override + { + //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... + return hstSelCol.data()[ievt]; + } +}; + +// Old test with multi-channel disabled #466 +struct CPUTestNoMultiChannel : public CPUTest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return false; } + + // Constructor + CPUTestNoMultiChannel( const std::string& refFileName ) + : CPUTest( refFileName ) {} // suffix .txt + + // Destructor + virtual ~CPUTestNoMultiChannel() {} +}; + +// New test with multi-channel enabled #896 +struct CPUTestMultiChannel : public CPUTest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return true; } + + // Constructor + CPUTestMultiChannel( const std::string& refFileName ) + : CPUTest( refFileName + "2" ) {} // suffix .txt2 + + // Destructor + virtual ~CPUTestMultiChannel() {} +}; +#endif + +#ifdef MGONGPUCPP_GPUIMPL +struct CUDATest : public CUDA_CPU_TestBase +{ + // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) + // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] + CPPProcess process; + PinnedHostBufferRndNumMomenta hstRndMom; + PinnedHostBufferMomenta hstMomenta; + PinnedHostBufferGs hstGs; + PinnedHostBufferRndNumHelicity hstRndHel; + PinnedHostBufferRndNumColor hstRndCol; + PinnedHostBufferWeights hstWeights; + PinnedHostBufferChannelIds hstChannelIds; + PinnedHostBufferMatrixElements hstMatrixElements; + PinnedHostBufferSelectedHelicity hstSelHel; + PinnedHostBufferSelectedColor hstSelCol; + PinnedHostBufferHelicityMask hstIsGoodHel; + DeviceBufferRndNumMomenta devRndMom; + DeviceBufferChannelIds devChannelIds; + DeviceBufferMomenta devMomenta; + DeviceBufferGs devGs; + DeviceBufferRndNumHelicity devRndHel; + DeviceBufferRndNumColor devRndCol; + DeviceBufferWeights devWeights; + DeviceBufferMatrixElements devMatrixElements; + DeviceBufferSelectedHelicity devSelHel; + DeviceBufferSelectedColor devSelCol; + DeviceBufferHelicityMask devIsGoodHel; + std::unique_ptr pmek; + + // Create a process object + // Read param_card and set parameters + // ** WARNING EVIL EVIL ** + // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. + // Don't remove! + CUDATest( const std::string& refFileName ) + : CUDA_CPU_TestBase( refFileName ) + , process( /*verbose=*/false ) + , hstRndMom( nevt ) + , hstChannelIds( nevt ) + , hstMomenta( nevt ) + , hstGs( nevt ) + , hstRndHel( nevt ) + , hstRndCol( nevt ) + , hstWeights( nevt ) + , hstMatrixElements( nevt ) + , hstSelHel( nevt ) + , hstSelCol( nevt ) + , hstIsGoodHel( CPPProcess::ncomb ) + , devRndMom( nevt ) + , devChannelIds( nevt ) + , devMomenta( nevt ) + , devGs( nevt ) + , devRndHel( nevt ) + , devRndCol( nevt ) + , devWeights( nevt ) + , devMatrixElements( nevt ) + , devSelHel( nevt ) + , devSelCol( nevt ) + , devIsGoodHel( CPPProcess::ncomb ) + , pmek( new MatrixElementKernelDevice( devMomenta, devGs, devRndHel, devRndCol, devChannelIds, devMatrixElements, devSelHel, devSelCol, gpublocks, gputhreads ) ) + { + // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? + // FIXME: the CPPProcess should really be a singleton? + process.initProc( "../../Cards/param_card.dat" ); + } + + virtual ~CUDATest() {} + + void prepareRandomNumbers( unsigned int iiter ) override + { + // Random numbers for momenta + CommonRandomNumberKernel rnk( hstRndMom ); + rnk.seedGenerator( 1337 + iiter ); + rnk.generateRnarray(); + copyDeviceFromHost( devRndMom, hstRndMom ); + // Random numbers for helicity and color selection (fix #931) + CommonRandomNumberKernel rnk2( hstRndHel ); + rnk2.seedGenerator( 1338 + iiter ); + rnk2.generateRnarray(); + copyDeviceFromHost( devRndHel, hstRndHel ); + CommonRandomNumberKernel rnk3( hstRndCol ); + rnk3.seedGenerator( 1339 + iiter ); + rnk3.generateRnarray(); + copyDeviceFromHost( devRndCol, hstRndCol ); + } + + void prepareMomenta( fptype energy ) override + { + RamboSamplingKernelDevice rsk( energy, devRndMom, devMomenta, devWeights, gpublocks, gputhreads ); + // --- 2a. Fill in momenta of initial state particles on the device + rsk.getMomentaInitial(); + // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device + // (i.e. map random numbers to final-state particle momenta for each of nevt events) + rsk.getMomentaFinal(); + // --- 2c. CopyDToH Weights + copyHostFromDevice( hstWeights, devWeights ); + // --- 2d. CopyDToH Momenta + copyHostFromDevice( hstMomenta, devMomenta ); + } + + void runSigmaKin( std::size_t iiter ) override + { + constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) + for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; + copyDeviceFromHost( devGs, hstGs ); // BUG FIX #566 + setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 + copyDeviceFromHost( devChannelIds, hstChannelIds ); + if( iiter == 0 ) pmek->computeGoodHelicities(); + pmek->computeMatrixElements( useChannelIds() ); + copyHostFromDevice( hstMatrixElements, devMatrixElements ); + copyHostFromDevice( hstSelHel, devSelHel ); + copyHostFromDevice( hstSelCol, devSelCol ); + } + + fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override + { + assert( ipar < npar ); + assert( ip4 < np4 ); + return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); + } + + fptype getMatrixElement( std::size_t ievt ) const override + { + return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); + } + + int getChannelId( std::size_t ievt ) const override + { + return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); + } + + int getSelectedHelicity( std::size_t ievt ) const override + { + //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... + return hstSelHel.data()[ievt]; + } + + int getSelectedColor( std::size_t ievt ) const override + { + //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... + return hstSelCol.data()[ievt]; + } +}; + +// Old test with multi-channel disabled #466 +struct CUDATestNoMultiChannel : public CUDATest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return false; } + + // Constructor + CUDATestNoMultiChannel( const std::string& refFileName ) + : CUDATest( refFileName ) {} // suffix .txt + + // Destructor + virtual ~CUDATestNoMultiChannel() {} +}; + +// New test with multi-channel enabled #896 +struct CUDATestMultiChannel : public CUDATest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return true; } + + // Constructor + CUDATestMultiChannel( const std::string& refFileName ) + : CUDATest( refFileName + "2" ) {} // suffix .txt2 + + // Destructor + virtual ~CUDATestMultiChannel() {} +}; +#endif /* clang-format off */ + +// AV July 2024 much simpler class structure without the presently-unnecessary googletest templates +// This is meant as a workaround to prevent not-understood segfault #907 when adding a second test +// Note: instantiate test2 first and test1 second to ensure that the channelid printout from the dtors comes from test1 first and test2 second +#ifdef MGONGPUCPP_GPUIMPL +// CUDA test drivers +CUDATestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID2( s ) s##_GPU_MULTICHANNEL +CUDATestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID1( s ) s##_GPU_NOMULTICHANNEL +#else +// CPU test drivers +CPUTestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID2( s ) s##_CPU_MULTICHANNEL +CPUTestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID1( s ) s##_CPU_NOMULTICHANNEL +#endif +// Madgraph tests +MadgraphTest mgTest2( driver2 ); +MadgraphTest mgTest1( driver1 ); +// Instantiate Google test 1 +#define XTESTID1( s ) TESTID1( s ) +TEST( XTESTID1( MG_EPOCH_PROCESS_ID ), compareMomAndME ) +{ +#ifdef MGONGPU_CHANNELID_DEBUG + driver1.pmek->setTagForNevtProcessedByChannel( "(no multichannel)" ); +#endif + mgTest1.CompareMomentaAndME( *this ); +} +// Instantiate Google test 2 +#define XTESTID2( s ) TESTID2( s ) +TEST( XTESTID2( MG_EPOCH_PROCESS_ID ), compareMomAndME ) +{ +#ifdef MGONGPU_CHANNELID_DEBUG + driver2.pmek->setTagForNevtProcessedByChannel( "(channelid array)" ); +#endif + mgTest2.CompareMomentaAndME( *this ); +} +/* clang-format on */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc new file mode 100644 index 0000000000..74c4902dc5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc @@ -0,0 +1,517 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. +// ---------------------------------------------------------------------------- +// Use ./runTest.exe --gtest_filter=*misc to run only testmisc.cc tests +// ---------------------------------------------------------------------------- + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#include "constexpr_math.h" +#include "epoch_process_id.h" +#include "valgrind.h" + +#include + +//#include +//#include // needs C++20... https://stackoverflow.com/a/65347016 +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +#define TESTID( s ) s##_GPU_MISC +#else +#define TESTID( s ) s##_CPU_MISC +#endif + +#define XTESTID( s ) TESTID( s ) + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#ifdef MGONGPU_CPPSIMD /* clang-format off */ +#define EXPECT_TRUE_sv( cond ) { bool_v mask( cond ); EXPECT_TRUE( maskand( mask ) ); } +#else +#define EXPECT_TRUE_sv( cond ) { EXPECT_TRUE( cond ); } +#endif /* clang-format on */ + + inline const std::string + boolTF( const bool& b ) + { + return ( b ? "T" : "F" ); + } + +#ifdef MGONGPU_CPPSIMD + inline const std::string + boolTF( const bool_v& v ) + { + std::stringstream out; + out << "{ " << ( v[0] ? "T" : "F" ); + for( int i = 1; i < neppV; i++ ) out << ", " << ( v[i] ? "T" : "F" ); + out << " }"; + return out.str(); + } +#endif +} + +TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + //-------------------------------------------------------------------------- + + EXPECT_TRUE( true ); + + //-------------------------------------------------------------------------- + + // Vector initialization for fptype_sv + { + fptype_sv f{ 0 }; + EXPECT_TRUE_sv( f == 0 ); + } + { + fptype_sv f = fptype_sv{ 0 }; + EXPECT_TRUE_sv( f == 0 ); + } + + // Vector initialization for fptype_sv - demonstrate bug #339 in older cxmake implementation + { + fptype_sv f{ 1 }; + //std::cout << f << std::endl << boolTF( f == 1 ) << std::endl; + //EXPECT_TRUE_sv( f == 1 ); // this fails for vectors! TFFF +#ifndef MGONGPU_CPPSIMD + EXPECT_TRUE_sv( f == 1 ); // this succeds: T +#else + EXPECT_TRUE( ( f == 1 )[0] ); // this succeds: TFFF[0] + EXPECT_TRUE( ( f[0] == 1 ) ); + for( int i = 1; i < neppV; i++ ) + { + EXPECT_TRUE( !( ( f == 1 )[i] ) ); // this succeds: FTTT[i>=1] + EXPECT_TRUE( ( f[i] == 0 ) ); // equals 0, not 1 + } +#endif + } + +#ifdef MGONGPU_CPPSIMD + // Vector initialization for cxtype_sv - demonstrate fix for bug #339 + { + fptype_sv f1 = fptype_v{ 0 } + 1; + EXPECT_TRUE_sv( f1 == 1 ); + cxtype_v c12 = cxmake( f1, 2 ); + //std::cout << c12 << std::endl << boolTF( c12.real() == 1 ) << std::endl << boolTF( c12.imag() == 2 ) << std::endl; + EXPECT_TRUE_sv( c12.real() == 1 ); + EXPECT_TRUE_sv( c12.imag() == 2 ); + cxtype_v c21 = cxmake( 2, f1 ); + //std::cout << c21 << std::endl << boolTF( c21.real() == 2 ) << std::endl << boolTF( c21.imag() == 1 ) << std::endl; + EXPECT_TRUE_sv( c21.real() == 2 ); + EXPECT_TRUE_sv( c21.imag() == 1 ); + } +#endif + + // Vector initialization for cxtype_sv + { + cxtype_sv c = cxzero_sv(); + EXPECT_TRUE_sv( c.real() == 0 ); + EXPECT_TRUE_sv( c.imag() == 0 ); + } + { + cxtype_sv c = cxmake( 1, fptype_sv{ 0 } ); // here was a bug #339 + EXPECT_TRUE_sv( c.real() == 1 ); + EXPECT_TRUE_sv( c.imag() == 0 ); + } + { + cxtype_sv c = cxmake( fptype_sv{ 0 }, 1 ); // here was a bug #339 + EXPECT_TRUE_sv( c.real() == 0 ); + EXPECT_TRUE_sv( c.imag() == 1 ); + } + + // Array initialization for cxtype_sv array (example: jamp_sv in CPPProcess.cc) + { + cxtype_sv array[2] = {}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "= {}" is missing!) + //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; + EXPECT_TRUE_sv( array[0].real() == 0 ); + EXPECT_TRUE_sv( array[0].imag() == 0 ); + EXPECT_TRUE_sv( array[1].real() == 0 ); + EXPECT_TRUE_sv( array[1].imag() == 0 ); + } + + // Alternative array initialization for cxtype_sv array (example: was used for outwf in testxxx.cc) + { + cxtype_sv array[2]{}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "{}" is missing!) + //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; + EXPECT_TRUE_sv( array[0].real() == 0 ); + EXPECT_TRUE_sv( array[0].imag() == 0 ); + EXPECT_TRUE_sv( array[1].real() == 0 ); + EXPECT_TRUE_sv( array[1].imag() == 0 ); + } + + //-------------------------------------------------------------------------- + + // Scalar complex references + { + using namespace mgOnGpu; + // Refs to f1, f2 + fptype f1 = 1; + fptype f2 = 2; + cxtype_ref r12( f1, f2 ); // copy refs + //cxtype_ref r12a( r12 ); //deleted + cxtype_ref r12a( cxtype_ref( f1, f2 ) ); // copy refs + //cxtype_ref r12b = r12; // deleted + cxtype_ref r12b = cxtype_ref( f1, f2 ); // copy refs + EXPECT_TRUE( cxtype( r12 ).real() == 1 ); + EXPECT_TRUE( cxtype( r12 ).imag() == 2 ); + EXPECT_TRUE( cxtype( r12a ).real() == 1 ); + EXPECT_TRUE( cxtype( r12a ).imag() == 2 ); + EXPECT_TRUE( cxtype( r12b ).real() == 1 ); + EXPECT_TRUE( cxtype( r12b ).imag() == 2 ); + // Refs to f1c, f2c + fptype f1c = 0; + fptype f2c = 0; + cxtype_ref r12c( f1c, f2c ); + EXPECT_TRUE( cxtype( r12c ).real() == 0 ); + EXPECT_TRUE( cxtype( r12c ).imag() == 0 ); + //r12c = r12; // deleted + r12c = cxtype( r12 ); // copy values + EXPECT_TRUE( cxtype( r12c ).real() == 1 ); + EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); + // Update f1, f2 + f1 = 10; + f2 = 20; + EXPECT_TRUE( cxtype( r12 ).real() == 10 ); + EXPECT_TRUE( cxtype( r12 ).imag() == 20 ); + EXPECT_TRUE( cxtype( r12a ).real() == 10 ); + EXPECT_TRUE( cxtype( r12a ).imag() == 20 ); + EXPECT_TRUE( cxtype( r12b ).real() == 10 ); + EXPECT_TRUE( cxtype( r12b ).imag() == 20 ); + EXPECT_TRUE( cxtype( r12c ).real() == 1 ); // points to f1c, not to f1 + EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); // points to f2c, not to f2 + } + + // Vector complex references + { + using namespace mgOnGpu; + // Refs to f1, f2 + fptype_sv f1 = fptype_sv{ 0 } + 1; + fptype_sv f2 = fptype_sv{ 0 } + 2; + cxtype_sv_ref r12( f1, f2 ); // copy refs + //cxtype_sv_ref r12a( r12 ); //deleted + cxtype_sv_ref r12a( cxtype_sv_ref( f1, f2 ) ); // copy refs + //cxtype_sv_ref r12b = r12; // deleted + cxtype_sv_ref r12b = cxtype_sv_ref( f1, f2 ); // copy refs + EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 2 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 2 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 2 ); + // Refs to f1c, f2c + fptype_sv f1c = fptype_sv{ 0 }; + fptype_sv f2c = fptype_sv{ 0 }; + cxtype_sv_ref r12c( f1c, f2c ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 0 ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 0 ); + //r12c = r12; // deleted + r12c = cxtype_sv( r12 ); // copy values + EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); + // Update f1, f2 + f1 = fptype_sv{ 0 } + 10; + f2 = fptype_sv{ 0 } + 20; + EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 10 ); + EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 20 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 10 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 20 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 10 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 20 ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); // points to f1c, not to f1 + EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); // points to f2c, not to f2 + } + + //-------------------------------------------------------------------------- + + // Boolean vector (mask) times FP vector + /* + // From https://github.com/madgraph5/madgraph4gpu/issues/765#issuecomment-1853672838 + channelids_sv = CHANNEL_ACCESS::kernelAccess( pchannelIds ); // the 4 channels in the SIMD vector + bool_sv mask_sv = ( channelids_sv == 1 ); + numerators_sv += mask_sv * cxabs2( amp_sv[0] ); + if( pchannelIds != nullptr ) denominators_sv += cxabs2( amp_sv[0] ); + */ + { + typedef bool_sv test_int_sv; // defined as scalar_or_vector of long int (FPTYPE=double) or int (FPTYPE=float) + test_int_sv channelids0_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) + test_int_sv channelids1_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) + fptype_sv absamp0_sv{}; // mimic cxabs2( amp_sv[0] ) + fptype_sv absamp1_sv{}; // mimic cxabs2( amp_sv[0] ) +#ifdef MGONGPU_CPPSIMD + for( int i = 0; i < neppV; i++ ) + { + channelids0_sv[i] = i; // 0123 + channelids1_sv[i] = i; // 1234 + absamp0_sv[i] = 10. + i; // 10. 11. 12. 13. + absamp1_sv[i] = 11. + i; // 11. 12. 13. 14. + } +#else + channelids0_sv = 0; + channelids1_sv = 1; + absamp0_sv = 10.; + absamp1_sv = 11.; +#endif + bool_sv mask0_sv = ( channelids0_sv % 2 == 0 ); // even channels 0123 -> TFTF (1010) + bool_sv mask1_sv = ( channelids1_sv % 2 == 0 ); // even channels 1234 -> FTFT (0101) + constexpr fptype_sv fpZERO_sv{}; // 0000 + //fptype_sv numerators0_sv = mask0_sv * absamp0_sv; // invalid operands to binary * ('__vector(4) long int' and '__vector(4) double') + fptype_sv numerators0_sv = fpternary( mask0_sv, absamp0_sv, fpZERO_sv ); // equivalent to "mask0_sv * absamp0_sv" + fptype_sv numerators1_sv = fpternary( mask1_sv, absamp1_sv, fpZERO_sv ); // equivalent to "mask1_sv * absamp1_sv" +#ifdef MGONGPU_CPPSIMD + //std::cout << "numerators0_sv: " << numerators0_sv << std::endl; + //std::cout << "numerators1_sv: " << numerators1_sv << std::endl; + for( int i = 0; i < neppV; i++ ) + { + // Values of numerators0_sv: 10.*1 11.*0 12.*1 13.*0 + if( channelids0_sv[i] % 2 == 0 ) // even channels + EXPECT_TRUE( numerators0_sv[i] == ( 10. + i ) ); + else // odd channels + EXPECT_TRUE( numerators0_sv[i] == 0. ); + // Values of numerators1_sv: 11.*0 12.*1 13.*0 14.*1 + if( channelids1_sv[i] % 2 == 0 ) // even channels + EXPECT_TRUE( numerators1_sv[i] == ( 11. + i ) ); + else // odd channels + EXPECT_TRUE( numerators1_sv[i] == 0. ); + } +#else + // Values of numerators0_sv: 10.*1 + EXPECT_TRUE( numerators0_sv == 10. ); + // Values of numerators1_sv: 11.*0 + EXPECT_TRUE( numerators1_sv == 0. ); +#endif + } + + //-------------------------------------------------------------------------- + + // Test constexpr floor + EXPECT_TRUE( constexpr_floor( 1.5 ) == 1 ); + EXPECT_TRUE( constexpr_floor( 0.5 ) == 0 ); + EXPECT_TRUE( constexpr_floor( -0.5 ) == -1 ); + EXPECT_TRUE( constexpr_floor( -1.5 ) == -2 ); + + // Test constexpr pow + EXPECT_TRUE( constexpr_pow( 10, 0 ) == 1 ); + EXPECT_TRUE( constexpr_pow( 10, 1 ) == 10 ); + EXPECT_TRUE( constexpr_pow( 10, 2 ) == 100 ); + EXPECT_NEAR( constexpr_pow( 10, -1 ), 0.1, 0.1 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10, -1 ) = " << constexpr_pow( 10, -1 ); + EXPECT_NEAR( constexpr_pow( 10, -2 ), 0.01, 0.01 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10, -2 ) = " << constexpr_pow( 10, -2 ); + EXPECT_NEAR( constexpr_pow( 100, 0.5 ), 10, 10 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 100, 0.5 ) = " << constexpr_pow( 100, 0.5 ); + EXPECT_NEAR( constexpr_pow( 100, -0.5 ), 0.1, 0.1 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 100, -0.5 ) = " << constexpr_pow( 100, -0.5 ); + EXPECT_NEAR( constexpr_pow( 10000, 0.25 ), 10, 10 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10000, 0.25 ) = " << constexpr_pow( 10000, 0.25 ); + EXPECT_NEAR( constexpr_pow( 10000, -0.25 ), 0.1, 0.1 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10000, -0.25 ) = " << constexpr_pow( 10000, -0.25 ); + +#ifndef __aarch64__ // TO BE UNDERSTOOD? DISABLE CONSTEXPR_SQRT TESTS ON AARCH64 (#1064) + // Distance from the horizontal or vertical axis (i.e. from 0, pi/2, pi, or 3pi/2) + auto distance4 = []( const long double xx ) + { + const long double xx2 = mapIn0to2Pi( xx ); // in [0,2*pi) + const long double xx3 = xx2 - constexpr_floor( xx2 / constexpr_pi_by_2 ) * constexpr_pi_by_2; // in [0,pi/2) + const long double d0 = xx3; // distance from 0 + const long double d1 = constexpr_pi_by_2 - xx3; // distance from pi/2 + return ( d0 < d1 ? d0 : d1 ); + }; + + // Test constexpr sin, cos, tan - specific, problematic, points + auto testSinCosTanX = []( const long double xx, const double tolerance0, const bool debug = false, const long long istep = -999999999 ) + { + const double x = (double)xx; + const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 + if( debug ) + { + //std::cout << std::setprecision(40) << "testSinCosTanX: xx= " << xx << std::endl; + //std::cout << std::setprecision(40) << " x= " << x << std::endl; + } + //std::cout << std::setprecision(40) << "xx - 3pi/2 " << xx - 3 * constexpr_pi_by_2 << std::endl; + //int width = 46; + //char buf[128]; + //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)xx ); + //std::cout << std::setprecision(40) << "testSinCosTanX: xx=" << buf << std::endl; + //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)x ); + //std::cout << std::setprecision(40) << " x= " << buf << std::endl; + EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::abs( std::sin( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::abs( std::cos( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; +#ifndef __aarch64__ + if( !RUNNING_ON_VALGRIND ) + { + EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::abs( std::tan( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + } + else +#endif + { + // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) + const long double ctanx = constexpr_tan( x ); + const long double taninf = 4E14; // declare tan(x) as "infinity" if above this threshold + if( ctanx > -taninf && ctanx < taninf ) + EXPECT_NEAR( std::tan( x ), ctanx, std::abs( std::tan( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + else + { + // Allow tan(x)=-inf if ctan(x)=+inf and viceversa + EXPECT_GT( std::abs( std::tan( x ) ), taninf ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + /* + // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) + if( ctanx > 0 ) + EXPECT_GT( std::tan( x ), taninf ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + else + EXPECT_LT( std::tan( x ), -taninf ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + */ + } + } + std::cout << std::setprecision( 6 ); // default + }; + testSinCosTanX( constexpr_pi, 1E-3, true ); // from math.h + testSinCosTanX( (long double)3.141592653589793238462643383279502884L, 1E-3, true ); // from math.h + testSinCosTanX( 4.712388980384687897640105802565813064575L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) + testSinCosTanX( 3 * constexpr_pi_by_2 - 1.96e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x!=xx) + testSinCosTanX( 3 * constexpr_pi_by_2 - 1.9601e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) + + // Test constexpr sin, cos, tan - 8 points on (or close to) the boundaries of the 8 sectors of [0,2*pi] + auto testSinCosTan8 = [testSinCosTanX]( const double deltax, const double tolerance ) + { + for( int ioff = -1; ioff < 2; ioff++, ioff++ ) // -1, 1 + { + const bool debug = false; + const int nstep = 8; + for( int istep = 0; istep < nstep + 1; istep++ ) + { + long double x0 = deltax * ioff; + long double x1 = deltax * ioff + 2 * constexpr_pi; + double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) + testSinCosTanX( x, tolerance, debug, istep ); + } + } + }; + + // Use much lower tolerance when testing on the boundaries of the 8 sectors of [0,2*pi] + // Use progressively stricter tolerances as you move away from the boundaries of the 8 sectors of [0,2*pi] + testSinCosTan8( 0, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... + testSinCosTan8( 1E-15, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... + testSinCosTan8( 1E-14, 1E-04 ); // fails with 1E-05 + testSinCosTan8( 1E-12, 1E-06 ); // fails with 1E-07 + testSinCosTan8( 1E-09, 1E-09 ); // fails with 1E-10 + testSinCosTan8( 1E-06, 1E-12 ); // fails with 1E-13 + testSinCosTan8( 1E-03, 1E-14 ); // fails with 1E-16: could use 1E-14 but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) + testSinCosTan8( 1E-02, 1E-14 ); // never fails? could use 1E-99(?) but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) + + // Test constexpr sin, cos, tan - N points almost randomly with a varying tolerance + auto testSinCosTanN = [distance4]( const int nstep, const double x0, const double x1 ) + { + auto toleranceForX = [distance4]( const double x ) + { + const double d4 = distance4( x ); + if( d4 < 1E-14 ) + return 1E-03; // NB: absolute distance limited to 1E-14 anyway even if relative tolerance is 1E-3... + else if( d4 < 1E-13 ) + return 1E-04; + else if( d4 < 1E-12 ) + return 1E-05; + else if( d4 < 1E-11 ) + return 1E-06; + else if( d4 < 1E-10 ) + return 1E-07; + else if( d4 < 1E-09 ) + return 1E-08; + else if( d4 < 1E-08 ) + return 1E-09; + else if( d4 < 1E-07 ) + return 1E-10; + else if( d4 < 1E-06 ) + return 1E-11; + else if( d4 < 1E-05 ) + return 1E-12; + else if( d4 < 1E-04 ) + return 1E-13; + else + return 1E-14; // play it safe even if the agreement might even be better? + }; + for( int istep = 0; istep < nstep + 1; istep++ ) + { + double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) + const double tolerance0 = toleranceForX( x ); + const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 + EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::max( std::abs( std::sin( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::max( std::abs( std::cos( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); +#ifndef __aarch64__ + if( !RUNNING_ON_VALGRIND ) + { + EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + } + else +#endif + { + // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) + const long double ctanx = constexpr_tan( x ); + const long double taninf = 4E14; // declare tan(x) as "infinity if above this threshold + if( ctanx > -taninf && ctanx < taninf ) + EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + else + { + // Allow tan(x)=-inf if ctan(x)=+inf and viceversa + EXPECT_GT( std::abs( std::tan( x ) ), taninf ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + /* + // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) + if( ctanx > 0 ) + EXPECT_GT( std::tan( x ), taninf ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + else + EXPECT_LT( std::tan( x ), -taninf ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + */ + } + } + } + }; + testSinCosTanN( 100, -4 * constexpr_pi, 6 * constexpr_pi ); // this was failing at 3*pi/2 (now fixed by absolute tolerance 3E-15) + testSinCosTanN( 10000, -constexpr_pi_by_2, 5 * constexpr_pi_by_2 ); + + // Test constexpr atan + { + const double tolerance = 1E-12; + const int nstep = 1000; + for( int istep = 0; istep < nstep + 1; istep++ ) + { + long double x0 = -5, x1 = +5; + double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) + EXPECT_NEAR( std::atan( x ), constexpr_atan( x ), std::abs( std::atan( x ) * tolerance ) ) + << "x=" << x << ", istep=" << istep; + } + } +#endif + //-------------------------------------------------------------------------- +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc new file mode 100644 index 0000000000..4fcf7d26b6 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc @@ -0,0 +1,456 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. +// ---------------------------------------------------------------------------- +// Use ./runTest.exe --gtest_filter=*xxx to run only testxxx.cc tests +// ---------------------------------------------------------------------------- + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "HelAmps_%(model_name)s.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessWavefunctions.h" +#include "MemoryBuffers.h" +#include "epoch_process_id.h" + +#include + +#include +#include +#include // for signal and SIGFPE (see https://stackoverflow.com/a/17473528) +#include +#include +#include +#include +#ifdef MGONGPUCPP_GPUIMPL +#define TESTID( s ) s##_GPU_XXX +#else +#define TESTID( s ) s##_CPU_XXX +#endif + +#define XTESTID( s ) TESTID( s ) + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + std::string fpeHandlerMessage = "unknown"; + int fpeHandlerIevt = -1; + inline void fpeHandlerTestxxx( int /*sig*/ ) + { +#ifdef MGONGPUCPP_GPUIMPL + std::cerr << "Floating Point Exception (GPU): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; +#else + std::cerr << "Floating Point Exception (CPU neppV=" << neppV << "): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; +#endif + exit( 1 ); + } +} + +TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif +#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) + auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); +#endif + constexpr bool dumpEvents = false; // dump the expected output of the test? + constexpr bool testEvents = !dumpEvents; // run the test? + constexpr fptype toleranceXXXs = std::is_same::value ? 1.E-15 : 1.E-5; + // Constant parameters + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + constexpr int np4 = CPPProcess::np4; + const int nevt = 32; // 12 independent tests plus 20 duplicates (need a multiple of 16 for floats '512z') + assert( nevt %% neppM == 0 ); // nevt must be a multiple of neppM + assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV + // Fill in the input momenta +#ifdef MGONGPUCPP_GPUIMPL + mg5amcGpu::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] +#else + mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] +#endif /* clang-format off */ + // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! + const fptype par0[np4 * nevt] = // AOS[nevt][np4] + { + 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) + 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) + 500, 180, 240, 400, // #3 (m=0 pT>0 pz>0) + 500, 180, 240, -400, // #4 (m=0 pT>0 pz<0) + 500, 0, 0, 500, // #5 DUPLICATE == #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, -500, // #6 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) + 500, 300, 400, 0, // #7 DUPLICATE == #2 (m=0 pT>0 pz=0) + 500, 180, 240, 400, // #8 DUPLICATE == #3 (m=0 pT>0 pz>0) + 500, 180, 240, -400, // #9 DUPLICATE == #4 (m=0 pT>0 pz<0) + 500, 0, 0, 500, // #10 DUPLICATE == #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, -500, // #11 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) + 500, 300, 400, 0, // #12 DUPLICATE == #2 (m=0 pT>0 pz=0) + 500, 180, 240, 400, // #13 DUPLICATE == #3 (m=0 pT>0 pz>0) + 500, 180, 240, -400, // #14 DUPLICATE == #4 (m=0 pT>0 pz<0) + 500, 0, 0, 500, // #15 DUPLICATE == #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, 0, // #16 (m=50>0 pT=0 pz=0) + 500, 0, 0, 300, // #17 (m=40>0 pT=0 pz>0) + 500, 0, 0, -300, // #18 (m=40>0 pT=0 pz<0) + 500, 180, 240, 0, // #19 (m=40>0 pT>0 pz=0) + 500, -240, -180, 0, // #20 (m=40>0 pT>0 pz=0) + 500, 180, 192, 144, // #21 (m=40>0 pT>0 pz>0) + 500, 180, 192, -144, // #22 (m=40>0 pT>0 pz<0) + 500, 0, 0, 0, // #23 DUPLICATE == #16 (m=50>0 pT=0 pz=0) + 500, 0, 0, 300, // #24 DUPLICATE == #17 (m=40>0 pT=0 pz>0) + 500, 0, 0, -300, // #25 DUPLICATE == #18 (m=40>0 pT=0 pz<0) + 500, 180, 240, 0, // #26 DUPLICATE == #19 (m=40>0 pT>0 pz=0) + 500, -240, -180, 0, // #27 DUPLICATE == #20 (m=40>0 pT>0 pz=0) + 500, 180, 192, 144, // #28 DUPLICATE == #21 (m=40>0 pT>0 pz>0) + 500, 180, 192, -144, // #29 DUPLICATE == #22 (m=40>0 pT>0 pz<0) + 500, 0, 0, 0, // #30 DUPLICATE == #16 (m=50>0 pT=0 pz=0) + 500, 0, 0, 300 // #31 DUPLICATE == #17 (m=40>0 pT=0 pz>0) + }; /* clang-format on */ + // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) + // See https://en.cppreference.com/w/c/language/array_initialization#Notes + fptype mass0[nevt] = {}; + bool ispzgt0[nevt] = {}; + bool ispzlt0[nevt] = {}; + bool isptgt0[nevt] = {}; + for( int ievt = 0; ievt < nevt; ievt++ ) + { + const fptype p0 = par0[ievt * np4 + 0]; + const fptype p1 = par0[ievt * np4 + 1]; + const fptype p2 = par0[ievt * np4 + 2]; + const fptype p3 = par0[ievt * np4 + 3]; + volatile fptype m2 = fpmax( p0 * p0 - p1 * p1 - p2 * p2 - p3 * p3, 0 ); // see #736 + if( m2 > 0 ) + mass0[ievt] = fpsqrt( (fptype)m2 ); + else + mass0[ievt] = 0; + ispzgt0[ievt] = ( p3 > 0 ); + ispzlt0[ievt] = ( p3 < 0 ); + isptgt0[ievt] = ( p1 != 0 ) || ( p2 != 0 ); + } + const int ipar0 = 0; // use only particle0 for this test + for( int ievt = 0; ievt < nevt; ievt++ ) + { + for( int ip4 = 0; ip4 < np4; ip4++ ) + { + MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA + } + } + // Expected output wavefunctions + std::vector> expwfs; +#include "testxxx_cc_ref.txt" // expwfs.push_back( {...} ); + std::string dumpFileName = "testxxx_cc_ref.txt.new"; + // Compute the output wavefunctions + // Dump new reference file if requested + constexpr int nw6 = CPPProcess::nw6; // dimensions of each wavefunction (HELAS KEK 91-11): e.g. 6 for e+ e- -> mu+ mu- (fermions and vectors) + int itest = 0; // index on the expected output vector + std::ofstream dumpFile; + if( dumpEvents ) + { + dumpFile.open( dumpFileName, std::ios::trunc ); + dumpFile << " // Copyright (C) 2020-2024 CERN and UCLouvain." << std::endl + << " // Licensed under the GNU Lesser General Public License (version 3 or later)." << std::endl + << " // Created by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin." << std::endl + << " // Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin." << std::endl; + } + // Lambda function for dumping wavefunctions + auto dumpwf6 = [&]( std::ostream& out, const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) + { + out << std::setprecision( 15 ) << std::scientific; + out << " expwfs.push_back( {"; + out << " // ---------" << std::endl; + for( int iw6 = 0; iw6 < nw6; iw6++ ) + { +#ifdef MGONGPU_CPPSIMD + const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration +#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK + out << std::setw( 26 ) << cxreal( wf[iw6][ieppV] ) << ", "; + out << std::setw( 22 ) << cximag( wf[iw6][ieppV] ); +#else + out << std::setw( 26 ) << wf[iw6].real()[ieppV] << ", "; + out << std::setw( 22 ) << wf[iw6].imag()[ieppV]; +#endif +#else + out << std::setw( 26 ) << wf[iw6].real(); + out << ", " << std::setw( 22 ) << wf[iw6].imag(); +#endif + if( iw6 < nw6 - 1 ) + out << ", "; + else + out << " } );"; + out << " // itest=" << itest << ": " << xxx << "#" << ievt; + out << " nsp=" << nsp << " mass=" << (int)mass << std::endl; + } + out << std::defaultfloat; + }; + // Lambda function for testing wavefunctions (1) + auto testwf6 = [&]( const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) + { + if( dumpEvents ) dumpwf6( dumpFile, wf, xxx, ievt, nsp, mass ); + if( testEvents ) + { + std::array& expwf = expwfs[itest]; + //std::cout << "Testing " << std::setw(3) << itest << ": " << xxx << " #" << ievt << std::endl; + ////for ( int iw6 = 0; iw60) can be used without triggering FPEs (#701) + // This is done by filling the full SIMD vector with the value of ievt, which was already tested to respect the relevant assumptions + for( int jevt = 0; jevt < nevt; jevt++ ) + for( int ip4 = 0; ip4 < np4; ip4++ ) + MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), jevt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA + } + }; + // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) + // See https://en.cppreference.com/w/c/language/array_initialization#Notes + cxtype_sv outwfI[6] = {}; // last result of ixxxxx (mass==0) + cxtype_sv outwfO[6] = {}; // last result of oxxxxx (mass==0) + cxtype_sv outwf[6] = {}; + cxtype_sv outwf3[6] = {}; // NB: only 3 are filled by sxxxxx, but 6 are compared! + fptype* fp_outwfI = reinterpret_cast( outwfI ); // proof of concept for using fptype* in the interface + fptype* fp_outwfO = reinterpret_cast( outwfO ); // proof of concept for using fptype* in the interface + fptype* fp_outwf = reinterpret_cast( outwf ); // proof of concept for using fptype* in the interface + fptype* fp_outwf3 = reinterpret_cast( outwf3 ); // proof of concept for using fptype* in the interface + const int nhel = 1; + // *** START OF TESTING LOOP + for( auto nsp: { -1, +1 } ) // antifermion/fermion (or initial/final for scalar and vector) + { + for( int ievt = 0; ievt < nevt; ievt++ ) + { +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + if( debug ) + { + std::cout << std::endl; + std::cout << "nsp=" << nsp << " ievt=" << ievt << ": "; + for( int ip4 = 0; ip4 < np4; ip4++ ) std::cout << par0[ievt * np4 + ip4] << ", "; + std::cout << std::endl; + } + const int ipagV = ievt / neppV; // #event vector in this iteration + const fptype* ievt0Momenta = MemoryAccessMomenta::ieventAccessRecordConst( hstMomenta.data(), ipagV * neppV ); + // Test ixxxxx - NO ASSUMPTIONS + { + prepareTest( "ixxxxx", ievt ); + const fptype fmass = mass0[ievt]; + ixxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfI, ipar0 ); + testwf6( outwfI, "ixxxxx", ievt, nsp, fmass ); + ixxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfI, ipar0 ); + testwf6( outwfI, "ixxxxx", ievt, nsp, -fmass ); + } + // Test ipzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) + { + prepareTest( "ipzxxx", ievt ); + ipzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfI, "ipzxxx", ievt ); + testwf6( outwf, "ipzxxx", ievt, nsp, 0 ); + } + // Test imzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) + { + prepareTest( "imzxxx", ievt ); + imzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfI, "imzxxx", ievt ); + testwf6( outwf, "imzxxx", ievt, nsp, 0 ); + } + // Test ixzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) + if( mass0[ievt] == 0 && isptgt0[ievt] ) + { + prepareTest( "ixzxxx", ievt ); + ixzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfI, "ixzxxx", ievt ); + testwf6( outwf, "ixzxxx", ievt, nsp, 0 ); + } + // Test vxxxxx - NO ASSUMPTIONS + { + prepareTest( "vxxxxx", ievt ); + const fptype vmass = mass0[ievt]; + vxxxxx( ievt0Momenta, vmass, nhel, nsp, fp_outwf, ipar0 ); + testwf6( outwf, "vxxxxx", ievt, nsp, vmass ); + vxxxxx( ievt0Momenta, -vmass, nhel, nsp, fp_outwf, ipar0 ); + testwf6( outwf, "vxxxxx", ievt, nsp, -vmass ); + } + // Test sxxxxx - NO ASSUMPTIONS + { + prepareTest( "sxxxxx", ievt ); + const fptype smass = mass0[ievt]; + sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass>0") + testwf6( outwf3, "sxxxxx", ievt, nsp, smass ); + sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass<0") + testwf6( outwf3, "sxxxxx", ievt, nsp, -smass ); + } + // Test oxxxxx - NO ASSUMPTIONS + { + prepareTest( "oxxxxx", ievt ); + const fptype fmass = mass0[ievt]; + oxxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfO, ipar0 ); + testwf6( outwfO, "oxxxxx", ievt, nsp, fmass ); + oxxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfO, ipar0 ); + testwf6( outwfO, "oxxxxx", ievt, nsp, -fmass ); + } + // Test opzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) + { + prepareTest( "opzxxx", ievt ); + opzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfO, "opzxxx", ievt ); + testwf6( outwf, "opzxxx", ievt, nsp, 0 ); + } + // Test omzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) + { + prepareTest( "omzxxx", ievt ); + omzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfO, "omzxxx", ievt ); + testwf6( outwf, "omzxxx", ievt, nsp, 0 ); + } + // Test oxzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) + if( mass0[ievt] == 0 && isptgt0[ievt] ) + { + prepareTest( "oxzxxx", ievt ); + oxzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfO, "oxzxxx", ievt ); + testwf6( outwf, "oxzxxx", ievt, nsp, 0 ); + } + } + } + // *** END OF TESTING LOOP + if( dumpEvents ) + { + dumpFile.close(); + std::cout << "INFO: New reference data dumped to file '" << dumpFileName << "'" << std::endl; + } +#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) + signal( SIGFPE, fpeHandlerDefault ); +#endif +} + +//========================================================================== + +// Reset the GPU after ALL tests have gone out of scope +// (This was needed to avoid leaks in profilers, but compute-sanitizer reports no leaks, is it STILL needed?) +// ========= NB: resetting the GPU too early causes segfaults that are very difficult to debug #907 ========= +// Try to use atexit (https://stackoverflow.com/a/14610501) but this still crashes! +// ********* FIXME? avoid CUDA API calls in destructors? (see https://stackoverflow.com/a/16982503) ********* +void +myexit() +{ +#ifdef MGONGPUCPP_GPUIMPL + //checkGpu( gpuDeviceReset() ); // FIXME??? this still crashes! should systematically avoid CUDA calls in all destructors? +#endif +} + +// Main function (see https://google.github.io/googletest/primer.html#writing-the-main-function) +// (NB: the test executables are now separate for C++ and CUDA, therefore main must be included all the time) +// (NB: previously, '#ifndef MGONGPUCPP_GPUIMPL' was ensuring that main was only included once while linking both C++ and CUDA tests) +int +main( int argc, char** argv ) +{ + atexit( myexit ); + testing::InitGoogleTest( &argc, argv ); + int status = RUN_ALL_TESTS(); + return status; +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc new file mode 100644 index 0000000000..d19c93bb9b --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc @@ -0,0 +1,667 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: T. Heimel (Nov 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro (2026). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "umami.h" + +#include "CPPProcess.h" +#include "GpuRuntime.h" +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" + +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +using namespace mg5amcGpu; +#else +using namespace mg5amcCpu; +#endif + +namespace +{ + + void* initialize_impl( + const fptype* momenta, + const fptype* couplings, + const unsigned int* flavor_indices, + fptype* matrix_elements, +#ifdef MGONGPUCPP_GPUIMPL + fptype* color_jamps, +#endif + fptype* numerators, + fptype* denominators, + std::size_t count ) + { + bool is_good_hel[CPPProcess::ncomb]; + sigmaKin_getGoodHel( + momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, +#ifdef MGONGPUCPP_GPUIMPL + color_jamps, +#endif + is_good_hel, + count ); + sigmaKin_setGoodHel( is_good_hel ); + return nullptr; + } + + void initialize( + const fptype* momenta, + const fptype* couplings, + const unsigned int* flavor_indices, + fptype* matrix_elements, +#ifdef MGONGPUCPP_GPUIMPL + fptype* color_jamps, +#endif + fptype* numerators, + fptype* denominators, + std::size_t count ) + { + // static local initialization is called exactly once in a thread-safe way + static void* dummy = initialize_impl( momenta, couplings, flavor_indices, matrix_elements, +#ifdef MGONGPUCPP_GPUIMPL + color_jamps, +#endif + numerators, + denominators, + count ); + } + +#ifdef MGONGPUCPP_GPUIMPL + __device__ +#endif + void + transpose_momenta( const double* momenta_in, fptype* momenta_out, std::size_t i_event_in, std::size_t i_event_out, std::size_t stride ) + { + std::size_t page_size = MemoryAccessMomentaBase::neppM; + std::size_t i_page = i_event_out / page_size; + std::size_t i_vector = i_event_out % page_size; + + for( std::size_t i_part = 0; i_part < CPPProcess::npar; ++i_part ) + { + for( std::size_t i_mom = 0; i_mom < 4; ++i_mom ) + { + momenta_out[i_page * CPPProcess::npar * 4 * page_size + + i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( CPPProcess::npar * i_mom + i_part ) + i_event_in]; + } + } + } + +#ifdef MGONGPUCPP_GPUIMPL + + __global__ void copy_inputs( + const double* momenta_in, + const double* helicity_random_in, + const double* color_random_in, + const double* diagram_random_in, + const double* alpha_s_in, + const unsigned int* flavor_indices_in, + fptype* momenta, + fptype* helicity_random, + fptype* color_random, + fptype* diagram_random, + fptype* g_s, + unsigned int* flavor_indices, + std::size_t count, + std::size_t stride, + std::size_t offset ) + { + std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; + if( i_event >= count ) return; + + transpose_momenta( &momenta_in[offset], momenta, i_event, i_event, stride ); + diagram_random[i_event] = diagram_random_in ? diagram_random_in[i_event + offset] : 0.5; + helicity_random[i_event] = helicity_random_in ? helicity_random_in[i_event + offset] : 0.5; + color_random[i_event] = color_random_in ? color_random_in[i_event + offset] : 0.5; + g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; + } + + __global__ void copy_outputs( + fptype* denominators, + fptype* numerators, + fptype* matrix_elements, + unsigned int* diagram_index, + int* color_index, + int* helicity_index, + double* m2_out, + double* amp2_out, + int* diagram_out, + int* color_out, + int* helicity_out, + std::size_t count, + std::size_t stride, + std::size_t offset ) + { + std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; + if( i_event >= count ) return; + + if( m2_out ) m2_out[i_event + offset] = matrix_elements[i_event]; + if( amp2_out ) + { + double denominator = denominators[i_event]; + for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_event * CPPProcess::ndiagrams + i_diag] / denominator; + } + } + if( diagram_out ) diagram_out[i_event + offset] = diagram_index[i_event] - 1; + if( color_out ) color_out[i_event + offset] = color_index[i_event] - 1; + if( helicity_out ) helicity_out[i_event + offset] = helicity_index[i_event] - 1; + } + +#endif // MGONGPUCPP_GPUIMPL + + struct InterfaceInstance + { + bool initialized = false; + }; + + std::vector g_externalMasses; + +} + +extern "C" +{ + UmamiStatus umami_get_meta( UmamiMetaKey meta_key, void* result ) + { + switch( meta_key ) + { + case UMAMI_META_DEVICE: + { + UmamiDevice& device = *static_cast( result ); +#ifdef MGONGPUCPP_GPUIMPL +#ifdef __CUDACC__ + device = UMAMI_DEVICE_CUDA; +#elif defined( __HIPCC__ ) + device = UMAMI_DEVICE_HIP; +#endif +#else + device = UMAMI_DEVICE_CPU; +#endif + break; + } + case UMAMI_META_PARTICLE_COUNT: + *static_cast( result ) = CPPProcess::npar; + break; + case UMAMI_META_DIAGRAM_COUNT: + *static_cast( result ) = CPPProcess::ndiagrams; + break; + case UMAMI_META_HELICITY_COUNT: + *static_cast( result ) = CPPProcess::ncomb; + break; + case UMAMI_META_COLOR_COUNT: + return UMAMI_ERROR_UNSUPPORTED_META; + case UMAMI_META_MASSES: + { + if( g_externalMasses.size() != (size_t)CPPProcess::npar ) return UMAMI_ERROR_UNINITIALIZED_META; + + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + static_cast( result )[ipar] = g_externalMasses[ipar]; + break; + } + default: + return UMAMI_ERROR_UNSUPPORTED_META; + } + return UMAMI_SUCCESS; + } + + UmamiStatus umami_supported_inputs( bool const** supported, int* count ) + { + // MOMENTA, ALPHA_S, FLAVOR_INDEX, RANDOM_COLOR, RANDOM_HELICITY, RANDOM_DIAGRAM, + // HELICITY_INDEX=false, DIAGRAM_INDEX=true, CHANNEL_INDEX=false + static const bool data[UMAMI_INPUT_KEY_COUNT] = { true, true, true, true, true, true, false, true }; + *supported = data; + *count = UMAMI_INPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_required_inputs( bool const** required, int* count ) + { + static const bool data[UMAMI_INPUT_KEY_COUNT] = { true }; // MOMENTA only + *required = data; + *count = UMAMI_INPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_supported_outputs( bool const** supported, int* count ) + { + // MATRIX_ELEMENT, DIAGRAM_AMP2, COLOR_INDEX, HELICITY_INDEX, DIAGRAM_INDEX, GPU_STREAM +#ifdef MGONGPUCPP_GPUIMPL + static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true, true }; +#else + static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true }; +#endif + *supported = data; + *count = UMAMI_OUTPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_initialize( UmamiHandle* handle, char const* param_card_path ) + { + CPPProcess process; + process.initProc( param_card_path ); + + const std::vector& masses = process.getMasses(); + g_externalMasses.assign( masses.begin(), masses.end() ); + + auto instance = new InterfaceInstance(); + *handle = instance; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_set_parameter( + [[maybe_unused]] UmamiHandle handle, + [[maybe_unused]] char const* name, + [[maybe_unused]] double parameter_real, + [[maybe_unused]] double parameter_imag ) + { + return UMAMI_ERROR_NOT_IMPLEMENTED; + } + + UmamiStatus umami_get_parameter( + [[maybe_unused]] UmamiHandle handle, + [[maybe_unused]] char const* name, + [[maybe_unused]] double* parameter_real, + [[maybe_unused]] double* parameter_imag ) + { + return UMAMI_ERROR_NOT_IMPLEMENTED; + } + + UmamiStatus umami_matrix_element( + UmamiHandle handle, + size_t count, + size_t stride, + size_t offset, + size_t input_count, + UmamiInputKey const* input_keys, + void const* const* inputs, + size_t output_count, + UmamiOutputKey const* output_keys, + void* const* outputs ) + { + const double* momenta_in = nullptr; + const double* alpha_s_in = nullptr; + const unsigned int* flavor_indices_in = nullptr; + const double* random_color_in = nullptr; + const double* random_helicity_in = nullptr; + const double* random_diagram_in = nullptr; + [[maybe_unused]] const int* diagram_in = nullptr; // TODO: unused + + for( std::size_t i = 0; i < input_count; ++i ) + { + const void* input = inputs[i]; + switch( input_keys[i] ) + { + case UMAMI_IN_MOMENTA: + momenta_in = static_cast( input ); + break; + case UMAMI_IN_ALPHA_S: + alpha_s_in = static_cast( input ); + break; + case UMAMI_IN_FLAVOR_INDEX: + flavor_indices_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_COLOR: + random_color_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_HELICITY: + random_helicity_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_DIAGRAM: + random_diagram_in = static_cast( input ); + break; + case UMAMI_IN_HELICITY_INDEX: + return UMAMI_ERROR_UNSUPPORTED_INPUT; + case UMAMI_IN_DIAGRAM_INDEX: + diagram_in = static_cast( input ); + break; + default: + return UMAMI_ERROR_UNSUPPORTED_INPUT; + } + } + if( !momenta_in ) return UMAMI_ERROR_MISSING_INPUT; + +#ifdef MGONGPUCPP_GPUIMPL + gpuStream_t gpu_stream = nullptr; +#endif + double* m2_out = nullptr; + double* amp2_out = nullptr; + int* diagram_out = nullptr; + int* color_out = nullptr; + int* helicity_out = nullptr; + for( std::size_t i = 0; i < output_count; ++i ) + { + void* output = outputs[i]; + switch( output_keys[i] ) + { + case UMAMI_OUT_MATRIX_ELEMENT: + m2_out = static_cast( output ); + break; + case UMAMI_OUT_DIAGRAM_AMP2: + amp2_out = static_cast( output ); + break; + case UMAMI_OUT_COLOR_INDEX: + color_out = static_cast( output ); + break; + case UMAMI_OUT_HELICITY_INDEX: + helicity_out = static_cast( output ); + break; + case UMAMI_OUT_DIAGRAM_INDEX: + diagram_out = static_cast( output ); + break; +#ifdef MGONGPUCPP_GPUIMPL + case UMAMI_OUT_GPU_STREAM: + gpu_stream = static_cast( output ); + break; +#endif + default: + return UMAMI_ERROR_UNSUPPORTED_OUTPUT; + } + } + +#ifdef MGONGPUCPP_GPUIMPL + std::size_t n_threads = 256; + std::size_t n_blocks = ( count + n_threads - 1 ) / n_threads; + std::size_t rounded_count = n_blocks * n_threads; + + fptype *momenta, *couplings, *g_s, *helicity_random, *color_random, *diagram_random, *color_jamps; + fptype *matrix_elements, *numerators, *denominators, *ghel_matrix_elements, *ghel_jamps; + int *helicity_index, *color_index; + unsigned int *flavor_indices, *diagram_index; + + std::size_t n_coup = mg5amcGpu::Parameters_dependentCouplings::ndcoup; + std::array, 16> ptrs_and_sizes = {{ + {reinterpret_cast(&momenta), rounded_count * CPPProcess::npar * 4 * sizeof( fptype )}, + {reinterpret_cast(&couplings), rounded_count * n_coup * 2 * sizeof( fptype )}, + {reinterpret_cast(&g_s), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&flavor_indices), rounded_count * sizeof( unsigned int )}, + {reinterpret_cast(&helicity_random), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&color_random), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&diagram_random), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&matrix_elements), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&diagram_index), rounded_count * sizeof( unsigned int )}, + {reinterpret_cast(&color_jamps), rounded_count * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, + {reinterpret_cast(&numerators), rounded_count * CPPProcess::ndiagrams * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&denominators), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&helicity_index), rounded_count * sizeof( int )}, + {reinterpret_cast(&color_index), rounded_count * sizeof( int )}, + {reinterpret_cast(&ghel_matrix_elements), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&ghel_jamps), rounded_count * CPPProcess::ncomb * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, + }}; + std::size_t total_size = 0; + constexpr std::size_t MAX_SIZE = std::max(sizeof(fptype), sizeof(int)); + for (auto [ptr, size] : ptrs_and_sizes) { + std::size_t aligned_size = (size + MAX_SIZE - 1) / MAX_SIZE * MAX_SIZE; + total_size += aligned_size; + } + uint8_t* buffer; + // we can consider caching this between matrix element calls + gpuMallocAsync( &buffer, total_size, gpu_stream ); + std::size_t buf_offset = 0; + for (auto [ptr, size] : ptrs_and_sizes) { + std::size_t aligned_size = (size + 7) / 8 * 8; + *ptr = buffer + buf_offset; + buf_offset += aligned_size; + } + + copy_inputs<<>>( + momenta_in, + random_helicity_in, + random_color_in, + random_diagram_in, + alpha_s_in, + flavor_indices_in, + momenta, + helicity_random, + color_random, + diagram_random, + g_s, + flavor_indices, + count, + stride, + offset ); + computeDependentCouplings<<>>( g_s, couplings ); + checkGpu( gpuPeekAtLastError() ); + + InterfaceInstance* instance = static_cast( handle ); + if( !instance->initialized ) + { + initialize( + momenta, couplings, flavor_indices, matrix_elements, color_jamps, numerators, denominators, rounded_count ); + instance->initialized = true; + } + + sigmaKin( + momenta, + couplings, + flavor_indices, + helicity_random, + color_random, + nullptr, + diagram_random, + matrix_elements, + helicity_index, + color_index, + color_jamps, + numerators, + denominators, + diagram_index, + false, + ghel_matrix_elements, + ghel_jamps, + nullptr, + nullptr, + &gpu_stream, + true, + n_blocks, + n_threads ); + + copy_outputs<<>>( + denominators, + numerators, + matrix_elements, + diagram_index, + color_index, + helicity_index, + m2_out, + amp2_out, + diagram_out, + color_out, + helicity_out, + count, + stride, + offset ); + checkGpu( gpuPeekAtLastError() ); + + gpuFreeAsync( buffer, gpu_stream ); +#else // MGONGPUCPP_GPUIMPL + constexpr std::size_t vector_size = MemoryAccessMomentaBase::neppM; + // need to round to round to double page size for some reason + constexpr std::size_t page_size2 = 2 * vector_size; + std::vector permutation; + std::size_t rounded_count; + + constexpr std::size_t flavor_count = CPPProcess::nmaxflavor; + HostBufferBase flavor_indices( ((count + page_size2 - 1) / page_size2 + flavor_count) * page_size2 ); + bool sort_flavors = vector_size > 1 && flavor_count > 1 && flavor_indices_in; + if ( sort_flavors ) + { + permutation.resize(count); + std::size_t voffset = 0; + std::size_t vector_indices[flavor_count] = {}; + std::size_t vector_counts[flavor_count] = {}; + // determine permutation of inputs such that all entries in a SIMD vector + // have the same flavor index + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + unsigned int flav = flavor_indices_in[i_event + offset]; + auto& vcount = vector_counts[flav]; + auto& vindex = vector_indices[flav]; + if ( vcount == 0 ) + { + vindex = voffset * page_size2; + for ( std::size_t i = 0; i < page_size2; ++i) { + flavor_indices[voffset * page_size2 + i] = flav; + } + voffset += 1; + } + permutation[i_event] = vindex + vcount; + vcount = (vcount + 1) % page_size2; + } + rounded_count = voffset * page_size2; + } else { + rounded_count = ( count + page_size2 - 1 ) / page_size2 * page_size2; + } + + HostBufferBase momenta( rounded_count * CPPProcess::npar * 4 ); + HostBufferBase couplings( rounded_count * mg5amcCpu::Parameters_dependentCouplings::ndcoup * 2 ); + HostBufferBase g_s( rounded_count ); + HostBufferBase helicity_random( rounded_count ); + HostBufferBase color_random( rounded_count ); + HostBufferBase diagram_random( rounded_count ); + HostBufferBase matrix_elements( rounded_count ); + HostBufferBase diagram_index( rounded_count ); + HostBufferBase numerators( rounded_count * CPPProcess::ndiagrams ); + HostBufferBase denominators( rounded_count ); + HostBufferBase helicity_index( rounded_count ); + HostBufferBase color_index( rounded_count ); + if ( sort_flavors ) { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_sorted = permutation[i_event]; + transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_sorted, stride ); + helicity_random[i_sorted] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; + color_random[i_sorted] = random_color_in ? random_color_in[i_event + offset] : 0.5; + diagram_random[i_sorted] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; + g_s[i_sorted] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + } + } else { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_event, stride ); + helicity_random[i_event] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; + color_random[i_event] = random_color_in ? random_color_in[i_event + offset] : 0.5; + diagram_random[i_event] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; + g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; + } + for ( std::size_t i_event = count; i_event < rounded_count; ++i_event ) { + flavor_indices[i_event] = 0; + } + } + computeDependentCouplings( g_s.data(), couplings.data(), rounded_count ); + + InterfaceInstance* instance = static_cast( handle ); + if( !instance->initialized ) + { + initialize( + momenta.data(), + couplings.data(), + flavor_indices.data(), + matrix_elements.data(), + numerators.data(), + denominators.data(), + rounded_count ); + instance->initialized = true; + } + + sigmaKin( + momenta.data(), + couplings.data(), + flavor_indices.data(), + helicity_random.data(), + color_random.data(), + nullptr, + diagram_random.data(), + matrix_elements.data(), + helicity_index.data(), + color_index.data(), + numerators.data(), + denominators.data(), + diagram_index.data(), + false, + rounded_count ); + + if ( sort_flavors ) + { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_sorted = permutation[i_event]; + std::size_t page_size = MemoryAccessMomentaBase::neppM; + std::size_t i_page = i_sorted / page_size; + std::size_t i_vector = i_sorted % page_size; // vector lane + + double denominator = denominators[i_sorted]; + if( m2_out != nullptr ) + { + m2_out[i_event + offset] = matrix_elements[i_sorted]; + } + if( amp2_out != nullptr ) + { + for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; + } + } + if( diagram_out != nullptr ) + { + diagram_out[i_event + offset] = diagram_index[i_sorted] - 1; + } + if( color_out != nullptr ) + { + color_out[i_event + offset] = color_index[i_sorted] - 1; + } + if( helicity_out != nullptr ) + { + helicity_out[i_event + offset] = helicity_index[i_sorted] - 1; + } + } + } else { + std::size_t page_size = MemoryAccessMomentaBase::neppM; + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_page = i_event / page_size; + std::size_t i_vector = i_event % page_size; + + double denominator = denominators[i_event]; + if( m2_out != nullptr ) + { + m2_out[i_event + offset] = matrix_elements[i_event]; + } + if( amp2_out != nullptr ) + { + for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; + } + } + if( diagram_out != nullptr ) + { + diagram_out[i_event + offset] = diagram_index[i_event] - 1; + } + if( color_out != nullptr ) + { + color_out[i_event + offset] = color_index[i_event] - 1; + } + if( helicity_out != nullptr ) + { + helicity_out[i_event + offset] = helicity_index[i_event] - 1; + } + } + } +#endif // MGONGPUCPP_GPUIMPL + return UMAMI_SUCCESS; + } + + UmamiStatus umami_free( UmamiHandle handle ) + { + InterfaceInstance* instance = static_cast( handle ); + delete instance; + return UMAMI_SUCCESS; + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/Bridge.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/Bridge.h new file mode 100644 index 0000000000..8417e3ad15 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/Bridge.h @@ -0,0 +1,633 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Nov 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2021-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef BRIDGE_H +#define BRIDGE_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" // for CPPProcess +#include "CrossSectionKernels.h" // for flagAbnormalMEs +#include "MatrixElementKernels.h" // for MatrixElementKernelHost, MatrixElementKernelDevice +#include "MemoryAccessMomenta.h" // for MemoryAccessMomenta::neppM +#include "MemoryBuffers.h" // for HostBufferMomenta, DeviceBufferMomenta etc + +//#ifdef __HIPCC__ +//#include // see +//https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 #else #include +// // bypass this completely to ease portability on LUMI #803 #endif + +#include // bypass std::filesystem #803 + +#include +#include +#include +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + /** + * A base class for a class whose pointer is passed between Fortran and C++. + * This is not really necessary, but it allows minimal type checks on all such + * pointers. + */ + struct CppObjectInFortran + { + CppObjectInFortran() {} + virtual ~CppObjectInFortran() {} + }; + + //-------------------------------------------------------------------------- + /** + * A templated class for calling the CUDA/C++ matrix element calculations of the + * event generation workflow. The FORTRANFPTYPE template parameter indicates the + * precision of the Fortran momenta from MadEvent (float or double). The + * precision of the matrix element calculation is hardcoded in the fptype + * typedef in CUDA/C++. + * + * The Fortran momenta passed in are in the form of + * DOUBLE PRECISION P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) + * where the dimensions are , , + * . In memory, this is stored in a way that C reads as an array + * P_MULTI[nevtF][nparF][np4F]. The CUDA/C++ momenta are stored as an + * array[npagM][npar][np4][neppM] with nevt=npagM*neppM. The Bridge is + * configured to store nevt==nevtF events in CUDA/C++. It also checks that + * Fortran and C++ parameters match, nparF==npar and np4F==np4. + * + * The cpu/gpu sequences take FORTRANFPTYPE* (not fptype*) momenta/MEs. + * This allows mixing double in MadEvent Fortran with float in CUDA/C++ + * sigmaKin. In the fcheck_sa.f test, Fortran uses double while CUDA/C++ may use + * double or float. In the check_sa "--bridge" test, everything is implemented + * in fptype (double or float). + */ + template + class Bridge final : public CppObjectInFortran + { + public: + /** + * Constructor + * + * @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran array + * loops (VECSIZE_USED <= VECSIZE_MEMMAX) + * @param nparF (NEXTERNAL, nexternal.inc) number of external particles in + * Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + * @param np4F number of momenta components, usually 4, in Fortran arrays + * (KEPT FOR SANITY CHECKS ONLY) + */ + Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F ); + + /** + * Destructor + */ + virtual ~Bridge() {} + + // Delete copy/move constructors and assignment operators + Bridge( const Bridge& ) = delete; + Bridge( Bridge&& ) = delete; + Bridge& operator=( const Bridge& ) = delete; + Bridge& operator=( Bridge&& ) = delete; + +#ifdef MGONGPUCPP_GPUIMPL + /** + * Set the gpublocks and gputhreads for the gpusequence - throws if evnt != + * gpublocks*gputhreads (this is needed for BridgeKernel tests rather than for + * actual production use in Fortran) + * + * @param gpublocks number of gpublocks + * @param gputhreads number of gputhreads + */ + void set_gpugrid( const int gpublocks, const int gputhreads ); + + /** + * Sequence to be executed for the Cuda matrix element calculation + * + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant + * alphas) + * @param iflavorVec the index of the flavor combination + * @param rndhel the pointer to the input random numbers for helicity + * selection + * @param rndcol the pointer to the input random numbers for color selection + * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 + * to n + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void gpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); +#else + /** + * Sequence to be executed for the vectorized CPU matrix element calculation + * + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant + * alphas) + * @param iflavorVec the index of the flavor combination + * @param rndhel the pointer to the input random numbers for helicity + * selection + * @param rndcol the pointer to the input random numbers for color selection + * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 + * to n + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void cpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); +#endif + + // Return the number of good helicities (-1 initially when they have not yet + // been calculated) + int nGoodHel() const { return m_nGoodHel; } + + // Return the total number of helicities (expose cudacpp ncomb in the Bridge + // interface to Fortran) + constexpr int nTotHel() const { return CPPProcess::ncomb; } + + private: + unsigned int m_nevt; // number of events + int m_nGoodHel; // the number of good helicities (-1 initially when they have + // not yet been calculated) + +#ifdef MGONGPUCPP_GPUIMPL + int m_gputhreads; // number of gpu threads (default set from number of + // events, can be modified) + int m_gpublocks; // number of gpu blocks (default set from number of events, + // can be modified) + DeviceBuffer m_devMomentaF; + DeviceBufferMomenta m_devMomentaC; + DeviceBufferGs m_devGs; + DeviceBufferIflavorVec m_devIflavorVec; + DeviceBufferRndNumHelicity m_devRndHel; + DeviceBufferRndNumColor m_devRndCol; + DeviceBufferMatrixElements m_devMEs; + DeviceBufferSelectedHelicity m_devSelHel; + DeviceBufferSelectedColor m_devSelCol; + DeviceBufferChannelIds m_devChannelIds; + PinnedHostBufferIflavorVec m_hstIflavorVec; + PinnedHostBufferGs m_hstGs; + PinnedHostBufferRndNumHelicity m_hstRndHel; + PinnedHostBufferRndNumColor m_hstRndCol; + PinnedHostBufferMatrixElements m_hstMEs; + PinnedHostBufferSelectedHelicity m_hstSelHel; + PinnedHostBufferSelectedColor m_hstSelCol; + PinnedHostBufferChannelIds m_hstChannelIds; + std::unique_ptr m_pmek; + // static constexpr int s_gputhreadsmin = 16; // minimum number of gpu threads + // (TEST VALUE FOR MADEVENT) + static constexpr int s_gputhreadsmin = + 32; // minimum number of gpu threads (DEFAULT) +#else + HostBufferMomenta m_hstMomentaC; + HostBufferGs m_hstGs; + HostBufferIflavorVec m_hstIflavorVec; + HostBufferRndNumHelicity m_hstRndHel; + HostBufferRndNumColor m_hstRndCol; + HostBufferMatrixElements m_hstMEs; + HostBufferSelectedHelicity m_hstSelHel; + HostBufferSelectedColor m_hstSelCol; + HostBufferChannelIds m_hstChannelIds; + std::unique_ptr m_pmek; +#endif + }; + + //-------------------------------------------------------------------------- + // + // Forward declare transposition methods + // + +#ifdef MGONGPUCPP_GPUIMPL + + template + __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); + +#endif // MGONGPUCPP_GPUIMPL + + template + void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); + + template + void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ); + + //-------------------------------------------------------------------------- + // + // Implementations of member functions of class Bridge + // + + template + Bridge::Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F) + : m_nevt( nevtF ), m_nGoodHel( -1 ) +#ifdef MGONGPUCPP_GPUIMPL + , m_gputhreads( 256 ) // default number of gpu threads + , m_gpublocks( m_nevt / m_gputhreads ) // this ensures m_nevt <= m_gpublocks*m_gputhreads + , m_devMomentaF( m_nevt ) + , m_devMomentaC( m_nevt ) + , m_devIflavorVec( m_nevt ) + , m_devGs( m_nevt ) + , m_devRndHel( m_nevt ) + , m_devRndCol( m_nevt ) + , m_devMEs( m_nevt ) + , m_devSelHel( m_nevt ) + , m_devSelCol( m_nevt ) + , m_devChannelIds( m_nevt ) +#else + , m_hstMomentaC( m_nevt ) +#endif + , m_hstGs( m_nevt ) + , m_hstIflavorVec( m_nevt ) + , m_hstRndHel( m_nevt ) + , m_hstRndCol( m_nevt ) + , m_hstMEs( m_nevt ) + , m_hstSelHel( m_nevt ) + , m_hstSelCol( m_nevt ) + , m_hstChannelIds( m_nevt ) + , m_pmek( nullptr ) + { + if( nparF != CPPProcess::npar ) + throw std::runtime_error( "Bridge constructor: npar mismatch" ); + if( np4F != CPPProcess::np4 ) + throw std::runtime_error( "Bridge constructor: np4 mismatch" ); +#ifdef MGONGPUCPP_GPUIMPL + if( ( m_nevt < s_gputhreadsmin ) || ( m_nevt % s_gputhreadsmin != 0 ) ) + throw std::runtime_error( + "Bridge constructor: nevt should be a multiple of " + + std::to_string( s_gputhreadsmin ) ); + while( m_nevt != m_gpublocks * m_gputhreads ) + { + m_gputhreads /= 2; + if( m_gputhreads < s_gputhreadsmin ) + throw std::logic_error( + "Bridge constructor: FIXME! cannot choose gputhreads" ); // this + // should + // never + // happen! + m_gpublocks = m_nevt / m_gputhreads; + } +#ifdef MGONGPUCPP_VERBOSE + std::cout << "WARNING! Instantiate device Bridge (nevt=" << m_nevt + << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads + << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" + << std::endl; +#endif + m_pmek.reset( new MatrixElementKernelDevice( + m_devMomentaC, m_devGs, m_devIflavorVec, m_devRndHel, m_devRndCol, m_devChannelIds, m_devMEs, m_devSelHel, m_devSelCol, m_gpublocks, m_gputhreads) ); +#else +#ifdef MGONGPUCPP_VERBOSE + std::cout << "WARNING! Instantiate host Bridge (nevt=" << m_nevt << ")" + << std::endl; +#endif + m_pmek.reset( new MatrixElementKernelHost( + m_hstMomentaC, m_hstGs, m_hstIflavorVec, m_hstRndHel, m_hstRndCol, m_hstChannelIds, m_hstMEs, m_hstSelHel, m_hstSelCol, m_nevt ) ); +#endif // MGONGPUCPP_GPUIMPL + // Create a process object, read param card and set parameters + // FIXME: the process instance can happily go out of scope because it is only + // needed to read parameters? + // FIXME: the CPPProcess should really be a singleton? what if fbridgecreate + // is called from several Fortran threads? + CPPProcess process( /*verbose=*/false ); + std::string paramCard = + "../Cards/param_card.dat"; // ZW: change default param_card.dat location + // to one dir down + /* +#ifdef __HIPCC__ + if( !std::experimental::filesystem::exists( paramCard ) ) paramCard = "../" + +paramCard; #else if( !std::filesystem::exists( paramCard ) ) paramCard = "../" + +paramCard; #endif + */ + // struct stat dummybuffer; // bypass std::filesystem #803 + // if( !( stat( paramCard.c_str(), &dummyBuffer ) == 0 ) ) paramCard = "../" + + // paramCard; // + auto fileExists = []( std::string& fileName ) + { + struct stat buffer; + return stat( fileName.c_str(), &buffer ) == 0; + }; + size_t paramCardCheck = 2; // ZW: check for paramCard up to 2 directories up + for( size_t k = 0; k < paramCardCheck; ++k ) + { + if( fileExists( paramCard ) ) break; // bypass std::filesystem #803 + paramCard = "../" + paramCard; + } + process.initProc( paramCard ); + } + +#ifdef MGONGPUCPP_GPUIMPL + template + void Bridge::set_gpugrid( const int gpublocks, + const int gputhreads ) + { + if( m_nevt != gpublocks * gputhreads ) + throw std::runtime_error( + "Bridge: gpublocks*gputhreads must equal m_nevt in set_gpugrid" ); + m_gpublocks = gpublocks; + m_gputhreads = gputhreads; +#ifdef MGONGPUCPP_VERBOSE + std::cout << "WARNING! Set grid in Bridge (nevt=" << m_nevt + << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads + << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" + << std::endl; +#endif + m_pmek->setGrid( m_gpublocks, m_gputhreads ); + } +#endif + +#ifdef MGONGPUCPP_GPUIMPL + template + void Bridge::gpu_sequence( const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool goodHelOnly ) + { + constexpr int neppM = MemoryAccessMomenta::neppM; + if constexpr( neppM == 1 && std::is_same_v ) + { + gpuMemcpy( m_devMomentaC.data(), momenta, m_devMomentaC.bytes(), gpuMemcpyHostToDevice ); + } + else + { + gpuMemcpy( m_devMomentaF.data(), momenta, m_devMomentaF.bytes(), gpuMemcpyHostToDevice ); + const int thrPerEvt = + CPPProcess::npar * + CPPProcess::np4; // AV: transpose alg does 1 element per thread (NOT 1 + // event per thread) + // const int thrPerEvt = 1; // AV: try new alg with 1 event per thread... + // this seems slower + gpuLaunchKernel( dev_transposeMomentaF2C, m_gpublocks * thrPerEvt, m_gputhreads, m_devMomentaF.data(), m_devMomentaC.data(), m_nevt ); + } + if constexpr( std::is_same_v ) + { + memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); + } + else + { + std::copy( gs, gs + m_nevt, m_hstGs.data() ); + std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); + std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); + } + const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); + if( useChannelIds ) + memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); + // else ... // no need to initialize m_hstChannel: it is allocated with + // gpuMallocHost and NOT initialized in PinnedHostBufferBase, but it is NOT + // used later on + // initialise iflavorVec + memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); + copyDeviceFromHost( m_devGs, m_hstGs ); + copyDeviceFromHost( m_devRndHel, m_hstRndHel ); + copyDeviceFromHost( m_devRndCol, m_hstRndCol ); + if( useChannelIds ) copyDeviceFromHost( m_devChannelIds, m_hstChannelIds ); + copyDeviceFromHost( m_devIflavorVec, m_hstIflavorVec ); + if( m_nGoodHel < 0 ) + { + m_nGoodHel = m_pmek->computeGoodHelicities(); + if( m_nGoodHel < 0 ) + throw std::runtime_error( + "Bridge gpu_sequence: computeGoodHelicities returned nGoodHel<0" ); + } + if( goodHelOnly ) return; + m_pmek->computeMatrixElements( useChannelIds ); + copyHostFromDevice( m_hstMEs, m_devMEs ); +#ifdef MGONGPUCPP_VERBOSE + flagAbnormalMEs( m_hstMEs.data(), m_nevt ); +#endif + copyHostFromDevice( m_hstSelHel, m_devSelHel ); + copyHostFromDevice( m_hstSelCol, m_devSelCol ); + if constexpr( std::is_same_v ) + { + memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); + memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); + memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); + } + else + { + std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); + std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); + std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); + } + } +#endif + +#ifndef MGONGPUCPP_GPUIMPL + template + void Bridge::cpu_sequence( const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool goodHelOnly ) + { + hst_transposeMomentaF2C( momenta, m_hstMomentaC.data(), m_nevt ); + if constexpr( std::is_same_v ) + { + memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); + } + else + { + std::copy( gs, gs + m_nevt, m_hstGs.data() ); + std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); + std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); + } + const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); + if( useChannelIds ) + memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); + // else ... // no need to initialize m_hstChannel: it is allocated and default + // initialized in HostBufferBase (and it is not used later on anyway) + // initialise iflavorVec + memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); + if( m_nGoodHel < 0 ) + { + m_nGoodHel = m_pmek->computeGoodHelicities(); + if( m_nGoodHel < 0 ) + throw std::runtime_error( + "Bridge cpu_sequence: computeGoodHelicities returned nGoodHel<0" ); + } + if( goodHelOnly ) return; + m_pmek->computeMatrixElements( useChannelIds ); +#ifdef MGONGPUCPP_VERBOSE + flagAbnormalMEs( m_hstMEs.data(), m_nevt ); +#endif + if constexpr( std::is_same_v ) + { + memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); + memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); + memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); + } + else + { + std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); + std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); + std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); + } + } +#endif + + //-------------------------------------------------------------------------- + // + // Implementations of transposition methods + // - FORTRAN arrays: P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) ==> + // p_multi[nevtF][nparF][np4F] in C++ (AOS) + // - C++ array: momenta[npagM][npar][np4][neppM] with nevt=npagM*neppM (AOSOA) + // + +#ifdef MGONGPUCPP_GPUIMPL + template + __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool oldImplementation = true; // default: use old implementation + if constexpr( oldImplementation ) + { + // SR initial implementation + constexpr int part = CPPProcess::npar; + constexpr int mome = CPPProcess::np4; + constexpr int strd = MemoryAccessMomenta::neppM; + int pos = blockDim.x * blockIdx.x + threadIdx.x; + int arrlen = nevt * part * mome; + if( pos < arrlen ) + { + int page_i = pos / ( strd * mome * part ); + int rest_1 = pos % ( strd * mome * part ); + int part_i = rest_1 / ( strd * mome ); + int rest_2 = rest_1 % ( strd * mome ); + int mome_i = rest_2 / strd; + int strd_i = rest_2 % strd; + int inpos = ( page_i * strd + strd_i ) // event number + * ( part * mome ) // event size (pos of event) + + part_i * mome // particle inside event + + mome_i; // momentum inside particle + out[pos] = in[inpos]; // F2C (Fortran to C) + } + } + else + { + // AV attempt another implementation with 1 event per thread: this seems + // slower... F-style: AOS[nevtF][nparF][np4F] C-style: + // AOSOA[npagM][npar][np4][neppM] with nevt=npagM*neppM + constexpr int npar = CPPProcess::npar; + constexpr int np4 = CPPProcess::np4; + constexpr int neppM = MemoryAccessMomenta::neppM; + assert( nevt % neppM == + 0 ); // number of events is not a multiple of neppM??? + int ievt = blockDim.x * blockIdx.x + threadIdx.x; + int ipagM = ievt / neppM; + int ieppM = ievt % neppM; + for( int ip4 = 0; ip4 < np4; ip4++ ) + for( int ipar = 0; ipar < npar; ipar++ ) + { + int cpos = ipagM * npar * np4 * neppM + ipar * np4 * neppM + + ip4 * neppM + ieppM; + int fpos = ievt * npar * np4 + ipar * np4 + ip4; + out[cpos] = in[fpos]; // F2C (Fortran to C) + } + } + } +#endif + + template + void hst_transposeMomenta( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool oldImplementation = false; // default: use new implementation + if constexpr( oldImplementation ) + { + // SR initial implementation + constexpr unsigned int part = CPPProcess::npar; + constexpr unsigned int mome = CPPProcess::np4; + constexpr unsigned int strd = MemoryAccessMomenta::neppM; + unsigned int arrlen = nevt * part * mome; + for( unsigned int pos = 0; pos < arrlen; ++pos ) + { + unsigned int page_i = pos / ( strd * mome * part ); + unsigned int rest_1 = pos % ( strd * mome * part ); + unsigned int part_i = rest_1 / ( strd * mome ); + unsigned int rest_2 = rest_1 % ( strd * mome ); + unsigned int mome_i = rest_2 / strd; + unsigned int strd_i = rest_2 % strd; + unsigned int inpos = ( page_i * strd + strd_i ) // event number + * ( part * mome ) // event size (pos of event) + + part_i * mome // particle inside event + + mome_i; // momentum inside particle + if constexpr( F2C ) // needs c++17 and cuda >=11.2 (#333) + out[pos] = in[inpos]; // F2C (Fortran to C) + else + out[inpos] = in[pos]; // C2F (C to Fortran) + } + } + else + { + // AV attempt another implementation: this is slightly faster (better c++ + // pipelining?) [NB! this is not a transposition, it is an AOS to AOSOA + // conversion: if neppM=1, a memcpy is enough] F-style: + // AOS[nevtF][nparF][np4F] C-style: AOSOA[npagM][npar][np4][neppM] with + // nevt=npagM*neppM + constexpr unsigned int npar = CPPProcess::npar; + constexpr unsigned int np4 = CPPProcess::np4; + constexpr unsigned int neppM = MemoryAccessMomenta::neppM; + if constexpr( neppM == 1 && std::is_same_v ) + { + memcpy( out, in, nevt * npar * np4 * sizeof( Tin ) ); + } + else + { + const unsigned int npagM = nevt / neppM; + assert( nevt % neppM == + 0 ); // number of events is not a multiple of neppM??? + for( unsigned int ipagM = 0; ipagM < npagM; ipagM++ ) + for( unsigned int ip4 = 0; ip4 < np4; ip4++ ) + for( unsigned int ipar = 0; ipar < npar; ipar++ ) + for( unsigned int ieppM = 0; ieppM < neppM; ieppM++ ) + { + unsigned int ievt = ipagM * neppM + ieppM; + unsigned int cpos = ipagM * npar * np4 * neppM + + ipar * np4 * neppM + ip4 * neppM + ieppM; + unsigned int fpos = ievt * npar * np4 + ipar * np4 + ip4; + if constexpr( F2C ) + out[cpos] = in[fpos]; // F2C (Fortran to C) + else + out[fpos] = in[cpos]; // C2F (C to Fortran) + } + } + } + } + + template + void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool F2C = true; + hst_transposeMomenta( in, out, nevt ); + } + + template + void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool F2C = false; + hst_transposeMomenta( in, out, nevt ); + } + + //-------------------------------------------------------------------------- +} // namespace mg5amcGpu +#endif // BRIDGE_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.cc new file mode 100644 index 0000000000..65c97616d8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.cc @@ -0,0 +1,164 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "BridgeKernels.h" + +#include "GpuAbstraction.h" +#include "MemoryAccessMomenta.h" + +#include + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) + constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- + + //-------------------------------------------------------------------------- + + BridgeKernelBase::BridgeKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( nevt ) + , m_bridge( nevt, npar, np4 ) + { + if( m_momenta.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: momenta must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: matrixElements must be a host array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +#ifndef MGONGPUCPP_GPUIMPL +namespace mg5amcCpu +{ + + //-------------------------------------------------------------------------- + + BridgeKernelHost::BridgeKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: Gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt) + : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, nevt) + , m_fortranMomenta( nevt ) + { + } + + //-------------------------------------------------------------------------- + + void BridgeKernelHost::transposeInputMomentaC2F() + { + hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); + } + + //-------------------------------------------------------------------------- + + int BridgeKernelHost::computeGoodHelicities() + { + constexpr bool goodHelOnly = true; + constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering + m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + return m_bridge.nGoodHel(); + } + + //-------------------------------------------------------------------------- + + void BridgeKernelHost::computeMatrixElements( const bool useChannelIds ) + { + constexpr bool goodHelOnly = false; + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + + //-------------------------------------------------------------------------- + + BridgeKernelDevice::BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: Gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads) + : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, gpublocks * gputhreads) + , m_fortranMomenta( nevt() ) + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + if( m_gpublocks == 0 ) throw std::runtime_error( "BridgeKernelDevice: gpublocks must be > 0" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "BridgeKernelDevice: gputhreads must be > 0" ); + m_bridge.set_gpugrid( gpublocks, gputhreads ); + } + + //-------------------------------------------------------------------------- + + void BridgeKernelDevice::transposeInputMomentaC2F() + { + hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); + } + + //-------------------------------------------------------------------------- + + int BridgeKernelDevice::computeGoodHelicities() + { + constexpr bool goodHelOnly = true; + constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering + m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + return m_bridge.nGoodHel(); + } + + //-------------------------------------------------------------------------- + + void BridgeKernelDevice::computeMatrixElements( const bool useChannelIds ) + { + constexpr bool goodHelOnly = false; + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.h new file mode 100644 index 0000000000..2d873486a8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.h @@ -0,0 +1,146 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally originally by: A. Valassi (Jan 2022). +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef BRIDGEKERNELS_H +#define BRIDGEKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "Bridge.h" +#include "MatrixElementKernels.h" +#include "MemoryBuffers.h" + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // A Bridge wrapper base class encapsulating matrix element calculations on a CPU host + class BridgeKernelBase : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + BridgeKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~BridgeKernelBase() {} + + // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge + virtual void transposeInputMomentaC2F() = 0; + + protected: + + // The wrapped bridge + Bridge m_bridge; + }; + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + // A Bridge wrapper class encapsulating matrix element calculations on a CPU host + class BridgeKernelHost final : public BridgeKernelBase + { + public: + + // Constructor from existing input and output buffers + BridgeKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~BridgeKernelHost() {} + + // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge + void transposeInputMomentaC2F() override final; + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // The buffer for the input momenta, transposed to Fortran array indexing + HostBufferMomenta m_fortranMomenta; + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A Bridge wrapper class encapsulating matrix element calculations on a GPU device + class BridgeKernelDevice : public BridgeKernelBase + { + public: + + // Constructor from existing input and output buffers + BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads); + + // Destructor + virtual ~BridgeKernelDevice() {} + + // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge + void transposeInputMomentaC2F() override final; + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The buffer for the input momenta, transposed to Fortran array indexing + PinnedHostBufferMomenta m_fortranMomenta; + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + }; +#endif + + //-------------------------------------------------------------------------- +} +#endif // BRIDGEKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc new file mode 100644 index 0000000000..16671b5686 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc @@ -0,0 +1,39 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "CommonRandomNumbers.h" +#include "GpuAbstraction.h" +#include "MemoryBuffers.h" +#include "RandomNumberKernels.h" + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + CommonRandomNumberKernel::CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ) + : RandomNumberKernelBase( rnarray ) + , m_seed( 20211220 ) + { + if( m_rnarray.isOnDevice() ) + throw std::runtime_error( "CommonRandomNumberKernel on host with a device random number array" ); + } + + //-------------------------------------------------------------------------- + + void CommonRandomNumberKernel::generateRnarray() + { + std::vector rnd = CommonRandomNumbers::generate( m_rnarray.size(), m_seed ); // NB: generate as double (HARDCODED) + std::copy( rnd.begin(), rnd.end(), m_rnarray.data() ); // NB: copy may imply a double-to-float conversion + } + + //-------------------------------------------------------------------------- +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc new file mode 100644 index 0000000000..7a3e0e0519 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc @@ -0,0 +1,238 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "CrossSectionKernels.h" + +#include "GpuAbstraction.h" +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessWeights.h" +#include "MemoryBuffers.h" + +#include + +// ****************************************************************************************** +// *** NB: Disabling fast math is essential here, otherwise results are undefined *** +// *** NB: This file CrossSectionKernels.cc IS BUILT WITH -fno-fast-math in the Makefile! *** +// *** NB: Attempts with __attribute__((optimize("-fno-fast-math"))) were unsatisfactory *** +// ****************************************************************************************** + +inline bool +fp_is_nan( const fptype& fp ) +{ + //#pragma clang diagnostic push + //#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) + return std::isnan( fp ); // always false for clang in fast math mode (tautological compare)? + //#pragma clang diagnostic pop +} + +inline bool +fp_is_abnormal( const fptype& fp ) +{ + if( fp_is_nan( fp ) ) return true; + if( fp != fp ) return true; + return false; +} + +inline bool +fp_is_zero( const fptype& fp ) +{ + if( fp == 0 ) return true; + return false; +} + +// See https://en.cppreference.com/w/cpp/numeric/math/FP_categories +inline const char* +fp_show_class( const fptype& fp ) +{ + switch( std::fpclassify( fp ) ) + { + case FP_INFINITE: return "Inf"; + case FP_NAN: return "NaN"; + case FP_NORMAL: return "normal"; + case FP_SUBNORMAL: return "subnormal"; + case FP_ZERO: return "zero"; + default: return "unknown"; + } +} + +inline void +debug_me_is_abnormal( const fptype& me, size_t ievtALL ) +{ + std::cout << "DEBUG[" << ievtALL << "]" + << " ME=" << me + << " fpisabnormal=" << fp_is_abnormal( me ) + << " fpclass=" << fp_show_class( me ) + << " (me==me)=" << ( me == me ) + << " (me==me+1)=" << ( me == me + 1 ) + << " isnan=" << fp_is_nan( me ) + << " isfinite=" << std::isfinite( me ) + << " isnormal=" << std::isnormal( me ) + << " is0=" << ( me == 0 ) + << " is1=" << ( me == 1 ) + << " abs(ME)=" << std::abs( me ) + << " isnan=" << fp_is_nan( std::abs( me ) ) + << std::endl; +} + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ) + { + for( unsigned int ievt = 0; ievt < nevt; ievt++ ) + { + if( fp_is_abnormal( hstMEs[ievt] ) ) + { + std::cout << "WARNING! flagging abnormal ME for ievt=" << ievt << std::endl; + hstMEs[ievt] = std::sqrt( -1. ); + } + } + } + + //-------------------------------------------------------------------------- + + CrossSectionKernelHost::CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t nevt ) + : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) + , NumberOfEvents( nevt ) + { + if( m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: samplingWeights must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: matrixElements must be a host array" ); + if( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with samplingWeights" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelHost::updateEventStatistics( const bool debug ) + { + EventStatistics stats; // new statistics for the new nevt events + // FIRST PASS: COUNT ALL/ABN/ZERO EVENTS, COMPUTE MIN/MAX, COMPUTE REFS AS MEANS OF SIMPLE SUMS + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + const size_t ievtALL = m_iter * nevt() + ievt; + // The following events are abnormal in a run with "-p 2048 256 12 -d" + // - check.exe/commonrand: ME[310744,451171,3007871,3163868,4471038,5473927] with fast math + // - check.exe/curand: ME[578162,1725762,2163579,5407629,5435532,6014690] with fast math + // - gcheck.exe/curand: ME[596016,1446938] with fast math + // Debug NaN/abnormal issues + //if ( ievtALL == 310744 ) // this ME is abnormal both with and without fast math + // debug_me_is_abnormal( me, ievtALL ); + //if ( ievtALL == 5473927 ) // this ME is abnormal only with fast math + // debug_me_is_abnormal( me, ievtALL ); + stats.nevtALL++; + if( fp_is_abnormal( me ) ) + { + if( debug ) // only printed out with "-p -d" (matrixelementALL is not filled without -p) + std::cout << "WARNING! ME[" << ievtALL << "] is NaN/abnormal" << std::endl; + stats.nevtABN++; + continue; + } + if( fp_is_zero( me ) ) stats.nevtZERO++; + stats.minME = std::min( stats.minME, (double)me ); + stats.maxME = std::max( stats.maxME, (double)me ); + stats.minWG = std::min( stats.minWG, (double)wg ); + stats.maxWG = std::max( stats.maxWG, (double)wg ); + stats.sumMEdiff += me; // NB stats.refME is 0 here + stats.sumWGdiff += wg; // NB stats.refWG is 0 here + } + stats.refME = stats.meanME(); // draft ref + stats.refWG = stats.meanWG(); // draft ref + stats.sumMEdiff = 0; + stats.sumWGdiff = 0; + // SECOND PASS: IMPROVE MEANS FROM SUMS OF DIFFS TO PREVIOUS REF, UPDATE REF + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + if( fp_is_abnormal( me ) ) continue; + stats.sumMEdiff += ( me - stats.refME ); + stats.sumWGdiff += ( wg - stats.refWG ); + } + stats.refME = stats.meanME(); // final ref + stats.refWG = stats.meanWG(); // final ref + stats.sumMEdiff = 0; + stats.sumWGdiff = 0; + // THIRD PASS: COMPUTE STDDEV FROM SQUARED SUMS OF DIFFS TO REF + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + if( fp_is_abnormal( me ) ) continue; + stats.sqsMEdiff += std::pow( me - stats.refME, 2 ); + stats.sqsWGdiff += std::pow( wg - stats.refWG, 2 ); + } + // FOURTH PASS: UPDATE THE OVERALL STATS BY ADDING THE NEW STATS + m_stats += stats; + // Increment the iterations counter + m_iter++; + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + + /* + //-------------------------------------------------------------------------- + + CrossSectionKernelDevice::CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t gpublocks, + const size_t gputhreads ) + : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) + , NumberOfEvents( gpublocks*gputhreads ) + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + if ( ! m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: samplingWeights must be a device array" ); + if ( ! m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: matrixElements must be a device array" ); + if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0" ); + if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0" ); + if ( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with samplingWeights" ); + if ( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelDevice::setGrid( const size_t gpublocks, const size_t gputhreads ) + { + if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0 in setGrid" ); + if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0 in setGrid" ); + if ( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch in setGrid" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelDevice::updateEventStatistics( const bool debug ) + { + // Increment the iterations counter + m_iter++; + } + + //-------------------------------------------------------------------------- + */ + +} +#endif + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h new file mode 100644 index 0000000000..d1e29b7999 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h @@ -0,0 +1,139 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef CROSSSECTIONKERNELS_H +#define CROSSSECTIONKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "EventStatistics.h" +#include "MemoryBuffers.h" + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // Helper function for Bridge.h: must be compiled without fast math + // Iterate through all output MEs and replace any NaN/abnormal ones by sqrt(-1) + void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ); + + //-------------------------------------------------------------------------- + + // A base class encapsulating the calculation of event statistics on a CPU host or on a GPU device + class CrossSectionKernelBase //: virtual public ICrossSectionKernel + { + protected: + + // Constructor from existing input and output buffers + CrossSectionKernelBase( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats ) // output: event statistics + : m_samplingWeights( samplingWeights ) + , m_matrixElements( matrixElements ) + , m_stats( stats ) + , m_iter( 0 ) + { + // NB: do not initialise EventStatistics (you may be asked to update an existing result) + } + + public: + + // Destructor + virtual ~CrossSectionKernelBase() {} + + // Update event statistics + virtual void updateEventStatistics( const bool debug = false ) = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The buffer for the sampling weights + const BufferWeights& m_samplingWeights; + + // The buffer for the output matrix elements + const BufferMatrixElements& m_matrixElements; + + // The event statistics + EventStatistics& m_stats; + + // The number of iterations processed so far + size_t m_iter; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating the calculation of event statistics on a CPU host + class CrossSectionKernelHost final : public CrossSectionKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t nevt ); + + // Destructor + virtual ~CrossSectionKernelHost() {} + + // Update event statistics + void updateEventStatistics( const bool debug = false ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + }; + + //-------------------------------------------------------------------------- + + /* +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating the calculation of event statistics on a GPU device + class CrossSectionKernelDevice : public CrossSectionKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t gpublocks, + const size_t gputhreads ); + + // Destructor + virtual ~CrossSectionKernelDevice(){} + + // Reset gpublocks and gputhreads + void setGrid( const size_t gpublocks, const size_t gputhreads ); + + // Update event statistics + void updateEventStatistics( const bool debug=false ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + + }; +#endif + */ + + //-------------------------------------------------------------------------- +} +#endif // CROSSSECTIONKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc new file mode 100644 index 0000000000..0c8e6e6445 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc @@ -0,0 +1,136 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "GpuRuntime.h" +#include "MemoryBuffers.h" +#include "RandomNumberKernels.h" + +#include + +#ifndef MGONGPU_HAS_NO_CURAND /* clang-format off */ +// NB This must come AFTER mgOnGpuConfig.h which contains our definition of __global__ when MGONGPUCPP_GPUIMPL is not defined +#include "curand.h" +#define checkCurand( code ){ assertCurand( code, __FILE__, __LINE__ ); } +inline void assertCurand( curandStatus_t code, const char *file, int line, bool abort = true ) +{ + if ( code != CURAND_STATUS_SUCCESS ) + { + printf( "CurandAssert: %s:%d code=%d\n", file, line, code ); + if ( abort ) assert( code == CURAND_STATUS_SUCCESS ); + } +} +#endif /* clang-format on */ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- +#ifndef MGONGPU_HAS_NO_CURAND + CurandRandomNumberKernel::CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) + : RandomNumberKernelBase( rnarray ) + , m_isOnDevice( onDevice ) + { + if( m_isOnDevice ) + { +#ifdef MGONGPUCPP_GPUIMPL + if( !m_rnarray.isOnDevice() ) + throw std::runtime_error( "CurandRandomNumberKernel on device with a host random number array" ); +#else + throw std::runtime_error( "CurandRandomNumberKernel does not support CurandDevice on CPU host" ); +#endif + } + else + { + if( m_rnarray.isOnDevice() ) + throw std::runtime_error( "CurandRandomNumberKernel on host with a device random number array" ); + } + createGenerator(); + } + + //-------------------------------------------------------------------------- + + CurandRandomNumberKernel::~CurandRandomNumberKernel() + { + destroyGenerator(); + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::seedGenerator( const unsigned int seed ) + { + if( m_isOnDevice ) + { + destroyGenerator(); // workaround for #429 + createGenerator(); // workaround for #429 + } + //printf( "seedGenerator: seed %d\n", seed ); + checkCurand( curandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::createGenerator() + { + // [NB Timings are for GenRnGen host|device (cpp|cuda) generation of 256*32*1 events with nproc=1: rn(0) is host=0.0012s] + const curandRngType_t type = CURAND_RNG_PSEUDO_MTGP32; // 0.00082s | 0.00064s (FOR FAST TESTS) + //const curandRngType_t type = CURAND_RNG_PSEUDO_XORWOW; // 0.049s | 0.0016s + //const curandRngType_t type = CURAND_RNG_PSEUDO_MRG32K3A; // 0.71s | 0.0012s (better but slower, especially in c++) + //const curandRngType_t type = CURAND_RNG_PSEUDO_MT19937; // 21s | 0.021s + //const curandRngType_t type = CURAND_RNG_PSEUDO_PHILOX4_32_10; // 0.024s | 0.00026s (used to segfault?) + if( m_isOnDevice ) + { + checkCurand( curandCreateGenerator( &m_rnGen, type ) ); + } + else + { + checkCurand( curandCreateGeneratorHost( &m_rnGen, type ) ); + } + //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_LEGACY ) ); // fails with code=104 (see #429) + checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_BEST ) ); + //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_DYNAMIC ) ); // fails with code=104 (see #429) + //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_SEEDED ) ); // fails with code=104 (see #429) + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::destroyGenerator() + { + checkCurand( curandDestroyGenerator( m_rnGen ) ); + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::generateRnarray() + { +#if defined MGONGPU_FPTYPE_DOUBLE + checkCurand( curandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#elif defined MGONGPU_FPTYPE_FLOAT + checkCurand( curandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#endif + /* + printf( "\nCurandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); + fptype* data = m_rnarray.data(); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) + { + data = new fptype[m_rnarray.size()](); + checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); + } +#endif + for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) + printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) delete[] data; +#endif + */ + } + + //-------------------------------------------------------------------------- +#endif +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h new file mode 100644 index 0000000000..4a1fd8464d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h @@ -0,0 +1,175 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef EventStatistics_H +#define EventStatistics_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" // for npar (meGeVexponent) + +#include +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // The EventStatistics struct is used to accumulate running aggregates of event statistics. + // This will eventually include the process cross section and the process maximum weight: + // one important case of EventStatistics will then be the "gridpack" result set, which is + // the output of the "integration" step and the input to "unweighted event generation" step. + // The current implementation only includes statistics for matrix elements (ME) and sampling weights (WG); + // in first approximation, the process cross section and maximum weight are just the mean ME and maximum ME, + // but eventually the sampling weights WG (e.g. from Rambo) must also be taken into account in the calculation. + // The implementation uses differences to reference values to improve numerical precision. + struct EventStatistics + { + public: + size_t nevtALL; // total number of events used + size_t nevtABN; // number of events used, where ME is abnormal (nevtABN <= nevtALL) + size_t nevtZERO; // number of not-abnormal events used, where ME is zero (nevtZERO <= nevtOK) + double minME; // minimum matrix element + double maxME; // maximum matrix element + double minWG; // minimum sampling weight + double maxWG; // maximum sampling weight + double refME; // "reference" matrix element (normally the current mean) + double refWG; // "reference" sampling weight (normally the current mean) + double sumMEdiff; // sum of diff to ref for matrix element + double sumWGdiff; // sum of diff to ref for sampling weight + double sqsMEdiff; // squared sum of diff to ref for matrix element + double sqsWGdiff; // squared sum of diff to ref for sampling weight + std::string tag; // a text tag for printouts + // Number of events used, where ME is not abnormal + size_t nevtOK() const { return nevtALL - nevtABN; } + // Mean matrix element + // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] + double meanME() const + { + return refME + ( nevtOK() > 0 ? sumMEdiff / nevtOK() : 0 ); + } + // Mean sampling weight + // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] + double meanWG() const + { + return refWG + ( nevtOK() > 0 ? sumWGdiff / nevtOK() : 0 ); + } + // Variance matrix element + // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] + double varME() const { return ( sqsMEdiff - std::pow( sumMEdiff, 2 ) / nevtOK() ) / nevtOK(); } + // Variance sampling weight + // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] + double varWG() const { return ( sqsWGdiff - std::pow( sumWGdiff, 2 ) / nevtOK() ) / nevtOK(); } + // Standard deviation matrix element + double stdME() const { return std::sqrt( varME() ); } + // Standard deviation sampling weight + double stdWG() const { return std::sqrt( varWG() ); } + // Update reference matrix element + void updateRefME( const double newRef ) + { + const double deltaRef = refME - newRef; + sqsMEdiff += deltaRef * ( 2 * sumMEdiff + nevtOK() * deltaRef ); + sumMEdiff += deltaRef * nevtOK(); + refME = newRef; + } + // Update reference sampling weight + void updateRefWG( const double newRef ) + { + const double deltaRef = refWG - newRef; + sqsWGdiff += deltaRef * ( 2 * sumWGdiff + nevtOK() * deltaRef ); + sumWGdiff += deltaRef * nevtOK(); + refWG = newRef; + } + // Constructor + EventStatistics() + : nevtALL( 0 ) + , nevtABN( 0 ) + , nevtZERO( 0 ) + , minME( std::numeric_limits::max() ) + , maxME( std::numeric_limits::lowest() ) + , minWG( std::numeric_limits::max() ) + , maxWG( std::numeric_limits::lowest() ) + , refME( 0 ) + , refWG( 0 ) + , sumMEdiff( 0 ) + , sumWGdiff( 0 ) + , sqsMEdiff( 0 ) + , sqsWGdiff( 0 ) + , tag( "" ) {} + // Combine two EventStatistics +#ifdef __clang__ + // Disable optimizations for this function in HIP (work around FPE crash #1003: originally using #if __HIP_CLANG_ONLY__) + // Disable optimizations for this function in clang tout court (work around FPE crash #1005: now using #ifdef __clang__) + // See https://clang.llvm.org/docs/LanguageExtensions.html#extensions-for-selectively-disabling-optimization + __attribute__( ( optnone ) ) +#endif + EventStatistics& + operator+=( const EventStatistics& stats ) + { + EventStatistics s1 = *this; // temporary copy + EventStatistics s2 = stats; // temporary copy + EventStatistics& sum = *this; + sum.nevtALL = s1.nevtALL + s2.nevtALL; + sum.nevtABN = s1.nevtABN + s2.nevtABN; + sum.nevtZERO = s1.nevtZERO + s2.nevtZERO; + sum.minME = std::min( s1.minME, s2.minME ); + sum.maxME = std::max( s1.maxME, s2.maxME ); + sum.minWG = std::min( s1.minWG, s2.minWG ); + sum.maxWG = std::max( s1.maxWG, s2.maxWG ); + sum.refME = ( s1.meanME() * s1.nevtOK() + s2.meanME() * s2.nevtOK() ) / sum.nevtOK(); // new mean ME + s1.updateRefME( sum.refME ); + s2.updateRefME( sum.refME ); + sum.sumMEdiff = s1.sumMEdiff + s2.sumMEdiff; + sum.sqsMEdiff = s1.sqsMEdiff + s2.sqsMEdiff; + sum.refWG = ( s1.meanWG() * s1.nevtOK() + s2.meanWG() * s2.nevtOK() ) / sum.nevtOK(); // new mean WG + s1.updateRefWG( sum.refWG ); + s2.updateRefWG( sum.refWG ); + sum.sumWGdiff = s1.sumWGdiff + s2.sumWGdiff; + sum.sqsWGdiff = s1.sqsWGdiff + s2.sqsWGdiff; + return sum; + } + // Printout + void printout( std::ostream& out ) const + { + const EventStatistics& s = *this; + constexpr int meGeVexponent = -( 2 * CPPProcess::npar - 8 ); + out << s.tag << "NumMatrixElems(notAbnormal) = " << s.nevtOK() << std::endl + << std::scientific // fixed format: affects all floats (default precision: 6) + << s.tag << "MeanMatrixElemValue = ( " << s.meanME() + << " +- " << s.stdME() / std::sqrt( s.nevtOK() ) << " ) GeV^" << meGeVexponent << std::endl // standard error + << s.tag << "[Min,Max]MatrixElemValue = [ " << s.minME + << " , " << s.maxME << " ] GeV^" << meGeVexponent << std::endl + << s.tag << "StdDevMatrixElemValue = ( " << s.stdME() + << std::string( 16, ' ' ) << " ) GeV^" << meGeVexponent << std::endl + << s.tag << "MeanWeight = ( " << s.meanWG() + << " +- " << s.stdWG() / std::sqrt( s.nevtOK() ) << std::endl // standard error + << s.tag << "[Min,Max]Weight = [ " << s.minWG + << " , " << s.maxWG << " ]" << std::endl + << s.tag << "StdDevWeight = ( " << s.stdWG() + << std::string( 16, ' ' ) << " )" << std::endl + << std::defaultfloat; // default format: affects all floats + } + }; + + //-------------------------------------------------------------------------- + + inline std::ostream& operator<<( std::ostream& out, const EventStatistics& s ) + { + s.printout( out ); + return out; + } + + //-------------------------------------------------------------------------- +} + +#endif // EventStatistics_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h new file mode 100644 index 0000000000..6f709dfe9a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h @@ -0,0 +1,168 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: J. Teig (Jul 2023) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2020-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MG5AMC_GPUABSTRACTION_H +#define MG5AMC_GPUABSTRACTION_H 1 + +#include "mgOnGpuConfig.h" + +#include + +//-------------------------------------------------------------------------- + +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) + +#ifndef MGONGPU_HAS_NO_BLAS +#include "cublas_v2.h" +#endif + +#define gpuError_t cudaError_t +#define gpuPeekAtLastError cudaPeekAtLastError +#define gpuGetErrorString cudaGetErrorString +#define gpuSuccess cudaSuccess + +#define gpuMallocHost( ptr, size ) checkGpu( cudaMallocHost( ptr, size ) ) +#define gpuMalloc( ptr, size ) checkGpu( cudaMalloc( ptr, size ) ) + +#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( cudaMemcpy( dstData, srcData, srcBytes, func ) ) +#define gpuMemset( data, value, bytes ) checkGpu( cudaMemset( data, value, bytes ) ) +#define gpuMemcpyHostToDevice cudaMemcpyHostToDevice +#define gpuMemcpyDeviceToHost cudaMemcpyDeviceToHost +#define gpuMemcpyDeviceToDevice cudaMemcpyDeviceToDevice +#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( cudaMemcpyToSymbol( type1, type2, size ) ) + +#define gpuFree( ptr ) checkGpu( cudaFree( ptr ) ) +#define gpuFreeHost( ptr ) checkGpu( cudaFreeHost( ptr ) ) + +#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( cudaGetSymbolAddress( devPtr, symbol ) ) + +#define gpuSetDevice cudaSetDevice +#define gpuDeviceSynchronize cudaDeviceSynchronize +#define gpuDeviceReset cudaDeviceReset + +#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) +//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> +#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) +#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) + +#define gpuStream_t cudaStream_t +#define gpuStreamCreate( pStream ) checkGpu( cudaStreamCreate( pStream ) ) +#define gpuStreamDestroy( stream ) checkGpu( cudaStreamDestroy( stream ) ) +#define gpuMallocAsync( ptr, size, stream ) checkGpu( cudaMallocAsync( ptr, size, stream ) ) +#define gpuFreeAsync( ptr, stream ) checkGpu( cudaFreeAsync( ptr, stream ) ) +#define gpuStreamSynchronize( stream ) checkGpu( cudaStreamSynchronize( stream ) ) + +#define gpuBlasStatus_t cublasStatus_t +#define GPUBLAS_STATUS_SUCCESS CUBLAS_STATUS_SUCCESS +#ifndef MGONGPU_HAS_NO_BLAS +#define gpuBlasHandle_t cublasHandle_t +#else +#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds +#endif +#define gpuBlasCreate cublasCreate +#define gpuBlasDestroy cublasDestroy +#define gpuBlasSetStream cublasSetStream + +#define gpuBlasSaxpy cublasSaxpy +#define gpuBlasSdot cublasSdot +#define gpuBlasSgemv cublasSgemv +#define gpuBlasSgemm cublasSgemm +#define gpuBlasSgemmStridedBatched cublasSgemmStridedBatched +#define gpuBlasDaxpy cublasDaxpy +#define gpuBlasDdot cublasDdot +#define gpuBlasDgemv cublasDgemv +#define gpuBlasDgemm cublasDgemm +#define gpuBlasDgemmStridedBatched cublasDgemmStridedBatched +#define GPUBLAS_OP_N CUBLAS_OP_N +#define GPUBLAS_OP_T CUBLAS_OP_T + +//-------------------------------------------------------------------------- + +#elif defined __HIPCC__ + +#ifndef MGONGPU_HAS_NO_BLAS +#include "hipblas/hipblas.h" +#endif + +#define gpuError_t hipError_t +#define gpuPeekAtLastError hipPeekAtLastError +#define gpuGetErrorString hipGetErrorString +#define gpuSuccess hipSuccess + +#define gpuMallocHost( ptr, size ) checkGpu( hipHostMalloc( ptr, size ) ) // HostMalloc better +#define gpuMalloc( ptr, size ) checkGpu( hipMalloc( ptr, size ) ) + +#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( hipMemcpy( dstData, srcData, srcBytes, func ) ) +#define gpuMemset( data, value, bytes ) checkGpu( hipMemset( data, value, bytes ) ) +#define gpuMemcpyHostToDevice hipMemcpyHostToDevice +#define gpuMemcpyDeviceToHost hipMemcpyDeviceToHost +#define gpuMemcpyDeviceToDevice hipMemcpyDeviceToDevice +#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( hipMemcpyToSymbol( type1, type2, size ) ) + +#define gpuFree( ptr ) checkGpu( hipFree( ptr ) ) +#define gpuFreeHost( ptr ) checkGpu( hipHostFree( ptr ) ) + +#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( hipGetSymbolAddress( devPtr, symbol ) ) + +#define gpuSetDevice hipSetDevice +#define gpuDeviceSynchronize hipDeviceSynchronize +#define gpuDeviceReset hipDeviceReset + +#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) +//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> +#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) +#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) + +#define gpuStream_t hipStream_t +#define gpuStreamCreate( pStream ) checkGpu( hipStreamCreate( pStream ) ) +#define gpuStreamDestroy( stream ) checkGpu( hipStreamDestroy( stream ) ) +#define gpuMallocAsync( ptr, size, stream ) checkGpu( hipMallocAsync( ptr, size, stream ) ) +#define gpuFreeAsync( ptr, stream ) checkGpu( hipFreeAsync( ptr, stream ) ) +#define gpuStreamSynchronize( stream ) checkGpu( hipStreamSynchronize( stream ) ) + +#define gpuBlasStatus_t hipblasStatus_t +#define GPUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS +#ifndef MGONGPU_HAS_NO_BLAS +#define gpuBlasHandle_t hipblasHandle_t +#else +#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds +#endif +#define gpuBlasCreate hipblasCreate +#define gpuBlasDestroy hipblasDestroy +#define gpuBlasSetStream hipblasSetStream + +#define gpuBlasSaxpy hipblasSaxpy +#define gpuBlasSdot hipblasSdot +#define gpuBlasSgemv hipblasSgemv +#define gpuBlasSgemm hipblasSgemm +#define gpuBlasSgemmStridedBatched hipblasSgemmStridedBatched +#define gpuBlasDaxpy hipblasDaxpy +#define gpuBlasDdot hipblasDdot +#define gpuBlasDgemv hipblasDgemv +#define gpuBlasDgemm hipblasDgemm +#define gpuBlasDgemmStridedBatched hipblasDgemmStridedBatched +#define GPUBLAS_OP_N HIPBLAS_OP_N +#define GPUBLAS_OP_T HIPBLAS_OP_T + +#endif + +//-------------------------------------------------------------------------- + +#ifdef MGONGPU_FPTYPE2_FLOAT +#define gpuBlasTaxpy gpuBlasSaxpy +#define gpuBlasTdot gpuBlasSdot +#define gpuBlasTgemv gpuBlasSgemv +#define gpuBlasTgemm gpuBlasSgemm +#define gpuBlasTgemmStridedBatched gpuBlasSgemmStridedBatched +#else +#define gpuBlasTaxpy gpuBlasDaxpy +#define gpuBlasTdot gpuBlasDdot +#define gpuBlasTgemv gpuBlasDgemv +#define gpuBlasTgemm gpuBlasDgemm +#define gpuBlasTgemmStridedBatched gpuBlasDgemmStridedBatched +#endif + +#endif // MG5AMC_GPUABSTRACTION_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h new file mode 100644 index 0000000000..2c8ae1a804 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h @@ -0,0 +1,102 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: J. Teig (Jun 2023, based on earlier work by S. Roiser) for the MG5aMC CUDACPP plugin. +// Further modified by: O. Mattelaer, S. Roiser, J. Teig, A. Valassi, Z. Wettersten (2020-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MG5AMC_GPURUNTIME_H +#define MG5AMC_GPURUNTIME_H 1 + +// MG5AMC on GPU uses the CUDA runtime API, not the lower level CUDA driver API +// See https://docs.nvidia.com/cuda/cuda-runtime-api/driver-vs-runtime-api.html#driver-vs-runtime-api + +#include "GpuAbstraction.h" + +#include + +//-------------------------------------------------------------------------- + +// See https://stackoverflow.com/a/14038590 +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ +#define checkGpu( code ) { assertGpu( code, __FILE__, __LINE__ ); } +inline void assertGpu( gpuError_t code, const char* file, int line, bool abort = true ) +{ + if( code != gpuSuccess ) + { + printf( "ERROR! assertGpu: '%s' (%d) in %s:%d\n", gpuGetErrorString( code ), code, file, line ); + if( abort ) assert( code == gpuSuccess ); + } +} +#endif /* clang-format on */ + +//-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ +#ifndef MGONGPU_HAS_NO_BLAS +#define checkGpuBlas( code ){ assertGpuBlas( code, __FILE__, __LINE__ ); } +inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, bool abort = true ) +{ + if ( code != GPUBLAS_STATUS_SUCCESS ) + { + printf( "ERROR! assertGpuBlas: '%d' in %s:%d\n", code, file, line ); + if( abort ) assert( code == GPUBLAS_STATUS_SUCCESS ); + } +} +#endif +#endif /* clang-format on */ + +//-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + // Instantiate a GpuRuntime at the beginnining of the application's main to + // invoke gpuSetDevice(0) in the constructor and book a gpuDeviceReset() call in the destructor + // *** FIXME! This will all need to be designed differently when going to multi-GPU nodes! *** + struct GpuRuntime final + { + GpuRuntime( const bool debug = true ) + : m_debug( debug ) { setUp( m_debug ); } + ~GpuRuntime() { tearDown( m_debug ); } + GpuRuntime( const GpuRuntime& ) = delete; + GpuRuntime( GpuRuntime&& ) = delete; + GpuRuntime& operator=( const GpuRuntime& ) = delete; + GpuRuntime& operator=( GpuRuntime&& ) = delete; + bool m_debug; + + // Set up CUDA application + // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** + // Calling cudaSetDevice on startup is useful to properly book-keep the time spent in CUDA initialization + static void setUp( const bool debug = false ) // ZW: changed debug default to false + { + // ** NB: it is useful to call cudaSetDevice, or cudaFree, to properly book-keep the time spent in CUDA initialization + // ** NB: otherwise, the first CUDA operation (eg a cudaMemcpyToSymbol in CPPProcess ctor) appears to take much longer! + /* + // [We initially added cudaFree(0) to "ease profile analysis" only because it shows up as a big recognizable block!] + // No explicit initialization is needed: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#initialization + // It is not clear what cudaFree(0) does at all: https://stackoverflow.com/questions/69967813/ + if ( debug ) std::cout << "__CudaRuntime: calling cudaFree(0)" << std::endl; + checkCuda( cudaFree( 0 ) ); // SLOW! + */ + // Replace cudaFree(0) by cudaSetDevice(0), even if it is not really needed either + // (but see https://developer.nvidia.com/blog/cuda-pro-tip-always-set-current-device-avoid-multithreading-bugs) + if( debug ) std::cout << "__GpuRuntime: calling GpuSetDevice(0)" << std::endl; + checkGpu( gpuSetDevice( 0 ) ); // SLOW! + } + + // Tear down CUDA application (call cudaDeviceReset) + // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** + // Calling cudaDeviceReset on shutdown is only needed for checking memory leaks in cuda-memcheck + // See https://docs.nvidia.com/cuda/cuda-memcheck/index.html#leak-checking + static void tearDown( const bool debug = false ) // ZW: changed debug default to false + { + if( debug ) std::cout << "__GpuRuntime: calling GpuDeviceReset()" << std::endl; + checkGpu( gpuDeviceReset() ); + } + }; +} +#endif + +//-------------------------------------------------------------------------- + +#endif // MG5AMC_GPURUNTIME_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc new file mode 100644 index 0000000000..b806da2e09 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc @@ -0,0 +1,146 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "mgOnGpuConfig.h" + +#include "GpuRuntime.h" +#include "MemoryBuffers.h" +#include "RandomNumberKernels.h" + +#include + +#ifndef MGONGPU_HAS_NO_HIPRAND /* clang-format off */ +#ifndef __HIP_PLATFORM_AMD__ +#define __HIP_PLATFORM_AMD__ 1 // enable hiprand for AMD (rocrand) +#endif +#include +#define checkHiprand( code ){ assertHiprand( code, __FILE__, __LINE__ ); } +inline void assertHiprand( hiprandStatus_t code, const char *file, int line, bool abort = true ) +{ + if ( code != HIPRAND_STATUS_SUCCESS ) + { + printf( "HiprandAssert: %s:%d code=%d\n", file, line, code ); + if ( abort ) assert( code == HIPRAND_STATUS_SUCCESS ); + } +} +#endif /* clang-format on */ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- +#ifndef MGONGPU_HAS_NO_HIPRAND + HiprandRandomNumberKernel::HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) + : RandomNumberKernelBase( rnarray ) + , m_isOnDevice( onDevice ) + { + if( m_isOnDevice ) + { +#ifdef MGONGPUCPP_GPUIMPL + if( !m_rnarray.isOnDevice() ) + throw std::runtime_error( "HiprandRandomNumberKernel on device with a host random number array" ); +#else + throw std::runtime_error( "HiprandRandomNumberKernel does not support HiprandDevice on CPU host" ); +#endif + } + else + { + if( m_rnarray.isOnDevice() ) + throw std::runtime_error( "HiprandRandomNumberKernel on host with a device random number array" ); + } + createGenerator(); + } + + //-------------------------------------------------------------------------- + + HiprandRandomNumberKernel::~HiprandRandomNumberKernel() + { + destroyGenerator(); + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::seedGenerator( const unsigned int seed ) + { + if( m_isOnDevice ) + { + destroyGenerator(); // workaround for #429 + createGenerator(); // workaround for #429 + } + //printf( "seedGenerator: seed %d\n", seed ); + checkHiprand( hiprandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::createGenerator() + { + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_DEFAULT; + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_XORWOW; + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MRG32K3A; + const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MTGP32; // same as curand; not implemented yet (code=1000) in host code + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MT19937; + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_PHILOX4_32_10; + if( m_isOnDevice ) + { + checkHiprand( hiprandCreateGenerator( &m_rnGen, type ) ); + } + else + { + // See https://github.com/ROCm/hipRAND/issues/76 + throw std::runtime_error( "HiprandRandomNumberKernel on host is not supported yet (hiprandCreateGeneratorHost is not implemented yet)" ); + //checkHiprand( hiprandCreateGeneratorHost( &m_rnGen, type ) ); // ALWAYS FAILS WITH CODE=1000 + } + // FIXME: hiprand ordering is not implemented yet + // See https://github.com/ROCm/hipRAND/issues/75 + /* + //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_LEGACY ) ); + checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_BEST ) ); + //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_DYNAMIC ) ); + //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_SEEDED ) ); + */ + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::destroyGenerator() + { + checkHiprand( hiprandDestroyGenerator( m_rnGen ) ); + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::generateRnarray() + { +#if defined MGONGPU_FPTYPE_DOUBLE + checkHiprand( hiprandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#elif defined MGONGPU_FPTYPE_FLOAT + checkHiprand( hiprandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#endif + /* + printf( "\nHiprandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); + fptype* data = m_rnarray.data(); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) + { + data = new fptype[m_rnarray.size()](); + checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); + } +#endif + for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) + printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) delete[] data; +#endif + */ + } + + //-------------------------------------------------------------------------- +#endif +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h new file mode 100644 index 0000000000..12b7d45adf --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h @@ -0,0 +1,339 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Hageboeck (Dec 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MADGRAPHTEST_H_ +#define MADGRAPHTEST_H_ 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" + +#include + +#include +#include +//#ifdef __HIPCC__ +//#include // see https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 +//#else +//#include // bypass this completely to ease portability on LUMI #803 +//#endif +#include +#include +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +using mg5amcGpu::CPPProcess; +#else +using mg5amcCpu::CPPProcess; +#endif + +namespace +{ + struct ReferenceData + { + std::vector>> momenta; + std::vector MEs; + std::vector ChanIds; + std::vector SelHels; + std::vector SelCols; + }; + + /// Read batches of reference data from a file and store them in a map. + std::map readReferenceData( const std::string& refFileName ) + { + std::cout << "INFO: Opening reference file " << refFileName << std::endl; + std::ifstream referenceFile( refFileName.c_str() ); + EXPECT_TRUE( referenceFile.is_open() ) << refFileName; + std::map referenceData; + unsigned int evtNo; + unsigned int batchNo; + for( std::string line; std::getline( referenceFile, line ); ) + { + std::stringstream lineStr( line ); + if( line.empty() || line[0] == '#' ) + { + continue; + } + else if( line.find( "Event" ) != std::string::npos ) + { + std::string dummy; + lineStr >> dummy >> evtNo >> dummy >> batchNo; + } + else if( line.find( "ME" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].MEs.size() ) + referenceData[batchNo].MEs.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].MEs[evtNo]; + } + else if( line.find( "ChanId" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].ChanIds.size() ) + referenceData[batchNo].ChanIds.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].ChanIds[evtNo]; + referenceData[batchNo].ChanIds[evtNo] = 0; // disable ChanId comparison if multichannel is not supported #976 + } + else if( line.find( "SelHel" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].SelHels.size() ) + referenceData[batchNo].SelHels.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].SelHels[evtNo]; + } + else if( line.find( "SelCol" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].SelCols.size() ) + referenceData[batchNo].SelCols.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].SelCols[evtNo]; + } + else + { + unsigned int particleIndex; + lineStr >> particleIndex; + if( evtNo <= referenceData[batchNo].momenta.size() ) + referenceData[batchNo].momenta.resize( evtNo + 1 ); + if( particleIndex <= referenceData[batchNo].momenta[evtNo].size() ) + referenceData[batchNo].momenta[evtNo].resize( particleIndex + 1 ); + auto& fourVec = referenceData[batchNo].momenta[evtNo][particleIndex]; + for( unsigned int i = 0; i < fourVec.size(); ++i ) + { + EXPECT_TRUE( lineStr.good() ); + lineStr >> fourVec[i]; + } + EXPECT_TRUE( lineStr.eof() ); + } + } + return referenceData; + } + +} + +/** + * Test driver providing a common interface for testing different implementations. + * Users need to implement: + * - Functions to retrieve matrix element and 4-momenta. These are used in the tests. + * - Driver functions that run the madgraph workflow. + */ +class TestDriverBase +{ + std::string m_refFileName; +public: + const unsigned int nparticle; + static constexpr unsigned int niter = 2; + static constexpr unsigned int gpublocks = 2; + static constexpr unsigned int gputhreads = 128; + static constexpr unsigned int nevt = gpublocks * gputhreads; + + TestDriverBase( unsigned int npart, const std::string& refFileName ) + : m_refFileName( refFileName ) + , nparticle( npart ) + { + } + TestDriverBase() = delete; + virtual ~TestDriverBase() {} + const std::string& getRefFileName() { return m_refFileName; } + + // ------------------------------------------------ + // Interface for retrieving info from madgraph + // ------------------------------------------------ + virtual fptype getMomentum( std::size_t evtNo, unsigned int particleNo, unsigned int component ) const = 0; + virtual fptype getMatrixElement( std::size_t evtNo ) const = 0; + virtual int getChannelId( std::size_t ievt ) const = 0; + virtual int getSelectedHelicity( std::size_t ievt ) const = 0; + virtual int getSelectedColor( std::size_t ievt ) const = 0; + + // ------------------------------------------------ + // Interface for steering madgraph run + // ------------------------------------------------ + virtual void prepareRandomNumbers( unsigned int iiter ) = 0; + virtual void prepareMomenta( fptype energy ) = 0; + virtual void runSigmaKin( std::size_t iiter ) = 0; + + /// Print the requested event into the stream. If the reference data has enough events, it will be printed as well. + void dumpParticles( std::ostream& stream, std::size_t ievt, unsigned int numParticles, unsigned int nDigit, const ReferenceData& referenceData ) const + { + const auto width = nDigit + 8; + for( unsigned int ipar = 0; ipar < numParticles; ipar++ ) + { + // NB: 'setw' affects only the next field (of any type) + stream << std::scientific // fixed format: affects all floats (default nDigit: 6) + << std::setprecision( nDigit ) + << std::setw( 4 ) << ipar + << std::setw( width ) << getMomentum( ievt, ipar, 0 ) + << std::setw( width ) << getMomentum( ievt, ipar, 1 ) + << std::setw( width ) << getMomentum( ievt, ipar, 2 ) + << std::setw( width ) << getMomentum( ievt, ipar, 3 ) + << "\n"; + if( ievt < referenceData.momenta.size() ) + { + stream << "ref" << ipar; + stream << std::setw( width ) << referenceData.momenta[ievt][ipar][0] + << std::setw( width ) << referenceData.momenta[ievt][ipar][1] + << std::setw( width ) << referenceData.momenta[ievt][ipar][2] + << std::setw( width ) << referenceData.momenta[ievt][ipar][3] + << "\n\n"; + } + stream << std::flush << std::defaultfloat; // default format: affects all floats + } + } +}; + +/** + * Test class that's defining all tests to run with a Madgraph workflow. + */ +class MadgraphTest +{ +public: + MadgraphTest( TestDriverBase& testDriverRef ) + : testDriver( &testDriverRef ) {} + ~MadgraphTest() {} + void CompareMomentaAndME( testing::Test& googleTest ) const; // NB: googleTest is ONLY needed for the HasFailure method... +private: + TestDriverBase* testDriver; // non-owning pointer +}; + +void +MadgraphTest::CompareMomentaAndME( testing::Test& googleTest ) const +{ + const fptype toleranceMomenta = std::is_same::value ? 1.E-10 : 4.E-2; // see #735 +#ifdef __APPLE__ + const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-2; // see #583 +#else + //const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 2.E-3; // fails smeft/hip #843 + const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-3; +#endif + constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) + // Dump events to a new reference file? + const char* dumpEventsC = getenv( "CUDACPP_RUNTEST_DUMPEVENTS" ); + const bool dumpEvents = ( dumpEventsC != 0 ) && ( std::string( dumpEventsC ) != "" ); + const std::string refFileName = testDriver->getRefFileName(); + /* +#ifdef __HIPCC__ + const std::string dumpFileName = std::experimental::filesystem::path( refFileName ).filename(); +#else + const std::string dumpFileName = std::filesystem::path( refFileName ).filename(); +#endif + */ + const std::string dumpFileName = refFileName; // bypass std::filesystem #803 + std::ofstream dumpFile; + if( dumpEvents ) + { + dumpFile.open( dumpFileName, std::ios::trunc ); + } + // Read reference data + std::map referenceData; + if( !dumpEvents ) + { + referenceData = readReferenceData( refFileName ); + } + ASSERT_FALSE( googleTest.HasFailure() ); // It doesn't make any sense to continue if we couldn't read the reference file. + // ************************************** + // *** START MAIN LOOP ON #ITERATIONS *** + // ************************************** + for( unsigned int iiter = 0; iiter < testDriver->niter; ++iiter ) + { + testDriver->prepareRandomNumbers( iiter ); + testDriver->prepareMomenta( energy ); + testDriver->runSigmaKin( iiter ); + // --- Run checks on all events produced in this iteration + for( std::size_t ievt = 0; ievt < testDriver->nevt && !googleTest.HasFailure(); ++ievt ) + { + if( dumpEvents ) + { + ASSERT_TRUE( dumpFile.is_open() ) << dumpFileName; + dumpFile << "Event " << std::setw( 8 ) << ievt << " " + << "Batch " << std::setw( 4 ) << iiter << "\n"; + testDriver->dumpParticles( dumpFile, ievt, testDriver->nparticle, 15, ReferenceData() ); + // Dump matrix element + dumpFile << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) + << testDriver->getMatrixElement( ievt ) << "\n" + << std::defaultfloat; + // Dump channelId + dumpFile << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n"; + // Dump selected helicity and color + dumpFile << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n"; + dumpFile << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" + << std::endl; // leave one line between events + continue; + } + // Check that we have the required reference data + ASSERT_GT( referenceData.size(), iiter ) + << "Don't have enough reference data for iteration " << iiter << ". Ref file:" << refFileName; + ASSERT_GT( referenceData[iiter].MEs.size(), ievt ) + << "Don't have enough reference MEs for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].ChanIds.size(), ievt ) + << "Don't have enough reference ChanIds for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].SelHels.size(), ievt ) + << "Don't have enough reference SelHels for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].SelCols.size(), ievt ) + << "Don't have enough reference SelCols for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].momenta.size(), ievt ) + << "Don't have enough reference momenta for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GE( referenceData[iiter].momenta[ievt].size(), testDriver->nparticle ) + << "Don't have enough reference particles for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + // This trace will help to understand the event that is being checked. + // It will only be printed in case of failures: + std::stringstream eventTrace; + eventTrace << "In comparing event " << ievt << " from iteration " << iiter << "\n"; + testDriver->dumpParticles( eventTrace, ievt, testDriver->nparticle, 15, referenceData[iiter] ); + eventTrace << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) + << testDriver->getMatrixElement( ievt ) << "\n" + << std::setw( 4 ) << "r.ME" << std::scientific << std::setw( 15 + 8 ) + << referenceData[iiter].MEs[ievt] << std::endl + << std::defaultfloat; + eventTrace << std::setw( 8 ) << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n" + << std::setw( 8 ) << "r.ChanId" << std::setw( 8 ) << referenceData[iiter].ChanIds[ievt] << std::endl; + eventTrace << std::setw( 8 ) << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n" + << std::setw( 8 ) << "r.SelHel" << std::setw( 8 ) << referenceData[iiter].SelHels[ievt] << std::endl; + eventTrace << std::setw( 8 ) << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" + << std::setw( 8 ) << "r.SelCol" << std::setw( 8 ) << referenceData[iiter].SelCols[ievt] << std::endl; + SCOPED_TRACE( eventTrace.str() ); + // Compare Momenta + for( unsigned int ipar = 0; ipar < testDriver->nparticle; ++ipar ) + { + std::stringstream momentumErrors; + for( unsigned int icomp = 0; icomp < CPPProcess::np4; ++icomp ) + { + const fptype pMadg = testDriver->getMomentum( ievt, ipar, icomp ); + const fptype pOrig = referenceData[iiter].momenta[ievt][ipar][icomp]; + //const fptype relDelta = fabs( ( pMadg - pOrig ) / pOrig ); // computing relDelta may lead to FPEs + const fptype delta = fabs( pMadg - pOrig ); + if( delta > toleranceMomenta * fabs( pOrig ) ) // better than "relDelta > toleranceMomenta" + { + momentumErrors << std::setprecision( 15 ) << std::scientific << "\nparticle " << ipar << "\tcomponent " << icomp + << "\n\t madGraph: " << std::setw( 22 ) << pMadg + << "\n\t reference: " << std::setw( 22 ) << pOrig + << "\n\t relative delta exceeds tolerance of " << toleranceMomenta; + } + } + ASSERT_TRUE( momentumErrors.str().empty() ) << momentumErrors.str(); + } + // Compare ME: + EXPECT_NEAR( testDriver->getMatrixElement( ievt ), + referenceData[iiter].MEs[ievt], + toleranceMEs * referenceData[iiter].MEs[ievt] ); + // Compare channelId + EXPECT_EQ( testDriver->getChannelId( ievt ), + referenceData[iiter].ChanIds[ievt] ); + // Compare selected helicity and color + EXPECT_EQ( testDriver->getSelectedHelicity( ievt ), + referenceData[iiter].SelHels[ievt] ); + EXPECT_EQ( testDriver->getSelectedColor( ievt ), + referenceData[iiter].SelCols[ievt] ); + } + } + if( dumpEvents ) + { + std::cout << "Event dump written to " << dumpFileName << std::endl; + } +} + +#endif /* MADGRAPHTEST_H_ */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc new file mode 100644 index 0000000000..872e4795e3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc @@ -0,0 +1,509 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "MatrixElementKernels.h" + +#include "CPPProcess.h" +#include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" + +#include // for fetestexcept +#include +#include + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + MatrixElementKernelBase::MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol ) // output: color selection + : m_momenta( momenta ) + , m_gs( gs ) + , m_iflavorVec( iflavorVec ) + , m_rndhel( rndhel ) + , m_rndcol( rndcol ) + , m_channelIds( channelIds ) + , m_matrixElements( matrixElements ) + , m_selhel( selhel ) + , m_selcol( selcol ) +#ifdef MGONGPU_CHANNELID_DEBUG + , m_nevtProcessedByChannel() + , m_tag() +#endif + { + //std::cout << "DEBUG: MatrixElementKernelBase ctor " << this << std::endl; +#ifdef MGONGPU_CHANNELID_DEBUG + for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + m_nevtProcessedByChannel[channelId] = 0; +#endif + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelBase::~MatrixElementKernelBase() + { + //std::cout << "DEBUG: MatrixElementKernelBase dtor " << this << std::endl; +#ifdef MGONGPU_CHANNELID_DEBUG + MatrixElementKernelBase::dumpNevtProcessedByChannel(); +#endif +#ifdef MGONGPUCPP_VERBOSE + MatrixElementKernelBase::dumpSignallingFPEs(); +#endif + } + + //-------------------------------------------------------------------------- + +#ifdef MGONGPU_CHANNELID_DEBUG + void MatrixElementKernelBase::updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ) + { + if( pHstChannelIds != nullptr ) + { + //std::cout << "DEBUG " << this << ": not nullptr " << nevt << std::endl; + for( unsigned int ievt = 0; ievt < nevt; ievt++ ) + { + const size_t channelId = pHstChannelIds[ievt]; // Fortran indexing + //assert( channelId > 0 ); + //assert( channelId < CPPProcess::ndiagrams ); + m_nevtProcessedByChannel[channelId]++; + } + } + else + { + //std::cout << "DEBUG " << this << ": nullptr " << std::endl; + m_nevtProcessedByChannel[0] += nevt; + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPU_CHANNELID_DEBUG + void MatrixElementKernelBase::dumpNevtProcessedByChannel() + { + size_t nevtProcessed = 0; + for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + nevtProcessed += m_nevtProcessedByChannel[channelId]; + std::ostringstream sstr; + sstr << " {"; + for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + { + if( m_nevtProcessedByChannel[channelId] > 0 ) + { + if( sstr.str() != " {" ) sstr << ","; + if( channelId == 0 ) + sstr << " no-multichannel"; + else + sstr << " " << channelId; + sstr << " : " << m_nevtProcessedByChannel[channelId]; + } + } + sstr << " }"; + std::cout << "DEBUG: MEK " << this; + if( m_tag != "" ) std::cout << " " << m_tag; + std::cout << " processed " << nevtProcessed << " events across " << CPPProcess::ndiagrams << " channels" << sstr.str() << std::endl; + } +#endif + + //-------------------------------------------------------------------------- + + void MatrixElementKernelBase::dumpSignallingFPEs() + { + // New strategy for issue #831: add a final report of FPEs + // Note: normally only underflow will be reported here (inexact is switched off because it would almost always signal; + // divbyzero, invalid and overflow are configured by feenablexcept to send a SIGFPE signal, and are normally fixed in the code) + // Note: this is now called in the individual destructors of MEK classes rather than in that of MatrixElementKernelBase(#837) + std::string fpes; + if( std::fetestexcept( FE_DIVBYZERO ) ) fpes += " FE_DIVBYZERO"; + if( std::fetestexcept( FE_INVALID ) ) fpes += " FE_INVALID"; + if( std::fetestexcept( FE_OVERFLOW ) ) fpes += " FE_OVERFLOW"; + if( std::fetestexcept( FE_UNDERFLOW ) ) fpes += " FE_UNDERFLOW"; + //if( std::fetestexcept( FE_INEXACT ) ) fpes += " FE_INEXACT"; // do not print this out: this would almost always signal! + if( fpes == "" ) + std::cout << "INFO: No Floating Point Exceptions have been reported" << std::endl; + else + std::cerr << "INFO: The following Floating Point Exceptions have been reported:" << fpes << std::endl; + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +#ifndef MGONGPUCPP_GPUIMPL +namespace mg5amcCpu +{ + + //-------------------------------------------------------------------------- + + MatrixElementKernelHost::MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt ) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( nevt ) + , m_couplings( nevt ) + , m_numerators( nevt * CPPProcess::ndiagrams ) + , m_denominators( nevt ) + { + //std::cout << "DEBUG: MatrixElementKernelHost::ctor " << this << std::endl; + if( m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: momenta must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: matrixElements must be a host array" ); + if( m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: channelIds must be a device array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with matrixElements" ); + if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with channelIds" ); + if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with iflavorVec" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MatrixElementKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Fail gently and avoid "Illegal instruction (core dumped)" if the host does not support the SIMD used in the ME calculation + // Note: this prevents a crash on pmpe04 but not on some github CI nodes? + // [NB: SIMD vectorization in mg5amc C++ code is only used in the ME calculation below MatrixElementKernelHost!] + if( !MatrixElementKernelHost::hostSupportsSIMD() ) + throw std::runtime_error( "Host does not support the SIMD implementation of MatrixElementKernelsHost" ); + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelHost::~MatrixElementKernelHost() + { + //std::cout << "DEBUG: MatrixElementKernelBase::dtor " << this << std::endl; + } + + //-------------------------------------------------------------------------- + + int MatrixElementKernelHost::computeGoodHelicities() + { + HostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); + // ... 0d1. Compute good helicity mask on the host + computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); + sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_numerators.data(), m_denominators.data(), hstIsGoodHel.data(), nevt() ); + // ... 0d2. Copy good helicity list to static memory on the host + // [FIXME! REMOVE THIS STATIC THAT BREAKS MULTITHREADING?] + return sigmaKin_setGoodHel( hstIsGoodHel.data() ); + } + + //-------------------------------------------------------------------------- + + void MatrixElementKernelHost::computeMatrixElements( const bool useChannelIds ) + { + computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_numerators.data(), m_denominators.data(), nullptr, true, nevt() ); +#ifdef MGONGPU_CHANNELID_DEBUG + //std::cout << "DEBUG: MatrixElementKernelHost::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; + MatrixElementKernelBase::updateNevtProcessedByChannel( pChannelIds, nevt() ); +#endif + } + + //-------------------------------------------------------------------------- + + // Does this host system support the SIMD used in the matrix element calculation? + bool MatrixElementKernelHost::hostSupportsSIMD( const bool verbose ) + { +#if defined __AVX512VL__ + bool known = true; + bool ok = __builtin_cpu_supports( "avx512vl" ); + const std::string tag = "skylake-avx512 (AVX512VL)"; +#elif defined __AVX2__ + bool known = true; + bool ok = __builtin_cpu_supports( "avx2" ); + const std::string tag = "haswell (AVX2)"; +#elif defined __SSE4_2__ +#ifdef __PPC__ + // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html + bool known = true; + bool ok = __builtin_cpu_supports( "vsx" ); + const std::string tag = "powerpc vsx (128bit as in SSE4.2)"; +#elif defined( __x86_64__ ) || defined( __i386__ ) + bool known = true; + bool ok = __builtin_cpu_supports( "sse4.2" ); + const std::string tag = "nehalem (SSE4.2)"; +#else // AV FIXME! Added by OM for Mac, should identify the correct __xxx__ flag that should be targeted + // DM now we have an explicit NEON target for ARM + bool known = false; // __builtin_cpu_supports is not supported + bool ok = true; // this is just an assumption! + const std::string tag = "simd arch not defined"; +#endif +#elif defined __ARM_NEON // consider using __BUILTIN_CPU_SUPPORTS__ + bool known = false; // __builtin_cpu_supports is not supported + // See https://stackoverflow.com/q/62783908 + // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu + bool ok = true; // this is just an assumption! + const std::string tag = "arm neon (128bit as in SSE4.2)"; +#else + bool known = true; + bool ok = true; + const std::string tag = "none"; +#endif + if( verbose ) + { + if( tag == "none" ) + std::cout << "INFO: The application does not require the host to support any AVX feature" << std::endl; + else if( ok && known ) + std::cout << "INFO: The application is built for " << tag << " and the host supports it" << std::endl; + else if( ok ) + std::cout << "WARNING: The application is built for " << tag << " but it is unknown if the host supports it" << std::endl; + else + std::cout << "ERROR! The application is built for " << tag << " but the host does not support it" << std::endl; + } + return ok; + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + + //-------------------------------------------------------------------------- + + MatrixElementKernelDevice::MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( gpublocks * gputhreads ) + , m_couplings( this->nevt() ) + , m_pHelMEs() + , m_pHelJamps() + , m_pHelNumerators() + , m_pHelDenominators() + , m_colJamp2s( CPPProcess::ncolor * this->nevt() ) +#ifdef MGONGPU_CHANNELID_DEBUG + , m_hstChannelIds( this->nevt() ) +#endif +#ifndef MGONGPU_HAS_NO_BLAS + , m_blasColorSum( false ) + , m_blasTf32Tensor( false ) + , m_pHelBlasTmp() + , m_blasHandle() +#endif + , m_helStreams() + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + //std::cout << "DEBUG: MatrixElementKernelDevice::ctor " << this << std::endl; + if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: momenta must be a device array" ); + if( !m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: matrixElements must be a device array" ); + if( !m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: channelIds must be a device array" ); // FIXME?! + if( !m_iflavorVec.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: iflavorVec must be a device array" ); + if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with matrixElements" ); + if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with channelIds" ); + if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with iflavorVec" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( m_gputhreads % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MatrixElementKernelHost: gputhreads should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Create the "one-helicity" jamp buffer that will be used for helicity filtering + m_pHelJamps.reset( new DeviceBufferSimple( CPPProcess::ncolor * mgOnGpu::nx2 * this->nevt() ) ); + // Create the "one-helicity" numerator and denominator buffers that will be used for helicity filtering + m_pHelNumerators.reset( new DeviceBufferSimple( this->nevt() * CPPProcess::ndiagrams ) ); + m_pHelDenominators.reset( new DeviceBufferSimple( this->nevt() ) ); + // Decide at runtime whether to use BLAS for color sums + // Decide at runtime whether TF32TENSOR math should be used in cuBLAS + static bool first = true; + if( first ) + { + first = false; + // Analyse environment variable CUDACPP_RUNTIME_BLASCOLORSUM + const char* blasEnv = getenv( "CUDACPP_RUNTIME_BLASCOLORSUM" ); + if( blasEnv && std::string( blasEnv ) != "" ) + { +#ifndef MGONGPU_HAS_NO_BLAS + m_blasColorSum = true; // fixme? eventually set default=true and decode "Y" and "N" choices? + std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty: enable BLAS" << std::endl; +#else + throw std::runtime_error( "Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty, but BLAS was disabled at build time" ); +#endif + } + else + { +#ifndef MGONGPU_HAS_NO_BLAS + std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is empty or not set: disable BLAS" << std::endl; +#else + std::cout << "INFO: BLAS was disabled at build time" << std::endl; +#endif + } +#ifndef MGONGPU_HAS_NO_BLAS +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) + // Analyse environment variable CUDACPP_RUNTIME_CUBLASTF32TENSOR + const char* blasEnv2 = getenv( "CUDACPP_RUNTIME_CUBLASTF32TENSOR" ); + if( blasEnv2 && std::string( blasEnv2 ) != "" ) + { + if( m_blasColorSum ) + { +#ifdef MGONGPU_FPTYPE2_FLOAT + m_blasTf32Tensor = true; + std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty: enable CUBLAS_TF32_TENSOR_OP_MATH" << std::endl; +#else + std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but color sums use FP64" << std::endl; +#endif + } + else + std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but BLAS was disabled at runtime" << std::endl; + } +#ifdef MGONGPU_FPTYPE2_FLOAT + else + { + if( m_blasColorSum ) + std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is empty or not set: keep cuBLAS math defaults" << std::endl; + } +#endif +#endif +#endif + } + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelDevice::~MatrixElementKernelDevice() + { + //std::cout << "DEBUG: MatrixElementKernelDevice::dtor " << this << std::endl; +#ifndef MGONGPU_HAS_NO_BLAS + if( m_blasHandle ) gpuBlasDestroy( m_blasHandle ); +#endif + for( int ihel = 0; ihel < CPPProcess::ncomb; ihel++ ) + { + if( m_helStreams[ihel] ) gpuStreamDestroy( m_helStreams[ihel] ); // do not destroy if nullptr + } + } + + //-------------------------------------------------------------------------- + + // FIXME! The relevance of this function should be reassessed (#543 and #902) + void MatrixElementKernelDevice::setGrid( const int /*gpublocks*/, const int /*gputhreads*/ ) + { + if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0 in setGrid" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0 in setGrid" ); + if( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch in setGrid" ); + } + + //-------------------------------------------------------------------------- + + int MatrixElementKernelDevice::computeGoodHelicities() + { + PinnedHostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); + // ... 0d1. Compute good helicity mask (a host variable) on the device + gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); + const int nevt = m_gpublocks * m_gputhreads; + sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_pHelJamps->data(), m_pHelNumerators->data(), m_pHelDenominators->data(), hstIsGoodHel.data(), nevt ); + // ... 0d3. Set good helicity list in host static memory + int nGoodHel = sigmaKin_setGoodHel( hstIsGoodHel.data() ); + assert( nGoodHel > 0 ); // SANITY CHECK: there should be at least one good helicity + // Create one GPU stream for each good helicity + for( int ighel = 0; ighel < nGoodHel; ighel++ ) + gpuStreamCreate( &m_helStreams[ighel] ); +#ifndef MGONGPU_HAS_NO_BLAS + // Create one cuBLAS/hipBLAS handle for each good helicity (attached to the default stream) + if( m_blasColorSum ) + { + checkGpuBlas( gpuBlasCreate( &m_blasHandle ) ); +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) + if( m_blasTf32Tensor ) + checkGpuBlas( cublasSetMathMode( m_blasHandle, CUBLAS_TF32_TENSOR_OP_MATH ) ); // enable TF32 tensor cores +#endif + } +#endif + // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) + m_pHelMEs.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); + // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) + // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) + m_pHelJamps.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); + // ... Create the "many-helicity" super-buffers of nGoodHel numerator and denominator buffers (dynamically allocated) + // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) + m_pHelNumerators.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ndiagrams * nevt ) ); + m_pHelDenominators.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); +#ifndef MGONGPU_HAS_NO_BLAS + // Create the "many-helicity" super-buffers of real/imag ncolor*nevt temporary buffers for cuBLAS/hipBLAS intermediate results in color_sum_blas +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Mixed precision mode: need two fptype2[ncolor*2*nevt] buffers and one fptype2[nevt] buffer per good helicity + if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ( 2 * CPPProcess::ncolor * mgOnGpu::nx2 + 1 ) * nevt ) ); +#else + // Standard single/double precision mode: need one fptype2[ncolor*2*nevt] buffer per good helicity + if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); +#endif +#endif + // Return the number of good helicities + return nGoodHel; + } + + //-------------------------------------------------------------------------- + + void MatrixElementKernelDevice::computeMatrixElements( const bool useChannelIds ) + { + gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); +#ifndef MGONGPU_HAS_NO_BLAS + fptype2* ghelAllBlasTmp = ( m_blasColorSum ? m_pHelBlasTmp->data() : nullptr ); + gpuBlasHandle_t* pBlasHandle = ( m_blasColorSum ? &m_blasHandle : nullptr ); +#else + fptype2* ghelAllBlasTmp = nullptr; + gpuBlasHandle_t* pBlasHandle = nullptr; +#endif + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_colJamp2s.data(), m_pHelNumerators->data(), m_pHelDenominators->data(), nullptr, true, m_pHelMEs->data(), m_pHelJamps->data(), ghelAllBlasTmp, pBlasHandle, m_helStreams, false, m_gpublocks, m_gputhreads ); +#ifdef MGONGPU_CHANNELID_DEBUG + //std::cout << "DEBUG: MatrixElementKernelDevice::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; + copyHostFromDevice( m_hstChannelIds, m_channelIds ); // FIXME?! + const unsigned int* pHstChannelIds = ( useChannelIds ? m_hstChannelIds.data() : nullptr ); + MatrixElementKernelBase::updateNevtProcessedByChannel( pHstChannelIds, nevt() ); +#endif + checkGpu( gpuPeekAtLastError() ); // is this needed? + checkGpu( gpuDeviceSynchronize() ); // probably not needed? but it avoids errors in sigmaKin above from appearing later on in random places... + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h new file mode 100644 index 0000000000..01041e43b7 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h @@ -0,0 +1,250 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MATRIXELEMENTKERNELS_H +#define MATRIXELEMENTKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "GpuAbstraction.h" +#include "MemoryBuffers.h" + +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // A base class encapsulating matrix element calculations on a CPU host or on a GPU device + class MatrixElementKernelBase //: virtual public IMatrixElementKernel + { + protected: + + // Constructor from existing input and output buffers + MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol); // output: color selection + + public: + + // Destructor + virtual ~MatrixElementKernelBase(); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + virtual int computeGoodHelicities() = 0; + + // Compute matrix elements + virtual void computeMatrixElements( const bool useChannelIds ) = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + // Dump signalling FPEs (#831 and #837) + static void dumpSignallingFPEs(); + +#ifdef MGONGPU_CHANNELID_DEBUG + // Add a MEK identifier for the channelId debug printout + void setTagForNevtProcessedByChannel( const std::string& tag ) { m_tag = tag; } + + protected: + // Update number of events processed by channel + void updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ); + + // Dump number of events processed by channel + void dumpNevtProcessedByChannel(); +#endif + + protected: + + // The buffer for the input momenta + const BufferMomenta& m_momenta; + + // The buffer for the gs to calculate the alphaS values + const BufferGs& m_gs; + + // The buffer for the flavor indices for the flavor combination + const BufferIflavorVec& m_iflavorVec; + + // The buffer for the random numbers for helicity selection + const BufferRndNumHelicity& m_rndhel; + + // The buffer for the random numbers for color selection + const BufferRndNumColor& m_rndcol; + + // The buffer for the channel ids for single-diagram enhancement + const BufferChannelIds& m_channelIds; + + // The buffer for the output matrix elements + BufferMatrixElements& m_matrixElements; + + // The buffer for the output helicity selection + BufferSelectedHelicity& m_selhel; + + // The buffer for the output color selection + BufferSelectedColor& m_selcol; + +#ifdef MGONGPU_CHANNELID_DEBUG + // The events-per-channel counter for debugging + std::map m_nevtProcessedByChannel; + + // The tag for events-per-channel debugging + std::string m_tag; +#endif + }; + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating matrix element calculations on a CPU host + class MatrixElementKernelHost final : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~MatrixElementKernelHost(); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // Does this host system support the SIMD used in the matrix element calculation? + // [NB: this is private, SIMD vectorization in mg5amc C++ code is currently only used in the ME calculations below MatrixElementKernelHost!] + static bool hostSupportsSIMD( const bool verbose = false ); // ZW: default verbose false + + private: + + // The buffer for the event-by-event couplings that depends on alphas QCD + HostBufferCouplings m_couplings; + + // The buffer for the event-by-event numerators of multichannel factors + HostBufferNumerators m_numerators; + + // The buffer for the event-by-event denominators of multichannel factors + HostBufferDenominators m_denominators; + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating matrix element calculations on a GPU device + class MatrixElementKernelDevice : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads ); + + // Destructor + virtual ~MatrixElementKernelDevice(); + + // Reset gpublocks and gputhreads + void setGrid( const int gpublocks, const int gputhreads ); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The buffer for the event-by-event couplings that depends on alphas QCD + DeviceBufferCouplings m_couplings; + + // The super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelMEs; + + // The super-buffer of nGoodHel jamp buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelJamps; + + // The super-buffer of nGoodHel numerator buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelNumerators; + + // The super-buffer of nGoodHel denominator buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelDenominators; + + // The super-buffer of ncolor jamp2 buffers + DeviceBufferSimple m_colJamp2s; + +#ifdef MGONGPU_CHANNELID_DEBUG + // The **host** buffer for the channelId array + // FIXME? MEKD should accept a host buffer as an argument instead of a device buffer, so that a second copy can be avoided? + PinnedHostBufferChannelIds m_hstChannelIds; +#endif + +#ifndef MGONGPU_HAS_NO_BLAS + // Decide at runtime whether to use BLAS for color sums + bool m_blasColorSum; + + // Decide at runtime whether TF32TENSOR math should be used in cuBLAS + bool m_blasTf32Tensor; + + // The super-buffer of nGoodHel cuBLAS/hipBLAS temporary buffers + std::unique_ptr m_pHelBlasTmp; + + // The cuBLAS/hipBLAS handle (a single one for all good helicities) + gpuBlasHandle_t m_blasHandle; +#endif + + // The array of GPU streams (one for each good helicity) + gpuStream_t m_helStreams[CPPProcess::ncomb]; // reserve ncomb streams (but only nGoodHel <= ncomb will be used) + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + }; +#endif + + //-------------------------------------------------------------------------- +} +#endif // MATRIXELEMENTKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h new file mode 100644 index 0000000000..f41e40bf82 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h @@ -0,0 +1,165 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessAmplitudes_H +#define MemoryAccessAmplitudes_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" + +#define MGONGPU_TRIVIAL_AMPLITUDES 1 + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + +#ifndef MGONGPU_TRIVIAL_AMPLITUDES + + // A class describing the internal layout of memory buffers for amplitudes + // This implementation uses an AOSOA[npagA][nx2][neppA] where nevt=npagA*neppA + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessAmplitudesBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the amplitude AOSOA memory buffer layout + static constexpr int neppA = 1; // AOS (just a test...) + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagA = ievt / neppA; // #event "A-page" + const int ieppA = ievt % neppA; // #event in the current event A-page + constexpr int ix2 = 0; + return &( buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA] ); // AOSOA[ipagA][ix2][ieppA] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ix2 ) + { + constexpr int ipagA = 0; + constexpr int ieppA = 0; + return buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA]; // AOSOA[ipagA][ix2][ieppA] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessAmplitudes : public MemoryAccessAmplitudesBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + +#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessAmplitudes + { + public: + +#ifndef MGONGPU_TRIVIAL_AMPLITUDES + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2 = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2Const = + KernelAccessHelper::template kernelAccessFieldConst; + +#else + + static __host__ __device__ inline cxtype_sv* + kernelAccess( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + static __host__ __device__ inline const cxtype_sv* + kernelAccessConst( const fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + +#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessAmplitudes HostAccessAmplitudes; + typedef KernelAccessAmplitudes DeviceAccessAmplitudes; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h new file mode 100644 index 0000000000..331e7f12c3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h @@ -0,0 +1,126 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Dec 2023, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessChannelIds_H +#define MemoryAccessChannelIds_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for channel ids + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessChannelIdsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] + static __host__ __device__ inline unsigned int* + ieventAccessRecord( unsigned int* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline unsigned int& + decodeRecord( unsigned int* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessChannelIds : public MemoryAccessChannelIdsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] + static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessChannelIds + { + public: + + // Expose selected functions from MemoryAccessChannelIds + static constexpr auto ieventAccessRecordConst = MemoryAccessChannelIds::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] + static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] + static __host__ __device__ inline const uint_sv& + kernelAccessConst( const unsigned int* buffer ) + { + const unsigned int& out = kernelAccessConst_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessChannelIds HostAccessChannelIds; + typedef KernelAccessChannelIds DeviceAccessChannelIds; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h new file mode 100644 index 0000000000..fd3fb80c65 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h @@ -0,0 +1,276 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessCouplings_H +#define MemoryAccessCouplings_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM +#include "MemoryBuffers.h" // for HostBufferCouplings::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for couplings + // This implementation uses an AOSOA[npagC][ndcoup][nx2][neppC] "super-buffer" where nevt=npagC*neppC + // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessCouplingsBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the coupling AOSOA memory buffer layout + static constexpr int neppC = MemoryAccessMomentaBase::neppM; // use the same AOSOA striding as for momenta + + // SANITY CHECK: check that neppC is a power of two + static_assert( ispoweroftwo( neppC ), "neppC is not a power of 2" ); + + //-------------------------------------------------------------------------- + // ** NB! A single super-buffer AOSOA[npagC][ndcoup][nx2][neppC] includes data for ndcoup different couplings ** + // ** NB! The ieventAccessRecord and kernelAccess functions refer to the buffer for one individual coupling ** + // ** NB! Use idcoupAccessBuffer to add a fixed offset and locate the buffer for one given individual coupling ** + //-------------------------------------------------------------------------- + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (non-const) ===> fptype* idcoupAccessBuffer( fptype* buffer, const int idcoup ) <===] + // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? + static __host__ __device__ inline fptype* + idcoupAccessBuffer( fptype* buffer, // input "super-buffer" + const int idcoup ) + { + constexpr int ipagC = 0; + constexpr int ieppC = 0; + constexpr int ix2 = 0; + // NB! this effectively adds an offset "idcoup * nx2 * neppC" + return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] + } + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (const) ===> const fptype* idcoupAccessBufferConst( const fptype* buffer, const int idcoup ) <===] + // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? + static __host__ __device__ inline const fptype* + idcoupAccessBufferConst( const fptype* buffer, // input "super-buffer" + const int idcoup ) + { + return idcoupAccessBuffer( const_cast( buffer ), idcoup ); + } + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of couplings that dependent on the running alphas QCD in this specific process + static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagC = ievt / neppC; // #event "C-page" + const int ieppC = ievt %% neppC; // #event in the current event C-page + constexpr int idcoup = 0; + constexpr int ix2 = 0; + return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ix2 ) + { + constexpr int ipagC = 0; + constexpr int ieppC = 0; + // NB! the offset "idcoup * nx2 * neppC" has been added in idcoupAccessBuffer + constexpr int idcoup = 0; + return buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC]; // AOSOA[ipagC][idcoup][ix2][ieppC] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessCouplings : public MemoryAccessCouplingsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessCouplings + { + public: + + // Expose selected functions from MemoryAccessCouplingsBase + static constexpr auto idcoupAccessBuffer = MemoryAccessCouplingsBase::idcoupAccessBuffer; + static constexpr auto idcoupAccessBufferConst = MemoryAccessCouplingsBase::idcoupAccessBufferConst; + + // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. + // For dependent (event-by-event, running-alphas) couplings the value is an AOSOA record [nx2][neppC] + // (real and imaginary SIMD lanes), so consecutive flavor slots are nx2*neppC fptype's apart. + static constexpr int flv_stride = MemoryAccessCouplingsBase::neppC * mgOnGpu::nx2; + + // Expose selected functions from MemoryAccessCouplings + static constexpr auto ieventAccessRecordConst = MemoryAccessCouplings::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2_s = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2Const_s = + KernelAccessHelper::template kernelAccessFieldConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccessIx2( fptype* buffer, + const int ix2 ) + { + fptype& out = kernelAccessIx2_s( buffer, ix2 ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays + constexpr int neppC = MemoryAccessCouplingsBase::neppC; + static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS + static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessIx2Const( const fptype* buffer, + const int ix2 ) + { + return kernelAccessIx2( const_cast( buffer ), ix2 ); + } + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessIx2Const( const fptype* buffer, + const int ix2 ) + { + const fptype& out = kernelAccessIx2Const_s( buffer, ix2 ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays + constexpr int neppC = MemoryAccessCouplingsBase::neppC; + static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS + static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non const, SCALAR OR VECTOR) ===> cxtype_sv_ref kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline cxtype_sv_ref + kernelAccess( fptype* buffer ) + { + /* + fptype_sv& real = kernelAccessIx2( buffer, 0 ); + fptype_sv& imag = kernelAccessIx2( buffer, 1 ); + printf( "C_ACCESS::kernelAccess: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + return cxtype_sv_ref( real, imag ); + */ + return cxtype_sv_ref( kernelAccessIx2( buffer, 0 ), + kernelAccessIx2( buffer, 1 ) ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] + static __host__ __device__ inline cxtype_sv + kernelAccessConst( const fptype* buffer ) + { + /* + const fptype_sv& real = kernelAccessIx2Const( buffer, 0 ); + const fptype_sv& imag = kernelAccessIx2Const( buffer, 1 ); + printf( "C_ACCESS::kernelAccessConst: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + return cxtype_sv( real, imag ); + */ + return cxtype_sv( kernelAccessIx2Const( buffer, 0 ), + kernelAccessIx2Const( buffer, 1 ) ); + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessCouplings HostAccessCouplings; + typedef KernelAccessCouplings DeviceAccessCouplings; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h new file mode 100644 index 0000000000..757de7b6f2 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h @@ -0,0 +1,89 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Apr 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessCouplingsFixed_H +#define MemoryAccessCouplingsFixed_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" +#include "mgOnGpuVectors.h" + +//#include "MemoryAccessHelpers.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for fixed couplings + // This implementation uses a STRUCT[ndcoup][nx2] "super-buffer" layout: in practice, the cIPC global array + // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling + // [If many implementations are used, a suffix _Sv1 should be appended to the class name] + class MemoryAccessCouplingsFixedBase //_Sv1 + { + public: + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (const) ===> const fptype* iicoupAccessBufferConst( const fptype* buffer, const int iicoup ) <===] + static __host__ __device__ inline const fptype* + iicoupAccessBufferConst( const fptype* buffer, // input "super-buffer": in practice, the cIPC global array + const int iicoup ) + { + constexpr int ix2 = 0; + // NB! this effectively adds an offset "iicoup * nx2" + return &( buffer[iicoup * nx2 + ix2] ); // STRUCT[idcoup][ix2] + } + + private: + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessCouplingsFixed + { + public: + + // Expose selected functions from MemoryAccessCouplingsFixedBase + static constexpr auto iicoupAccessBufferConst = MemoryAccessCouplingsFixedBase::iicoupAccessBufferConst; + + // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. + // For fixed (independent) couplings the value is a single scalar complex (real,imag): nx2 fptype's, broadcast across the SIMD vector. + static constexpr int flv_stride = mgOnGpu::nx2; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] + static __host__ __device__ inline const cxtype_sv + kernelAccessConst( const fptype* buffer ) + { + // TRIVIAL ACCESS to fixed-couplings buffers! + //return cxmake( fptype_sv{ buffer[0] }, fptype_sv{ buffer[1] } ); // NO! BUG #339! + const fptype_sv r_sv = fptype_sv{ 0 } + buffer[0]; + const fptype_sv i_sv = fptype_sv{ 0 } + buffer[1]; + return cxmake( r_sv, i_sv ); // ugly but effective + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessCouplingsFixed HostAccessCouplingsFixed; + typedef KernelAccessCouplingsFixed DeviceAccessCouplingsFixed; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h new file mode 100644 index 0000000000..205952e514 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h @@ -0,0 +1,31 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessDenominators_H +#define MemoryAccessDenominators_H 1 + +#include "MemoryAccessGs.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for denominators + // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs + + typedef KernelAccessGs HostAccessDenominators; + typedef KernelAccessGs DeviceAccessDenominators; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h new file mode 100644 index 0000000000..bfcb3e3222 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h @@ -0,0 +1,171 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessGs_H +#define MemoryAccessGs_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for Gs + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessGsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessGs : public MemoryAccessGsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessGs + { + public: + + // Expose selected functions from MemoryAccessGs + static constexpr auto ieventAccessRecord = MemoryAccessGs::ieventAccessRecord; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccess( fptype* buffer ) <===] + static constexpr auto kernelAccess_s = + KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non-const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccess( fptype* buffer ) + { + fptype& out = kernelAccess_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (SCALAR OR VECTOR) ===> fptype_sv* kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv* + kernelAccessP( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + static constexpr auto kernelAccessConst_s = + KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccess( const fptype* buffer ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessConst( const fptype* buffer ) + { + const fptype& out = kernelAccessConst_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessGs HostAccessGs; + typedef KernelAccessGs DeviceAccessGs; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h new file mode 100644 index 0000000000..8b5bd27f4d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h @@ -0,0 +1,158 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessHelpers_H +#define MemoryAccessHelpers_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +//---------------------------------------------------------------------------- + +// A templated helper class that includes the boilerplate code for MemoryAccess classes +template +class MemoryAccessHelper +{ +public: + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = T::ieventAccessRecord; + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline const FT* + ieventAccessRecordConst( const FT* buffer, + const int ievt ) + { + return ieventAccessRecord( const_cast( buffer ), ievt ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + static constexpr auto decodeRecord = T::decodeRecord; + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + decodeRecordConst( const FT* buffer, + Ts... args ) // variadic template + { + return T::decodeRecord( const_cast( buffer ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessField( fptype* buffer, const ievt, Ts... args ) <===] + template + static __host__ __device__ inline FT& + ieventAccessField( FT* buffer, + const int ievt, + Ts... args ) // variadic template + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + return T::decodeRecord( T::ieventAccessRecord( buffer, ievt ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessFieldConst( const fptype* buffer, const ievt, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + ieventAccessFieldConst( const FT* buffer, + const int ievt, + Ts... args ) // variadic template + { + return ieventAccessField( const_cast( buffer ), ievt, args... ); + } +}; + +//---------------------------------------------------------------------------- + +// A templated helper class that includes the boilerplate code for KernelAccess classes +template +class KernelAccessHelper : public MemoryAccessHelper +{ +public: + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non-const) ===> fptype* kernelAccessRecord( fptype* buffer ) <===] + static __host__ __device__ inline FT* + kernelAccessRecord( FT* buffer ) + { + if constexpr( !onDevice ) // requires c++17 also in CUDA (#333) + { + // FIXME #436: clarify that buffer includes all events on device, and only the record for an event subset on host! + // FIXME #436: am I not assuming that the following line is always identical to buffer for all access classes T? + return T::ieventAccessRecord( buffer, 0 ); + } + else + { +#ifdef MGONGPUCPP_GPUIMPL + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid + //printf( "kernelAccessRecord: ievt=%d threadId=%d\n", ievt, threadIdx.x ); + return T::ieventAccessRecord( buffer, ievt ); // NB fptype and fptype_sv coincide for CUDA +#else + throw std::runtime_error( "kernelAccessRecord on device is only implemented in CUDA" ); +#endif + } + } + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const) ===> const fptype* kernelAccessRecordConst( const fptype* buffer ) <===] + static __host__ __device__ inline const FT* + kernelAccessRecordConst( const FT* buffer ) + { + return kernelAccessRecord( const_cast( buffer ) ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessField( fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline FT& + kernelAccessField( FT* buffer, + Ts... args ) // variadic template + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + return T::decodeRecord( kernelAccessRecord( buffer ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessFieldConst( const fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + kernelAccessFieldConst( const FT* buffer, + Ts... args ) // variadic template + { + return kernelAccessField( const_cast( buffer ), args... ); + } + + //-------------------------------------------------------------------------- +}; + +#endif // MemoryAccessHelpers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h new file mode 100644 index 0000000000..ffc74b0f0c --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h @@ -0,0 +1,125 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: D. Massaro (Jan 2026, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessIflavorVec_H +#define MemoryAccessIflavorVec_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for channel ids + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessIflavorVecBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] + static __host__ __device__ inline unsigned int* + ieventAccessRecord( unsigned int* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline unsigned int& + decodeRecord( unsigned int* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessIflavorVec : public MemoryAccessIflavorVecBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] + static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessIflavorVec + { + public: + + // Expose selected functions from MemoryAccessIflavorVec + static constexpr auto ieventAccessRecordConst = MemoryAccessIflavorVec::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] + static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] + static __host__ __device__ inline const uint_sv& + kernelAccessConst( const unsigned int* buffer ) + { + const unsigned int& out = kernelAccessConst_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessIflavorVec HostAccessIflavorVec; + typedef KernelAccessIflavorVec DeviceAccessIflavorVec; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h new file mode 100644 index 0000000000..4236e20602 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h @@ -0,0 +1,147 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessMatrixElements_H +#define MemoryAccessMatrixElements_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for matrix elements + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessMatrixElementsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessMatrixElements : public MemoryAccessMatrixElementsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessMatrixElements + { + public: + + // Expose selected functions from MemoryAccessMatrixElements + static constexpr auto ieventAccessRecord = MemoryAccessMatrixElements::ieventAccessRecord; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccess_s( fptype* buffer ) <===] + static constexpr auto kernelAccess_s = + KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( const fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccess( fptype* buffer ) + { + fptype& out = kernelAccess_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + static constexpr auto kernelAccessConst = + KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessMatrixElements HostAccessMatrixElements; + typedef KernelAccessMatrixElements DeviceAccessMatrixElements; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h new file mode 100644 index 0000000000..3ef4d76fbd --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h @@ -0,0 +1,276 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessMomenta_H +#define MemoryAccessMomenta_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for momenta + // This implementation uses an AOSOA[npagM][npar][np4][neppM] where nevt=npagM*neppM + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessMomentaBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the momenta AOSOA memory buffer layout + // (these are all best kept as a compile-time constants: see issue #23) +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ + // ----------------------------------------------------------------------------------------------- + // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline + // --- This is relevant to ensure coalesced access to momenta in global memory + // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms + // ----------------------------------------------------------------------------------------------- + //static constexpr int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) + static constexpr int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 1.03E9 instead of 1.11E9 in eemumu) +#else + // ----------------------------------------------------------------------------------------------- + // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register + // --- This is relevant to ensure faster access to momenta from C++ memory cache lines + // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded + // --- In practice, neppR, neppM and neppV could now (in principle) all be different + // ----------------------------------------------------------------------------------------------- +#ifdef MGONGPU_CPPSIMD + static constexpr int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance + //static constexpr int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) + //static constexpr int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 4.66E6 instead of 5.09E9 in eemumu) + //static constexpr int neppM = MGONGPU_CPPSIMD*2; // FOR TESTS +#else + static constexpr int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) +#endif +#endif /* clang-format on */ + + // SANITY CHECK: check that neppM is a power of two + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a 4-momentum + static constexpr int np4 = CPPProcess::np4; + + // The number of particles in this physics process + static constexpr int npar = CPPProcess::npar; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagM = ievt / neppM; // #event "M-page" + const int ieppM = ievt % neppM; // #event in the current event M-page + constexpr int ip4 = 0; + constexpr int ipar = 0; + return &( buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM] ); // AOSOA[ipagM][ipar][ip4][ieppM] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ip4, const int ipar" and rename "Field" as "Ip4Ipar"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ip4, + const int ipar ) + { + constexpr int ipagM = 0; + constexpr int ieppM = 0; + return buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessMomenta : public MemoryAccessMomentaBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto decodeRecordIp4Ipar = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto decodeRecordIp4IparConst = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIp4Ipar( fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + static constexpr auto ieventAccessIp4Ipar = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + // DEFAULT VERSION + static constexpr auto ieventAccessIp4IparConst = + MemoryAccessHelper::template ieventAccessFieldConst; + + /* + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + // DEBUG VERSION WITH PRINTOUTS + static __host__ __device__ inline const fptype& + ieventAccessIp4IparConst( const fptype* buffer, + const int ievt, + const int ip4, + const int ipar ) + { + const fptype& out = MemoryAccessHelper::template ieventAccessFieldConst( buffer, ievt, ip4, ipar ); + printf( "ipar=%2d ip4=%2d ievt=%8d out=%8.3f\n", ipar, ip4, ievt, out ); + return out; + } + */ + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessMomenta + { + public: + + // Expose selected functions from MemoryAccessMomenta + static constexpr auto ieventAccessRecordConst = MemoryAccessMomenta::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIp4Ipar( fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto kernelAccessIp4Ipar = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // DEFAULT VERSION + static constexpr auto kernelAccessIp4IparConst_s = + KernelAccessHelper::template kernelAccessFieldConst; + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // DEBUG VERSION WITH PRINTOUTS + static __host__ __device__ inline const fptype& + kernelAccessIp4IparConst_s( const fptype* buffer, + const int ip4, + const int ipar ) + { + const fptype& out = KernelAccessHelper::template kernelAccessFieldConst( buffer, ip4, ipar ); + printf( "ipar=%2d ip4=%2d ievt='kernel' out=%8.3f\n", ipar, ip4, out ); + return out; + } + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> fptype_sv kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // FIXME? Eventually return by const reference and support aligned arrays only? + // FIXME? Currently return by value to support also unaligned and arbitrary arrays + static __host__ __device__ inline fptype_sv + kernelAccessIp4IparConst( const fptype* buffer, + const int ip4, + const int ipar ) + { + const fptype& out = kernelAccessIp4IparConst_s( buffer, ip4, ipar ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + constexpr int neppM = MemoryAccessMomentaBase::neppM; + constexpr bool useContiguousEventsIfPossible = true; // DEFAULT + //constexpr bool useContiguousEventsIfPossible = false; // FOR PERFORMANCE TESTS (treat as arbitrary array even if it is an AOSOA) + // Use c++17 "if constexpr": compile-time branching + if constexpr( useContiguousEventsIfPossible && ( neppM >= neppV ) && ( neppM % neppV == 0 ) ) + { + //constexpr bool skipAlignmentCheck = true; // FASTEST (SEGFAULTS IF MISALIGNED ACCESS, NEEDS A SANITY CHECK ELSEWHERE!) + constexpr bool skipAlignmentCheck = false; // DEFAULT: A BIT SLOWER BUT SAFER [ALLOWS MISALIGNED ACCESS] + if constexpr( skipAlignmentCheck ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! assume aligned AOSOA, skip check" << std::endl; first=false; } // SLOWER (5.06E6) + // FASTEST? (5.09E6 in eemumu 512y) + // This assumes alignment for momenta1d without checking - causes segmentation fault in reinterpret_cast if not aligned! + return mg5amcCpu::fptypevFromAlignedArray( out ); // use reinterpret_cast + } + else if( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! aligned AOSOA, reinterpret cast" << std::endl; first=false; } // SLOWER (5.00E6) + // DEFAULT! A tiny bit (<1%) slower because of the alignment check (5.07E6 in eemumu 512y) + // This explicitly checks buffer alignment to avoid segmentation faults in reinterpret_cast + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWER (4.93E6) + // A bit (1%) slower (5.05E6 in eemumu 512y) + // This does not require buffer alignment, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + return mg5amcCpu::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) + } + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! arbitrary array" << std::endl; first=false; } // SLOWER (5.08E6) + // ?!Used to be much slower, now a tiny bit faster for AOSOA?! (5.11E6 for AOSOA, 4.64E6 for AOS in eemumu 512y) + // This does not even require AOSOA with neppM>=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) + constexpr int ievt0 = 0; // just make it explicit in the code that buffer refers to a given ievt0 and decoderIeppV fetches event ievt0+ieppV + auto decoderIeppv = [buffer, ip4, ipar]( int ieppV ) + -> const fptype& + { return MemoryAccessMomenta::ieventAccessIp4IparConst( buffer, ievt0 + ieppV, ip4, ipar ); }; + return mg5amcCpu::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) + } +#endif + } + + // Is this a HostAccess or DeviceAccess class? + // [this is only needed for a warning printout in rambo.h for nparf==1 #358] + static __host__ __device__ inline constexpr bool + isOnDevice() + { + return onDevice; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessMomenta HostAccessMomenta; + typedef KernelAccessMomenta DeviceAccessMomenta; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h new file mode 100644 index 0000000000..c8d0343b7e --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h @@ -0,0 +1,31 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessNumerators_H +#define MemoryAccessNumerators_H 1 + +#include "MemoryAccessGs.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for numerators + // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs + + typedef KernelAccessGs HostAccessNumerators; + typedef KernelAccessGs DeviceAccessNumerators; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h new file mode 100644 index 0000000000..57f1d02081 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h @@ -0,0 +1,145 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessRandomNumbers_H +#define MemoryAccessRandomNumbers_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "MemoryAccessHelpers.h" + +#ifdef MGONGPUCPP_GPUIMPL +using mg5amcGpu::CPPProcess; +#else +using mg5amcCpu::CPPProcess; +#endif + +//---------------------------------------------------------------------------- + +// A class describing the internal layout of memory buffers for random numbers +// This implementation uses an AOSOA[npagR][nparf][np4][neppR] where nevt=npagR*neppR +// [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] +class MemoryAccessRandomNumbersBase //_AOSOAv1 +{ +public: /* clang-format off */ + + // Number of Events Per Page in the random number AOSOA memory buffer layout + // *** NB Different values of neppR lead to different physics results: the *** + // *** same 1d array is generated, but it is interpreted in different ways *** + static constexpr int neppR = 8; // HARDCODED TO GIVE ALWAYS THE SAME PHYSICS RESULTS! + //static constexpr int neppR = 1; // AOS (tests of sectors/requests) + +private: /* clang-format on */ + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a 4-momentum + static constexpr int np4 = CPPProcess::np4; + + // The number of final state particles in this physics process + static constexpr int nparf = CPPProcess::nparf; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagR = ievt / neppR; // #event "R-page" + const int ieppR = ievt % neppR; // #event in the current event R-page + constexpr int ip4 = 0; + constexpr int iparf = 0; + return &( buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR] ); // AOSOA[ipagR][iparf][ip4][ieppR] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ip4, const int iparf" and rename "Field" as "Ip4Iparf"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ip4, + const int iparf ) + { + constexpr int ipagR = 0; + constexpr int ieppR = 0; + return buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR]; // AOSOA[ipagR][iparf][ip4][ieppR] + } +}; + +//---------------------------------------------------------------------------- + +// A class providing access to memory buffers for a given event, based on explicit event numbers +// Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations +class MemoryAccessRandomNumbers : public MemoryAccessRandomNumbersBase +{ +public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto decodeRecordIp4Iparf = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto decodeRecordIp4IparfConst = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIp4Iparf( fptype* buffer, const ievt, const int ipar, const int iparf ) <===] + static constexpr auto ieventAccessIp4Iparf = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparfConst( const fptype* buffer, const ievt, const int ipar, const int iparf ) <===] + static constexpr auto ieventAccessIp4IparfConst = + MemoryAccessHelper::template ieventAccessFieldConst; +}; + +//---------------------------------------------------------------------------- + +// A class providing access to memory buffers for a given event, based on implicit kernel rules +// Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations +template +class KernelAccessRandomNumbers +{ +public: + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIp4Iparf( fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto kernelAccessIp4Iparf = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIp4IparfConst( const fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto kernelAccessIp4IparfConst = + KernelAccessHelper::template kernelAccessFieldConst; +}; + +//---------------------------------------------------------------------------- + +typedef KernelAccessRandomNumbers HostAccessRandomNumbers; +typedef KernelAccessRandomNumbers DeviceAccessRandomNumbers; + +//---------------------------------------------------------------------------- + +#endif // MemoryAccessRandomNumbers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h new file mode 100644 index 0000000000..8563d8f2f7 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h @@ -0,0 +1,138 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessVectors_H +#define MemoryAccessVectors_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#ifndef MGONGPUCPP_GPUIMPL +namespace mg5amcCpu // this is only needed for CPU SIMD vectorization +{ + +#ifdef MGONGPU_CPPSIMD + //-------------------------------------------------------------------------- + + // Cast one non-const fptype_v reference (one vector of neppV fptype values) from one non-const fptype reference (#435), + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned + inline fptype_v& fptypevFromAlignedArray( fptype& ref ) + { + return *reinterpret_cast( &ref ); + } + + inline uint_v& uintvFromAlignedArray( unsigned int& ref ) + { + return *reinterpret_cast( &ref ); + } + + // Cast one const fptype_v reference (one vector of neppV fptype values) from one const fptype reference, + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned + inline const fptype_v& fptypevFromAlignedArray( const fptype& ref ) + { + return *reinterpret_cast( &ref ); + } + + inline const uint_v& uintvFromAlignedArray( const unsigned int& ref ) + { + return *reinterpret_cast( &ref ); + } + + // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", but that the arrays are not aligned + inline fptype_v fptypevFromUnalignedArray( const fptype& ref ) + { +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (2) + *( &ref + 1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (4) + *( &ref + 1 ), + *( &ref + 2 ), + *( &ref + 3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (8) + *( &ref + 1 ), + *( &ref + 2 ), + *( &ref + 3 ), + *( &ref + 4 ), + *( &ref + 5 ), + *( &ref + 6 ), + *( &ref + 7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (16) + *( &ref + 1 ), + *( &ref + 2 ), + *( &ref + 3 ), + *( &ref + 4 ), + *( &ref + 5 ), + *( &ref + 6 ), + *( &ref + 7 ), + *( &ref + 8 ), + *( &ref + 9 ), + *( &ref + 10 ), + *( &ref + 11 ), + *( &ref + 12 ), + *( &ref + 13 ), + *( &ref + 14 ), + *( &ref + 15 ) }; +#else +#error Internal error! Unknown MGONGPU_CPPSIMD value +#endif + } + + // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, + // with no a priori assumption on how the input fptype array should be decoded + template + inline fptype_v fptypevFromArbitraryArray( Functor decoderIeppv ) + { +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (2) + decoderIeppv( 1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (4) + decoderIeppv( 1 ), + decoderIeppv( 2 ), + decoderIeppv( 3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (8) + decoderIeppv( 1 ), + decoderIeppv( 2 ), + decoderIeppv( 3 ), + decoderIeppv( 4 ), + decoderIeppv( 5 ), + decoderIeppv( 6 ), + decoderIeppv( 7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (16) + decoderIeppv( 1 ), + decoderIeppv( 2 ), + decoderIeppv( 3 ), + decoderIeppv( 4 ), + decoderIeppv( 5 ), + decoderIeppv( 6 ), + decoderIeppv( 7 ), + decoderIeppv( 8 ), + decoderIeppv( 9 ), + decoderIeppv( 10 ), + decoderIeppv( 11 ), + decoderIeppv( 12 ), + decoderIeppv( 13 ), + decoderIeppv( 14 ), + decoderIeppv( 15 ) }; +#else +#error Internal error! Unknown MGONGPU_CPPSIMD value +#endif + } + + //-------------------------------------------------------------------------- +#endif + +} // end namespace +#endif + +#endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h new file mode 100644 index 0000000000..14a37c7fea --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h @@ -0,0 +1,170 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessWavefunctions_H +#define MemoryAccessWavefunctions_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" + +#define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + +#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + // A class describing the internal layout of memory buffers for wavefunctions + // This implementation uses an AOSOA[npagW][nw6][nx2][neppW] where nevt=npagW*neppW + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessWavefunctionsBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the wavefunction AOSOA memory buffer layout + static constexpr int neppW = 1; // AOS (just a test...) + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a (fermion or vector) wavefunction + static constexpr int nw6 = mgOnGpu::nw6; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagW = ievt / neppW; // #event "W-page" + const int ieppW = ievt % neppW; // #event in the current event W-page + constexpr int iw6 = 0; + constexpr int ix2 = 0; + return &( buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW] ); // AOSOA[ipagW][iw6][ix2][ieppW] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int iw6, const int ix2" and rename "Field" as "Iw6Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int iw6, + const int ix2 ) + { + constexpr int ipagW = 0; + constexpr int ieppW = 0; + return buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW]; // AOSOA[ipagW][iw6][ix2][ieppW] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessWavefunctions : public MemoryAccessWavefunctionsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto decodeRecordIw6Ix2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto decodeRecordIw6Ix2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIw6Ix2( fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] + static constexpr auto ieventAccessIw6Ix2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIw6Ix2Const( const fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] + static constexpr auto ieventAccessIw6Ix2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + +#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessWavefunctions + { + public: + +#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIw6Ix2( fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto kernelAccessIw6Ix2 = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIw6Ix2Const( const fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto kernelAccessIw6Ix2Const = + KernelAccessHelper::template kernelAccessFieldConst; + +#else + + static __host__ __device__ inline cxtype_sv* + kernelAccess( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + static __host__ __device__ inline const cxtype_sv* + kernelAccessConst( const fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + +#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessWavefunctions HostAccessWavefunctions; + typedef KernelAccessWavefunctions DeviceAccessWavefunctions; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h new file mode 100644 index 0000000000..c56527f581 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h @@ -0,0 +1,150 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessWeights_H +#define MemoryAccessWeights_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for weights + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessWeightsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessWeights : public MemoryAccessWeightsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessWeights + { + public: + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] + // FINAL IMPLEMENTATION FOR CUDA 11.4 + static constexpr auto kernelAccess = + KernelAccessHelper::template kernelAccessField<>; + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] + // TEMPORARY HACK FOR CUDA 11.1 + static __host__ __device__ inline fptype& + kernelAccess( fptype* buffer ) + { + return KernelAccessHelper::template kernelAccessField<>( buffer ); + } + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + // FINAL IMPLEMENTATION FOR CUDA 11.4 + static constexpr auto kernelAccessConst = + KernelAccessHelper::template kernelAccessFieldConst<>; + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + // TEMPORARY HACK FOR CUDA 11.1 + static __host__ __device__ inline const fptype& + kernelAccessConst( const fptype* buffer ) + { + return KernelAccessHelper::template kernelAccessFieldConst<>( buffer ); + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessWeights HostAccessWeights; + typedef KernelAccessWeights DeviceAccessWeights; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h new file mode 100644 index 0000000000..d259aa7456 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h @@ -0,0 +1,603 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021, based on earlier work by S. Hageboeck) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryBuffers_H +#define MemoryBuffers_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "CPPProcess.h" +#include "GpuRuntime.h" +#include "Parameters.h" +#include "processConfig.h" + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + namespace MemoryBuffers + { + // Process-independent compile-time constants + static constexpr size_t np4 = CPPProcess::np4; + static constexpr size_t nw6 = CPPProcess::nw6; + static constexpr size_t nx2 = mgOnGpu::nx2; + // Process-dependent compile-time constants + static constexpr size_t nparf = CPPProcess::nparf; + static constexpr size_t npar = CPPProcess::npar; + static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; + static constexpr size_t ncolor = CPPProcess::ncolor; + } + + //-------------------------------------------------------------------------- + + // An abstract interface encapsulating a given number of events + class INumberOfEvents + { + public: + virtual ~INumberOfEvents() {} + virtual size_t nevt() const = 0; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating a given number of events + class NumberOfEvents : virtual public INumberOfEvents + { + public: + NumberOfEvents( const size_t nevt ) + : m_nevt( nevt ) {} + virtual ~NumberOfEvents() {} + virtual size_t nevt() const override { return m_nevt; } + private: + const size_t m_nevt; + }; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer (not necessarily an event buffer) + template + class BufferBase : virtual public INumberOfEvents + { + protected: + BufferBase( const size_t size, const bool onDevice ) + : m_size( size ), m_data( nullptr ), m_isOnDevice( onDevice ) {} + public: + virtual ~BufferBase() {} + T* data() { return m_data; } + const T* data() const { return m_data; } + T& operator[]( const size_t index ) { return m_data[index]; } + const T& operator[]( const size_t index ) const { return m_data[index]; } + size_t size() const { return m_size; } + size_t bytes() const { return m_size * sizeof( T ); } + bool isOnDevice() const { return m_isOnDevice; } + virtual size_t nevt() const override { throw std::runtime_error( "This BufferBase is not an event buffer" ); } + protected: + const size_t m_size; + T* m_data; + const bool m_isOnDevice; + }; + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + constexpr bool HostBufferALIGNED = false; // ismisaligned=false + constexpr bool HostBufferMISALIGNED = true; // ismisaligned=true + + // A class encapsulating a C++ host buffer + template + class HostBufferBase : public BufferBase + { + public: + HostBufferBase( const size_t size ) + : BufferBase( size, false ) + { + if constexpr( !ismisaligned ) + this->m_data = new( std::align_val_t( cppAlign ) ) T[size](); + else + this->m_data = new( std::align_val_t( cppAlign ) ) T[size + 1]() + 1; // TEST MISALIGNMENT! + } + virtual ~HostBufferBase() + { + if constexpr( !ismisaligned ) + ::operator delete[]( this->m_data, std::align_val_t( cppAlign ) ); + else + ::operator delete[]( ( this->m_data ) - 1, std::align_val_t( cppAlign ) ); // TEST MISALIGNMENT! + } + static constexpr bool isaligned() { return !ismisaligned; } + public: + static constexpr size_t cppAlign = mgOnGpu::cppAlign; + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA pinned host buffer + template + class PinnedHostBufferBase : public BufferBase + { + public: + PinnedHostBufferBase( const size_t size ) + : BufferBase( size, false ) + { + gpuMallocHost( &( this->m_data ), this->bytes() ); + } + virtual ~PinnedHostBufferBase() + { + gpuFreeHost( this->m_data ); + } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA device buffer + template + class DeviceBufferBase : public BufferBase + { + public: + DeviceBufferBase( const size_t size ) + : BufferBase( size, true ) + { + gpuMalloc( &( this->m_data ), this->bytes() ); + } + virtual ~DeviceBufferBase() + { + gpuFree( this->m_data ); + } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for a given number of events + template + class HostBuffer : public HostBufferBase, virtual private NumberOfEvents + { + public: + HostBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , HostBufferBase( sizePerEvent * nevt ) + { + //std::cout << "HostBuffer::ctor " << this << " " << nevt << std::endl; + } + virtual ~HostBuffer() + { + //std::cout << "HostBuffer::dtor " << this << std::endl; + } + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA pinned host buffer for a given number of events + template + class PinnedHostBuffer : public PinnedHostBufferBase, virtual private NumberOfEvents + { + public: + PinnedHostBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , PinnedHostBufferBase( sizePerEvent * nevt ) {} + virtual ~PinnedHostBuffer() {} + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA device buffer for a given number of events + template + class DeviceBuffer : public DeviceBufferBase, virtual protected NumberOfEvents + { + public: + DeviceBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , DeviceBufferBase( sizePerEvent * nevt ) + { + //std::cout << "DeviceBuffer::ctor " << this << " " << nevt << std::endl; + } + virtual ~DeviceBuffer() + { + //std::cout << "DeviceBuffer::dtor " << this << std::endl; + } + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a simple CUDA device buffer managed on an ad-hoc basis + typedef DeviceBuffer DeviceBufferSimple; + typedef DeviceBuffer DeviceBufferSimple2; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for momenta random numbers + typedef BufferBase BufferRndNumMomenta; + + // The size (number of elements) per event in a memory buffer for momenta random numbers + constexpr size_t sizePerEventRndNumMomenta = MemoryBuffers::np4 * MemoryBuffers::nparf; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for momenta random numbers + typedef HostBuffer HostBufferRndNumMomenta; +#else + // A class encapsulating a CUDA pinned host buffer for momenta random numbers + typedef PinnedHostBuffer PinnedHostBufferRndNumMomenta; + // A class encapsulating a CUDA device buffer for momenta random numbers + typedef DeviceBuffer DeviceBufferRndNumMomenta; +#endif + + //-------------------------------------------------------------------------- + + /* + // A base class encapsulating a memory buffer with ONE fptype per event + typedef BufferBase BufferOneFp; + + // The size (number of elements) per event in a memory buffer with ONE fptype per event + constexpr size_t sizePerEventOneFp = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer with ONE fptype per event + typedef HostBuffer HostBufferOneFp; +#else + // A class encapsulating a CUDA pinned host buffer for gs + typedef PinnedHostBuffer PinnedHostBufferOneFp; + // A class encapsulating a CUDA device buffer for gs + typedef DeviceBuffer DeviceBufferOneFp; +#endif + + // Memory buffers for Gs (related to the event-by-event strength of running coupling constant alphas QCD) + typedef BufferOneFp BufferGs; + typedef HostBufferOneFp HostBufferGs; + typedef PinnedHostBufferOneFp PinnedHostBufferGs; + typedef DeviceBufferOneFp DeviceBufferGs; + */ + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for Gs (related to the event-by-event strength of running coupling constant alphas QCD) + typedef BufferBase BufferGs; + + // The size (number of elements) per event in a memory buffer for Gs + constexpr size_t sizePerEventGs = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for gs + typedef HostBuffer HostBufferGs; +#else + // A class encapsulating a CUDA pinned host buffer for gs + typedef PinnedHostBuffer PinnedHostBufferGs; + // A class encapsulating a CUDA device buffer for gs + typedef DeviceBuffer DeviceBufferGs; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for numerators (of the multichannel single-diagram enhancement factors) + typedef BufferBase BufferNumerators; + + // The size (number of elements) per event in a memory buffer for numerators + // (should be equal to the number of diagrams in the process) + constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for numerators + typedef HostBuffer HostBufferNumerators; +#else + // A class encapsulating a CUDA pinned host buffer for numerators + typedef PinnedHostBuffer PinnedHostBufferNumerators; + // A class encapsulating a CUDA device buffer for numerators + typedef DeviceBuffer DeviceBufferNumerators; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for denominators (of the multichannel single-diagram enhancement factors) + typedef BufferBase BufferDenominators; + + // The size (number of elements) per event in a memory buffer for denominators + constexpr size_t sizePerEventDenominators = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for denominators + typedef HostBuffer HostBufferDenominators; +#else + // A class encapsulating a CUDA pinned host buffer for denominators + typedef PinnedHostBuffer PinnedHostBufferDenominators; + // A class encapsulating a CUDA device buffer for denominators + typedef DeviceBuffer DeviceBufferDenominators; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for couplings that depend on the event-by-event running coupling constant alphas QCD + typedef BufferBase BufferCouplings; + + // The size (number of elements) per event in a memory buffer for random numbers + constexpr size_t sizePerEventCouplings = MemoryBuffers::ndcoup * MemoryBuffers::nx2; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for couplings + typedef HostBuffer HostBufferCouplings; +#else + // A class encapsulating a CUDA pinned host buffer for couplings + typedef PinnedHostBuffer PinnedHostBufferCouplings; + // A class encapsulating a CUDA device buffer for couplings + typedef DeviceBuffer DeviceBufferCouplings; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for momenta + typedef BufferBase BufferMomenta; + + // The size (number of elements) per event in a memory buffer for momenta + constexpr size_t sizePerEventMomenta = MemoryBuffers::np4 * MemoryBuffers::npar; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for momenta + typedef HostBuffer HostBufferMomenta; + //typedef HostBuffer HostBufferMomenta; // TEST MISALIGNMENT! +#else + // A class encapsulating a CUDA pinned host buffer for momenta + typedef PinnedHostBuffer PinnedHostBufferMomenta; + // A class encapsulating a CUDA device buffer for momenta + typedef DeviceBuffer DeviceBufferMomenta; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for sampling weights + typedef BufferBase BufferWeights; + + // The size (number of elements) per event in a memory buffer for sampling weights + constexpr size_t sizePerEventWeights = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for sampling weights + typedef HostBuffer HostBufferWeights; +#else + // A class encapsulating a CUDA pinned host buffer for sampling weights + typedef PinnedHostBuffer PinnedHostBufferWeights; + // A class encapsulating a CUDA device buffer for sampling weights + typedef DeviceBuffer DeviceBufferWeights; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for matrix elements + typedef BufferBase BufferMatrixElements; + + // The size (number of elements) per event in a memory buffer for matrix elements + constexpr size_t sizePerEventMatrixElements = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for matrix elements + typedef HostBuffer HostBufferMatrixElements; +#else + // A class encapsulating a CUDA pinned host buffer for matrix elements + typedef PinnedHostBuffer PinnedHostBufferMatrixElements; + // A class encapsulating a CUDA device buffer for matrix elements + typedef DeviceBuffer DeviceBufferMatrixElements; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for the helicity mask + typedef BufferBase BufferHelicityMask; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for the helicity mask + typedef HostBufferBase HostBufferHelicityMask; +#else + // A class encapsulating a CUDA pinned host buffer for the helicity mask + typedef PinnedHostBufferBase PinnedHostBufferHelicityMask; + // A class encapsulating a CUDA device buffer for the helicity mask + typedef DeviceBufferBase DeviceBufferHelicityMask; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for wavefunctions + typedef BufferBase BufferWavefunctions; + + // The size (number of elements) per event in a memory buffer for wavefunctions + constexpr size_t sizePerEventWavefunctions = MemoryBuffers::nw6 * MemoryBuffers::nx2; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for wavefunctions + typedef HostBuffer HostBufferWavefunctions; +#else + // A class encapsulating a CUDA pinned host buffer for wavefunctions + typedef PinnedHostBuffer PinnedHostBufferWavefunctions; + // A class encapsulating a CUDA device buffer for wavefunctions + typedef DeviceBuffer DeviceBufferWavefunctions; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for helicity random numbers + typedef BufferBase BufferRndNumHelicity; + + // The size (number of elements) per event in a memory buffer for helicity random numbers + constexpr size_t sizePerEventRndNumHelicity = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for helicity random numbers + typedef HostBuffer HostBufferRndNumHelicity; +#else + // A class encapsulating a CUDA pinned host buffer for helicity random numbers + typedef PinnedHostBuffer PinnedHostBufferRndNumHelicity; + // A class encapsulating a CUDA device buffer for helicity random numbers + typedef DeviceBuffer DeviceBufferRndNumHelicity; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for color random numbers + typedef BufferBase BufferRndNumColor; + + // The size (number of elements) per event in a memory buffer for color random numbers + constexpr size_t sizePerEventRndNumColor = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for color random numbers + typedef HostBuffer HostBufferRndNumColor; +#else + // A class encapsulating a CUDA pinned host buffer for color random numbers + typedef PinnedHostBuffer PinnedHostBufferRndNumColor; + // A class encapsulating a CUDA device buffer for color random numbers + typedef DeviceBuffer DeviceBufferRndNumColor; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for channel ids + typedef BufferBase BufferChannelIds; + + // The size (number of elements) per event in a memory buffer for channel ids + constexpr size_t sizePerEventChannelId = 1; + +#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) + // A class encapsulating a C++ host buffer for channel ids + typedef HostBuffer HostBufferChannelIds; +#else + // A class encapsulating a CUDA pinned host buffer for channel ids + typedef PinnedHostBuffer PinnedHostBufferChannelIds; + // A class encapsulating a CUDA device buffer for channel ids + typedef DeviceBuffer DeviceBufferChannelIds; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for channel ids + typedef BufferBase BufferIflavorVec; + + // The size (number of elements) per event in a memory buffer for channel ids + constexpr size_t sizePerEventIflavorVec = 1; + +#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) + // A class encapsulating a C++ host buffer for channel ids + typedef HostBuffer HostBufferIflavorVec; +#else + // A class encapsulating a CUDA pinned host buffer for channel ids + typedef PinnedHostBuffer PinnedHostBufferIflavorVec; + // A class encapsulating a CUDA device buffer for channel ids + typedef DeviceBuffer DeviceBufferIflavorVec; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for helicity selection + typedef BufferBase BufferSelectedHelicity; + + // The size (number of elements) per event in a memory buffer for helicity selection + constexpr size_t sizePerEventSelectedHelicity = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for helicity selection + typedef HostBuffer HostBufferSelectedHelicity; +#else + // A class encapsulating a CUDA pinned host buffer for helicity selection + typedef PinnedHostBuffer PinnedHostBufferSelectedHelicity; + // A class encapsulating a CUDA device buffer for helicity selection + typedef DeviceBuffer DeviceBufferSelectedHelicity; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for color selection + typedef BufferBase BufferSelectedColor; + + // The size (number of elements) per event in a memory buffer for color selection + constexpr size_t sizePerEventSelectedColor = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for color selection + typedef HostBuffer HostBufferSelectedColor; +#else + // A class encapsulating a CUDA pinned host buffer for color selection + typedef PinnedHostBuffer PinnedHostBufferSelectedColor; + // A class encapsulating a CUDA device buffer for color selection + typedef DeviceBuffer DeviceBufferSelectedColor; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // The size (number of elements) per event in a memory buffer for jamps + constexpr size_t sizePerEventJamps = MemoryBuffers::ncolor * MemoryBuffers::nx2; + + // A class encapsulating a CUDA device buffer for color selection + typedef DeviceBuffer DeviceBufferJamps; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + template + void copyDeviceFromHost( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy + { + if( dst.size() != src.size() ) + { + std::ostringstream sstr; + sstr << "Size (#elements) mismatch in copyDeviceFromHost: dst=" << dst.size() << ", src=" << src.size(); + throw std::runtime_error( sstr.str() ); + } + if( dst.bytes() != src.bytes() ) + { + std::ostringstream sstr; + sstr << "Size (#bytes) mismatch in copyDeviceFromHost: dst=" << dst.bytes() << ", src=" << src.bytes(); + throw std::runtime_error( sstr.str() ); + } + // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array + gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyHostToDevice ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + template + void copyHostFromDevice( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy + { + if( dst.size() != src.size() ) + { + std::ostringstream sstr; + sstr << "Size (#elements) mismatch in copyHostFromDevice: dst=" << dst.size() << ", src=" << src.size(); + throw std::runtime_error( sstr.str() ); + } + if( dst.bytes() != src.bytes() ) + { + std::ostringstream sstr; + sstr << "Size (#bytes) mismatch in copyHostFromDevice: dst=" << dst.bytes() << ", src=" << src.bytes(); + throw std::runtime_error( sstr.str() ); + } + // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array + gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyDeviceToHost ); + } +#endif + + //-------------------------------------------------------------------------- +} + +#endif // MemoryBuffers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc new file mode 100644 index 0000000000..d40b1b4174 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc @@ -0,0 +1,246 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "RamboSamplingKernels.h" + +#include "GpuRuntime.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessRandomNumbers.h" +#include "MemoryAccessWeights.h" +#include "MemoryBuffers.h" +#include "rambo.h" // inline classic (massive) RAMBO, ported from standalone_cpp +#include "massless_rambo.h" // inline implementation of massless RAMBO algorithms and kernels + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + MasslessRamboSamplingKernelHost::MasslessRamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t nevt ) + : SamplingKernelBase( energy, rndmom, momenta, weights ) + , NumberOfEvents( nevt ) + { + if( m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: rndmom must be a host array" ); + if( m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: momenta must be a host array" ); + if( m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: weights must be a host array" ); + if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with rndmom" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with weights" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Sanity checks for memory access (random number buffer) + constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout + static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); + if( nevt % neppR != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppR=" << neppR; + throw std::runtime_error( sstr.str() ); + } + } + + //-------------------------------------------------------------------------- + + void + MasslessRamboSamplingKernelHost::getMomentaInitial() + { + constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; + // ** START LOOP ON IEVT ** + for( size_t ievt = 0; ievt < nevt(); ++ievt ) + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); + getMomentaInitial( m_energy, ievtMomenta ); + } + // ** END LOOP ON IEVT ** + } + + //-------------------------------------------------------------------------- + + void + MasslessRamboSamplingKernelHost::getMomentaFinal() + { + constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; + // ** START LOOP ON IEVT ** + for( size_t ievt = 0; ievt < nevt(); ++ievt ) + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + const fptype* ievtRndmom = MemoryAccessRandomNumbers::ieventAccessRecordConst( m_rndmom.data(), ievt ); + fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); + fptype* ievtWeights = MemoryAccessWeights::ieventAccessRecord( m_weights.data(), ievt ); + getMomentaFinal( m_energy, ievtRndmom, ievtMomenta, ievtWeights ); + } + // ** END LOOP ON IEVT ** + } + + //-------------------------------------------------------------------------- + + RamboSamplingKernelHost::RamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED + const std::vector& masses, // input: external-leg masses + const int ninitial, // input: #initial-state particles + const size_t nevt, // input: #events + BufferMomenta& momenta, // output: momenta + BufferWeights& weights ) // output: weights + : SamplingKernelBase( energy, rndmom, momenta, weights ) + , NumberOfEvents( nevt ) + , m_masses( masses.begin(), masses.end() ) + , m_ninitial( ninitial ) + { + if( m_momenta.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: momenta must be a host array" ); + if( m_weights.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: weights must be a host array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with weights" ); + + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "RamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + } + + //-------------------------------------------------------------------------- + + void + RamboSamplingKernelHost::getMomentaInitial() + { + // NOOP + } + + //-------------------------------------------------------------------------- + + void + RamboSamplingKernelHost::getMomentaFinal() + { + const int npar = (int)m_masses.size(); + // ** START LOOP ON IEVT ** + for( size_t ievt = 0; ievt < nevt(); ++ievt ) + { + // Clas. RAMBO returns [E,px,py,pz] vector per ex. particle + // own RNG, intial final once + // For reproducibility betwn fptype = FP32/FP64 generation in FP64 + double wgt = 0.; + const std::vector> point = + rambo::get_momenta( m_ninitial, (double)m_energy, m_masses, wgt ); + for( int ipar = 0; ipar < npar; ++ipar ) + for( int ip4 = 0; ip4 < 4; ++ip4 ) + MemoryAccessMomenta::ieventAccessIp4Ipar( m_momenta.data(), ievt, ip4, ipar ) = (fptype)point[ipar][ip4]; + MemoryAccessWeights::ieventAccess( m_weights.data(), ievt ) = (fptype)wgt; + } + // ** END LOOP ON IEVT ** + } + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + MasslessRamboSamplingKernelDevice::MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t gpublocks, + const size_t gputhreads ) + : SamplingKernelBase( energy, rndmom, momenta, weights ) + , NumberOfEvents( gpublocks * gputhreads ) + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + if( !m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: rndmom must be a device array" ); + if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: momenta must be a device array" ); + if( !m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: weights must be a device array" ); + if( m_gpublocks == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gpublocks must be > 0" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gputhreads must be > 0" ); + if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with rndmom" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with momenta" ); + if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with weights" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( m_gputhreads % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelHost: gputhreads should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Sanity checks for memory access (random number buffer) + constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout + static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); + if( m_gputhreads % neppR != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelDevice: gputhreads should be a multiple of neppR=" << neppR; + throw std::runtime_error( sstr.str() ); + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + getMomentaInitialDevice( const fptype energy, + fptype* momenta ) + { + constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; + return getMomentaInitial( energy, momenta ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + MasslessRamboSamplingKernelDevice::getMomentaInitial() + { + gpuLaunchKernel( getMomentaInitialDevice, m_gpublocks, m_gputhreads, m_energy, m_momenta.data() ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + getMomentaFinalDevice( const fptype energy, + const fptype* rndmom, + fptype* momenta, + fptype* wgts ) + { + constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; + return getMomentaFinal( energy, rndmom, momenta, wgts ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + MasslessRamboSamplingKernelDevice::getMomentaFinal() + { + gpuLaunchKernel( getMomentaFinalDevice, m_gpublocks, m_gputhreads, m_energy, m_rndmom.data(), m_momenta.data(), m_weights.data() ); + } +#endif + + //-------------------------------------------------------------------------- +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h new file mode 100644 index 0000000000..6244d55618 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h @@ -0,0 +1,173 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef RAMBOSAMPLINGKERNELS_H +#define RAMBOSAMPLINGKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryBuffers.h" + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // A base class encapsulating phase space sampling on a CPU host or on a GPU device + class SamplingKernelBase //: virtual public ISamplingKernel + { + protected: + + // Constructor from existing input and output buffers + SamplingKernelBase( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights ) // output: weights + : m_energy( energy ) + , m_rndmom( rndmom ) + , m_momenta( momenta ) + , m_weights( weights ) + { + } + + public: + + // Destructor + virtual ~SamplingKernelBase() {} + + // Get momenta of initial state particles + virtual void getMomentaInitial() = 0; + + // Get momenta of final state particles and weights + virtual void getMomentaFinal() = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The energy + const fptype m_energy; + + // The buffer for the input random numbers + const BufferRndNumMomenta& m_rndmom; + + // The buffer for the output momenta + BufferMomenta& m_momenta; + + // The buffer for the output weights + BufferWeights& m_weights; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating RAMBO phase space sampling on a CPU host + class MasslessRamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MasslessRamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t nevt ); + + // Destructor + virtual ~MasslessRamboSamplingKernelHost() {} + + // Get momenta of initial state particles + void getMomentaInitial() override final; + + // Get momenta of final state particles and weights + void getMomentaFinal() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + }; + + //-------------------------------------------------------------------------- + + // Compability port mirroring the massless momenta implementation + // For now own RNG internally (or keep to match Fortran) TODO + // rndmom just for interface (maybe delete later) TODO + class RamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents + { + public: + + RamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED + const std::vector& masses, // input: external-leg masses + const int ninitial, // input: #n initial-state particles + const size_t nevt, // input: #n events + BufferMomenta& momenta, // output: momenta + BufferWeights& weights); // output: weights + + virtual ~RamboSamplingKernelHost() {} + + // No-op, kept to match the massless + void getMomentaInitial() override final; + + // All the magic here + void getMomentaFinal() override final; + + bool isOnDevice() const override final { return false; } + + private: + + // The EXTERNAL masses + std::vector m_masses; + + // The number of inital particles + const int m_ninitial; + }; + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating RAMBO phase space sampling on a GPU device + class MasslessRamboSamplingKernelDevice final : public SamplingKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t gpublocks, + const size_t gputhreads ); + + // Destructor + virtual ~MasslessRamboSamplingKernelDevice() {} + + // Get momenta of initial state particles + void getMomentaInitial() override final; + + // Get momenta of final state particles and weights + void getMomentaFinal() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + }; +#endif + + //-------------------------------------------------------------------------- +} +#endif // RAMBOSAMPLINGKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h new file mode 100644 index 0000000000..0fb880bb36 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h @@ -0,0 +1,192 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef RANDOMNUMBERKERNELS_H +#define RANDOMNUMBERKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryBuffers.h" + +// Forward definition from curand.h (the full header is only needed in CurandRandomKernel.cc) +struct curandGenerator_st; + +// Forward definition from hiprand.h (the full header is only needed in HiprandRandomKernel.cc) +struct rocrand_generator_base_type; +typedef rocrand_generator_base_type hiprandGenerator_st; + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + /* + // An interface encapsulating random number generation on a CPU host or on a GPU device + class IRandomNumberKernel + { + public: + + // Destructor + virtual ~IRandomNumberKernel(){} + + // Seed the random number generator + virtual void seedGenerator( const unsigned int seed ) = 0; + + // Generate the random number array + virtual void generateRnarray() = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + }; + */ + + //-------------------------------------------------------------------------- + + // A base class encapsulating random number generation on a CPU host or on a GPU device + class RandomNumberKernelBase //: virtual public IRandomNumberKernel + { + + protected: + + // Constructor from an existing output buffer + RandomNumberKernelBase( BufferRndNumMomenta& rnarray ) + : m_rnarray( rnarray ) {} + + public: + + // Destructor + virtual ~RandomNumberKernelBase() {} + + // Seed the random number generator + virtual void seedGenerator( const unsigned int seed ) = 0; + + // Generate the random number array + virtual void generateRnarray() = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The buffer for the output random numbers + BufferRndNumMomenta& m_rnarray; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating common random number generation on a CPU host + class CommonRandomNumberKernel final : public RandomNumberKernelBase + { + public: + + // Constructor from an existing output buffer + CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ); + + // Destructor + ~CommonRandomNumberKernel() {} + + // Seed the random number generator + void seedGenerator( const unsigned int seed ) override final { m_seed = seed; }; + + // Generate the random number array + void generateRnarray() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // The generator seed + unsigned int m_seed; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating CURAND random number generation on a CPU host or on a GPU device + class CurandRandomNumberKernel final : public RandomNumberKernelBase + { + public: + + // Constructor from an existing output buffer + CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); + + // Destructor + ~CurandRandomNumberKernel(); + + // Seed the random number generator + void seedGenerator( const unsigned int seed ) override final; + + // Generate the random number array + void generateRnarray() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return m_isOnDevice; } + + private: + + // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void createGenerator(); + + // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void destroyGenerator(); + + private: + + // Is this a host or device kernel? + const bool m_isOnDevice; + + // The curand generator + // (NB: curand.h defines typedef curandGenerator_t as a pointer to forward-defined 'struct curandGenerator_st') + curandGenerator_st* m_rnGen; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating HIPRAND random number generation on a CPU host or on a GPU device + class HiprandRandomNumberKernel final : public RandomNumberKernelBase + { + public: + + // Constructor from an existing output buffer + HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); + + // Destructor + ~HiprandRandomNumberKernel(); + + // Seed the random number generator + void seedGenerator( const unsigned int seed ) override final; + + // Generate the random number array + void generateRnarray() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return m_isOnDevice; } + + private: + + // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void createGenerator(); + + // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void destroyGenerator(); + + private: + + // Is this a host or device kernel? + const bool m_isOnDevice; + + // The hiprand generator + // (NB: hiprand.h defines typedef hiprandGenerator_t as a pointer to forward-defined 'struct hiprandGenerator_st') + hiprandGenerator_st* m_rnGen; + }; + + //-------------------------------------------------------------------------- +} +#endif // RANDOMNUMBERKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc new file mode 100644 index 0000000000..68e93edb50 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc @@ -0,0 +1,1159 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: O. Mattelaer (Nov 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, D. Massaro, O. Mattelaer, S. Roiser, J. Teig, A. Thete, A. Valassi (2020-2026). +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== +// +// Standalone script for MadGraph7 standalone mode. +// Generates phase-space points with RAMBO and evaluates the matrix element +// through the UMAMI interface (umami.h). +// +// Two run modes: +// * matrix (default): evaluates one phase-space point (generated with the +// classic standalone RAMBO, so identical to the one of +// the Fortran/C++ standalone 'check' drivers at the +// same energy) and prints it together with the matrix +// element of every flavor combination. +// * perf : runs nblocks*nthreads*niter events on a single flavor +// and prints performance counters. +// +//========================================================================== + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "GpuAbstraction.h" +#include "GpuRuntime.h" +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" +#include "RamboSamplingKernels.h" +#include "RandomNumberKernels.h" +#include "epoch_process_id.h" +#include "read_slha.h" +#include "timermap.h" +#include "umami.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STRINGIFY( s ) #s +#define XSTRINGIFY( s ) STRINGIFY( s ) +#define SEP79 79 + +namespace +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + // Fixed physics inputs + fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV + constexpr unsigned long long kSeed = 20200805ULL; // reproducible RAMBO seed + + // Matrix-mode always runs 8 events on a single flavor index. + constexpr unsigned int kMatrixBlocks = 1; + constexpr unsigned int kMatrixThreads = 8; + + // Power of GeV of the matrix-element output; depends only on the number of external legs. + constexpr int kMEGeVExponent = -( 2 * CPPProcess::npar - 8 ); + + bool is_number( const char* s ) + { + const char* t = s; + while( *t != '\0' && isdigit( *t ) ) ++t; + return (int)strlen( s ) == t - s; + } + + // Accepts plain decimal numbers such as "1000" or "1000.0" (used for the + // optional energy argument of matrix mode). + bool is_float( const char* s ) + { + int ndots = 0; + const char* t = s; + while( *t != '\0' && ( isdigit( *t ) || ( *t == '.' && ndots++ == 0 ) ) ) ++t; + return t != s && (int)strlen( s ) == t - s; + } + + enum Mode { MODE_MATRIX, MODE_PERF }; + + enum RamboType { RAMBO_MASSIVE, RAMBO_MASSLESS }; + + // One external-particle list per LHE event, each particle stored as (E, px, py, pz). + using LheEvent = std::array, CPPProcess::npar>; + + bool read_lhe_events( const std::string& path, std::vector& events ) + { + constexpr int npar = CPPProcess::npar; + std::ifstream in( path ); + if( !in ) + { + std::cerr << "ERROR! cannot open LHE file '" << path << "'" << std::endl; + return false; + } + std::string line; + while( std::getline( in, line ) ) + { + if( line.find( "" ) == std::string::npos ) continue; + if( !std::getline( in, line ) ) break; + std::istringstream hdr( line ); + int nptcl = 0; + hdr >> nptcl; + if( nptcl != npar ) + { + std::cerr << "ERROR! LHE event has " << nptcl << " particles, expected " << npar << std::endl; + return false; + } + // particle lines: pdg status mother1 mother2 color1 color2 px py pz E m lifetime spin + LheEvent ev; + int ipar = 0; + while( ipar < npar && std::getline( in, line ) ) + { + if( line.empty() ) continue; + std::istringstream ls( line ); + long pdg; + int status, m1, m2, c1, c2; + double px, py, pz, E; + if( !( ls >> pdg >> status >> m1 >> m2 >> c1 >> c2 >> px >> py >> pz >> E ) ) + { + std::cerr << "ERROR! malformed LHE particle line: " << line << std::endl; + return false; + } + ev[ipar] = { E, px, py, pz }; + ++ipar; + } + if( ipar != npar ) + { + std::cerr << "ERROR! truncated LHE event (got " << ipar << " of " << npar << " particles)" << std::endl; + return false; + } + events.push_back( ev ); + } + if( events.empty() ) + { + std::cerr << "ERROR! no events found in '" << path << "'" << std::endl; + return false; + } + return true; + } + + int usage( const char* argv0, int ret = 1 ) + { + std::cout + << "Usage:\n" + << " " << argv0 << " [matrix] [-v|--verbose] []\n" + << " " << argv0 << " perf [-v|--verbose] [-f|--flavor ] [--rambo-massless]" + << " [-e|--events ] [<#blocksPerGrid> <#threadsPerBlock>] <#iterations>\n" + << " " << argv0 << " -p [opts] (legacy alias for `perf`)\n" + << "\n" + << "Subcommands:\n" + << " matrix (default) Evaluate one phase-space point (classic standalone\n" + << " RAMBO, identical to the Fortran/C++ 'check' drivers,\n" + << " Ecms = , default 1000 GeV) and print it with\n" + << " the matrix element for each flavor combination.\n" + << " With -v also prints backend/fptype/hardcodePARAM header.\n" + << " perf Run #blocks*#threads events over #iterations iterations\n" + << " on a single flavor index, then print performance counters.\n" + << " Always prints inputs + backend/fptype header.\n" + << " With -v also dumps every event's phase-space point and ME.\n" + << "\n" + << "Options:\n" + << " -e|--events (perf only) Read the external momenta from an LHE\n" + << " file instead of generating them with RAMBO. The events are\n" + << " processed in batches of #blocks*#threads; #iterations is\n" + << " ignored (derived from the number of events in the file).\n" + << "\n" + << "perf-mode defaults if positional args are omitted:\n" + << " #blocksPerGrid = 64, #threadsPerBlock = 256, #iterations = 1.\n"; + return ret; + } + + // AOSOA -> UMAMI SoA single-event helper. Layout reminder: + // AOSOA: aosoa[i_page * npar*4*neppM + ipar*4*neppM + ip4*neppM + i_vector] + // UMAMI: soa[ip4 * npar*nevt + ipar*nevt + ievt] + __host__ __device__ inline void + aosoa_to_umami_one( const fptype* aosoa, + double* soa, + std::size_t ievt, + std::size_t nevt ) + { + constexpr int npar = CPPProcess::npar; + for( int ipar = 0; ipar < npar; ++ipar ) + { + for( int ip4 = 0; ip4 < 4; ++ip4 ) + { + soa[(std::size_t)ip4 * npar * nevt + (std::size_t)ipar * nevt + ievt] = + (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, ip4, ipar ); + } + } + } + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + aosoa_to_umami_kernel( const fptype* aosoa, + double* soa, + std::size_t nevt ) + { + std::size_t ievt = blockDim.x * blockIdx.x + threadIdx.x; + if( ievt >= nevt ) return; + aosoa_to_umami_one( aosoa, soa, ievt, nevt ); + } +#endif + + const char* backend_label() + { +#ifdef __CUDACC__ + return "CUDA"; +#elif defined( __HIPCC__ ) + return "HIP"; +#else + return "CPP"; +#endif + } + + const char* fp_label() + { +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + return "MIXED"; +#elif defined MGONGPU_FPTYPE_DOUBLE + return "DOUBLE"; +#elif defined MGONGPU_FPTYPE_FLOAT + return "FLOAT"; +#else + return "UNKNOWN"; +#endif + } + + void print_run_header( std::ostream& os ) + { + os << "Process = " << XSTRINGIFY( MG_EPOCH_PROCESS_ID ) << "_" << backend_label() +#ifdef MGONGPU_HARDCODE_PARAM + << " [hardcodePARAM=1]" << std::endl +#else + << " [hardcodePARAM=0]" << std::endl +#endif + << "FP precision = " << fp_label() << std::endl + << "Random number generation = COMMON RANDOM HOST" << std::endl; + } + + void print_momenta_table( std::ostream& os, const fptype* aosoa, unsigned int ievt ) + { + auto constexpr prec = std::numeric_limits::digits10; + constexpr int npar = CPPProcess::npar; + os << std::string( SEP79, '-' ) << std::endl + << " n E px py pz" << std::endl; + for( int ipar = 0; ipar < npar; ++ipar ) + { + double E = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 0, ipar ); + double px = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 1, ipar ); + double py = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 2, ipar ); + double pz = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 3, ipar ); + os << std::scientific << std::setprecision( prec ) + << std::setw( 4 ) << ipar + 1 + << std::setw( prec + 10 ) << E + << std::setw( prec + 10 ) << px + << std::setw( prec + 10 ) << py + << std::setw( prec + 10 ) << pz + << std::endl + << std::defaultfloat; + } + os << std::string( SEP79, '-' ) << std::endl; + } + + // Run sigmaKin via UMAMI for `nevt` events and copy back the MEs. + // Both the momenta (UMAMI SoA layout) and the per-event flavor buffer must be set + // by the caller. On GPU the buffers are device pointers and `hstMEs` receives the + // host-side copy; on CPU `umamiMEs` is the output buffer. + bool run_umami( + UmamiHandle handle, + unsigned int nevt, + mgOnGpu::TimerMap& timermap, + double& wavetime, +#ifdef MGONGPUCPP_GPUIMPL + const DeviceBufferBase& devUmamiMomenta, + const DeviceBufferBase& devFlv, + DeviceBufferBase& devUmamiMEs, + std::vector& hstMEs +#else + const std::vector& umamiMomenta, + const std::vector& flvVec, + std::vector& umamiMEs +#endif + ) + { + constexpr unsigned int UmamiInKeyNum = 2; + timermap.start( "3a SigmaKin" ); + UmamiInputKey in_keys[UmamiInKeyNum] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX }; + UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; +#ifdef MGONGPUCPP_GPUIMPL + const void* inputs[UmamiInKeyNum] = { devUmamiMomenta.data(), devFlv.data() }; + void* outputs[1] = { devUmamiMEs.data() }; +#else + const void* inputs[UmamiInKeyNum] = { umamiMomenta.data(), flvVec.data() }; + void* outputs[1] = { umamiMEs.data() }; +#endif + UmamiStatus st = umami_matrix_element( + handle, nevt, nevt, 0, UmamiInKeyNum, in_keys, inputs, 1, out_keys, outputs ); + wavetime += timermap.stop(); + if( st != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; + return false; + } + +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "3b CpDTHmes" ); + gpuMemcpy( hstMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); + wavetime += timermap.stop(); +#endif + return true; + } + + // -------------------------------------------------------------------------- + // Classic MadGraph standalone RAMBO (RANMAR generator seeded with the fixed + // values 1802/9373), reproducing the exact phase-space point used by the + // Fortran and C++ standalone 'check' drivers at the same energy, so that + // matrix-mode output can be compared line by line across backends. + // Host-side only; ported from madgraph/iolibs/template_files/rambo.cc. + // -------------------------------------------------------------------------- + namespace classic_rambo + { + struct Random + { + double ranu[98]; + double ranc, rancd, rancm; + int iranmr, jranmr; + + // universal random number generator proposed by Marsaglia and Zaman + double ranmar() + { + double uni = ranu[iranmr] - ranu[jranmr]; + if( uni < 0 ) uni = uni + 1; + ranu[iranmr] = uni; + iranmr = iranmr - 1; + jranmr = jranmr - 1; + if( iranmr == 0 ) iranmr = 97; + if( jranmr == 0 ) jranmr = 97; + ranc = ranc - rancd; + if( ranc < 0 ) ranc = ranc + rancm; + uni = uni - ranc; + if( uni < 0 ) uni = uni + 1; + return uni; + } + + void rmarin( int ij, int kl ) + { + int i = ij / 177 % 177 + 2; + int j = ij % 177 + 2; + int k = ( kl / 169 ) % 178 + 1; + int l = kl % 169; + for( int ii = 1; ii < 98; ii++ ) + { + double s = 0; + double t = .5; + for( int jj = 1; jj < 25; jj++ ) + { + int m = ( ( i * j % 179 ) * k ) % 179; + i = j; + j = k; + k = m; + l = ( 53 * l + 1 ) % 169; + if( ( l * m ) % 64 >= 32 ) s = s + t; + t = .5 * t; + } + ranu[ii] = s; + } + ranc = 362436. / 16777216.; + rancd = 7654321. / 16777216.; + rancm = 16777213. / 16777216.; + iranmr = 97; + jranmr = 33; + } + }; + + inline double rn() + { + static Random rand; + static bool init = true; + if( init ) + { + init = false; + rand.rmarin( 1802, 9373 ); + } + double ran; + while( true ) + { + ran = rand.ranmar(); + if( ran > 1e-16 ) break; + } + return ran; + } + + // RAMBO: democratic multi-particle phase space generator (S.D. Ellis, + // R. Kleiss, W.J. Stirling); weights are logarithmic. + inline std::vector> + rambo( double et, const std::vector& xm, double& wt ) + { + const int n = (int)xm.size(); + std::vector> q( n, std::vector( 4 ) ); + std::vector> p( n, std::vector( 4 ) ); + std::vector z( n ), r( 4 ), b( 3 ), p2( n ), xm2( n ), e( n ), v( n ); + const double acc = 1e-14; + const int itmax = 6; + const double twopi = 8. * atan( 1. ); + const double po2log = log( twopi / 4. ); + + // factorials for the phase-space weight + z[1] = po2log; + for( int k = 2; k < n; k++ ) z[k] = z[k - 1] + po2log - 2. * log( double( k - 1 ) ); + for( int k = 2; k < n; k++ ) z[k] = z[k] - log( double( k ) ); + + if( n < 1 || n > 101 ) + { + std::cout << "Too few or many particles: " << n << std::endl; + exit( -1 ); + } + double xmt = 0.; + int nm = 0; + for( int i = 0; i < n; i++ ) + { + if( xm[i] != 0. ) nm = nm + 1; + xmt = xmt + std::abs( xm[i] ); + } + if( xmt > et ) + { + std::cout << "Too low energy: " << et << " needed " << xmt << std::endl; + exit( -1 ); + } + + // generate n massless momenta in infinite phase space + for( int i = 0; i < n; i++ ) + { + double r1 = rn(); + double c = 2. * r1 - 1.; + double s = sqrt( 1. - c * c ); + double f = twopi * rn(); + r1 = rn(); + double r2 = rn(); + q[i][0] = -log( r1 * r2 ); + q[i][3] = q[i][0] * c; + q[i][2] = q[i][0] * s * cos( f ); + q[i][1] = q[i][0] * s * sin( f ); + } + + // parameters of the conformal transformation + for( int k = 0; k < 4; k++ ) r[k] = 0.; + for( int i = 0; i < n; i++ ) + for( int k = 0; k < 4; k++ ) r[k] = r[k] + q[i][k]; + double rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); + for( int k = 1; k < 4; k++ ) b[k - 1] = -r[k] / rmas; + double g = r[0] / rmas; + double a = 1. / ( 1. + g ); + double x = et / rmas; + + // transform the q's conformally into the p's + for( int i = 0; i < n; i++ ) + { + double bq = b[0] * q[i][1] + b[1] * q[i][2] + b[2] * q[i][3]; + for( int k = 1; k < 4; k++ ) p[i][k] = x * ( q[i][k] + b[k - 1] * ( q[i][0] + a * bq ) ); + p[i][0] = x * ( g * q[i][0] + bq ); + } + + wt = po2log; + if( n != 2 ) wt = ( 2. * n - 4. ) * log( et ) + z[n - 1]; + + // massless case is done + if( nm == 0 ) return p; + + // massive particles: rescale the momenta by a factor x + double xmax = sqrt( 1. - pow( xmt / et, 2 ) ); + for( int i = 0; i < n; i++ ) + { + xm2[i] = pow( xm[i], 2 ); + p2[i] = pow( p[i][0], 2 ); + } + int iter = 0; + x = xmax; + double accu = et * acc; + while( true ) + { + double f0 = -et; + double g0 = 0.; + double x2 = x * x; + for( int i = 0; i < n; i++ ) + { + e[i] = sqrt( xm2[i] + x2 * p2[i] ); + f0 = f0 + e[i]; + g0 = g0 + p2[i] / e[i]; + } + if( std::abs( f0 ) <= accu ) break; + iter = iter + 1; + if( iter > itmax ) + { + std::cout << "Too many iterations without desired accuracy: " << itmax << std::endl; + break; + } + x = x - f0 / ( x * g0 ); + } + for( int i = 0; i < n; i++ ) + { + v[i] = x * p[i][0]; + for( int k = 1; k < 4; k++ ) p[i][k] = x * p[i][k]; + p[i][0] = e[i]; + } + + double wt2 = 1.; + double wt3 = 0.; + for( int i = 0; i < n; i++ ) + { + wt2 = wt2 * v[i] / e[i]; + wt3 = wt3 + pow( v[i], 2 ) / e[i]; + } + double wtm = ( 2. * n - 3. ) * log( x ) + log( wt2 / wt3 * et ); + wt = wt + wtm; + return p; + } + + // Auxiliary function changing convention between MadGraph5_aMC@NLO and + // RAMBO four-momenta (same as get_momenta in the standalone_cpp driver). + inline std::vector> + get_momenta( int ninitial, double energy, const std::vector& masses, double& wgt ) + { + const int nexternal = (int)masses.size(); + const int nfinal = nexternal - ninitial; + const double e2 = pow( energy, 2 ); + const double m1 = masses[0]; + + if( ninitial == 1 ) + { + std::vector> p( 1, std::vector( 4, 0. ) ); + p[0][0] = m1; + std::vector finalmasses( masses.begin() + 1, masses.end() ); + std::vector> p_rambo = rambo( m1, finalmasses, wgt ); + p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); + return p; + } + + if( ninitial != 2 ) + { + std::cout << "Rambo needs 1 or 2 incoming particles" << std::endl; + exit( -1 ); + } + + double etot = energy; + if( nfinal == 1 ) etot = m1; + const double m2 = masses[1]; + const double mom = sqrt( ( pow( e2, 2 ) - 2 * e2 * pow( m1, 2 ) + pow( m1, 4 ) - 2 * e2 * pow( m2, 2 ) - 2 * pow( m1, 2 ) * pow( m2, 2 ) + pow( m2, 4 ) ) / ( 4 * e2 ) ); + const double energy1 = sqrt( pow( mom, 2 ) + pow( m1, 2 ) ); + const double energy2 = sqrt( pow( mom, 2 ) + pow( m2, 2 ) ); + std::vector> p( 2, std::vector( 4, 0. ) ); + p[0][0] = energy1; + p[0][3] = mom; + p[1][0] = energy2; + p[1][3] = -mom; + + if( nfinal == 1 ) + { + p.push_back( std::vector( 4, 0. ) ); + p[2][0] = etot; + wgt = 1; + return p; + } + std::vector finalmasses( masses.begin() + 2, masses.end() ); + std::vector> p_rambo = rambo( etot, finalmasses, wgt ); + p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); + return p; + } + } + + // -------------------------------------------------------------------------- + // matrix mode: same PS point fed to every flavor combination, print event 0. + // The point is generated with the classic standalone RAMBO so it is + // identical to the one of the Fortran/C++ standalone 'check' drivers. + // -------------------------------------------------------------------------- + int run_matrix_mode( bool verbose ) + { + constexpr unsigned int nevt = kMatrixBlocks * kMatrixThreads; + const unsigned int nFlavors = CPPProcess::nmaxflavor; + + mgOnGpu::TimerMap timermap; + +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "00 GpuInit" ); + GpuRuntime gpuRuntime( false ); + + PinnedHostBufferRndNumMomenta hstRndmom( nevt ); + PinnedHostBufferMomenta hstMomenta( nevt ); + PinnedHostBufferWeights hstWeights( nevt ); + DeviceBufferRndNumMomenta devRndmom( nevt ); + DeviceBufferMomenta devMomenta( nevt ); + DeviceBufferWeights devWeights( nevt ); + DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + DeviceBufferBase devUmamiMEs( nevt ); + DeviceBufferBase devFlv( nevt ); + std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + std::vector flvVec( nevt ); + std::vector hstUmamiMEs( nevt ); +#else + HostBufferRndNumMomenta hstRndmom( nevt ); + HostBufferMomenta hstMomenta( nevt ); + HostBufferWeights hstWeights( nevt ); + std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + std::vector umamiMEs( nevt ); + std::vector flvVec( nevt ); +#endif + + UmamiHandle umami_handle = nullptr; + if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_initialize failed" << std::endl; + return 2; + } + + // Generate one shared phase-space point used by every flavor, with the + // classic standalone RAMBO so it matches the Fortran/C++ 'check' drivers. + CPPProcess process; + process.initProc( "../../Cards/param_card.dat" ); + double rambowgt = 0.; + + // Retrieve masses + int npar_meta = 0; + if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + std::vector massesD( npar_meta ); + if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + const std::vector masses( massesD.begin(), massesD.end() ); + + std::vector> point = + classic_rambo::get_momenta( CPPProcess::npari, (double)kEnergy, masses, rambowgt ); + + // alpha_s from the param card so the couplings match the Fortran/C++ + // 'check' drivers (UMAMI otherwise falls back to a hardcoded g_s). + SLHAReader slha( "../../Cards/param_card.dat", false ); + const double alphaS = slha.get_block_entry( "sminputs", 3, 1.180000e-01 ); + std::vector alphasVec( nevt, alphaS ); +#ifdef MGONGPUCPP_GPUIMPL + DeviceBufferBase devAlphaS( nevt ); + gpuMemcpy( devAlphaS.data(), alphasVec.data(), nevt * sizeof( double ), gpuMemcpyHostToDevice ); +#endif + + // Always massive RAMBO + std::unique_ptr prsk( + new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); + prsk->getMomentaInitial(); + prsk->getMomentaFinal(); + + // Fill the UMAMI SoA buffer with nevt copies of the same event: + // soa[ip4 * npar*nevt + ipar*nevt + ievt] + for( int ip4 = 0; ip4 < 4; ++ip4 ) + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + for( unsigned int ievt = 0; ievt < nevt; ++ievt ) + umamiMomenta[(std::size_t)ip4 * CPPProcess::npar * nevt + (std::size_t)ipar * nevt + ievt] = point[ipar][ip4]; +#ifdef MGONGPUCPP_GPUIMPL + gpuMemcpy( devUmamiMomenta.data(), umamiMomenta.data(), umamiMomenta.size() * sizeof( double ), gpuMemcpyHostToDevice ); + // Host only implementation now (copy) + copyDeviceFromHost( devMomenta, hstMomenta ); + gpuLaunchKernel( aosoa_to_umami_kernel, kMatrixBlocks, kMatrixThreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); + checkGpu( gpuPeekAtLastError() ); +#else + for( std::size_t ievt = 0; ievt < nevt; ++ievt ) + aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); +#endif + + if( verbose ) + { + std::cout << std::string( SEP79, '*' ) << std::endl; + print_run_header( std::cout ); + std::cout << std::string( SEP79, '*' ) << std::endl; + } + + std::cout << "Phase space point:" << std::endl + << std::string( SEP79, '-' ) << std::endl + << " n E px py pz" << std::endl; + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + { + std::cout << std::scientific << std::setprecision( 7 ) + << std::setw( 2 ) << ipar + 1 + << std::setw( 16 ) << point[ipar][0] + << std::setw( 16 ) << point[ipar][1] + << std::setw( 16 ) << point[ipar][2] + << std::setw( 16 ) << point[ipar][3] + << std::endl + << std::defaultfloat; + } + std::cout << std::string( SEP79, '-' ) << std::endl; + + for( unsigned int iflav = 0; iflav < nFlavors; ++iflav ) + { + std::fill( flvVec.begin(), flvVec.end(), iflav ); +#ifdef MGONGPUCPP_GPUIMPL + gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); +#endif + timermap.start( "3a SigmaKin" ); + UmamiInputKey in_keys[3] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S }; + UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; +#ifdef MGONGPUCPP_GPUIMPL + const void* inputs[3] = { devUmamiMomenta.data(), devFlv.data(), devAlphaS.data() }; + void* outputs[1] = { devUmamiMEs.data() }; +#else + const void* inputs[3] = { umamiMomenta.data(), flvVec.data(), alphasVec.data() }; + void* outputs[1] = { umamiMEs.data() }; +#endif + UmamiStatus st = umami_matrix_element( + umami_handle, nevt, nevt, 0, 3, in_keys, inputs, 1, out_keys, outputs ); + timermap.stop(); + if( st != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; + umami_free( umami_handle ); + return 3; + } +#ifdef MGONGPUCPP_GPUIMPL + gpuMemcpy( hstUmamiMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); + const double* mes = hstUmamiMEs.data(); +#else + const double* mes = umamiMEs.data(); +#endif + + std::cout << " PDG"; + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + std::cout << std::setw( 12 ) << CPPProcess::flavorPDG( iflav, ipar ); + std::cout << std::endl + << " Matrix element = " << std::scientific << std::setprecision( 16 ) + << mes[0] << " GeV^" << kMEGeVExponent << std::endl + << std::defaultfloat + << std::string( SEP79, '-' ) << std::endl; + } + + umami_free( umami_handle ); + return 0; + } + + // -------------------------------------------------------------------------- + // perf mode: nblocks*nthreads events per iteration on a single flavor. + // -------------------------------------------------------------------------- + int run_perf_mode( bool verbose, + unsigned int gpublocks, + unsigned int gputhreads, + unsigned int niter, + unsigned int flavorID, + RamboType ramboType, + const std::string& lheFile = "" ) + { + const unsigned int nevt = gpublocks * gputhreads; + + // LHE instead of generating. Processed in batches of nevt and + // niter is derived from the number of events read. + std::vector lheEvents; + if( !lheFile.empty() ) + { + if( !read_lhe_events( lheFile, lheEvents ) ) return 2; + niter = (unsigned int)( ( lheEvents.size() + nevt - 1 ) / nevt ); + std::cout << "Reading events from LHE file = " << lheFile + << " (" << lheEvents.size() << " events, " << niter + << " batches of " << nevt << ")" << std::endl; + } + + mgOnGpu::TimerMap timermap; + +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "00 GpuInit" ); + GpuRuntime gpuRuntime( false ); + + PinnedHostBufferRndNumMomenta hstRndmom( nevt ); + PinnedHostBufferMomenta hstMomenta( nevt ); + PinnedHostBufferWeights hstWeights( nevt ); + DeviceBufferRndNumMomenta devRndmom( nevt ); + DeviceBufferMomenta devMomenta( nevt ); + DeviceBufferWeights devWeights( nevt ); + DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + DeviceBufferBase devUmamiMEs( nevt ); + DeviceBufferBase devFlv( nevt ); + std::vector flvVec( nevt, flavorID ); + std::vector hstUmamiMEs( nevt ); + // perf-mode runs a single flavor, so the device-side flavor buffer is filled once. + gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); +#else + HostBufferRndNumMomenta hstRndmom( nevt ); + HostBufferMomenta hstMomenta( nevt ); + HostBufferWeights hstWeights( nevt ); + std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + std::vector umamiMEs( nevt ); + std::vector flvVec( nevt, flavorID ); +#endif + + std::unique_ptr prnk( + new CommonRandomNumberKernel( hstRndmom ) ); + + UmamiHandle umami_handle = nullptr; + if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_initialize failed" << std::endl; + return 2; + } + + // Retrieve masses + std::vector masses; + if( ramboType != RAMBO_MASSLESS) + { + int npar_meta = 0; + if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + std::vector massesD( npar_meta ); + if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + masses.assign( massesD.begin(), massesD.end() ); + } + + std::unique_ptr prsk; + if( ramboType != RAMBO_MASSLESS ) + { + // Massive host only (copy) + prsk.reset( new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); + } + else + { +#ifdef MGONGPUCPP_GPUIMPL + prsk.reset( new MasslessRamboSamplingKernelDevice( kEnergy, devRndmom, devMomenta, devWeights, gpublocks, gputhreads ) ); +#else + prsk.reset( new MasslessRamboSamplingKernelHost( kEnergy, hstRndmom, hstMomenta, hstWeights, nevt ) ); +#endif + } + + std::unique_ptr genrtimes( new double[niter] ); + std::unique_ptr rambtimes( new double[niter] ); + std::unique_ptr wavetimes( new double[niter] ); + + unsigned int nevtABN = 0; + unsigned int nevtZERO = 0; + double sumME = 0.; + double sumMEsq = 0.; + double minME = std::numeric_limits::infinity(); + double maxME = -std::numeric_limits::infinity(); + unsigned int nevtALL = 0; + + for( unsigned int iiter = 0; iiter < niter; ++iiter ) + { + double genrtime = 0; + double rambtime = 0; + unsigned int nreal = nevt; // number of real (non-padding) events in this batch + if( lheFile.empty() ) + { + timermap.start( "1a GenSeed " ); + prnk->seedGenerator( kSeed + iiter ); + genrtime += timermap.stop(); + timermap.start( "1b GenRnGen" ); + prnk->generateRnarray(); + genrtime += timermap.stop(); +#ifdef MGONGPUCPP_GPUIMPL + if( ramboType == RAMBO_MASSLESS ) + { + timermap.start( "1c CpHTDrnd" ); + copyDeviceFromHost( devRndmom, hstRndmom ); + genrtime += timermap.stop(); + } +#endif + + timermap.start( "2a RamboIni" ); + prsk->getMomentaInitial(); + rambtime += timermap.stop(); + timermap.start( "2b RamboFin" ); + prsk->getMomentaFinal(); + rambtime += timermap.stop(); +#ifdef MGONGPUCPP_GPUIMPL + // Massive host only (copy) + if( ramboType != RAMBO_MASSLESS ) + { + timermap.start( "2c CpHTDmom" ); + copyDeviceFromHost( devMomenta, hstMomenta ); + rambtime += timermap.stop(); + } +#endif + } + else + { + // Fill this batch from the LHE events (AOSOA layout, (E,px,py,pz) per leg). + // padded by repeating its last real event so the SIMD page is valid + // only the nreal real events are counted below. + timermap.start( "2e ReadLHE " ); + const std::size_t base = (std::size_t)iiter * nevt; + nreal = (unsigned int)std::min( nevt, lheEvents.size() - base ); + for( unsigned int ievt = 0; ievt < nevt; ++ievt ) + { + const std::size_t src = base + std::min( ievt, (std::size_t)nreal - 1 ); + const LheEvent& ev = lheEvents[src]; + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + for( int ip4 = 0; ip4 < 4; ++ip4 ) + MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar ) = (fptype)ev[ipar][ip4]; + } + rambtime += timermap.stop(); +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "2c CpHTDmom" ); + copyDeviceFromHost( devMomenta, hstMomenta ); + rambtime += timermap.stop(); +#endif + } + + timermap.start( "2d Aosoa2U " ); +#ifdef MGONGPUCPP_GPUIMPL + gpuLaunchKernel( aosoa_to_umami_kernel, gpublocks, gputhreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); + checkGpu( gpuPeekAtLastError() ); +#else + for( std::size_t ievt = 0; ievt < nevt; ++ievt ) + aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); +#endif + rambtime += timermap.stop(); + + double wavetime = 0; + if( !run_umami( umami_handle, nevt, timermap, wavetime, +#ifdef MGONGPUCPP_GPUIMPL + devUmamiMomenta, devFlv, devUmamiMEs, hstUmamiMEs +#else + umamiMomenta, flvVec, umamiMEs +#endif + ) ) + { + umami_free( umami_handle ); + return 3; + } + +#ifdef MGONGPUCPP_GPUIMPL + if( verbose ) + { + timermap.start( "3c CpDTHmom" ); + copyHostFromDevice( hstMomenta, devMomenta ); + wavetime += timermap.stop(); + } + const double* mes = hstUmamiMEs.data(); +#else + const double* mes = umamiMEs.data(); +#endif + + timermap.start( "4@ UpdtStat" ); + for( unsigned int ievt = 0; ievt < nreal; ++ievt ) + { + double me = mes[ievt]; + ++nevtALL; + if( !std::isfinite( me ) ) + ++nevtABN; + else if( me == 0. ) + ++nevtZERO; + sumME += me; + sumMEsq += me * me; + if( me < minME ) minME = me; + if( me > maxME ) maxME = me; + } + + genrtimes[iiter] = genrtime; + rambtimes[iiter] = rambtime; + wavetimes[iiter] = wavetime; + + if( verbose ) + { + std::cout << std::string( SEP79, '*' ) << std::endl + << "Iteration #" << iiter + 1 << " of " << niter << std::endl; + for( unsigned int ievt = 0; ievt < nreal; ++ievt ) + { + std::cout << "Event #" << ievt + 1 << std::endl; + print_momenta_table( std::cout, hstMomenta.data(), ievt ); + std::cout << " Matrix element = " << std::scientific << std::setprecision( 16 ) + << mes[ievt] << " GeV^" << kMEGeVExponent << std::endl + << std::defaultfloat + << std::string( SEP79, '-' ) << std::endl; + } + } + } + + double sumgtim = 0, sumrtim = 0, sumwtim = 0; + double minwtim = wavetimes[0], maxwtim = wavetimes[0]; + for( unsigned int i = 0; i < niter; ++i ) + { + sumgtim += genrtimes[i]; + sumrtim += rambtimes[i]; + sumwtim += wavetimes[i]; + minwtim = std::min( minwtim, wavetimes[i] ); + maxwtim = std::max( maxwtim, wavetimes[i] ); + } + double meanwtim = sumwtim / niter; + + unsigned int nevtGood = nevtALL - nevtABN; + double meanME = ( nevtGood > 0 ) ? sumME / nevtGood : 0.; + double varME = ( nevtGood > 0 ) ? sumMEsq / nevtGood - meanME * meanME : 0.; + double stdME = ( varME > 0 ) ? std::sqrt( varME ) : 0.; + + std::cout << std::string( SEP79, '*' ) << std::endl; + print_run_header( std::cout ); + std::cout << "NumBlocksPerGrid = " << gpublocks << std::endl + << "NumThreadsPerBlock = " << gputhreads << std::endl + << "NumIterations = " << niter << std::endl + << "FlavorIndex = " << flavorID << " / " << CPPProcess::nmaxflavor << std::endl + << std::string( SEP79, '-' ) << std::endl + << "NaN/abnormal MEs = " << nevtABN << std::endl + << "Zero MEs = " << nevtZERO << std::endl + << std::string( SEP79, '-' ) << std::endl + << "NumberOfEntries = " << niter << std::endl + << std::scientific + << "TotalTime[Rnd+Rmb+ME] (123) = ( " << sumgtim + sumrtim + sumwtim << " ) sec" << std::endl + << "TotalTime[Rambo+ME] (23) = ( " << sumrtim + sumwtim << " ) sec" << std::endl + << "TotalTime[RndNumGen] (1) = ( " << sumgtim << " ) sec" << std::endl + << "TotalTime[Rambo] (2) = ( " << sumrtim << " ) sec" << std::endl + << "TotalTime[MatrixElems] (3) = ( " << sumwtim << " ) sec" << std::endl + << "MeanTimeInMatrixElems = ( " << meanwtim << " ) sec" << std::endl + << "[Min,Max]TimeInMatrixElems = [ " << minwtim << " , " << maxwtim << " ] sec" << std::endl + << std::string( SEP79, '-' ) << std::endl + << "TotalEventsComputed = " << nevtALL << std::endl + << "EvtsPerSec[Rnd+Rmb+ME](123) = ( " << nevtALL / ( sumgtim + sumrtim + sumwtim ) << " ) sec^-1" << std::endl + << "EvtsPerSec[Rmb+ME] (23) = ( " << nevtALL / ( sumrtim + sumwtim ) << " ) sec^-1" << std::endl + << "EvtsPerSec[MatrixElems] (3) = ( " << nevtALL / sumwtim << " ) sec^-1" << std::endl + << std::defaultfloat + << std::string( SEP79, '*' ) << std::endl + << "MeanMatrixElemValue = ( " << meanME << " +- " << stdME / std::sqrt( (double)std::max( 1u, nevtGood ) ) + << " ) GeV^" << kMEGeVExponent << std::endl + << "[Min,Max]MatrixElemValue = [ " << minME << " , " << maxME << " ] GeV^" << kMEGeVExponent << std::endl + << std::string( SEP79, '*' ) << std::endl; + timermap.dump(); + std::cout << std::string( SEP79, '*' ) << std::endl; + + umami_free( umami_handle ); + return 0; + } +} + +int main( int argc, char** argv ) +{ + + Mode mode = MODE_MATRIX; + RamboType ramboType = RAMBO_MASSIVE; // default + bool ramboTypeSet = false; + bool verbose = false; + unsigned int flavorID = 0; + unsigned int gpublocks = 64; + unsigned int gputhreads = 256; + unsigned int niter = 1; + unsigned int numvec[3] = { 0, 0, 0 }; + int nnum = 0; + std::string lheFile; // -e/--events: read momenta from this LHE file (perf mode only) + + // Optional leading subcommand (no leading dash). + int firstArg = 1; + if( firstArg < argc ) + { + std::string a = argv[firstArg]; + if( a == "matrix" ) { mode = MODE_MATRIX; ++firstArg; } + else if( a == "perf" ) { mode = MODE_PERF; ++firstArg; } + } + + double energyArg = -1.; + + for( int argn = firstArg; argn < argc; ++argn ) + { + std::string arg = argv[argn]; + if( arg == "--verbose" || arg == "-v" ) + verbose = true; + else if( arg == "--performance" || arg == "-p" ) + mode = MODE_PERF; // legacy alias + else if( ( arg == "--flavor" || arg == "-f" ) && argn + 1 < argc && is_number( argv[argn + 1] ) ) + flavorID = strtoul( argv[++argn], nullptr, 0 ); + else if( arg == "--rambo-massless" ) + { + std::string r = argv[++argn]; + ramboType = RAMBO_MASSLESS; + ramboTypeSet = true; + } + else if( ( arg == "--events" || arg == "-e" ) && argn + 1 < argc ) + { + lheFile = argv[++argn]; + mode = MODE_PERF; // reading events from file only makes sense in perf mode + } + else if( is_number( argv[argn] ) && nnum < 3 ) + { + numvec[nnum++] = strtoul( argv[argn], nullptr, 0 ); + if( energyArg < 0 ) energyArg = atof( argv[argn] ); + } + else if( is_float( argv[argn] ) && energyArg < 0 ) + { + // decimal number: only meaningful as the matrix-mode energy + energyArg = atof( argv[argn] ); + } + else + return usage( argv[0] ); + } +//ENERGY CHANGE FOR THE MATRIX MODE +// (default 1000 GeV as for the Fortran/C++ standalone 'check' drivers; +// can be overridden with a single positional argument) + if( mode == MODE_MATRIX ) kEnergy = ( energyArg > 0 ) ? energyArg : 1000.; + + if( mode == MODE_MATRIX ) + { + if( ramboType == RAMBO_MASSLESS ) + { + std::cerr << "ERROR: matrix mode only supports the classic RAMBO (-r c)." << std::endl; + return usage( argv[0] ); + } + if( nnum > 1 ) + { + std::cerr << "WARNING: extra positional args are ignored in matrix mode " + << "(dimensions are fixed at " << kMatrixBlocks << " " << kMatrixThreads << " 1)." + << std::endl; + } + return run_matrix_mode( verbose ); + } + + // perf mode + if( nnum == 3 ) + { + gpublocks = numvec[0]; + gputhreads = numvec[1]; + niter = numvec[2]; + } + else if( nnum == 1 ) + { + niter = numvec[0]; + } + else if( nnum != 0 ) + { + return usage( argv[0] ); + } + if( niter == 0 && lheFile.empty() ) return usage( argv[0] ); // niter is derived from the file in LHE mode + + if( flavorID >= CPPProcess::nmaxflavor ) + { + std::cerr << "ERROR: flavor index " << flavorID + << " is out of range [0, " << CPPProcess::nmaxflavor << ")." << std::endl; + return 1; + } + + return run_perf_mode( verbose, gpublocks, gputhreads, niter, flavorID, ramboType, lheFile ); +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc new file mode 100644 index 0000000000..30c6799932 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc @@ -0,0 +1,434 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "color_sum.h" + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessMatrixElements.h" + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors + + //-------------------------------------------------------------------------- + + // *** COLOR MATRIX BELOW *** +%(color_matrix_lines)s + +#ifdef MGONGPUCPP_GPUIMPL + // The normalized color matrix (divide each column by denom) + template + struct NormalizedColorMatrix + { + constexpr __host__ __device__ NormalizedColorMatrix() + : value() + { + for( int icol = 0; icol < ncolor; icol++ ) + for( int jcol = 0; jcol < ncolor; jcol++ ) + value[icol * ncolor + jcol] = colorMatrix[icol][jcol] / colorDenom[icol]; + } + T value[ncolor * ncolor]; + }; + // The fptype2 version is the default used by kernels (supporting mixed floating point mode also in blas) + static __device__ fptype2 s_pNormalizedColorMatrix2[ncolor * ncolor]; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void createNormalizedColorMatrix() + { + static bool first = true; + if( first ) + { + first = false; + constexpr NormalizedColorMatrix normalizedColorMatrix2; + gpuMemcpyToSymbol( s_pNormalizedColorMatrix2, normalizedColorMatrix2.value, ncolor * ncolor * sizeof( fptype2 ) ); + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + void + color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity + const int ievt0 ) // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) + { + // Pre-compute a constexpr triangular color matrix properly normalized #475 + struct TriangularNormalizedColorMatrix + { + // See https://stackoverflow.com/a/34465458 + __host__ __device__ constexpr TriangularNormalizedColorMatrix() + : value() + { + for( int icol = 0; icol < ncolor; icol++ ) + { + // Diagonal terms + value[icol][icol] = colorMatrix[icol][icol] / colorDenom[icol]; + // Off-diagonal terms + for( int jcol = icol + 1; jcol < ncolor; jcol++ ) + value[icol][jcol] = 2 * colorMatrix[icol][jcol] / colorDenom[icol]; + } + } + fptype2 value[ncolor][ncolor]; + }; + static constexpr auto cf2 = TriangularNormalizedColorMatrix(); + // Use the property that M is a real matrix (see #475): + // we can rewrite the quadratic form (A-iB)(M)(A+iB) as AMA - iBMA + iBMA + BMB = AMA + BMB + // In addition, on C++ use the property that M is symmetric (see #475), + // and also use constexpr to compute "2*" and "/colorDenom[icol]" once and for all at compile time: + // we gain (not a factor 2...) in speed here as we only loop over the up diagonal part of the matrix. + // Strangely, CUDA is slower instead, so keep the old implementation for the moment. + fptype_sv deltaMEs = { 0 }; +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype_sv deltaMEs_next = { 0 }; + // Mixed mode: merge two neppV vectors into one neppV2 vector + fptype2_sv jampR_sv[ncolor]; + fptype2_sv jampI_sv[ncolor]; + for( int icol = 0; icol < ncolor; icol++ ) + { + jampR_sv[icol] = fpvmerge( cxreal( allJamp_sv[icol] ), cxreal( allJamp_sv[ncolor + icol] ) ); + jampI_sv[icol] = fpvmerge( cximag( allJamp_sv[icol] ), cximag( allJamp_sv[ncolor + icol] ) ); + } +#else + const cxtype_sv* jamp_sv = allJamp_sv; +#endif + // Loop over icol + for( int icol = 0; icol < ncolor; icol++ ) + { + // Diagonal terms +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype2_sv& jampRi_sv = jampR_sv[icol]; + fptype2_sv& jampIi_sv = jampI_sv[icol]; +#else + fptype2_sv jampRi_sv = (fptype2_sv)( cxreal( jamp_sv[icol] ) ); + fptype2_sv jampIi_sv = (fptype2_sv)( cximag( jamp_sv[icol] ) ); +#endif + fptype2_sv ztempR_sv = cf2.value[icol][icol] * jampRi_sv; + fptype2_sv ztempI_sv = cf2.value[icol][icol] * jampIi_sv; + // Loop over jcol + for( int jcol = icol + 1; jcol < ncolor; jcol++ ) + { + // Off-diagonal terms +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype2_sv& jampRj_sv = jampR_sv[jcol]; + fptype2_sv& jampIj_sv = jampI_sv[jcol]; +#else + fptype2_sv jampRj_sv = (fptype2_sv)( cxreal( jamp_sv[jcol] ) ); + fptype2_sv jampIj_sv = (fptype2_sv)( cximag( jamp_sv[jcol] ) ); +#endif + ztempR_sv += cf2.value[icol][jcol] * jampRj_sv; + ztempI_sv += cf2.value[icol][jcol] * jampIj_sv; + } + fptype2_sv deltaMEs2 = ( jampRi_sv * ztempR_sv + jampIi_sv * ztempI_sv ); // may underflow #831 +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + deltaMEs += fpvsplit0( deltaMEs2 ); + deltaMEs_next += fpvsplit1( deltaMEs2 ); +#else + deltaMEs += deltaMEs2; +#endif + } + // *** STORE THE RESULTS *** + using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv += deltaMEs; // fix #435 +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype* MEs_next = E_ACCESS::ieventAccessRecord( allMEs, ievt0 + neppV ); + fptype_sv& MEs_sv_next = E_ACCESS::kernelAccess( MEs_next ); + MEs_sv_next += deltaMEs_next; +#endif + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity + const int nGoodHel, // input: number of good helicities + const int nevtIfAllHelicities ) // input: zero in single-helicity mode, number of events in multi-helicity mode + { + if (nevtIfAllHelicities) { + int ighel = blockIdx.y; + allMEs = allMEs + ighel * nevtIfAllHelicities; // MEs for one specific helicity ighel + allJamps = allJamps + ighel * nevtIfAllHelicities; // Jamps for one specific helicity ighel + } + using J_ACCESS = DeviceAccessJamp; + fptype jampR[ncolor]; + fptype jampI[ncolor]; + for( int icol = 0; icol < ncolor; icol++ ) + { + constexpr int ihel0 = 0; // the input buffer allJamps already points to a specific helicity + cxtype jamp = J_ACCESS::kernelAccessIcolIhelNhelConst( allJamps, icol, ihel0, nGoodHel ); + jampR[icol] = jamp.real(); + jampI[icol] = jamp.imag(); + } + // Loop over icol + fptype deltaMEs = { 0 }; + for( int icol = 0; icol < ncolor; icol++ ) + { + fptype2 ztempR = { 0 }; + fptype2 ztempI = { 0 }; + fptype2 jampRi = jampR[icol]; + fptype2 jampIi = jampI[icol]; + // OLD IMPLEMENTATION (ihel3: symmetric square matrix) - Loop over all jcol + //for( int jcol = 0; jcol < ncolor; jcol++ ) + //{ + // fptype2 jampRj = jampR[jcol]; + // fptype2 jampIj = jampI[jcol]; + // ztempR += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix + // ztempI += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix + //} + // NEW IMPLEMENTATION #475 (ihel3p1: triangular lower diagonal matrix) - Loop over jcol < icol + ztempR += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampRi; // use fptype2 version of color matrix + ztempI += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampIi; // use fptype2 version of color matrix + for( int jcol = 0; jcol < icol; jcol++ ) + { + fptype2 jampRj = jampR[jcol]; + fptype2 jampIj = jampI[jcol]; + ztempR += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix + ztempI += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix + } + deltaMEs += ztempR * jampRi; + deltaMEs += ztempI * jampIi; + } + // *** STORE THE RESULTS *** + using E_ACCESS = DeviceAccessMatrixElements; // non-trivial access: buffer includes all events + // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) + E_ACCESS::kernelAccess( allMEs ) += deltaMEs; // fix #435 + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +#ifndef MGONGPU_HAS_NO_BLAS +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + __global__ void + convertD2F_Jamps( fptype2* allJampsFpt2, // output: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel + const fptype* allJamps, // input: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel + const int nhel ) // input: number of good helicities nGoodHel + { + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + constexpr int ihel = 0; // the input buffer allJamps already points to a specific helicity + // NB! From a functional point of view, any striding will be ok here as long as ncolor*2*nevt elements are all correctly copied! + // NB! Just in case this may be better for performance reasons, however, the same striding as in compute_jamps and cuBLAS is used here + for( int ix2 = 0; ix2 < mgOnGpu::nx2; ix2++ ) + for( int icol = 0; icol < ncolor; icol++ ) + allJampsFpt2[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] = + allJamps[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt]; + } +#endif +#endif +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +#ifndef MGONGPU_HAS_NO_BLAS +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + __global__ void + convertF2D_MEs( fptype* allMEs, // output: allMEs[nevt] for one specific helicity + const fptype2* allMEsFpt2 ) // input: allMEs[nevt] for one specific helicity + { + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + allMEs[ievt] = allMEsFpt2[ievt]; + } +#endif +#endif +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ +#ifndef MGONGPU_HAS_NO_BLAS + void + color_sum_blas( fptype* ghelAllMEs, // output: allMEs super-buffer[nhel][nevt], add |M|^2 separately for each helicity + const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nhel][nevt] for nhel good helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nhel good helicities + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) +#else + gpuStream_t* /*ghelStreams*/, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) +#endif + const int nhel, // input: number of good helicities (nhel == nGoodHel) + const int gpublocks, // input: cuda gpublocks + const int gputhreads ) // input: cuda gputhreads + { + const int nevt = gpublocks * gputhreads; + + // Get the address associated with the normalized color matrix in device memory + static fptype2* devNormColMat = nullptr; + if( !devNormColMat ) gpuGetSymbolAddress( (void**)&devNormColMat, s_pNormalizedColorMatrix2 ); + +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Mixed precision mode: need two fptype2[2*ncolor*nhel*nevt] buffers and one fptype2[nhel*nevt] buffers for the nhel helicities + fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of first fptype2[ncolor*2*nhel*nevt] buffer + fptype2* ghelAllJampsFpt2 = ghelAllBlasTmp + ncolor * mgOnGpu::nx2 * nhel * nevt; // start of second fptype2[ncolor*2*nhel*nevt] buffer + fptype2* ghelAllMEsFpt2 = ghelAllBlasTmp + 2 * ncolor * mgOnGpu::nx2 * nhel * nevt; // start of fptype2[nhel*nevt] buffer + // Convert jamps from double to float + for( int ighel = 0; ighel < nhel; ighel++ ) + { + const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // jamps for a single helicity ihel + fptype2* hAllJampsFpt2 = ghelAllJampsFpt2 + ighel * nevt; // jamps for a single helicity ihel + gpuLaunchKernelStream( convertD2F_Jamps, gpublocks, gputhreads, ghelStreams[ighel], hAllJampsFpt2, hAllJamps, nhel ); + } + // Real and imaginary components + const fptype2* ghelAllJampsReal = ghelAllJampsFpt2; + const fptype2* ghelAllJampsImag = ghelAllJampsFpt2 + ncolor * nhel * nevt; +#else + // Standard single or double precision mode: need one fptype2[ncolor*2*nhel*nevt] buffer + static_assert( std::is_same::value ); + fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of fptype2[ncolor*2*nhel*nevt] buffer + fptype2* ghelAllMEsFpt2 = ghelAllMEs; + // Real and imaginary components + const fptype2* ghelAllJampsReal = ghelAllJamps; // this is not a cast (the two types are identical) + const fptype2* ghelAllJampsImag = ghelAllJamps + ncolor * nhel * nevt; // this is not a cast (the two types are identical) +#endif + // Real and imaginary components + fptype2* ghelAllZtempReal = ghelAllZtempBoth; + fptype2* ghelAllZtempImag = ghelAllZtempBoth + ncolor * nhel * nevt; + + // Note: striding for cuBLAS from DeviceAccessJamp: + // - ghelAllJamps(icol,ihel,ievt).real is ghelAllJamps[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] + // - ghelAllJamps(icol,ihel,ievt).imag is ghelAllJamps[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] + + // Step 1: Compute Ztemp[ncolor][nhel*nevt] = ColorMatrix[ncolor][ncolor] * JampsVector[ncolor][nhel*nevt] for both real and imag + // In this case alpha=1 and beta=0: the operation is Ztemp = alpha * ColorMatrix * JampsVector + beta * Ztemp + fptype2 alpha1 = 1; + fptype2 beta1 = 0; + const int ncolorM = ncolor; + const int nevtN = nhel*nevt; + const int ncolorK = ncolor; + checkGpuBlas( gpuBlasTgemm( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose ColMat + GPUBLAS_OP_T, // transpose JampsV (new1) + ncolorM, nevtN, ncolorK, + &alpha1, + devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK + ghelAllJampsReal, nevtN, // JampsV is nevtN x ncolorK + &beta1, + ghelAllZtempReal, ncolorM ) ); // Ztemp is ncolorM x nevtN + checkGpuBlas( gpuBlasTgemm( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose ColMat + GPUBLAS_OP_T, // transpose JampsV (new1) + ncolorM, nevtN, ncolorK, + &alpha1, + devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK + ghelAllJampsImag, nevtN, // JampsV is nevtN x ncolorK (new1) + &beta1, + ghelAllZtempImag, ncolorM ) ); // Ztemp is ncolorM x nevtN + + // Step 2: For each ievt, compute the dot product of JampsVector[ncolor][ievt] dot tmp[ncolor][ievt] + // In this case alpha=1 and beta=1: the operation is ME = alpha * ( Tmp dot JampsVector ) + beta * ME + // Use cublasSgemmStridedBatched to perform these batched dot products in one call + fptype2 alpha2 = 1; + fptype2 beta2 = 1; + checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose JampsV (new1) + GPUBLAS_OP_N, // do not transpose Tmp + 1, 1, ncolor, // result is 1x1 (dot product) + &alpha2, + ghelAllJampsReal, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column + ghelAllZtempReal, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column + &beta2, + ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) + nevtN ) ); // there are nevtN (nhel*nevt) "batches" + checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose JampsV (new1) + GPUBLAS_OP_N, // do not transpose Tmp + 1, 1, ncolor, // result is 1x1 (dot product) + &alpha2, + ghelAllJampsImag, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column (new1) + ghelAllZtempImag, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column + &beta2, + ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) + nevtN ) ); // there are nevt (nhel*nevt) "batches" + +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Convert MEs from float to double + for( int ighel = 0; ighel < nhel; ighel++ ) + { + fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for a single helicity ihel + fptype2* hAllMEsFpt2 = ghelAllMEsFpt2 + ighel * nevt; // MEs for a single helicity ihel + gpuLaunchKernelStream( convertF2D_MEs, gpublocks, gputhreads, ghelStreams[ighel], hAllMEs, hAllMEsFpt2 ); + } +#endif + } +#endif /* clang-format on */ +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) + const int nGoodHel, // input: number of good helicities + const int gpublocks, // input: cuda gpublocks + const int gputhreads, // input: cuda gputhreads + const bool processAllHelicities ) // input: if true, use blockIdx.y to index helicities + { + const int nevt = gpublocks * gputhreads; + // CASE 1: KERNEL + if( !pBlasHandle ) + { + assert( ghelAllBlasTmp == nullptr ); // sanity check for HASBLAS=hasNoBlas or CUDACPP_RUNTIME_BLASCOLORSUM not set + if (processAllHelicities) { + gpuLaunchKernel2D( color_sum_kernel, gpublocks, nGoodHel, gputhreads, ghelStreams[0], ghelAllMEs, ghelAllJamps, nGoodHel, nevt ); + } else { + // Loop over helicities + for( int ighel = 0; ighel < nGoodHel; ighel++ ) + { + fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for one specific helicity ighel + const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // Jamps for one specific helicity ighel + gpuStream_t hStream = ghelStreams[ighel]; + gpuLaunchKernelStream( color_sum_kernel, gpublocks, gputhreads, hStream, hAllMEs, hAllJamps, nGoodHel, 0 ); + } + } + } + // CASE 2: BLAS + else + { +#ifdef MGONGPU_HAS_NO_BLAS + assert( false ); // sanity check: no path to this statement for HASBLAS=hasNoBlas +#else + if (processAllHelicities) { + assert( false ); // BLAS in async mode not supported for now + } else { + checkGpu( gpuDeviceSynchronize() ); // do not start the BLAS color sum for all helicities until the loop over helicities has completed + // Reset the tmp buffer +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( 2 * ncolor * mgOnGpu::nx2 + 1 ) * sizeof( fptype2 ) ); +#else + gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( ncolor * mgOnGpu::nx2 ) * sizeof( fptype2 ) ); +#endif + // Delegate the color sum to BLAS for + color_sum_blas( ghelAllMEs, ghelAllJamps, ghelAllBlasTmp, pBlasHandle, ghelStreams, nGoodHel, gpublocks, gputhreads ); + } +#endif + } + } +#endif + + //-------------------------------------------------------------------------- + +} // end namespace diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h new file mode 100644 index 0000000000..347184c4e1 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h @@ -0,0 +1,105 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef COLOR_SUM_H +#define COLOR_SUM_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#include "CPPProcess.h" +#include "GpuAbstraction.h" + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + class DeviceAccessJamp + { + public: + static __device__ inline cxtype_ref + kernelAccessIcolIhelNhel( fptype* buffer, const int icol, const int ihel, const int nhel ) + { + const int ncolor = CPPProcess::ncolor; // the number of leading colors + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) + //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" + // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) + // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS + //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" + // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) + return cxtype_ref( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], + buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); + } + static __device__ inline const cxtype + kernelAccessIcolIhelNhelConst( const fptype* buffer, const int icol, const int ihel, const int nhel ) + { + const int ncolor = CPPProcess::ncolor; // the number of leading colors + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) + //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" + // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) + // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS + //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" + // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) + return cxtype( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], + buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); + } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void createNormalizedColorMatrix(); +#endif + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + void + color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity + const int ievt0 ); // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) + const int nGoodHel, // input: number of good helicities + const int gpublocks, // input: cuda gpublocks + const int gputhreads, // input: cuda gputhreads + const bool processAllHelicities); // input: if true, use blockIdx.y to index helicities +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity + const int nGoodHel, // input: number of good helicities + const int nevtIfAllHelicities); // input: zero in single-helicity mode, number of events in multi-helicity mode +#endif + + //-------------------------------------------------------------------------- +} + +#endif // COLOR_SUM_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h new file mode 100644 index 0000000000..027f1aa44e --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h @@ -0,0 +1,68 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: O. Mattelaer, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef COLORAMPS_H +#define COLORAMPS_H 1 + +#include "CPPProcess.h" + + +namespace mgOnGpu +{ + // Summary of numbering and indexing conventions for the relevant concepts (see issue #826 and PR #852) + // - Diagram number (no variable) in [0, N_diagrams-1]: all values are allowed (N_diagrams distinct values) + // It follows the same C-style indexing of MadSpace + // => this number is displayed for information before each block of code in CPPProcess.cc + // - Channel number ("channelId" in C, CHANNEL_ID in F) in [1, N_channels]: not all values are allowed (N_config <= N_channels <= N_diagrams distinct values) + // *** NB channelId is a diagram number: but ALL diagrams > N_channels, and also some < N_channels, do not have an associated SDE config number (#919) *** + // => this number (with F indexing as in ps/pdf output) is passed around as an API argument between cudacpp functions + // Note: the old API passes around a single CHANNEL_ID (and uses CHANNEL_ID=0 to indicate no-multichannel mode, but this is not used in coloramps.h), + // while the new API passes around an array of CHANNEL_ID's (and uses a NULL array pointer to indicate no-multichannel mode) + // - Channel number in C indexing: "channelID - 1" + // => this number (with C indexing) is used as the index of the channel2iconfig array below + // - Config number ("iconfig" in C, ICONFIG in F) in [1, N_config]: all values are allowed (N_config <= N_channels <= N_diagrams distinct values) + // - Config number in C indexing: "iconfig - 1" + // => this number (with C indexing) is used as the index of the icolamp array below + + // The number of channels in the channel2iconfig array below + // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** + constexpr unsigned int nchannels = %(nb_diag)i; +#ifdef MGONGPUCPP_GPUIMPL + static_assert( nchannels <= mg5amcGpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 +#else + static_assert( nchannels <= mg5amcCpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 +#endif + + // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) + // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) + // This array has N_diagrams elements, but only N_config <= N_diagrams valid values (iconfig>0) + // (NB: this array is created on the host in C++ code and on the device in GPU code, but a host copy is also needed in runTest #917) + __device__ constexpr int channel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed +%(channelc2iconfig_lines)s + }; + + // Host copy of the channel2iconfig array (this is needed in runTest #917) +#ifndef MGONGPUCPP_GPUIMPL + constexpr const int* hostChannel2iconfig = channel2iconfig; +#else + constexpr int hostChannel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed +%(channelc2iconfig_lines)s + }; +#endif + + // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) + constexpr unsigned int nconfigSDE = %(nb_channel)s; + + // Map iconfig to the mask of allowed colors (e.g. "colormask = icolamp[iconfig - 1]": input index uses C indexing) + // This array has N_config <= N_diagrams elements + // (NB: this array is created on the host in C++ code and on the device in GPU code) + __device__ constexpr bool icolamp[%(nb_channel)s][%(nb_color)s] = { // note: a trailing comma in the initializer list is allowed +%(is_LC)s + }; + +} + +#endif // COLORAMPS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h new file mode 100644 index 0000000000..32c1bbb963 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h @@ -0,0 +1,335 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Feb 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef constexpr_math_h +#define constexpr_math_h 1 + +#include "mgOnGpuConfig.h" + +#include +#include +#include +#include + +// SWITCH ON/OFF DEBUGGING +#undef CONSTEXPR_MATH_DEBUG // no-debug +//#define CONSTEXPR_MATH_DEBUG 1 // debug + +// FOR DEBUGGING +#ifdef CONSTEXPR_MATH_DEBUG +#define CONSTEXPRMATHFUN inline +#define CONSTEXPRMATHVAR const +#else +#define CONSTEXPRMATHFUN constexpr +#define CONSTEXPRMATHVAR constexpr +#endif + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) + constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) + { + return curr == prev ? curr : sqrtNewtonRaphson( xx, 0.5 * ( curr + xx / curr ), curr ); + } + constexpr long double constexpr_sqrt( const long double xx ) + { + return xx >= 0 // && x < std::numeric_limits::infinity() // avoid -Wtautological-constant-compare warning in fast math + ? sqrtNewtonRaphson( xx, xx, 0 ) + : std::numeric_limits::quiet_NaN(); + } + + // SQRT constants + constexpr long double constexpr_sqrt2 = constexpr_sqrt( 2 ); + + // Constexpr implementation of floor (see https://stackoverflow.com/a/66146159) + constexpr int constexpr_floor( const long double xx ) + { + const int i = static_cast( xx ); + return xx < i ? i - 1 : i; + } + + // Constexpr implementation of pow + constexpr long double constexpr_pow( const long double base, const long double exp, const bool requireExpGe0 = false ) + { + // NB(1): this iterative integer implementation of constexpr_pow requires exponent >= 0 + if( requireExpGe0 ) assert( exp >= 0 ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + if( exp < 0 ) return 1. / constexpr_pow( base, -exp, true ); + // NB(2): this iterative integer implementation of constexpr_pow requires an integer exponent, excexpt for special cases (1/2, 1/4) + if( exp == 0.5 ) return constexpr_sqrt( base ); + if( exp == 0.25 ) return constexpr_sqrt( constexpr_sqrt( base ) ); + const int iexp = constexpr_floor( exp ); + assert( static_cast( iexp ) == exp ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + // Iterative implementation of pow if exp is a non negative integer + return iexp == 0 ? 1 : base * constexpr_pow( base, iexp - 1 ); + } + + // PI constants + // NB1: M_PIl from from cmath is not defined on Mac + // NB2: std::numbers::pi needs c++20 but we are still using c++17 + // NB3: I could use my constexpr_atan(1)*4... but a literal is better? + //constexpr long double constexpr_pi = M_PIl; // pi + //constexpr long double constexpr_pi_by_2 = M_PI_2l; // pi/2 + //constexpr long double constexpr_pi_by_4 = M_PI_4l; // pi/4 + constexpr long double constexpr_pi = 3.141592653589793238462643383279502884L; // same as M_PIl in gcc + constexpr long double constexpr_pi_by_2 = 1.570796326794896619231321691639751442L; // same as M_PI_2l in gcc + constexpr long double constexpr_pi_by_4 = 0.785398163397448309615660845819875721L; // same as M_PI_4l in gcc + static_assert( constexpr_pi_by_4 * 4 == constexpr_pi ); + static_assert( constexpr_pi_by_4 * 2 == constexpr_pi_by_2 ); + static_assert( constexpr_pi_by_2 * 2 == constexpr_pi ); + + // Constexpr implementation of sin for 0= constexpr_pi_by_4 ) std::cout << "sinTaylor xx=" << xx << std::endl; +#endif + assert( xx >= 0 && "The argument of sinTaylor is lower than the expected range [0,pi/4)" ); + assert( xx < constexpr_pi_by_4 && "The argument of sinTaylor is higher than the expected range [0,pi/4)" ); + long double sinx = 0; + int ipow = 1; + long double delta = xx; + while( true ) + { + long double sinxlast = sinx; + sinx += delta; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + if( sinx == sinxlast ) break; + // Next iteration + ipow += 2; + delta *= -xx * xx / ( ipow - 1 ) / ipow; + } +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + return sinx; + } + +#ifdef CONSTEXPR_MATH_DEBUG + // Debug printouts for trig functions + inline void debug_constexpr_trig( const long double xx, size_t call ) + { + CONSTEXPRMATHVAR long double xxminuspi = xx - constexpr_pi; + CONSTEXPRMATHVAR long double twopiminusxx = 2 * constexpr_pi - xx; + std::cout << std::setprecision( 40 ); + std::cout << "constexpr_sin_quad call=" << call << std::endl + << " xx=" << xx << std::endl + << " (xx-pi)=" << xxminuspi << std::endl + << " (2pi-xx)=" << twopiminusxx << std::endl; + std::cout << std::setprecision( 6 ); + if( xx < 0 ) // (-inf, 0) + std::cout << " -- case 1 (xx < 0)" << std::endl; + else if( xx == 0 ) // [0] *** NEW + std::cout << " -- case 2 (xx == 0)" << std::endl; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + std::cout << " -- case 3 (xx < pi/4)" << std::endl; + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + std::cout << " -- case 4 (xx == pi/4)" << std::endl; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + std::cout << " -- case 5 (xx < pi/2)" << std::endl; + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + std::cout << " -- case 6 (xx == pi/2)" << std::endl; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + std::cout << " -- case 7 (xx < 3*pi/4)" << std::endl; + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + std::cout << " -- case 8 (xx == 3*pi/4)" << std::endl; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + std::cout << " -- case 9 (xx < pi)" << std::endl; + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + std::cout << " -- case 10 (xx == pi)" << std::endl; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + std::cout << " -- case 11 (xx < 2*pi)" << std::endl; + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + std::cout << " -- case 12 (xx == 2*pi)" << std::endl; + else // (2*pi, +inf) + std::cout << " -- case 13 (xx > 2*pi)" << std::endl; + } +#endif + + // Mapping to [0,2*pi) range (long double signature) + constexpr long double mapIn0to2Pi( const long double xx ) + { + return xx - constexpr_floor( xx / 2 / constexpr_pi ) * 2 * constexpr_pi; + } + + // Constexpr implementation of cos (long double signature) + CONSTEXPRMATHFUN long double constexpr_cos_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 1; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx ), 2 ) ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return sinTaylor( constexpr_pi_by_2 - xx ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 0; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return -sinTaylor( xx - constexpr_pi_by_2 ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return -1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return -constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi - xx ), 2 ) ); + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + return -1; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return constexpr_cos_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 1; + else // (2*pi, +inf) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of cos (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_cos( const double x ) + { + return constexpr_cos_quad( x ); + } + + // Constexpr implementation of sin (long double signature) + CONSTEXPRMATHFUN long double constexpr_sin_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 0; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return sinTaylor( xx ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi_by_2 - xx ), 2 ) ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 1; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx - constexpr_pi_by_2 ), 2 ) ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return sinTaylor( constexpr_pi - xx ); + else if( xx == constexpr_pi ) // [pi] *** NEW (1st fix #903 infinite recursion) + return 0; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return -constexpr_sin_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 0; + else // (2*pi, +inf) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of sin (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_sin( const double x ) + { + return constexpr_sin_quad( x ); + } + + // Constexpr implementation of tan (long double signature) + CONSTEXPRMATHFUN long double constexpr_tan_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } + if( xx < 0 ) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + else if( xx < 2 * constexpr_pi ) // [0, 2*pi) + return constexpr_sin_quad( xx, assume0to2Pi ) / constexpr_cos_quad( xx, assume0to2Pi ); + else // [8/4*pi, +inf) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of tan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_tan( const double x ) + { + return constexpr_tan_quad( x ); + } + + // Constexpr implementation of atan for -1= -1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + assert( xx < 1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + long double atanx = 0; + int ipow = 1; + long double xpow = xx; + while( true ) + { + long double atanxlast = atanx; + atanx += xpow / ipow; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", xpow=" << xpow << ", atanx=" << atanx << std::endl; // for debugging (not constexpr) +#endif + if( atanx == atanxlast ) break; + // Next iteration + ipow += 2; + xpow *= -xx * xx; + } + return atanx; + } + + // Constexpr implementation of atan (long double signature) + CONSTEXPRMATHFUN long double constexpr_atan_quad( const long double xx ) + { + if( xx > 1 ) + return constexpr_pi_by_2 - atanTaylor( 1 / xx ); + else if( xx == 1 ) + return constexpr_pi_by_4; + else if( xx > -1 ) + return atanTaylor( xx ); + else if( xx == -1 ) + return -constexpr_pi_by_4; + else // if( xx < -1 ) + return -constexpr_pi_by_2 - atanTaylor( 1 / xx ); + } + + // Constexpr implementation of atan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_atan( const double x ) + { + return constexpr_atan_quad( x ); + } +} + +#endif // constexpr_math_h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_hel_amps_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_hel_amps_h.inc new file mode 100644 index 0000000000..8dde0fff4b --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_hel_amps_h.inc @@ -0,0 +1,90 @@ +// Copyright (C) 2010 The ALOHA Development team and Contributors. +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Sep 2010) for the MG5aMC backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== +// This file has been automatically generated for %(output_name)s by +%(info_lines)s +//========================================================================== + +#ifndef HelAmps_%(model_name)s_H +#define HelAmps_%(model_name)s_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#include "Parameters.h" + +#include +//#include +//#include +//#include +//#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + + // ALOHA-style object for easy flavor consolidation and non-template API + struct ALOHAOBJ { + + static constexpr int np4 = 4; // dimensions of 4-momenta (E,px,py,pz) + static constexpr int nw6 = %(nwave)d; // dimensions of each wavefunction (notice, this is +1 in case of FD gauge) + fptype_sv * pvec; + fptype * w; + int flv_index; + + __host__ __device__ ALOHAOBJ() {} + __host__ __device__ ALOHAOBJ(fptype_sv * pvec_sv, cxtype_sv * w_sv, int flv = -1) + : pvec(pvec_sv), w(reinterpret_cast(w_sv)), flv_index(flv) {} + }; + + struct FLV_COUPLING_VIEW { + + const int* const partner1; + const int* const partner2; + const fptype* const value; + + __host__ __device__ + FLV_COUPLING_VIEW(const int* p1, const int* p2, const fptype* v) + : partner1(p1), partner2(p2), value(v) {} + }; + + // FSTRIDE is the number of fptype's used to store one flavor slot of the value buffer: + // - independent (fixed) flavored couplings: FSTRIDE = nx2 = 2 (a single scalar complex, broadcast across the SIMD vector) + // - dependent (event-by-event, running-alphas) flavored couplings: FSTRIDE = nx2*neppC (an AOSOA SIMD record) + // It must match C_ACCESS::flv_stride of the access type the consuming vertex routine is instantiated with. + template + class FLV_COUPLING_ARRAY { + + static_assert(SIZE >= 0, "flvCOUPs SIZE must be non-negative"); + static_assert(STRIDE > 0, "flvCOUPs STRIDE must be positive"); + static_assert(FSTRIDE > 0, "flvCOUPs FSTRIDE must be positive"); + const int* const partner1; + const int* const partner2; + const fptype* const value; + + public: + __host__ __device__ + FLV_COUPLING_ARRAY(const int* p1, const int* p2, const fptype* v) + : partner1(p1), partner2(p2), value(v) {} + + __host__ __device__ + FLV_COUPLING_VIEW operator[](const int i) const { + return FLV_COUPLING_VIEW{ + partner1 + i*STRIDE, + partner2 + i*STRIDE, + value + i*FSTRIDE*STRIDE + }; + } + }; +%(function_declarations)s diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_cc.inc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_cc.inc new file mode 100644 index 0000000000..4650fc76c8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_cc.inc @@ -0,0 +1,101 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== +// This file has been automatically generated for CUDA/C++ standalone by +%(info_lines)s +//========================================================================== + +#include "Parameters.h" + +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +using namespace mg5amcGpu; +#else +using namespace mg5amcCpu; +#endif + +#ifndef MGONGPU_HARDCODE_PARAM + +// Initialize static instance +Parameters* Parameters::instance = 0; + +// Function to get static instance - only one instance per program +Parameters* +Parameters::getInstance() +{ + if( instance == 0 ) + instance = new Parameters(); + return instance; +} + +void +Parameters::setIndependentParameters( SLHAReader& slha ) +{ + zero = 0; // define "zero" + ZERO = 0; // define "zero" + std::vector indices( 2, 0 ); // prepare a vector for indices + %(set_independent_parameters)s +} + +void +Parameters::setIndependentCouplings() +{ + %(set_independent_couplings)s + %(set_flv_couplings)s +} + +/* +void +Parameters::setDependentParameters() // now computed event-by-event (running alphas #373) +{ + %(set_dependent_parameters)s +} + +void +Parameters::setDependentCouplings() // now computed event-by-event (running alphas #373) +{ + %(set_dependent_couplings)s +} +*/ + +#endif + +// Routines for printing out parameters +void +Parameters::printIndependentParameters() +{ + std::cout << "model parameters independent of event kinematics:" << std::endl; + std::cout << "(Warning: aS in the runcard is ignored because event-by-event Gs are hardcoded or retrieved from Fortran)" << std::endl; + %(print_independent_parameters)s +} + +void +Parameters::printIndependentCouplings() +{ + std::cout << "model couplings independent of event kinematics:" << std::endl; + %(print_independent_couplings)s +} + +/* +void +Parameters::printDependentParameters() // now computed event-by-event (running alphas #373) +{ + std::cout << "model parameters dependent on event kinematics:" << std::endl; + %(print_dependent_parameters)s +} + +void +Parameters::printDependentCouplings() // now computed event-by-event (running alphas #373) +{ + std::cout << "model couplings dependent on event kinematics:" << std::endl; + %(print_dependent_couplings)s +} +*/ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_h.inc new file mode 100644 index 0000000000..c13f8ebe2f --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_h.inc @@ -0,0 +1,264 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. +// ========================================================================== +// This file has been automatically generated for CUDA/C++ standalone by +%(info_lines)s +//========================================================================== + +#ifndef Parameters_H +#define Parameters_H + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" +#include "mgOnGpuVectors.h" + +#include "constexpr_math.h" + +//========================================================================== + +// AV Jan 2024 (PR #625): this ugly #define was the only way I found to avoid creating arrays[nBsm] in CPPProcess.cc if nBsm is 0 +// The problem is that nBsm is determined when generating Parameters.h, which happens after CPPProcess.cc has already been generated +// For simplicity, keep this code hardcoded also for SM processes (a nullptr is needed as in the case nBsm == 0) +%(bsmdefine)s + +#ifndef MGONGPU_HARDCODE_PARAM%(eftwarn0)s + +#include "read_slha.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // Define FLV_COUPLING struct + struct FLV_COUPLING { + static constexpr int max_flavor = %(max_flavor)d; + + int partner1[max_flavor]; + int partner2[max_flavor]; + cxtype* value[max_flavor]; // because it will refer already calculated couplings + + FLV_COUPLING() { + for (int i = 0; i < max_flavor; ++i) { + partner1[i] = -1; + partner2[i] = -1; + value[i] = nullptr; // flavor slots with no coupling stay null so the + // cIPF_value setup (value[j] ? *value[j] : 0) does + // not dereference an uninitialised pointer + } + } + }; + + class Parameters + { + public: + + static Parameters* getInstance(); + + // Define "zero" + double zero, ZERO; + + %(independent_parameters)s + + %(independent_couplings)s + + %(dependent_parameters)s + + %(dependent_couplings)s + + %(flavor_independent_couplings)s + + %(flavor_dependent_couplings)s + + // Set parameters that are unchanged during the run + void setIndependentParameters( SLHAReader& slha ); + + // Set couplings that are unchanged during the run + void setIndependentCouplings(); + + // Set parameters that are changed event by event + //void setDependentParameters(); // now computed event-by-event (running alphas #373) + + // Set couplings that are changed event by event + //void setDependentCouplings(); // now computed event-by-event (running alphas #373) + + // Print parameters that are unchanged during the run + void printIndependentParameters(); + + // Print couplings that are unchanged during the run + void printIndependentCouplings(); + + // Print parameters that are changed event by event + //void printDependentParameters(); // now computed event-by-event (running alphas #373) + + // Print couplings that are changed event by event + //void printDependentCouplings(); // now computed event-by-event (running alphas #373) + + // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; + static constexpr int nBsmIndepParam = %(nbsmip)i; + %(hasbsmip)sdouble mdl_bsmIndepParam[nBsmIndepParam]; + + private: + + static Parameters* instance; + }; + +} // end namespace mg5amcGpu/mg5amcCpu + +#else%(eftwarn1)s + +#include +#include + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // Hardcoded constexpr physics parameters + namespace Parameters // keep the same name rather than HardcodedParameters for simplicity + { + // Model parameters independent of aS + constexpr double zero = 0; + constexpr double ZERO = 0; + %(hardcoded_independent_parameters)s + // Model couplings independent of aS + %(hardcoded_independent_couplings)s + // Model parameters dependent on aS + %(hardcoded_dependent_parameters)s + // Model couplings dependent on aS + %(hardcoded_dependent_couplings)s + // Print parameters that are unchanged during the run + void printIndependentParameters(); + + // Print couplings that are unchanged during the run + void printIndependentCouplings(); + + // Print parameters that are changed event by event + //void printDependentParameters(); // now computed event-by-event (running alphas #373) + + // Print couplings that are changed event by event + //void printDependentCouplings(); // now computed event-by-event (running alphas #373) + + // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; + constexpr int nBsmIndepParam = %(nbsmip)i; + %(hasbsmip)s__device__ constexpr double mdl_bsmIndepParam[nBsmIndepParam] = { %(bsmip)s }; + } + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif + +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + namespace Parameters_dependentCouplings + { + constexpr size_t ndcoup = %(ndcoup)d; // #couplings that vary event by event because they depend on the running alphas QCD +%(idcoup)s + struct DependentCouplings_sv + { +%(dcoupdecl)s + }; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" // e.g. <> +#pragma GCC diagnostic ignored "-Wunused-variable" // e.g. <> +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> +#ifdef MGONGPUCPP_GPUIMPL +#pragma nv_diagnostic push +#pragma nv_diag_suppress 177 // e.g. <> +#endif + __host__ __device__ inline const DependentCouplings_sv computeDependentCouplings_fromG( const fptype_sv& G_sv, const double* bsmIndepParamPtr ) + { +#ifdef MGONGPU_HARDCODE_PARAM + using namespace Parameters; +#else%(eftspecial0)s +#endif + // NB: hardcode cxtype cI(0,1) instead of cxtype (or hardcoded cxsmpl) mdl_complexi (which exists in Parameters) because: + // (1) mdl_complexi is always (0,1); (2) mdl_complexi is undefined in device code; (3) need cxsmpl conversion to cxtype in code below + const cxtype cI( 0., 1. ); + DependentCouplings_sv out; +#if not( defined MGONGPU_CPPSIMD && defined MGONGPU_FPTYPE_FLOAT ) + // Couplings are (scalar, or vector of) doubles, or scalar floats - default implementation + { + const fptype_sv& G = G_sv; + // Model parameters dependent on aS +%(dcoupsetdpar)s + // Model couplings dependent on aS +%(dcoupsetdcoup)s + } +#else + // Couplings are VECTORS OF FLOATS: #439 special handling is needed (variable Gs are vector floats, fixed parameters are scalar doubles) + // Use an explicit loop to avoid <> + // Problems may come e.g. in EFTs from multiplying a vector float (related to aS-dependent G) by a scalar double (aS-independent parameters) + // (NB in pure SM processes this special handling is not needed, but we keep it here for simplicity, see PR #824)%(dcoupoutfptypev2)s + for( int i = 0; i < neppV; i++ ) + { + const fptype& G = G_sv[i]; + // Model parameters dependent on aS +%(dcoupsetdpar2)s + // Model couplings dependent on aS + %(dcoupsetdcoup2)s + }%(dcoupoutdcoup2)s +#endif + return out; + } +#ifdef MGONGPUCPP_GPUIMPL +#pragma GCC diagnostic pop +#pragma nv_diagnostic pop +#endif + } + + //========================================================================== + + namespace Parameters_independentCouplings + { + constexpr size_t nicoup = %(nicoup)d; // #couplings that are fixed for all events because they do not depend on the running alphas QCD +%(iicoup)s + } + + //========================================================================== + +#pragma GCC diagnostic push +#ifndef __clang__ +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> +#endif + // Compute the output couplings (e.g. gc10 and gc11) from the input gs + template + __device__ inline void + G2COUP( const fptype gs[], + fptype couplings[], + const double* bsmIndepParamPtr ) + { + mgDebug( 0, __FUNCTION__ ); + using namespace Parameters_dependentCouplings; + const fptype_sv& gs_sv = G_ACCESS::kernelAccessConst( gs ); + DependentCouplings_sv couplings_sv = computeDependentCouplings_fromG( gs_sv, bsmIndepParamPtr ); +%(dcoupaccessbuffer)s%(dcoupkernelaccess)s%(dcoupcompute)s + mgDebug( 1, __FUNCTION__ ); + return; + } +#pragma GCC diagnostic pop + +} // end namespace mg5amcGpu/mg5amcCpu + +//========================================================================== + +#endif // Parameters_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc new file mode 100644 index 0000000000..05c3b3e0c4 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc @@ -0,0 +1,161 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Oct 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "Bridge.h" +#include "CPPProcess.h" +#include "GpuRuntime.h" + +extern "C" +{ + /** + * The namespace where the Bridge class is taken from. + * + * In the current implementation, two separate shared libraries are created for the GPU/CUDA and CPU/C++ implementations. + * Actually, two shared libraries for GPU and CPU are created for each of the five SIMD implementations on CPUs (none, sse4, avx2, 512y, 512z). + * A single fcreatebridge_ symbol is created in each library with the same name, connected to the appropriate Bridge on CPU or GPU. + * The Fortran MadEvent code is always the same: the choice whether to use a CPU or GPU implementation is done by linking the appropriate library. + * As the names of the two CPU/GPU libraries are the same in the five SIMD implementations, the choice of SIMD is done by setting LD_LIBRARY_PATH. + * + * In a future implementation, a single heterogeneous shared library may be created, with the same interface. + * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. + * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. + */ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + /** + * The floating point precision used in Fortran arrays. + * This is presently hardcoded to double precision (REAL*8). + */ + using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays + //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays + + /** + * Create a Bridge and return its pointer. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param nevtF the pointer to the number of events in the Fortran arrays + * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + */ + void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) + { +#ifdef MGONGPUCPP_GPUIMPL + GpuRuntime::setUp(); +#endif + // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) + // FIXME: disable OMP in Bridge when called from Fortran + *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); + } + + /** + * Delete a Bridge. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + */ + void fbridgedelete_( CppObjectInFortran** ppbridge ) + { + Bridge* pbridge = dynamic_cast*>( *ppbridge ); + if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); + delete pbridge; +#ifdef MGONGPUCPP_GPUIMPL + GpuRuntime::tearDown(); +#endif + } + + /** + * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant alphas) + * @param rndhel the pointer to the input random numbers for helicity selection + * @param rndcol the pointer to the input random numbers for color selection + * @param channelIds the pointer to the input channels i.e. Feynman diagrams to enhance (1 to n: 0 is an invalid value!) + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void fbridgesequence_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ) + { + Bridge* pbridge = dynamic_cast*>( *ppbridge ); + //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); + if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); +#ifdef MGONGPUCPP_GPUIMPL + // Use the device/GPU implementation in the CUDA library + // (there is also a host implementation in this library) + pbridge->gpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); +#else + // Use the host/CPU implementation in the C++ library + // (there is no device implementation in this library) + pbridge->cpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); +#endif + } + + /** + * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++, without multi-channel mode. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant alphas) + * @param rndhel the pointer to the input random numbers for helicity selection + * @param rndcol the pointer to the input random numbers for color selection + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ) + { + //printf("fbridgesequence_nomultichannel_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); + fbridgesequence_( ppbridge, momenta, gs, iflavorVec, rndhel, rndcol, nullptr, mes, selhel, selcol, pgoodHelOnly ); + } + + /** + * Retrieve the number of good helicities for helicity filtering in the Bridge. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param pngoodhel the pointer to the output number of good helicities + * @param pntothel the pointer to the output total number of helicities + */ + void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, + unsigned int* pngoodhel, + unsigned int* pntothel ) + { + Bridge* pbridge = dynamic_cast*>( *ppbridge ); + if( pbridge == 0 ) throw std::runtime_error( "fbridgegetngoodhel_: invalid Bridge address" ); + *pngoodhel = pbridge->nGoodHel(); + *pntothel = pbridge->nTotHel(); + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h new file mode 100644 index 0000000000..e676df0a33 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h @@ -0,0 +1,52 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: Z. Wettersten (Oct 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, A. Thete, A. Valassi (2025). + +#include "Bridge.h" +#include "CPPProcess.h" +#include "GpuRuntime.h" + +#ifndef _FBRIDGE_H_ +#define _FBRIDGE_H_ + +extern "C" +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + using FORTRANFPTYPE = double; + + void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ); + + void fbridgedelete_( CppObjectInFortran** ppbridge ); + + void fbridgesequence_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ); + + void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ); + + void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, unsigned int* pngoodhel, unsigned int* pntothel ); +} +#endif // _FBRIDGE_H_ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc new file mode 100644 index 0000000000..8a4468fb3c --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc @@ -0,0 +1,166 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Feb 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "mgOnGpuConfig.h" + +#include "Bridge.h" +#include "CPPProcess.h" +#include "MemoryBuffers.h" +#include "RamboSamplingKernels.h" +#include "RandomNumberKernels.h" + +//-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + template + class Sampler final : public CppObjectInFortran + { + public: + // Constructor + // @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran arrays + // @param nparF (NEXTERNAL, nexternal.inc) number of external particles in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) + // @param np4F number of momenta components, usually 4, in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) + Sampler( int nevtF, int nparF, int np4F ); + // Destructor + virtual ~Sampler() {} + // Delete copy/move constructors and assignment operators + Sampler( const Sampler& ) = delete; + Sampler( Sampler&& ) = delete; + Sampler& operator=( const Sampler& ) = delete; + Sampler& operator=( Sampler&& ) = delete; + // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta + void samplerHostSequence( FORTRANFPTYPE* fortranMomenta ); + private: + const int m_nevt; // The number of events in each iteration + int m_iiter; // The iteration counter (for random number seeding) +#ifndef MGONGPUCPP_GPUIMPL + HostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers + HostBufferMomenta m_hstMomenta; // Memory buffers for momenta + HostBufferWeights m_hstWeights; // Memory buffers for sampling weights +#else + PinnedHostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers + PinnedHostBufferMomenta m_hstMomenta; // Memory buffers for momenta + PinnedHostBufferWeights m_hstWeights; // Memory buffers for sampling weights +#endif + std::unique_ptr m_prnk; // The appropriate RandomNumberKernel + std::unique_ptr m_prsk; // The appropriate SamplingKernel + // HARDCODED DEFAULTS + static constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) + }; + + template + Sampler::Sampler( int nevtF, int nparF, int np4F ) + : m_nevt( nevtF ) + , m_iiter( 0 ) + , m_hstRndmom( nevtF ) + , m_hstMomenta( nevtF ) + , m_hstWeights( nevtF ) + , m_prnk( new CommonRandomNumberKernel( m_hstRndmom ) ) + , m_prsk( new RamboSamplingKernelHost( energy, m_hstRndmom, m_hstMomenta, m_hstWeights, nevtF ) ) + { + if( nparF != CPPProcess::npar ) throw std::runtime_error( "Sampler constructor: npar mismatch" ); + if( np4F != CPPProcess::np4 ) throw std::runtime_error( "Sampler constructor: np4 mismatch" ); + std::cout << "WARNING! Instantiate host Sampler (nevt=" << m_nevt << ")" << std::endl; + } + + // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta + template + void Sampler::samplerHostSequence( FORTRANFPTYPE* fortranMomenta ) + { + std::cout << "Iteration #" << m_iiter + 1 << std::endl; + // === STEP 1 OF 3 + // --- 1a. Seed rnd generator (to get same results on host and device in curand) + // [NB This should not be necessary using the host API: "Generation functions + // can be called multiple times on the same generator to generate successive + // blocks of results. For pseudorandom generators, multiple calls to generation + // functions will yield the same result as a single call with a large size."] + // *** NB! REMEMBER THAT THE FORTRAN SAMPLER ALWAYS USES COMMON RANDOM NUMBERS! *** + constexpr unsigned long long seed = 20200805; + m_prnk->seedGenerator( seed + m_iiter ); + m_iiter++; + // --- 1b. Generate all relevant numbers to build nevt events (i.e. nevt phase space points) on the host + m_prnk->generateRnarray(); + //std::cout << "Got random numbers" << std::endl; + // === STEP 2 OF 3 + // --- 2a. Fill in momenta of initial state particles on the device + m_prsk->getMomentaInitial(); + //std::cout << "Got initial momenta" << std::endl; + // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device + // (i.e. map random numbers to final-state particle momenta for each of nevt events) + m_prsk->getMomentaFinal(); + //std::cout << "Got final momenta" << std::endl; + // --- 2c. TransposeC2F + hst_transposeMomentaC2F( m_hstMomenta.data(), fortranMomenta, m_nevt ); + } +} + +//-------------------------------------------------------------------------- + +extern "C" +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + /** + * The floating point precision used in Fortran arrays. + * This is presently hardcoded to double precision (REAL*8). + */ + using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays + //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays + + /** + * Create a Sampler and return its pointer. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) + * @param nevtF the pointer to the number of events in the Fortran arrays + * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + */ + void fsamplercreate_( CppObjectInFortran** ppsampler, const int* pnevtF, const int* pnparF, const int* pnp4F ) + { + *ppsampler = new Sampler( *pnevtF, *pnparF, *pnp4F ); + } + + /** + * Delete a Sampler. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) + */ + void fsamplerdelete_( CppObjectInFortran** ppsampler ) + { + Sampler* psampler = dynamic_cast*>( *ppsampler ); + if( psampler == 0 ) throw std::runtime_error( "fsamplerdelete_: invalid Sampler address" ); + delete psampler; + } + + /** + * Execute the matrix-element calculation "sequence" via a Sampler on GPU/CUDA or CUDA/C++. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) + * @param momenta the pointer to the input 4-momenta + * @param mes the pointer to the output matrix elements + */ + void fsamplersequence_( CppObjectInFortran** ppsampler, FORTRANFPTYPE* momenta ) + { + Sampler* psampler = dynamic_cast*>( *ppsampler ); + if( psampler == 0 ) throw std::runtime_error( "fsamplersequence_: invalid Sampler address" ); + // Use the host/CPU implementation (there is no device implementation) + psampler->samplerHostSequence( momenta ); + } +} + +//-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h new file mode 100644 index 0000000000..bcd3990bed --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h @@ -0,0 +1,200 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: S. Roiser (Feb 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== + +#ifndef MASSLESS_RAMBO_H +#define MASSLESS_RAMBO_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +#include "CPPProcess.h" + +#include +#include +#include + +// Simplified rambo version for 2 to N (with N>=2) processes with massless particles +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +namespace massless_rambo { + + constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) + constexpr int npari = CPPProcess::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- + constexpr int nparf = CPPProcess::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- + constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- + + //-------------------------------------------------------------------------- + + // Fill in the momenta of the initial particles + // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] + template + __host__ __device__ void + ramboGetMomentaInitial( const fptype energy, // input: energy + fptype* momenta ) // output: momenta for one event or for a set of events + { + const fptype energy1 = energy / 2; + const fptype energy2 = energy / 2; + const fptype mom = energy / 2; + M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 0 ) = energy1; + M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 0 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 0 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 0 ) = mom; + M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 1 ) = energy2; + M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 1 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 1 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 1 ) = -mom; + } + + //-------------------------------------------------------------------------- + + // Fill in the momenta of the final particles using the RAMBO algorithm + // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] + template + __host__ __device__ void + ramboGetMomentaFinal( const fptype energy, // input: energy + const fptype* rndmom, // input: random numbers in [0,1] for one event or for a set of events + fptype* momenta, // output: momenta for one event or for a set of events + fptype* wgts ) // output: weights for one event or for a set of events + { + /**************************************************************************** + * rambo * + * ra(ndom) m(omenta) b(eautifully) o(rganized) * + * * + * a democratic multi-particle phase space generator * + * authors: s.d. ellis, r. kleiss, w.j. stirling * + * this is version 1.0 - written by r. kleiss * + * -- adjusted by hans kuijf, weights are logarithmic (1990-08-20) * + * -- adjusted by madgraph@sheffield_gpu_hackathon team (2020-07-29) * + * * + ****************************************************************************/ + + // output weight + fptype& wt = W_ACCESS::kernelAccess( wgts ); + + // AV special case nparf==1 (issue #358) + if constexpr( nparf == 1 ) + { + static bool first = true; + if( first ) + { +#ifdef MGONGPUCPP_GPUIMPL + if constexpr( M_ACCESS::isOnDevice() ) // avoid + { + const int ievt0 = 0; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid + if( ievt == ievt0 ) + printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); + } + else +#endif + { + printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); + } + first = false; + } + const int iparf = 0; + for( int i4 = 0; i4 < np4; i4++ ) + { + M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = 0; + for( int ipari = 0; ipari < npari; ipari++ ) + { + M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) += M_ACCESS::kernelAccessIp4Ipar( momenta, i4, ipari ); + } + } + wt = 1; + return; + } + + // initialization step: factorials for the phase space weight + const fptype twopi = 8. * atan( 1. ); + const fptype po2log = log( twopi / 4. ); + fptype z[nparf]; + if constexpr( nparf > 1 ) // avoid build warning on clang (related to #358) + z[1] = po2log; + for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = z[kpar - 1] + po2log - 2. * log( fptype( kpar - 1 ) ); + for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = ( z[kpar] - log( fptype( kpar ) ) ); + + // generate n massless momenta in infinite phase space + fptype q[nparf][np4]; + for( int iparf = 0; iparf < nparf; iparf++ ) + { + const fptype r1 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 0, iparf ); + const fptype r2 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 1, iparf ); + const fptype r3 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 2, iparf ); + const fptype r4 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 3, iparf ); + const fptype c = 2. * r1 - 1.; + const fptype s = sqrt( 1. - c * c ); + const fptype f = twopi * r2; + q[iparf][0] = -log( r3 * r4 ); + q[iparf][3] = q[iparf][0] * c; + q[iparf][2] = q[iparf][0] * s * cos( f ); + q[iparf][1] = q[iparf][0] * s * sin( f ); + } + + // calculate the parameters of the conformal transformation + fptype r[np4]; + fptype b[np4 - 1]; + for( int i4 = 0; i4 < np4; i4++ ) r[i4] = 0.; + for( int iparf = 0; iparf < nparf; iparf++ ) + { + for( int i4 = 0; i4 < np4; i4++ ) r[i4] = r[i4] + q[iparf][i4]; + } + const fptype rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); + for( int i4 = 1; i4 < np4; i4++ ) b[i4 - 1] = -r[i4] / rmas; + const fptype g = r[0] / rmas; + const fptype a = 1. / ( 1. + g ); + const fptype x0 = energy / rmas; + + // transform the q's conformally into the p's (i.e. the 'momenta') + for( int iparf = 0; iparf < nparf; iparf++ ) + { + fptype bq = b[0] * q[iparf][1] + b[1] * q[iparf][2] + b[2] * q[iparf][3]; + for( int i4 = 1; i4 < np4; i4++ ) + { + M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = x0 * ( q[iparf][i4] + b[i4 - 1] * ( q[iparf][0] + a * bq ) ); + } + M_ACCESS::kernelAccessIp4Ipar( momenta, 0, iparf + npari ) = x0 * ( g * q[iparf][0] + bq ); + } + + // calculate weight (NB return log of weight) + wt = po2log; + if( nparf != 2 ) wt = ( 2. * nparf - 4. ) * log( energy ) + z[nparf - 1]; + +#ifndef MGONGPUCPP_GPUIMPL + // issue warnings if weight is too small or too large + static int iwarn[5] = { 0, 0, 0, 0, 0 }; + if( wt < -180. ) + { + if( iwarn[0] <= 5 ) std::cout << "Too small wt, risk for underflow: " << wt << std::endl; + iwarn[0] = iwarn[0] + 1; + } + if( wt > 174. ) + { + if( iwarn[1] <= 5 ) std::cout << "Too large wt, risk for overflow: " << wt << std::endl; + iwarn[1] = iwarn[1] + 1; + } +#endif + + // return for weighted massless momenta + // nothing else to do in this event if all particles are massless (nm==0) + + return; + } + + //-------------------------------------------------------------------------- +} +} + +#endif // MASSLESS_RAMBO_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h new file mode 100644 index 0000000000..09c76a6bd5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h @@ -0,0 +1,296 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jul 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCONFIG_H +#define MGONGPUCONFIG_H 1 + +// Is this a GPU (CUDA, HIP) or CPU implementation? +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#define MGONGPUCPP_GPUIMPL cuda +#elif defined __HIPCC__ +#define MGONGPUCPP_GPUIMPL hip +#include "hip/hip_runtime.h" // needed for blockDim, blockIdx, threadIdx: better in mgOnGpuConfig.h than in GpuAbstraction.h +#else +#undef MGONGPUCPP_GPUIMPL +#endif + +// Make sure that __HIP_PLATFORM_NVIDIA__ is undefined +// (__HIP_PLATFORM_AMD__ is defined by hipcc or in HiprandRandomNumberKernel.cc) +#undef __HIP_PLATFORM_NVIDIA__ // disable hiprand for NVidia (curand) + +// ** NB1 Throughputs (e.g. 6.8E8) are events/sec for "./gcheck.exe -p 65536 128 12" +// ** NB2 Baseline on b7g47n0004 fluctuates (probably depends on load on other VMs) + +// Choose if curand is supported for generating random numbers +// For HIP, by default, do not allow curand to be used (hiprand or common random numbers will be used instead) +// For both CUDA and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_CURAND +// (there exist CUDA installations, e.g. using the HPC package, which do not include curand - see PR #784 and #785) +#if defined __HIPCC__ +#define MGONGPU_HAS_NO_CURAND 1 +#else +//#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#undef MGONGPU_HAS_NO_CURAND // default +////#define MGONGPU_HAS_NO_CURAND 1 +//#else +//#undef MGONGPU_HAS_NO_CURAND // default +////#define MGONGPU_HAS_NO_CURAND 1 +//#endif +#endif + +// Choose if hiprand is supported for generating random numbers +// For CUDA, by default, do not allow hiprand to be used (curand or common random numbers will be used instead) +// For both HIP and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_HIPRAND +// (there may exist HIP installations which do not include hiprand?) +#if defined __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#define MGONGPU_HAS_NO_HIPRAND 1 +#else +//#ifdef __HIPCC__ +//#undef MGONGPU_HAS_NO_HIPRAND // default +////#define MGONGPU_HAS_NO_HIPRAND 1 +//#else +//#undef MGONGPU_HAS_NO_HIPRAND // default +////#define MGONGPU_HAS_NO_HIPRAND 1 +//#endif +#endif + +// Choose floating point precision (for everything but color algebra #537) +// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE_FLOAT, nothing happens (issue #167) +#if not defined MGONGPU_FPTYPE_DOUBLE and not defined MGONGPU_FPTYPE_FLOAT +// Floating point precision (CHOOSE ONLY ONE) +#define MGONGPU_FPTYPE_DOUBLE 1 // default +//#define MGONGPU_FPTYPE_FLOAT 1 // 2x faster +#endif + +// Choose floating point precision (for color algebra alone #537) +// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE2_FLOAT, nothing happens (issue #167) +#if not defined MGONGPU_FPTYPE2_DOUBLE and not defined MGONGPU_FPTYPE2_FLOAT +// Floating point precision (CHOOSE ONLY ONE) +#define MGONGPU_FPTYPE2_DOUBLE 1 // default +//#define MGONGPU_FPTYPE2_FLOAT 1 // 2x faster +#endif + +// Choose whether to inline all HelAmps functions +// This optimization can gain almost a factor 4 in C++, similar to -flto (issue #229) +// By default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_INLINE_HELAMPS +//#undef MGONGPU_INLINE_HELAMPS // default +////#define MGONGPU_INLINE_HELAMPS 1 + +// Choose whether to hardcode the cIPD physics parameters rather than reading them from user cards +// This optimization can gain 20%% in CUDA in eemumu (issue #39) +// By default, do not hardcode, but allow this macro to be set from outside with e.g. -DMGONGPU_HARDCODE_PARAM +// ** NB: The option to use hardcoded cIPD physics parameters is supported again even now when alphas is running (#373) +// ** NB: Note however that it now only refers to cIPD parameters (cIPC parameters are always accessed through global memory) +//#undef MGONGPU_HARDCODE_PARAM // default +////#define MGONGPU_HARDCODE_PARAM 1 + +/* clang-format off */ +// Complex type in CUDA: thrust or cucomplex or cxsmpl (CHOOSE ONLY ONE) +// (NB THIS IS MGONGPU_*CU*CXTYPE_xxx) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#define MGONGPU_CUCXTYPE_THRUST 1 // default (~1.15E9/double, ~3.2E9/float) +//#define MGONGPU_CUCXTYPE_CUCOMPLEX 1 // ~10 percent slower (1.03E9/double, ~2.8E9/float) +//#define MGONGPU_CUCXTYPE_CXSMPL 1 // ~10 percent slower (1.00E9/double, ~2.9E9/float) + +// Complex type in HIP: cxsmpl (ONLY ONE OPTION POSSIBLE? #810) +// (NB THIS IS MGONGPU_*HIP*CXTYPE_xxx) +#elif defined __HIPCC__ +#define MGONGPU_HIPCXTYPE_CXSMPL 1 // default for HIP + +// Complex type in C++: std::complex or cxsmpl (CHOOSE ONLY ONE) +// (NB THIS IS MGONGPU_*CPP*CXTYPE_xxx) +#else +//#define MGONGPU_CPPCXTYPE_STDCOMPLEX 1 // ~8 percent slower on float, same on double (5.1E6/double, 9.4E6/float) +#define MGONGPU_CPPCXTYPE_CXSMPL 1 // new default (5.1E6/double, 10.2E6/float) +#endif + +// Choose if cuBLAS and hipBLAS are supported for generating random numbers +// For both CUDA and HIP, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_BLAS +// (there may exist CUDA/HIP installations, e.g. using the HPC package, which do not include cuBLAS/hipBLAS?) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#undef MGONGPU_HAS_NO_BLAS // default +////#define MGONGPU_HAS_NO_BLAS 1 +#elif defined __HIPCC__ +//#undef MGONGPU_HAS_NO_BLAS // default +////#define MGONGPU_HAS_NO_BLAS 1 +#else +#define MGONGPU_HAS_NO_BLAS 1 +#endif + +// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#undef MGONGPU_NSIGHT_DEBUG // default in CUDA +//#define MGONGPU_NSIGHT_DEBUG 1 // CURRENTLY NO LONGER SUPPORTED! +#else +#undef MGONGPU_NSIGHT_DEBUG // only option in HIP or C++ +#endif /* clang-format on */ + +// SANITY CHECKS (floating point precision for everything but color algebra #537) +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE_DOUBLE or defined MGONGPU_FPTYPE_FLOAT +#endif + +// SANITY CHECKS (floating point precision for color algebra alone #537) +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE2_DOUBLE or defined MGONGPU_FPTYPE2_FLOAT +#endif +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You cannot use double precision for color algebra and single precision elsewhere +#endif + +// SANITY CHECKS (CUDA complex number implementation) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CUCOMPLEX +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX for CUDA +#elif defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CXSMPL for CUDA +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX and defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE OF MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL for CUDA +#endif +#endif + +// SANITY CHECKS (C++ complex number implementation) +#ifndef MGONGPUCPP_GPUIMPL +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX and defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL for C++ +#endif +#endif + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu +{ + + // --- Type definitions + + // Floating point type (for everything but color algebra #537): fptype +#if defined MGONGPU_FPTYPE_DOUBLE + typedef double fptype; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE_FLOAT + typedef float fptype; // single precision (4 bytes, fp32) +#endif + + // Floating point type (for color algebra alone #537): fptype2 +#if defined MGONGPU_FPTYPE2_DOUBLE + typedef double fptype2; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE2_FLOAT + typedef float fptype2; // single precision (4 bytes, fp32) +#endif + + // --- Platform-specific software implementation details + + // Maximum number of blocks per grid + // ** NB Some arrays of pointers will be allocated statically to fit all these blocks + // ** (the actual memory for each block will then be allocated dynamically only for existing blocks) + //const int nbpgMAX = 2048; + + // Maximum number of threads per block + //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? + const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" + + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) + // Only needed for C++ code but can be enforced also in NVCC builds of C++ code using CUDA>=11.2 and C++17 (#318, #319, #333) +#ifndef MGONGPUCPP_GPUIMPL + constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) +#endif + +} + +// Expose typedefs and operators outside the namespace +using mgOnGpu::fptype; +using mgOnGpu::fptype2; + +// Undefine ARM_NEON (hack for cppnone on Apple silicon ARM) +#ifdef MGONGPU_NOARMNEON +#undef __ARM_NEON +#endif + +// C++ SIMD vectorization width (this will be used to set neppV) +#ifdef MGONGPUCPP_GPUIMPL // CUDA and HIP implementations have no SIMD +#undef MGONGPU_CPPSIMD +#elif defined __AVX512VL__ && defined MGONGPU_PVW512 // C++ "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 8 +#else +#define MGONGPU_CPPSIMD 16 +#endif +#elif defined __AVX512VL__ // C++ "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif +#elif defined __AVX2__ // C++ "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif +#elif defined __SSE4_2__ // C++ "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [Power9 default] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 +#else +#define MGONGPU_CPPSIMD 4 +#endif +#elif defined __ARM_NEON // C++ "sse4" ARM NEON (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [ARM default] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 +#else +#define MGONGPU_CPPSIMD 4 +#endif +#else // C++ "none" i.e. no SIMD +#undef MGONGPU_CPPSIMD +#endif + +/* clang-format off */ +// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation [NB: CURRENTLY NO LONGER SUPPORTED!] +// Arguments (not used so far): text is __FUNCTION__, code is 0 (start) or 1 (end) +//#if defined __CUDACC__ && defined MGONGPU_NSIGHT_DEBUG // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#define mgDebugDeclare() __shared__ float mgDebugCounter[mgOnGpu::ntpbMAX]; +//#define mgDebugInitialise() { mgDebugCounter[threadIdx.x] = 0; } +//#define mgDebug( code, text ) { mgDebugCounter[threadIdx.x] += 1; } +//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%%f\n", mgDebugCounter[threadIdx.x] ); } +//#else +#define mgDebugDeclare() /*noop*/ +#define mgDebugInitialise() /*noop*/ +#define mgDebug( code, text ) /*noop*/ +#define mgDebugFinalise() /*noop*/ +//#endif /* clang-format on */ + +// Define empty CUDA/HIP declaration specifiers for C++ +#ifndef MGONGPUCPP_GPUIMPL +#define __global__ +#define __host__ +#define __device__ +#endif + +// For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two (https://stackoverflow.com/a/108360) +inline constexpr bool +ispoweroftwo( int n ) +{ + return ( n > 0 ) && !( n & ( n - 1 ) ); +} + +// Compiler version support (#96): require nvcc from CUDA >= 11.2, e.g. to use C++17 (see #333) +#ifdef __NVCC__ +#if( __CUDACC_VER_MAJOR__ < 11 ) || ( __CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ < 2 ) +#error Unsupported CUDA version: please use CUDA >= 11.2 +#endif +#endif + +// Compiler version support (#96): require clang >= 11 +#if defined __clang__ +#if( __clang_major__ < 11 ) +#error Unsupported clang version: please use clang >= 11 +#endif +// Compiler version support (#96): require gcc >= 9.3, e.g. for some OMP issues (see #269) +// [NB skip this check for the gcc toolchain below clang or icx (TEMPORARY? #355)] +#elif defined __GNUC__ +#if( __GNUC__ < 9 ) || ( __GNUC__ == 9 && __GNUC_MINOR__ < 3 ) +#error Unsupported gcc version: please gcc >= 9.3 +#endif +#endif + +#endif // MGONGPUCONFIG_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h new file mode 100644 index 0000000000..d6e1d02e05 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h @@ -0,0 +1,745 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022, based on earlier work by D. Smith) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCXTYPES_H +#define MGONGPUCXTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +#include + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) HEADERS +//========================================================================== + +#include + +// Complex type in cuda: thrust or cucomplex or cxsmpl +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_THRUST +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) +#include +#pragma clang diagnostic pop +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX +#include +#elif not defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL +#endif +// Complex type in HIP: cxsmpl +#elif defined __HIPCC__ +#if not defined MGONGPU_HIPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_HIPCXTYPE_CXSMPL +#endif +#else +// Complex type in c++ or HIP: std::complex or cxsmpl +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX +#include +#elif not defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL +#endif +#endif + +//========================================================================== +// COMPLEX TYPES: INSTRUMENTED CUCOMPLEX CLASS (cucomplex) +//========================================================================== + +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_CUCOMPLEX +namespace mg5amcGpu +{ +#if defined MGONGPU_FPTYPE_DOUBLE + class cucomplex + { + public: + __host__ __device__ cucomplex( const double& r = 0, const double& i = 0 ) + : m_ri( make_cuDoubleComplex( r, i ) ) {} + __host__ __device__ constexpr cucomplex( const cuDoubleComplex& ri ) + : m_ri( ri ) {} + //__host__ __device__ operator cuDoubleComplex&() { return m_ri; } + __host__ __device__ constexpr operator cuDoubleComplex() const { return m_ri; } + __host__ __device__ double real() const { return cuCreal( m_ri ); } + __host__ __device__ double imag() const { return cuCimag( m_ri ); } + inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) + { + m_ri = cuCadd( m_ri, c ); + return *this; + } + inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) + { + m_ri = cuCsub( m_ri, c ); + return *this; + } + private: + cuDoubleComplex m_ri; + }; +#elif defined MGONGPU_FPTYPE_FLOAT + class cucomplex + { + public: + __host__ __device__ cucomplex( const float& r = 0, const float& i = 0 ) + : m_ri( make_cuFloatComplex( r, i ) ) {} + __host__ __device__ constexpr cucomplex( const cuFloatComplex& ri ) + : m_ri( ri ) {} + //__host__ __device__ operator cuFloatComplex&() { return m_ri; } + __host__ __device__ constexpr operator cuFloatComplex() const { return m_ri; } + __host__ __device__ float real() const { return cuCrealf( m_ri ); } + __host__ __device__ float imag() const { return cuCimagf( m_ri ); } + inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) + { + m_ri = cuCaddf( m_ri, c ); + return *this; + } + inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) + { + m_ri = cuCsubf( m_ri, c ); + return *this; + } + private: + cuFloatComplex m_ri; + }; +#endif +} +#endif +#endif + +//========================================================================== +// COMPLEX TYPES: SIMPLE COMPLEX CLASS (cxsmpl) +//========================================================================== + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu /* clang-format off */ +{ + // The number of floating point types in a complex type (real, imaginary) + constexpr int nx2 = 2; + + // --- Type definition (simple complex type derived from cxtype_v) + template + class cxsmpl + { + public: + __host__ __device__ constexpr cxsmpl() : m_real( 0 ), m_imag( 0 ) {} + cxsmpl( const cxsmpl& ) = default; + cxsmpl( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl( const FP& r, const FP& i = 0 ) : m_real( r ), m_imag( i ) {} + __host__ __device__ constexpr cxsmpl( const std::complex& c ) : m_real( c.real() ), m_imag( c.imag() ) {} + cxsmpl& operator=( const cxsmpl& ) = default; + cxsmpl& operator=( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl& operator+=( const cxsmpl& c ) { m_real += c.real(); m_imag += c.imag(); return *this; } + __host__ __device__ constexpr cxsmpl& operator-=( const cxsmpl& c ) { m_real -= c.real(); m_imag -= c.imag(); return *this; } + __host__ __device__ constexpr const FP& real() const { return m_real; } + __host__ __device__ constexpr const FP& imag() const { return m_imag; } + template __host__ __device__ constexpr operator cxsmpl() const { return cxsmpl( m_real, m_imag ); } +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef MGONGPU_CUCXTYPE_THRUST + template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX + __host__ __device__ constexpr operator mg5amcGpu::cucomplex() const { return mg5amcGpu::cucomplex( m_real, m_imag ); } +#endif +#else +#ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX + template __host__ __device__ constexpr operator std::complex() const { return std::complex( m_real, m_imag ); } +#endif +#endif + private: + FP m_real, m_imag; // RI + }; + + template + constexpr // (NB: now valid code? in the past this failed as "a constexpr function cannot have a nonliteral return type mgOnGpu::cxsmpl") + inline __host__ __device__ cxsmpl + conj( const cxsmpl& c ) + { + return cxsmpl( c.real(), -c.imag() ); + } +} /* clang-format on */ + +// Expose the cxsmpl class outside the namespace +using mgOnGpu::cxsmpl; + +// Printout to stream for user defined types +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + template + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxsmpl& c ) + { + //out << std::complex( c.real(), c.imag() ); + out << "(" << c.real() << ", " << c.imag() << ")"; // add a space after the comma + return out; + } + + // Operators for cxsmpl + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl a ) + { + return a; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a ) + { + return cxsmpl( -a.real(), -a.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() + b.real(), a.imag() + b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) + b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() - b.real(), a.imag() - b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) - b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const double& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const double& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const cxsmpl& b ) + { + FP bnorm = b.real() * b.real() + b.imag() * b.imag(); + return cxsmpl( ( a.real() * b.real() + a.imag() * b.imag() ) / bnorm, + ( a.imag() * b.real() - a.real() * b.imag() ) / bnorm ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) / b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const FP& b ) + { + return a + cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const FP& b ) + { + return a - cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const FP& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const FP& b ) + { + return a / cxsmpl( b, 0 ); + } +} + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // --- Type definitions (complex type: cxtype) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_THRUST + typedef thrust::complex cxtype; +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX + typedef cucomplex cxtype; +#else + typedef cxsmpl cxtype; +#endif +#else // c++ +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX + typedef std::complex cxtype; +#else + typedef cxsmpl cxtype; +#endif +#endif + + // SANITY CHECK: memory access may be based on casts of fptype[2] to cxtype (e.g. for wavefunctions) + static_assert( sizeof( cxtype ) == mgOnGpu::nx2 * sizeof( fptype ), "sizeof(cxtype) is not 2*sizeof(fptype)" ); +} + +// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) +// DO NOT expose typedefs and operators outside the namespace +//using mgOnGpu::cxtype; + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL + + //------------------------------ + // CUDA or C++ - using cxsmpl + //------------------------------ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // cxsmpl constructor + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return c.real(); // cxsmpl::real() + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return c.imag(); // cxsmpl::imag() + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( cxsmpl ) + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +#endif // #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL + + //========================================================================== + +#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST // cuda + thrust (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) + + //------------------------------ + // CUDA - using thrust::complex + //------------------------------ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // thrust::complex constructor + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return c.real(); // thrust::complex::real() + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return c.imag(); // thrust::complex::imag() + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( thrust::complex ) + } + + inline __host__ __device__ const cxtype& + cxmake( const cxtype& c ) + { + return c; + } + +#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST + + //========================================================================== + +#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX // cuda + cucomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) + + //------------------------------ + // CUDA - using cuComplex + //------------------------------ + +#if defined MGONGPU_FPTYPE_DOUBLE // cuda + cucomplex + double + + //+++++++++++++++++++++++++ + // cuDoubleComplex ONLY + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return make_cuDoubleComplex( r, i ); + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return cuCreal( c ); // returns by value + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return cuCimag( c ); // returns by value + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const cxtype& b ) + { + return cuCadd( a, b ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const cxtype& b ) + { + return cuCsub( a, b ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const cxtype& b ) + { + return cuCmul( a, b ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const cxtype& b ) + { + return cuCdiv( a, b ); + } + + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + //out << std::complex( cxreal( c ), cximag( c ) ); + out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma + return out; + } + +#elif defined MGONGPU_FPTYPE_FLOAT // cuda + cucomplex + float + + //+++++++++++++++++++++++++ + // cuFloatComplex ONLY + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return make_cuFloatComplex( r, i ); + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return cuCrealf( c ); // returns by value + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return cuCimagf( c ); // returns by value + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const cxtype& b ) + { + return cuCaddf( a, b ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const cxtype& b ) + { + return cuCsubf( a, b ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const cxtype& b ) + { + return cuCmulf( a, b ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const cxtype& b ) + { + return cuCdivf( a, b ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cucomplex (cast double-to-float) + { + return cxmake( (fptype)c.real(), (fptype)c.imag() ); + } + + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + //out << std::complex( cxreal( c ), cximag( c ) ); + out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma + return out; + } + +#endif + + //+++++++++++++++++++++++++ + // cuDoubleComplex OR + // cuFloatComplex + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + operator+( const cxtype a ) + { + return a; + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a ) + { + return cxmake( -cxreal( a ), -cximag( a ) ); + } + + inline __host__ __device__ cxtype + operator+( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) + b; + } + + inline __host__ __device__ cxtype + operator-( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) - b; + } + + inline __host__ __device__ cxtype + operator*( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) * b; + } + + inline __host__ __device__ cxtype + operator/( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) / b; + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const fptype& b ) + { + return a + cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const fptype& b ) + { + return a - cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const fptype& b ) + { + return a * cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const fptype& b ) + { + return a / cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return cxmake( cxreal( c ), -cximag( c ) ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cucomplex (float-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX + + //========================================================================== + +#if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX // c++/hip + stdcomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) + + //------------------------------ + // C++ - using std::complex + //------------------------------ + + inline cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // std::complex constructor + } + + inline fptype + cxreal( const cxtype& c ) + { + return c.real(); // std::complex::real() + } + + inline fptype + cximag( const cxtype& c ) + { + return c.imag(); // std::complex::imag() + } + + inline cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( std::complex ) + } + + inline const cxtype& + cxmake( const cxtype& c ) // std::complex to std::complex (float-to-float or double-to-double) + { + return c; + } + +#if defined MGONGPU_FPTYPE_FLOAT + inline cxtype + cxmake( const std::complex& c ) // std::complex to std::complex (cast double-to-float) + { + return cxmake( (fptype)c.real(), (fptype)c.imag() ); + } +#endif + +#endif // #if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX + + //========================================================================== + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +} // end namespace mg5amcGpu/mg5amcCpu + +//========================================================================== +// COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] + // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined + // It is now always included in the code because it is needed also to access an fptype wavefunction buffer as a cxtype + class cxtype_ref + { + public: + cxtype_ref() = delete; + cxtype_ref( const cxtype_ref& ) = delete; + cxtype_ref( cxtype_ref&& ) = default; // copy const refs + __host__ __device__ cxtype_ref( fptype& r, fptype& i ) + : m_preal( &r ), m_pimag( &i ) {} // copy (create from) const refs + cxtype_ref& operator=( const cxtype_ref& ) = delete; + //__host__ __device__ cxtype_ref& operator=( cxtype_ref&& c ) {...} // REMOVED! Should copy refs or copy values? No longer needed in cxternary + __host__ __device__ cxtype_ref& operator=( const cxtype& c ) + { + *m_preal = cxreal( c ); + *m_pimag = cximag( c ); + return *this; + } // copy (assign) non-const values + __host__ __device__ operator cxtype() const { return cxmake( *m_preal, *m_pimag ); } + private: + fptype* const m_preal; // const pointer to non-const fptype R + fptype* const m_pimag; // const pointer to non-const fptype I + }; + + // Printout to stream for user defined types + inline __host__ __device__ std::ostream& + operator<<( std::ostream& out, const cxtype_ref& c ) + { + out << (cxtype)c; + return out; + } + +} // end namespace mg5amcGpu/mg5amcCpu + +//========================================================================== + +#endif // MGONGPUCXTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h new file mode 100644 index 0000000000..a2489d05d5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h @@ -0,0 +1,102 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUFPTYPES_H +#define MGONGPUFPTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include +#include + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL // cuda +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //========================================================================== + +#ifdef MGONGPUCPP_GPUIMPL // cuda + + //------------------------------ + // Floating point types - Cuda + //------------------------------ + + /* + inline __host__ __device__ fptype + fpmax( const fptype& a, const fptype& b ) + { + return max( a, b ); + } + + inline __host__ __device__ fptype + fpmin( const fptype& a, const fptype& b ) + { + return min( a, b ); + } + */ + + inline __host__ __device__ const fptype& + fpmax( const fptype& a, const fptype& b ) + { + return ( ( b < a ) ? a : b ); + } + + inline __host__ __device__ const fptype& + fpmin( const fptype& a, const fptype& b ) + { + return ( ( a < b ) ? a : b ); + } + + inline __host__ __device__ fptype + fpsqrt( const fptype& f ) + { +#if defined MGONGPU_FPTYPE_FLOAT + // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__SINGLE.html + return sqrtf( f ); +#else + // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__DOUBLE.html + return sqrt( f ); +#endif + } + +#endif // #ifdef MGONGPUCPP_GPUIMPL + + //========================================================================== + +#ifndef MGONGPUCPP_GPUIMPL + + //------------------------------ + // Floating point types - C++ + //------------------------------ + + inline const fptype& + fpmax( const fptype& a, const fptype& b ) + { + return std::max( a, b ); + } + + inline const fptype& + fpmin( const fptype& a, const fptype& b ) + { + return std::min( a, b ); + } + + inline fptype + fpsqrt( const fptype& f ) + { + return std::sqrt( f ); + } + +#endif // #ifndef MGONGPUCPP_GPUIMPL + + //========================================================================== + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h new file mode 100644 index 0000000000..74d93f05a3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h @@ -0,0 +1,932 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Nov 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi, Z. Wettersten (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUVECTORS_H +#define MGONGPUVECTORS_H 1 + +#include "mgOnGpuCxtypes.h" +#include "mgOnGpuFptypes.h" + +#include + +//========================================================================== + +//------------------------------ +// Vector types - C++ +//------------------------------ + +#ifdef __clang__ +// If set: return a pair of (fptype&, fptype&) by non-const reference in cxtype_v::operator[] +// This is forbidden in clang ("non-const reference cannot bind to vector element") +// See also https://stackoverflow.com/questions/26554829 +//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // clang test (compilation fails also on clang 12.0, issue #182) +#undef MGONGPU_HAS_CPPCXTYPEV_BRK // clang default +#elif defined __INTEL_COMPILER +//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // icc default? +#undef MGONGPU_HAS_CPPCXTYPEV_BRK // icc test +#else +#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // gcc default +//#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) +#endif + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#ifdef MGONGPU_CPPSIMD + + const int neppV = MGONGPU_CPPSIMD; + + // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) + static_assert( mgOnGpu::cppAlign % ( neppV * sizeof( fptype ) ) == 0 ); + + // SANITY CHECK: check that neppV is a power of two + static_assert( ispoweroftwo( neppV ), "neppV is not a power of 2" ); + + // --- Type definition (using vector compiler extensions: need -march=...) + // For gcc: https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html + // For clang: https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors +#ifdef __clang__ + typedef fptype fptype_v __attribute__( ( ext_vector_type( neppV ) ) ); // RRRR +#else + typedef fptype fptype_v __attribute__( ( vector_size( neppV * sizeof(fptype) ), aligned( neppV * sizeof(fptype) ) ) ); // RRRR +#endif + + // Mixed fptypes #537: float for color algebra and double elsewhere +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int neppV2 = MGONGPU_CPPSIMD * 2; + static_assert( mgOnGpu::cppAlign % ( neppV2 * sizeof( fptype2 ) ) == 0 ); + static_assert( ispoweroftwo( neppV2 ), "neppV2 is not a power of 2" ); +#ifdef __clang__ + typedef fptype2 fptype2_v __attribute__( ( ext_vector_type( neppV2 ) ) ); // RRRRRRRR +#else + typedef fptype2 fptype2_v __attribute__( ( vector_size( neppV2 * sizeof( fptype2 ) ), aligned( neppV2 * sizeof( fptype2 ) ) ) ); // RRRRRRRR +#endif +#else + typedef fptype_v fptype2_v; +#endif + + // --- Type definition (using vector compiler extensions: need -march=...) + class cxtype_v // no need for "class alignas(2*sizeof(fptype_v)) cxtype_v" + { + public: + // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) + // See https://en.cppreference.com/w/c/language/array_initialization#Notes + cxtype_v() + : m_real{ 0 }, m_imag{ 0 } {} // RRRR=0000 IIII=0000 + cxtype_v( const cxtype_v& ) = default; + cxtype_v( cxtype_v&& ) = default; + cxtype_v( const fptype_v& r, const fptype_v& i ) + : m_real( r ), m_imag( i ) {} + cxtype_v( const fptype_v& r ) + : m_real( r ), m_imag{ 0 } {} // IIII=0000 + cxtype_v( const fptype& r ) + : m_real( fptype_v{} + r ), m_imag{ 0 } {} // IIII=0000 + cxtype_v& operator=( const cxtype_v& ) = default; + cxtype_v& operator=( cxtype_v&& ) = default; + cxtype_v& operator+=( const cxtype_v& c ) + { + m_real += c.real(); + m_imag += c.imag(); + return *this; + } + cxtype_v& operator-=( const cxtype_v& c ) + { + m_real -= c.real(); + m_imag -= c.imag(); + return *this; + } +#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK + // NB: THIS IS THE FUNDAMENTAL DIFFERENCE BETWEEN MGONGPU_HAS_CPPCXTYPEV_BRK DEFINED AND NOT DEFINED + // NB: the alternative "clang" implementation is simpler: it simply does not have any bracket operator[] + //cxtype_ref operator[]( size_t i ) const { return cxtype_ref( m_real[i], m_imag[i] ); } // gcc14.2 build fails #1004 + cxtype_ref operator[]( size_t i ) { return cxtype_ref( m_real[i], m_imag[i] ); } + cxtype operator[]( size_t i ) const { return cxtype( m_real[i], m_imag[i] ); } +#endif + const fptype_v& real() const + { + return m_real; + } + const fptype_v& imag() const { return m_imag; } + private: + fptype_v m_real, m_imag; // RRRRIIII + }; + + // --- Type definition (using vector compiler extensions: need -march=...) +#ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors + typedef unsigned int uint_v __attribute__( ( ext_vector_type( neppV ) ) ); +#if defined MGONGPU_FPTYPE_DOUBLE + typedef long int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb +#elif defined MGONGPU_FPTYPE_FLOAT + typedef int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb +#endif +#else // gcc + typedef unsigned int uint_v __attribute__( ( vector_size( neppV * sizeof( unsigned int ) ), aligned( neppV * sizeof( unsigned int ) ) ) ); +#if defined MGONGPU_FPTYPE_DOUBLE + typedef long int bool_v __attribute__( ( vector_size( neppV * sizeof( long int ) ), aligned( neppV * sizeof( long int ) ) ) ); // bbbb +#elif defined MGONGPU_FPTYPE_FLOAT + typedef int bool_v __attribute__( ( vector_size( neppV * sizeof( int ) ), aligned( neppV * sizeof( int ) ) ) ); // bbbb +#endif +#endif + +#else // i.e #ifndef MGONGPU_CPPSIMD (this includes #ifdef MGONGPUCPP_GPUIMPL) + + const int neppV = 1; + +#endif // #ifdef MGONGPU_CPPSIMD +} + +//-------------------------------------------------------------------------- + +// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) +// DO NOT expose typedefs outside the namespace +//using mgOnGpu::neppV; +//#ifdef MGONGPU_CPPSIMD +//using mgOnGpu::fptype_v; +//using mgOnGpu::fptype2_v; +//using mgOnGpu::cxtype_v; +//using mgOnGpu::bool_v; +//#endif + +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#ifndef MGONGPUCPP_GPUIMPL + + // Printout to stream for user defined types + +#ifndef MGONGPU_CPPCXTYPE_CXSMPL // operator<< for cxsmpl has already been defined! + inline std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + out << "[" << cxreal( c ) << "," << cximag( c ) << "]"; + //out << cxreal(c) << "+i" << cximag(c); + return out; + } +#endif + + /* +#ifdef MGONGPU_CPPSIMD + inline std::ostream& + operator<<( std::ostream& out, const bool_v& v ) + { + out << "{ " << v[0]; + for ( int i=1; i 0 ) outi = fpsqrt( (fptype)v[i] ); + out[i] = outi; + } + return out; + } + + inline fptype_v + fpsqrt( const fptype_v& v ) + { + // See https://stackoverflow.com/questions/18921049/gcc-vector-extensions-sqrt + fptype_v out = {}; // avoid warning 'out' may be used uninitialized: see #594 + for( int i = 0; i < neppV; i++ ) out[i] = fpsqrt( v[i] ); + return out; + } +#endif + + /* +#ifdef MGONGPU_CPPSIMD + inline fptype_v + fpvmake( const fptype v[neppV] ) + { + fptype_v out = {}; // see #594 + for ( int i=0; i + +#ifdef MGONGPUCPP_GPUIMPL +using namespace mg5amcGpu; +#else +using namespace mg5amcCpu; +#endif + +struct CUDA_CPU_TestBase : public TestDriverBase +{ + static constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static constexpr int np4 = CPPProcess::np4; + static constexpr int npar = CPPProcess::npar; + static_assert( gputhreads % neppM == 0, "ERROR! #threads/block should be a multiple of neppM" ); + static_assert( gputhreads <= mgOnGpu::ntpbMAX, "ERROR! #threads/block should be <= ntpbMAX" ); + CUDA_CPU_TestBase( const std::string& refFileName ) + : TestDriverBase( npar, refFileName ) {} + // Does this test use channelIds? + virtual bool useChannelIds() const = 0; + // Set channelId array (in the same way for CUDA and CPU tests) + static constexpr unsigned int warpSize = 32; // FIXME: add a sanity check in madevent that this is the minimum? (would need to expose this from cudacpp to madevent) + static void setChannelIds( BufferChannelIds& hstChannelIds, std::size_t iiter ) + { + static const char* debugC = getenv( "CUDACPP_RUNTEST_DEBUG" ); + static const bool debug = ( debugC != 0 ) && ( std::string( debugC ) != "" ); + // Fill channelIds for multi-channel tests #896 + // (NB: these are only used if useChannelIds == true) + // TEMPORARY(0): debug multichannel tests with channelId=1 for all events + //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1; + // TEMPORARY(1): debug multichannel tests with channelId=1,2,..,ndiag,1,2,..ndiag,... (every event gets a different channel, no warps) + //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1 + i % CPPProcess::ndiagrams; + // ALMOST FINAL test implementation: 1111222233331111... (every 32-event warp gets a different channel) + // FINAL(?) test implementation: 2222333344442222... (every 32-event warp gets a different channel, skip those without associated iconfig #917) + static_assert( nevt % warpSize == 0, "ERROR! nevt should be a multiple of warpSize" ); + constexpr unsigned int nWarp = nevt / warpSize; + for( unsigned int iWarp = 0; iWarp < nWarp; ++iWarp ) + { + //const unsigned int channelId = 1 + ( iWarp + iiter * nWarp ) % CPPProcess::ndiagrams; // bug #917 + const int iconfig = 1 + ( iWarp + iiter * nWarp ) % mgOnGpu::nconfigSDE; + unsigned int channelId = 0; + //for( unsigned int idiagram = 1; idiagram < CPPProcess::ndiagrams; idiagram++ ) // two bugs #920 and #919 + for( unsigned int idiagram = 0; idiagram < mgOnGpu::nchannels; idiagram++ ) // fix #920 and work around #919 + { + if( mgOnGpu::hostChannel2iconfig[idiagram] == iconfig ) + { + channelId = idiagram + 1; // fix #917 (NB add +1 because channelId uses F indexing) + break; + } + } + assert( channelId > 0 ); // sanity check that the channelId for the given iconfig was found + if( debug ) std::cout << "CUDA_CPU_TestBase::setChannelIds: iWarp=" << iWarp << ", iconfig=" << iconfig << ", channelId=" << channelId << std::endl; + for( unsigned int i = 0; i < warpSize; ++i ) + hstChannelIds[iWarp * warpSize + i] = channelId; + } + } +}; + +#ifndef MGONGPUCPP_GPUIMPL +struct CPUTest : public CUDA_CPU_TestBase +{ + // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) + // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] + CPPProcess process; + HostBufferRndNumMomenta hstRndMom; + HostBufferChannelIds hstChannelIds; + HostBufferMomenta hstMomenta; + HostBufferGs hstGs; + HostBufferRndNumHelicity hstRndHel; + HostBufferRndNumColor hstRndCol; + HostBufferWeights hstWeights; + HostBufferMatrixElements hstMatrixElements; + HostBufferSelectedHelicity hstSelHel; + HostBufferSelectedColor hstSelCol; + HostBufferHelicityMask hstIsGoodHel; + std::unique_ptr pmek; + + // Create a process object + // Read param_card and set parameters + // ** WARNING EVIL EVIL ** + // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. + // Don't remove! + CPUTest( const std::string& refFileName ) + : CUDA_CPU_TestBase( refFileName ) + , process( /*verbose=*/false ) + , hstRndMom( nevt ) + , hstChannelIds( nevt ) + , hstMomenta( nevt ) + , hstGs( nevt ) + , hstRndHel( nevt ) + , hstRndCol( nevt ) + , hstWeights( nevt ) + , hstMatrixElements( nevt ) + , hstSelHel( nevt ) + , hstSelCol( nevt ) + , hstIsGoodHel( CPPProcess::ncomb ) + , pmek( new MatrixElementKernelHost( hstMomenta, hstGs, hstRndHel, hstRndCol, hstChannelIds, hstMatrixElements, hstSelHel, hstSelCol, nevt ) ) + { + // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? + // FIXME: the CPPProcess should really be a singleton? + process.initProc( "../../Cards/param_card.dat" ); + } + + virtual ~CPUTest() {} + + void prepareRandomNumbers( unsigned int iiter ) override + { + // Random numbers for momenta + CommonRandomNumberKernel rnk( hstRndMom ); + rnk.seedGenerator( 1337 + iiter ); + rnk.generateRnarray(); + // Random numbers for helicity and color selection (fix #931) + CommonRandomNumberKernel rnk2( hstRndHel ); + rnk2.seedGenerator( 1338 + iiter ); + rnk2.generateRnarray(); + CommonRandomNumberKernel rnk3( hstRndCol ); + rnk3.seedGenerator( 1339 + iiter ); + rnk3.generateRnarray(); + } + + void prepareMomenta( fptype energy ) override + { + RamboSamplingKernelHost rsk( energy, hstRndMom, hstMomenta, hstWeights, nevt ); + // --- 2a. Fill in momenta of initial state particles on the device + rsk.getMomentaInitial(); + // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device + // (i.e. map random numbers to final-state particle momenta for each of nevt events) + rsk.getMomentaFinal(); + } + + void runSigmaKin( std::size_t iiter ) override + { + constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) + for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; + setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 + if( iiter == 0 ) pmek->computeGoodHelicities(); + pmek->computeMatrixElements( useChannelIds() ); + } + + fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override + { + assert( ipar < npar ); + assert( ip4 < np4 ); + return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); + } + + fptype getMatrixElement( std::size_t ievt ) const override + { + return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); + } + + int getChannelId( std::size_t ievt ) const override + { + return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); + } + + int getSelectedHelicity( std::size_t ievt ) const override + { + //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... + return hstSelHel.data()[ievt]; + } + + int getSelectedColor( std::size_t ievt ) const override + { + //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... + return hstSelCol.data()[ievt]; + } +}; + +// Old test with multi-channel disabled #466 +struct CPUTestNoMultiChannel : public CPUTest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return false; } + + // Constructor + CPUTestNoMultiChannel( const std::string& refFileName ) + : CPUTest( refFileName ) {} // suffix .txt + + // Destructor + virtual ~CPUTestNoMultiChannel() {} +}; + +// New test with multi-channel enabled #896 +struct CPUTestMultiChannel : public CPUTest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return true; } + + // Constructor + CPUTestMultiChannel( const std::string& refFileName ) + : CPUTest( refFileName + "2" ) {} // suffix .txt2 + + // Destructor + virtual ~CPUTestMultiChannel() {} +}; +#endif + +#ifdef MGONGPUCPP_GPUIMPL +struct CUDATest : public CUDA_CPU_TestBase +{ + // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) + // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] + CPPProcess process; + PinnedHostBufferRndNumMomenta hstRndMom; + PinnedHostBufferMomenta hstMomenta; + PinnedHostBufferGs hstGs; + PinnedHostBufferRndNumHelicity hstRndHel; + PinnedHostBufferRndNumColor hstRndCol; + PinnedHostBufferWeights hstWeights; + PinnedHostBufferChannelIds hstChannelIds; + PinnedHostBufferMatrixElements hstMatrixElements; + PinnedHostBufferSelectedHelicity hstSelHel; + PinnedHostBufferSelectedColor hstSelCol; + PinnedHostBufferHelicityMask hstIsGoodHel; + DeviceBufferRndNumMomenta devRndMom; + DeviceBufferChannelIds devChannelIds; + DeviceBufferMomenta devMomenta; + DeviceBufferGs devGs; + DeviceBufferRndNumHelicity devRndHel; + DeviceBufferRndNumColor devRndCol; + DeviceBufferWeights devWeights; + DeviceBufferMatrixElements devMatrixElements; + DeviceBufferSelectedHelicity devSelHel; + DeviceBufferSelectedColor devSelCol; + DeviceBufferHelicityMask devIsGoodHel; + std::unique_ptr pmek; + + // Create a process object + // Read param_card and set parameters + // ** WARNING EVIL EVIL ** + // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. + // Don't remove! + CUDATest( const std::string& refFileName ) + : CUDA_CPU_TestBase( refFileName ) + , process( /*verbose=*/false ) + , hstRndMom( nevt ) + , hstChannelIds( nevt ) + , hstMomenta( nevt ) + , hstGs( nevt ) + , hstRndHel( nevt ) + , hstRndCol( nevt ) + , hstWeights( nevt ) + , hstMatrixElements( nevt ) + , hstSelHel( nevt ) + , hstSelCol( nevt ) + , hstIsGoodHel( CPPProcess::ncomb ) + , devRndMom( nevt ) + , devChannelIds( nevt ) + , devMomenta( nevt ) + , devGs( nevt ) + , devRndHel( nevt ) + , devRndCol( nevt ) + , devWeights( nevt ) + , devMatrixElements( nevt ) + , devSelHel( nevt ) + , devSelCol( nevt ) + , devIsGoodHel( CPPProcess::ncomb ) + , pmek( new MatrixElementKernelDevice( devMomenta, devGs, devRndHel, devRndCol, devChannelIds, devMatrixElements, devSelHel, devSelCol, gpublocks, gputhreads ) ) + { + // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? + // FIXME: the CPPProcess should really be a singleton? + process.initProc( "../../Cards/param_card.dat" ); + } + + virtual ~CUDATest() {} + + void prepareRandomNumbers( unsigned int iiter ) override + { + // Random numbers for momenta + CommonRandomNumberKernel rnk( hstRndMom ); + rnk.seedGenerator( 1337 + iiter ); + rnk.generateRnarray(); + copyDeviceFromHost( devRndMom, hstRndMom ); + // Random numbers for helicity and color selection (fix #931) + CommonRandomNumberKernel rnk2( hstRndHel ); + rnk2.seedGenerator( 1338 + iiter ); + rnk2.generateRnarray(); + copyDeviceFromHost( devRndHel, hstRndHel ); + CommonRandomNumberKernel rnk3( hstRndCol ); + rnk3.seedGenerator( 1339 + iiter ); + rnk3.generateRnarray(); + copyDeviceFromHost( devRndCol, hstRndCol ); + } + + void prepareMomenta( fptype energy ) override + { + RamboSamplingKernelDevice rsk( energy, devRndMom, devMomenta, devWeights, gpublocks, gputhreads ); + // --- 2a. Fill in momenta of initial state particles on the device + rsk.getMomentaInitial(); + // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device + // (i.e. map random numbers to final-state particle momenta for each of nevt events) + rsk.getMomentaFinal(); + // --- 2c. CopyDToH Weights + copyHostFromDevice( hstWeights, devWeights ); + // --- 2d. CopyDToH Momenta + copyHostFromDevice( hstMomenta, devMomenta ); + } + + void runSigmaKin( std::size_t iiter ) override + { + constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) + for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; + copyDeviceFromHost( devGs, hstGs ); // BUG FIX #566 + setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 + copyDeviceFromHost( devChannelIds, hstChannelIds ); + if( iiter == 0 ) pmek->computeGoodHelicities(); + pmek->computeMatrixElements( useChannelIds() ); + copyHostFromDevice( hstMatrixElements, devMatrixElements ); + copyHostFromDevice( hstSelHel, devSelHel ); + copyHostFromDevice( hstSelCol, devSelCol ); + } + + fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override + { + assert( ipar < npar ); + assert( ip4 < np4 ); + return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); + } + + fptype getMatrixElement( std::size_t ievt ) const override + { + return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); + } + + int getChannelId( std::size_t ievt ) const override + { + return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); + } + + int getSelectedHelicity( std::size_t ievt ) const override + { + //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... + return hstSelHel.data()[ievt]; + } + + int getSelectedColor( std::size_t ievt ) const override + { + //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... + return hstSelCol.data()[ievt]; + } +}; + +// Old test with multi-channel disabled #466 +struct CUDATestNoMultiChannel : public CUDATest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return false; } + + // Constructor + CUDATestNoMultiChannel( const std::string& refFileName ) + : CUDATest( refFileName ) {} // suffix .txt + + // Destructor + virtual ~CUDATestNoMultiChannel() {} +}; + +// New test with multi-channel enabled #896 +struct CUDATestMultiChannel : public CUDATest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return true; } + + // Constructor + CUDATestMultiChannel( const std::string& refFileName ) + : CUDATest( refFileName + "2" ) {} // suffix .txt2 + + // Destructor + virtual ~CUDATestMultiChannel() {} +}; +#endif /* clang-format off */ + +// AV July 2024 much simpler class structure without the presently-unnecessary googletest templates +// This is meant as a workaround to prevent not-understood segfault #907 when adding a second test +// Note: instantiate test2 first and test1 second to ensure that the channelid printout from the dtors comes from test1 first and test2 second +#ifdef MGONGPUCPP_GPUIMPL +// CUDA test drivers +CUDATestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID2( s ) s##_GPU_MULTICHANNEL +CUDATestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID1( s ) s##_GPU_NOMULTICHANNEL +#else +// CPU test drivers +CPUTestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID2( s ) s##_CPU_MULTICHANNEL +CPUTestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID1( s ) s##_CPU_NOMULTICHANNEL +#endif +// Madgraph tests +MadgraphTest mgTest2( driver2 ); +MadgraphTest mgTest1( driver1 ); +// Instantiate Google test 1 +#define XTESTID1( s ) TESTID1( s ) +TEST( XTESTID1( MG_EPOCH_PROCESS_ID ), compareMomAndME ) +{ +#ifdef MGONGPU_CHANNELID_DEBUG + driver1.pmek->setTagForNevtProcessedByChannel( "(no multichannel)" ); +#endif + mgTest1.CompareMomentaAndME( *this ); +} +// Instantiate Google test 2 +#define XTESTID2( s ) TESTID2( s ) +TEST( XTESTID2( MG_EPOCH_PROCESS_ID ), compareMomAndME ) +{ +#ifdef MGONGPU_CHANNELID_DEBUG + driver2.pmek->setTagForNevtProcessedByChannel( "(channelid array)" ); +#endif + mgTest2.CompareMomentaAndME( *this ); +} +/* clang-format on */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc new file mode 100644 index 0000000000..74c4902dc5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc @@ -0,0 +1,517 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. +// ---------------------------------------------------------------------------- +// Use ./runTest.exe --gtest_filter=*misc to run only testmisc.cc tests +// ---------------------------------------------------------------------------- + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#include "constexpr_math.h" +#include "epoch_process_id.h" +#include "valgrind.h" + +#include + +//#include +//#include // needs C++20... https://stackoverflow.com/a/65347016 +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +#define TESTID( s ) s##_GPU_MISC +#else +#define TESTID( s ) s##_CPU_MISC +#endif + +#define XTESTID( s ) TESTID( s ) + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#ifdef MGONGPU_CPPSIMD /* clang-format off */ +#define EXPECT_TRUE_sv( cond ) { bool_v mask( cond ); EXPECT_TRUE( maskand( mask ) ); } +#else +#define EXPECT_TRUE_sv( cond ) { EXPECT_TRUE( cond ); } +#endif /* clang-format on */ + + inline const std::string + boolTF( const bool& b ) + { + return ( b ? "T" : "F" ); + } + +#ifdef MGONGPU_CPPSIMD + inline const std::string + boolTF( const bool_v& v ) + { + std::stringstream out; + out << "{ " << ( v[0] ? "T" : "F" ); + for( int i = 1; i < neppV; i++ ) out << ", " << ( v[i] ? "T" : "F" ); + out << " }"; + return out.str(); + } +#endif +} + +TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + //-------------------------------------------------------------------------- + + EXPECT_TRUE( true ); + + //-------------------------------------------------------------------------- + + // Vector initialization for fptype_sv + { + fptype_sv f{ 0 }; + EXPECT_TRUE_sv( f == 0 ); + } + { + fptype_sv f = fptype_sv{ 0 }; + EXPECT_TRUE_sv( f == 0 ); + } + + // Vector initialization for fptype_sv - demonstrate bug #339 in older cxmake implementation + { + fptype_sv f{ 1 }; + //std::cout << f << std::endl << boolTF( f == 1 ) << std::endl; + //EXPECT_TRUE_sv( f == 1 ); // this fails for vectors! TFFF +#ifndef MGONGPU_CPPSIMD + EXPECT_TRUE_sv( f == 1 ); // this succeds: T +#else + EXPECT_TRUE( ( f == 1 )[0] ); // this succeds: TFFF[0] + EXPECT_TRUE( ( f[0] == 1 ) ); + for( int i = 1; i < neppV; i++ ) + { + EXPECT_TRUE( !( ( f == 1 )[i] ) ); // this succeds: FTTT[i>=1] + EXPECT_TRUE( ( f[i] == 0 ) ); // equals 0, not 1 + } +#endif + } + +#ifdef MGONGPU_CPPSIMD + // Vector initialization for cxtype_sv - demonstrate fix for bug #339 + { + fptype_sv f1 = fptype_v{ 0 } + 1; + EXPECT_TRUE_sv( f1 == 1 ); + cxtype_v c12 = cxmake( f1, 2 ); + //std::cout << c12 << std::endl << boolTF( c12.real() == 1 ) << std::endl << boolTF( c12.imag() == 2 ) << std::endl; + EXPECT_TRUE_sv( c12.real() == 1 ); + EXPECT_TRUE_sv( c12.imag() == 2 ); + cxtype_v c21 = cxmake( 2, f1 ); + //std::cout << c21 << std::endl << boolTF( c21.real() == 2 ) << std::endl << boolTF( c21.imag() == 1 ) << std::endl; + EXPECT_TRUE_sv( c21.real() == 2 ); + EXPECT_TRUE_sv( c21.imag() == 1 ); + } +#endif + + // Vector initialization for cxtype_sv + { + cxtype_sv c = cxzero_sv(); + EXPECT_TRUE_sv( c.real() == 0 ); + EXPECT_TRUE_sv( c.imag() == 0 ); + } + { + cxtype_sv c = cxmake( 1, fptype_sv{ 0 } ); // here was a bug #339 + EXPECT_TRUE_sv( c.real() == 1 ); + EXPECT_TRUE_sv( c.imag() == 0 ); + } + { + cxtype_sv c = cxmake( fptype_sv{ 0 }, 1 ); // here was a bug #339 + EXPECT_TRUE_sv( c.real() == 0 ); + EXPECT_TRUE_sv( c.imag() == 1 ); + } + + // Array initialization for cxtype_sv array (example: jamp_sv in CPPProcess.cc) + { + cxtype_sv array[2] = {}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "= {}" is missing!) + //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; + EXPECT_TRUE_sv( array[0].real() == 0 ); + EXPECT_TRUE_sv( array[0].imag() == 0 ); + EXPECT_TRUE_sv( array[1].real() == 0 ); + EXPECT_TRUE_sv( array[1].imag() == 0 ); + } + + // Alternative array initialization for cxtype_sv array (example: was used for outwf in testxxx.cc) + { + cxtype_sv array[2]{}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "{}" is missing!) + //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; + EXPECT_TRUE_sv( array[0].real() == 0 ); + EXPECT_TRUE_sv( array[0].imag() == 0 ); + EXPECT_TRUE_sv( array[1].real() == 0 ); + EXPECT_TRUE_sv( array[1].imag() == 0 ); + } + + //-------------------------------------------------------------------------- + + // Scalar complex references + { + using namespace mgOnGpu; + // Refs to f1, f2 + fptype f1 = 1; + fptype f2 = 2; + cxtype_ref r12( f1, f2 ); // copy refs + //cxtype_ref r12a( r12 ); //deleted + cxtype_ref r12a( cxtype_ref( f1, f2 ) ); // copy refs + //cxtype_ref r12b = r12; // deleted + cxtype_ref r12b = cxtype_ref( f1, f2 ); // copy refs + EXPECT_TRUE( cxtype( r12 ).real() == 1 ); + EXPECT_TRUE( cxtype( r12 ).imag() == 2 ); + EXPECT_TRUE( cxtype( r12a ).real() == 1 ); + EXPECT_TRUE( cxtype( r12a ).imag() == 2 ); + EXPECT_TRUE( cxtype( r12b ).real() == 1 ); + EXPECT_TRUE( cxtype( r12b ).imag() == 2 ); + // Refs to f1c, f2c + fptype f1c = 0; + fptype f2c = 0; + cxtype_ref r12c( f1c, f2c ); + EXPECT_TRUE( cxtype( r12c ).real() == 0 ); + EXPECT_TRUE( cxtype( r12c ).imag() == 0 ); + //r12c = r12; // deleted + r12c = cxtype( r12 ); // copy values + EXPECT_TRUE( cxtype( r12c ).real() == 1 ); + EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); + // Update f1, f2 + f1 = 10; + f2 = 20; + EXPECT_TRUE( cxtype( r12 ).real() == 10 ); + EXPECT_TRUE( cxtype( r12 ).imag() == 20 ); + EXPECT_TRUE( cxtype( r12a ).real() == 10 ); + EXPECT_TRUE( cxtype( r12a ).imag() == 20 ); + EXPECT_TRUE( cxtype( r12b ).real() == 10 ); + EXPECT_TRUE( cxtype( r12b ).imag() == 20 ); + EXPECT_TRUE( cxtype( r12c ).real() == 1 ); // points to f1c, not to f1 + EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); // points to f2c, not to f2 + } + + // Vector complex references + { + using namespace mgOnGpu; + // Refs to f1, f2 + fptype_sv f1 = fptype_sv{ 0 } + 1; + fptype_sv f2 = fptype_sv{ 0 } + 2; + cxtype_sv_ref r12( f1, f2 ); // copy refs + //cxtype_sv_ref r12a( r12 ); //deleted + cxtype_sv_ref r12a( cxtype_sv_ref( f1, f2 ) ); // copy refs + //cxtype_sv_ref r12b = r12; // deleted + cxtype_sv_ref r12b = cxtype_sv_ref( f1, f2 ); // copy refs + EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 2 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 2 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 2 ); + // Refs to f1c, f2c + fptype_sv f1c = fptype_sv{ 0 }; + fptype_sv f2c = fptype_sv{ 0 }; + cxtype_sv_ref r12c( f1c, f2c ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 0 ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 0 ); + //r12c = r12; // deleted + r12c = cxtype_sv( r12 ); // copy values + EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); + // Update f1, f2 + f1 = fptype_sv{ 0 } + 10; + f2 = fptype_sv{ 0 } + 20; + EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 10 ); + EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 20 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 10 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 20 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 10 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 20 ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); // points to f1c, not to f1 + EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); // points to f2c, not to f2 + } + + //-------------------------------------------------------------------------- + + // Boolean vector (mask) times FP vector + /* + // From https://github.com/madgraph5/madgraph4gpu/issues/765#issuecomment-1853672838 + channelids_sv = CHANNEL_ACCESS::kernelAccess( pchannelIds ); // the 4 channels in the SIMD vector + bool_sv mask_sv = ( channelids_sv == 1 ); + numerators_sv += mask_sv * cxabs2( amp_sv[0] ); + if( pchannelIds != nullptr ) denominators_sv += cxabs2( amp_sv[0] ); + */ + { + typedef bool_sv test_int_sv; // defined as scalar_or_vector of long int (FPTYPE=double) or int (FPTYPE=float) + test_int_sv channelids0_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) + test_int_sv channelids1_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) + fptype_sv absamp0_sv{}; // mimic cxabs2( amp_sv[0] ) + fptype_sv absamp1_sv{}; // mimic cxabs2( amp_sv[0] ) +#ifdef MGONGPU_CPPSIMD + for( int i = 0; i < neppV; i++ ) + { + channelids0_sv[i] = i; // 0123 + channelids1_sv[i] = i; // 1234 + absamp0_sv[i] = 10. + i; // 10. 11. 12. 13. + absamp1_sv[i] = 11. + i; // 11. 12. 13. 14. + } +#else + channelids0_sv = 0; + channelids1_sv = 1; + absamp0_sv = 10.; + absamp1_sv = 11.; +#endif + bool_sv mask0_sv = ( channelids0_sv % 2 == 0 ); // even channels 0123 -> TFTF (1010) + bool_sv mask1_sv = ( channelids1_sv % 2 == 0 ); // even channels 1234 -> FTFT (0101) + constexpr fptype_sv fpZERO_sv{}; // 0000 + //fptype_sv numerators0_sv = mask0_sv * absamp0_sv; // invalid operands to binary * ('__vector(4) long int' and '__vector(4) double') + fptype_sv numerators0_sv = fpternary( mask0_sv, absamp0_sv, fpZERO_sv ); // equivalent to "mask0_sv * absamp0_sv" + fptype_sv numerators1_sv = fpternary( mask1_sv, absamp1_sv, fpZERO_sv ); // equivalent to "mask1_sv * absamp1_sv" +#ifdef MGONGPU_CPPSIMD + //std::cout << "numerators0_sv: " << numerators0_sv << std::endl; + //std::cout << "numerators1_sv: " << numerators1_sv << std::endl; + for( int i = 0; i < neppV; i++ ) + { + // Values of numerators0_sv: 10.*1 11.*0 12.*1 13.*0 + if( channelids0_sv[i] % 2 == 0 ) // even channels + EXPECT_TRUE( numerators0_sv[i] == ( 10. + i ) ); + else // odd channels + EXPECT_TRUE( numerators0_sv[i] == 0. ); + // Values of numerators1_sv: 11.*0 12.*1 13.*0 14.*1 + if( channelids1_sv[i] % 2 == 0 ) // even channels + EXPECT_TRUE( numerators1_sv[i] == ( 11. + i ) ); + else // odd channels + EXPECT_TRUE( numerators1_sv[i] == 0. ); + } +#else + // Values of numerators0_sv: 10.*1 + EXPECT_TRUE( numerators0_sv == 10. ); + // Values of numerators1_sv: 11.*0 + EXPECT_TRUE( numerators1_sv == 0. ); +#endif + } + + //-------------------------------------------------------------------------- + + // Test constexpr floor + EXPECT_TRUE( constexpr_floor( 1.5 ) == 1 ); + EXPECT_TRUE( constexpr_floor( 0.5 ) == 0 ); + EXPECT_TRUE( constexpr_floor( -0.5 ) == -1 ); + EXPECT_TRUE( constexpr_floor( -1.5 ) == -2 ); + + // Test constexpr pow + EXPECT_TRUE( constexpr_pow( 10, 0 ) == 1 ); + EXPECT_TRUE( constexpr_pow( 10, 1 ) == 10 ); + EXPECT_TRUE( constexpr_pow( 10, 2 ) == 100 ); + EXPECT_NEAR( constexpr_pow( 10, -1 ), 0.1, 0.1 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10, -1 ) = " << constexpr_pow( 10, -1 ); + EXPECT_NEAR( constexpr_pow( 10, -2 ), 0.01, 0.01 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10, -2 ) = " << constexpr_pow( 10, -2 ); + EXPECT_NEAR( constexpr_pow( 100, 0.5 ), 10, 10 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 100, 0.5 ) = " << constexpr_pow( 100, 0.5 ); + EXPECT_NEAR( constexpr_pow( 100, -0.5 ), 0.1, 0.1 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 100, -0.5 ) = " << constexpr_pow( 100, -0.5 ); + EXPECT_NEAR( constexpr_pow( 10000, 0.25 ), 10, 10 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10000, 0.25 ) = " << constexpr_pow( 10000, 0.25 ); + EXPECT_NEAR( constexpr_pow( 10000, -0.25 ), 0.1, 0.1 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10000, -0.25 ) = " << constexpr_pow( 10000, -0.25 ); + +#ifndef __aarch64__ // TO BE UNDERSTOOD? DISABLE CONSTEXPR_SQRT TESTS ON AARCH64 (#1064) + // Distance from the horizontal or vertical axis (i.e. from 0, pi/2, pi, or 3pi/2) + auto distance4 = []( const long double xx ) + { + const long double xx2 = mapIn0to2Pi( xx ); // in [0,2*pi) + const long double xx3 = xx2 - constexpr_floor( xx2 / constexpr_pi_by_2 ) * constexpr_pi_by_2; // in [0,pi/2) + const long double d0 = xx3; // distance from 0 + const long double d1 = constexpr_pi_by_2 - xx3; // distance from pi/2 + return ( d0 < d1 ? d0 : d1 ); + }; + + // Test constexpr sin, cos, tan - specific, problematic, points + auto testSinCosTanX = []( const long double xx, const double tolerance0, const bool debug = false, const long long istep = -999999999 ) + { + const double x = (double)xx; + const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 + if( debug ) + { + //std::cout << std::setprecision(40) << "testSinCosTanX: xx= " << xx << std::endl; + //std::cout << std::setprecision(40) << " x= " << x << std::endl; + } + //std::cout << std::setprecision(40) << "xx - 3pi/2 " << xx - 3 * constexpr_pi_by_2 << std::endl; + //int width = 46; + //char buf[128]; + //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)xx ); + //std::cout << std::setprecision(40) << "testSinCosTanX: xx=" << buf << std::endl; + //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)x ); + //std::cout << std::setprecision(40) << " x= " << buf << std::endl; + EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::abs( std::sin( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::abs( std::cos( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; +#ifndef __aarch64__ + if( !RUNNING_ON_VALGRIND ) + { + EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::abs( std::tan( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + } + else +#endif + { + // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) + const long double ctanx = constexpr_tan( x ); + const long double taninf = 4E14; // declare tan(x) as "infinity" if above this threshold + if( ctanx > -taninf && ctanx < taninf ) + EXPECT_NEAR( std::tan( x ), ctanx, std::abs( std::tan( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + else + { + // Allow tan(x)=-inf if ctan(x)=+inf and viceversa + EXPECT_GT( std::abs( std::tan( x ) ), taninf ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + /* + // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) + if( ctanx > 0 ) + EXPECT_GT( std::tan( x ), taninf ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + else + EXPECT_LT( std::tan( x ), -taninf ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + */ + } + } + std::cout << std::setprecision( 6 ); // default + }; + testSinCosTanX( constexpr_pi, 1E-3, true ); // from math.h + testSinCosTanX( (long double)3.141592653589793238462643383279502884L, 1E-3, true ); // from math.h + testSinCosTanX( 4.712388980384687897640105802565813064575L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) + testSinCosTanX( 3 * constexpr_pi_by_2 - 1.96e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x!=xx) + testSinCosTanX( 3 * constexpr_pi_by_2 - 1.9601e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) + + // Test constexpr sin, cos, tan - 8 points on (or close to) the boundaries of the 8 sectors of [0,2*pi] + auto testSinCosTan8 = [testSinCosTanX]( const double deltax, const double tolerance ) + { + for( int ioff = -1; ioff < 2; ioff++, ioff++ ) // -1, 1 + { + const bool debug = false; + const int nstep = 8; + for( int istep = 0; istep < nstep + 1; istep++ ) + { + long double x0 = deltax * ioff; + long double x1 = deltax * ioff + 2 * constexpr_pi; + double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) + testSinCosTanX( x, tolerance, debug, istep ); + } + } + }; + + // Use much lower tolerance when testing on the boundaries of the 8 sectors of [0,2*pi] + // Use progressively stricter tolerances as you move away from the boundaries of the 8 sectors of [0,2*pi] + testSinCosTan8( 0, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... + testSinCosTan8( 1E-15, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... + testSinCosTan8( 1E-14, 1E-04 ); // fails with 1E-05 + testSinCosTan8( 1E-12, 1E-06 ); // fails with 1E-07 + testSinCosTan8( 1E-09, 1E-09 ); // fails with 1E-10 + testSinCosTan8( 1E-06, 1E-12 ); // fails with 1E-13 + testSinCosTan8( 1E-03, 1E-14 ); // fails with 1E-16: could use 1E-14 but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) + testSinCosTan8( 1E-02, 1E-14 ); // never fails? could use 1E-99(?) but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) + + // Test constexpr sin, cos, tan - N points almost randomly with a varying tolerance + auto testSinCosTanN = [distance4]( const int nstep, const double x0, const double x1 ) + { + auto toleranceForX = [distance4]( const double x ) + { + const double d4 = distance4( x ); + if( d4 < 1E-14 ) + return 1E-03; // NB: absolute distance limited to 1E-14 anyway even if relative tolerance is 1E-3... + else if( d4 < 1E-13 ) + return 1E-04; + else if( d4 < 1E-12 ) + return 1E-05; + else if( d4 < 1E-11 ) + return 1E-06; + else if( d4 < 1E-10 ) + return 1E-07; + else if( d4 < 1E-09 ) + return 1E-08; + else if( d4 < 1E-08 ) + return 1E-09; + else if( d4 < 1E-07 ) + return 1E-10; + else if( d4 < 1E-06 ) + return 1E-11; + else if( d4 < 1E-05 ) + return 1E-12; + else if( d4 < 1E-04 ) + return 1E-13; + else + return 1E-14; // play it safe even if the agreement might even be better? + }; + for( int istep = 0; istep < nstep + 1; istep++ ) + { + double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) + const double tolerance0 = toleranceForX( x ); + const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 + EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::max( std::abs( std::sin( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::max( std::abs( std::cos( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); +#ifndef __aarch64__ + if( !RUNNING_ON_VALGRIND ) + { + EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + } + else +#endif + { + // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) + const long double ctanx = constexpr_tan( x ); + const long double taninf = 4E14; // declare tan(x) as "infinity if above this threshold + if( ctanx > -taninf && ctanx < taninf ) + EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + else + { + // Allow tan(x)=-inf if ctan(x)=+inf and viceversa + EXPECT_GT( std::abs( std::tan( x ) ), taninf ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + /* + // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) + if( ctanx > 0 ) + EXPECT_GT( std::tan( x ), taninf ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + else + EXPECT_LT( std::tan( x ), -taninf ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + */ + } + } + } + }; + testSinCosTanN( 100, -4 * constexpr_pi, 6 * constexpr_pi ); // this was failing at 3*pi/2 (now fixed by absolute tolerance 3E-15) + testSinCosTanN( 10000, -constexpr_pi_by_2, 5 * constexpr_pi_by_2 ); + + // Test constexpr atan + { + const double tolerance = 1E-12; + const int nstep = 1000; + for( int istep = 0; istep < nstep + 1; istep++ ) + { + long double x0 = -5, x1 = +5; + double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) + EXPECT_NEAR( std::atan( x ), constexpr_atan( x ), std::abs( std::atan( x ) * tolerance ) ) + << "x=" << x << ", istep=" << istep; + } + } +#endif + //-------------------------------------------------------------------------- +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc new file mode 100644 index 0000000000..4fcf7d26b6 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc @@ -0,0 +1,456 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. +// ---------------------------------------------------------------------------- +// Use ./runTest.exe --gtest_filter=*xxx to run only testxxx.cc tests +// ---------------------------------------------------------------------------- + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "HelAmps_%(model_name)s.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessWavefunctions.h" +#include "MemoryBuffers.h" +#include "epoch_process_id.h" + +#include + +#include +#include +#include // for signal and SIGFPE (see https://stackoverflow.com/a/17473528) +#include +#include +#include +#include +#ifdef MGONGPUCPP_GPUIMPL +#define TESTID( s ) s##_GPU_XXX +#else +#define TESTID( s ) s##_CPU_XXX +#endif + +#define XTESTID( s ) TESTID( s ) + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + std::string fpeHandlerMessage = "unknown"; + int fpeHandlerIevt = -1; + inline void fpeHandlerTestxxx( int /*sig*/ ) + { +#ifdef MGONGPUCPP_GPUIMPL + std::cerr << "Floating Point Exception (GPU): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; +#else + std::cerr << "Floating Point Exception (CPU neppV=" << neppV << "): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; +#endif + exit( 1 ); + } +} + +TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif +#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) + auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); +#endif + constexpr bool dumpEvents = false; // dump the expected output of the test? + constexpr bool testEvents = !dumpEvents; // run the test? + constexpr fptype toleranceXXXs = std::is_same::value ? 1.E-15 : 1.E-5; + // Constant parameters + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + constexpr int np4 = CPPProcess::np4; + const int nevt = 32; // 12 independent tests plus 20 duplicates (need a multiple of 16 for floats '512z') + assert( nevt %% neppM == 0 ); // nevt must be a multiple of neppM + assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV + // Fill in the input momenta +#ifdef MGONGPUCPP_GPUIMPL + mg5amcGpu::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] +#else + mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] +#endif /* clang-format off */ + // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! + const fptype par0[np4 * nevt] = // AOS[nevt][np4] + { + 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) + 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) + 500, 180, 240, 400, // #3 (m=0 pT>0 pz>0) + 500, 180, 240, -400, // #4 (m=0 pT>0 pz<0) + 500, 0, 0, 500, // #5 DUPLICATE == #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, -500, // #6 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) + 500, 300, 400, 0, // #7 DUPLICATE == #2 (m=0 pT>0 pz=0) + 500, 180, 240, 400, // #8 DUPLICATE == #3 (m=0 pT>0 pz>0) + 500, 180, 240, -400, // #9 DUPLICATE == #4 (m=0 pT>0 pz<0) + 500, 0, 0, 500, // #10 DUPLICATE == #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, -500, // #11 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) + 500, 300, 400, 0, // #12 DUPLICATE == #2 (m=0 pT>0 pz=0) + 500, 180, 240, 400, // #13 DUPLICATE == #3 (m=0 pT>0 pz>0) + 500, 180, 240, -400, // #14 DUPLICATE == #4 (m=0 pT>0 pz<0) + 500, 0, 0, 500, // #15 DUPLICATE == #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, 0, // #16 (m=50>0 pT=0 pz=0) + 500, 0, 0, 300, // #17 (m=40>0 pT=0 pz>0) + 500, 0, 0, -300, // #18 (m=40>0 pT=0 pz<0) + 500, 180, 240, 0, // #19 (m=40>0 pT>0 pz=0) + 500, -240, -180, 0, // #20 (m=40>0 pT>0 pz=0) + 500, 180, 192, 144, // #21 (m=40>0 pT>0 pz>0) + 500, 180, 192, -144, // #22 (m=40>0 pT>0 pz<0) + 500, 0, 0, 0, // #23 DUPLICATE == #16 (m=50>0 pT=0 pz=0) + 500, 0, 0, 300, // #24 DUPLICATE == #17 (m=40>0 pT=0 pz>0) + 500, 0, 0, -300, // #25 DUPLICATE == #18 (m=40>0 pT=0 pz<0) + 500, 180, 240, 0, // #26 DUPLICATE == #19 (m=40>0 pT>0 pz=0) + 500, -240, -180, 0, // #27 DUPLICATE == #20 (m=40>0 pT>0 pz=0) + 500, 180, 192, 144, // #28 DUPLICATE == #21 (m=40>0 pT>0 pz>0) + 500, 180, 192, -144, // #29 DUPLICATE == #22 (m=40>0 pT>0 pz<0) + 500, 0, 0, 0, // #30 DUPLICATE == #16 (m=50>0 pT=0 pz=0) + 500, 0, 0, 300 // #31 DUPLICATE == #17 (m=40>0 pT=0 pz>0) + }; /* clang-format on */ + // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) + // See https://en.cppreference.com/w/c/language/array_initialization#Notes + fptype mass0[nevt] = {}; + bool ispzgt0[nevt] = {}; + bool ispzlt0[nevt] = {}; + bool isptgt0[nevt] = {}; + for( int ievt = 0; ievt < nevt; ievt++ ) + { + const fptype p0 = par0[ievt * np4 + 0]; + const fptype p1 = par0[ievt * np4 + 1]; + const fptype p2 = par0[ievt * np4 + 2]; + const fptype p3 = par0[ievt * np4 + 3]; + volatile fptype m2 = fpmax( p0 * p0 - p1 * p1 - p2 * p2 - p3 * p3, 0 ); // see #736 + if( m2 > 0 ) + mass0[ievt] = fpsqrt( (fptype)m2 ); + else + mass0[ievt] = 0; + ispzgt0[ievt] = ( p3 > 0 ); + ispzlt0[ievt] = ( p3 < 0 ); + isptgt0[ievt] = ( p1 != 0 ) || ( p2 != 0 ); + } + const int ipar0 = 0; // use only particle0 for this test + for( int ievt = 0; ievt < nevt; ievt++ ) + { + for( int ip4 = 0; ip4 < np4; ip4++ ) + { + MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA + } + } + // Expected output wavefunctions + std::vector> expwfs; +#include "testxxx_cc_ref.txt" // expwfs.push_back( {...} ); + std::string dumpFileName = "testxxx_cc_ref.txt.new"; + // Compute the output wavefunctions + // Dump new reference file if requested + constexpr int nw6 = CPPProcess::nw6; // dimensions of each wavefunction (HELAS KEK 91-11): e.g. 6 for e+ e- -> mu+ mu- (fermions and vectors) + int itest = 0; // index on the expected output vector + std::ofstream dumpFile; + if( dumpEvents ) + { + dumpFile.open( dumpFileName, std::ios::trunc ); + dumpFile << " // Copyright (C) 2020-2024 CERN and UCLouvain." << std::endl + << " // Licensed under the GNU Lesser General Public License (version 3 or later)." << std::endl + << " // Created by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin." << std::endl + << " // Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin." << std::endl; + } + // Lambda function for dumping wavefunctions + auto dumpwf6 = [&]( std::ostream& out, const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) + { + out << std::setprecision( 15 ) << std::scientific; + out << " expwfs.push_back( {"; + out << " // ---------" << std::endl; + for( int iw6 = 0; iw6 < nw6; iw6++ ) + { +#ifdef MGONGPU_CPPSIMD + const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration +#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK + out << std::setw( 26 ) << cxreal( wf[iw6][ieppV] ) << ", "; + out << std::setw( 22 ) << cximag( wf[iw6][ieppV] ); +#else + out << std::setw( 26 ) << wf[iw6].real()[ieppV] << ", "; + out << std::setw( 22 ) << wf[iw6].imag()[ieppV]; +#endif +#else + out << std::setw( 26 ) << wf[iw6].real(); + out << ", " << std::setw( 22 ) << wf[iw6].imag(); +#endif + if( iw6 < nw6 - 1 ) + out << ", "; + else + out << " } );"; + out << " // itest=" << itest << ": " << xxx << "#" << ievt; + out << " nsp=" << nsp << " mass=" << (int)mass << std::endl; + } + out << std::defaultfloat; + }; + // Lambda function for testing wavefunctions (1) + auto testwf6 = [&]( const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) + { + if( dumpEvents ) dumpwf6( dumpFile, wf, xxx, ievt, nsp, mass ); + if( testEvents ) + { + std::array& expwf = expwfs[itest]; + //std::cout << "Testing " << std::setw(3) << itest << ": " << xxx << " #" << ievt << std::endl; + ////for ( int iw6 = 0; iw60) can be used without triggering FPEs (#701) + // This is done by filling the full SIMD vector with the value of ievt, which was already tested to respect the relevant assumptions + for( int jevt = 0; jevt < nevt; jevt++ ) + for( int ip4 = 0; ip4 < np4; ip4++ ) + MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), jevt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA + } + }; + // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) + // See https://en.cppreference.com/w/c/language/array_initialization#Notes + cxtype_sv outwfI[6] = {}; // last result of ixxxxx (mass==0) + cxtype_sv outwfO[6] = {}; // last result of oxxxxx (mass==0) + cxtype_sv outwf[6] = {}; + cxtype_sv outwf3[6] = {}; // NB: only 3 are filled by sxxxxx, but 6 are compared! + fptype* fp_outwfI = reinterpret_cast( outwfI ); // proof of concept for using fptype* in the interface + fptype* fp_outwfO = reinterpret_cast( outwfO ); // proof of concept for using fptype* in the interface + fptype* fp_outwf = reinterpret_cast( outwf ); // proof of concept for using fptype* in the interface + fptype* fp_outwf3 = reinterpret_cast( outwf3 ); // proof of concept for using fptype* in the interface + const int nhel = 1; + // *** START OF TESTING LOOP + for( auto nsp: { -1, +1 } ) // antifermion/fermion (or initial/final for scalar and vector) + { + for( int ievt = 0; ievt < nevt; ievt++ ) + { +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + if( debug ) + { + std::cout << std::endl; + std::cout << "nsp=" << nsp << " ievt=" << ievt << ": "; + for( int ip4 = 0; ip4 < np4; ip4++ ) std::cout << par0[ievt * np4 + ip4] << ", "; + std::cout << std::endl; + } + const int ipagV = ievt / neppV; // #event vector in this iteration + const fptype* ievt0Momenta = MemoryAccessMomenta::ieventAccessRecordConst( hstMomenta.data(), ipagV * neppV ); + // Test ixxxxx - NO ASSUMPTIONS + { + prepareTest( "ixxxxx", ievt ); + const fptype fmass = mass0[ievt]; + ixxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfI, ipar0 ); + testwf6( outwfI, "ixxxxx", ievt, nsp, fmass ); + ixxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfI, ipar0 ); + testwf6( outwfI, "ixxxxx", ievt, nsp, -fmass ); + } + // Test ipzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) + { + prepareTest( "ipzxxx", ievt ); + ipzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfI, "ipzxxx", ievt ); + testwf6( outwf, "ipzxxx", ievt, nsp, 0 ); + } + // Test imzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) + { + prepareTest( "imzxxx", ievt ); + imzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfI, "imzxxx", ievt ); + testwf6( outwf, "imzxxx", ievt, nsp, 0 ); + } + // Test ixzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) + if( mass0[ievt] == 0 && isptgt0[ievt] ) + { + prepareTest( "ixzxxx", ievt ); + ixzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfI, "ixzxxx", ievt ); + testwf6( outwf, "ixzxxx", ievt, nsp, 0 ); + } + // Test vxxxxx - NO ASSUMPTIONS + { + prepareTest( "vxxxxx", ievt ); + const fptype vmass = mass0[ievt]; + vxxxxx( ievt0Momenta, vmass, nhel, nsp, fp_outwf, ipar0 ); + testwf6( outwf, "vxxxxx", ievt, nsp, vmass ); + vxxxxx( ievt0Momenta, -vmass, nhel, nsp, fp_outwf, ipar0 ); + testwf6( outwf, "vxxxxx", ievt, nsp, -vmass ); + } + // Test sxxxxx - NO ASSUMPTIONS + { + prepareTest( "sxxxxx", ievt ); + const fptype smass = mass0[ievt]; + sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass>0") + testwf6( outwf3, "sxxxxx", ievt, nsp, smass ); + sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass<0") + testwf6( outwf3, "sxxxxx", ievt, nsp, -smass ); + } + // Test oxxxxx - NO ASSUMPTIONS + { + prepareTest( "oxxxxx", ievt ); + const fptype fmass = mass0[ievt]; + oxxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfO, ipar0 ); + testwf6( outwfO, "oxxxxx", ievt, nsp, fmass ); + oxxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfO, ipar0 ); + testwf6( outwfO, "oxxxxx", ievt, nsp, -fmass ); + } + // Test opzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) + { + prepareTest( "opzxxx", ievt ); + opzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfO, "opzxxx", ievt ); + testwf6( outwf, "opzxxx", ievt, nsp, 0 ); + } + // Test omzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) + { + prepareTest( "omzxxx", ievt ); + omzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfO, "omzxxx", ievt ); + testwf6( outwf, "omzxxx", ievt, nsp, 0 ); + } + // Test oxzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) + if( mass0[ievt] == 0 && isptgt0[ievt] ) + { + prepareTest( "oxzxxx", ievt ); + oxzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfO, "oxzxxx", ievt ); + testwf6( outwf, "oxzxxx", ievt, nsp, 0 ); + } + } + } + // *** END OF TESTING LOOP + if( dumpEvents ) + { + dumpFile.close(); + std::cout << "INFO: New reference data dumped to file '" << dumpFileName << "'" << std::endl; + } +#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) + signal( SIGFPE, fpeHandlerDefault ); +#endif +} + +//========================================================================== + +// Reset the GPU after ALL tests have gone out of scope +// (This was needed to avoid leaks in profilers, but compute-sanitizer reports no leaks, is it STILL needed?) +// ========= NB: resetting the GPU too early causes segfaults that are very difficult to debug #907 ========= +// Try to use atexit (https://stackoverflow.com/a/14610501) but this still crashes! +// ********* FIXME? avoid CUDA API calls in destructors? (see https://stackoverflow.com/a/16982503) ********* +void +myexit() +{ +#ifdef MGONGPUCPP_GPUIMPL + //checkGpu( gpuDeviceReset() ); // FIXME??? this still crashes! should systematically avoid CUDA calls in all destructors? +#endif +} + +// Main function (see https://google.github.io/googletest/primer.html#writing-the-main-function) +// (NB: the test executables are now separate for C++ and CUDA, therefore main must be included all the time) +// (NB: previously, '#ifndef MGONGPUCPP_GPUIMPL' was ensuring that main was only included once while linking both C++ and CUDA tests) +int +main( int argc, char** argv ) +{ + atexit( myexit ); + testing::InitGoogleTest( &argc, argv ); + int status = RUN_ALL_TESTS(); + return status; +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc new file mode 100644 index 0000000000..d19c93bb9b --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc @@ -0,0 +1,667 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: T. Heimel (Nov 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro (2026). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "umami.h" + +#include "CPPProcess.h" +#include "GpuRuntime.h" +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" + +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +using namespace mg5amcGpu; +#else +using namespace mg5amcCpu; +#endif + +namespace +{ + + void* initialize_impl( + const fptype* momenta, + const fptype* couplings, + const unsigned int* flavor_indices, + fptype* matrix_elements, +#ifdef MGONGPUCPP_GPUIMPL + fptype* color_jamps, +#endif + fptype* numerators, + fptype* denominators, + std::size_t count ) + { + bool is_good_hel[CPPProcess::ncomb]; + sigmaKin_getGoodHel( + momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, +#ifdef MGONGPUCPP_GPUIMPL + color_jamps, +#endif + is_good_hel, + count ); + sigmaKin_setGoodHel( is_good_hel ); + return nullptr; + } + + void initialize( + const fptype* momenta, + const fptype* couplings, + const unsigned int* flavor_indices, + fptype* matrix_elements, +#ifdef MGONGPUCPP_GPUIMPL + fptype* color_jamps, +#endif + fptype* numerators, + fptype* denominators, + std::size_t count ) + { + // static local initialization is called exactly once in a thread-safe way + static void* dummy = initialize_impl( momenta, couplings, flavor_indices, matrix_elements, +#ifdef MGONGPUCPP_GPUIMPL + color_jamps, +#endif + numerators, + denominators, + count ); + } + +#ifdef MGONGPUCPP_GPUIMPL + __device__ +#endif + void + transpose_momenta( const double* momenta_in, fptype* momenta_out, std::size_t i_event_in, std::size_t i_event_out, std::size_t stride ) + { + std::size_t page_size = MemoryAccessMomentaBase::neppM; + std::size_t i_page = i_event_out / page_size; + std::size_t i_vector = i_event_out % page_size; + + for( std::size_t i_part = 0; i_part < CPPProcess::npar; ++i_part ) + { + for( std::size_t i_mom = 0; i_mom < 4; ++i_mom ) + { + momenta_out[i_page * CPPProcess::npar * 4 * page_size + + i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( CPPProcess::npar * i_mom + i_part ) + i_event_in]; + } + } + } + +#ifdef MGONGPUCPP_GPUIMPL + + __global__ void copy_inputs( + const double* momenta_in, + const double* helicity_random_in, + const double* color_random_in, + const double* diagram_random_in, + const double* alpha_s_in, + const unsigned int* flavor_indices_in, + fptype* momenta, + fptype* helicity_random, + fptype* color_random, + fptype* diagram_random, + fptype* g_s, + unsigned int* flavor_indices, + std::size_t count, + std::size_t stride, + std::size_t offset ) + { + std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; + if( i_event >= count ) return; + + transpose_momenta( &momenta_in[offset], momenta, i_event, i_event, stride ); + diagram_random[i_event] = diagram_random_in ? diagram_random_in[i_event + offset] : 0.5; + helicity_random[i_event] = helicity_random_in ? helicity_random_in[i_event + offset] : 0.5; + color_random[i_event] = color_random_in ? color_random_in[i_event + offset] : 0.5; + g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; + } + + __global__ void copy_outputs( + fptype* denominators, + fptype* numerators, + fptype* matrix_elements, + unsigned int* diagram_index, + int* color_index, + int* helicity_index, + double* m2_out, + double* amp2_out, + int* diagram_out, + int* color_out, + int* helicity_out, + std::size_t count, + std::size_t stride, + std::size_t offset ) + { + std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; + if( i_event >= count ) return; + + if( m2_out ) m2_out[i_event + offset] = matrix_elements[i_event]; + if( amp2_out ) + { + double denominator = denominators[i_event]; + for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_event * CPPProcess::ndiagrams + i_diag] / denominator; + } + } + if( diagram_out ) diagram_out[i_event + offset] = diagram_index[i_event] - 1; + if( color_out ) color_out[i_event + offset] = color_index[i_event] - 1; + if( helicity_out ) helicity_out[i_event + offset] = helicity_index[i_event] - 1; + } + +#endif // MGONGPUCPP_GPUIMPL + + struct InterfaceInstance + { + bool initialized = false; + }; + + std::vector g_externalMasses; + +} + +extern "C" +{ + UmamiStatus umami_get_meta( UmamiMetaKey meta_key, void* result ) + { + switch( meta_key ) + { + case UMAMI_META_DEVICE: + { + UmamiDevice& device = *static_cast( result ); +#ifdef MGONGPUCPP_GPUIMPL +#ifdef __CUDACC__ + device = UMAMI_DEVICE_CUDA; +#elif defined( __HIPCC__ ) + device = UMAMI_DEVICE_HIP; +#endif +#else + device = UMAMI_DEVICE_CPU; +#endif + break; + } + case UMAMI_META_PARTICLE_COUNT: + *static_cast( result ) = CPPProcess::npar; + break; + case UMAMI_META_DIAGRAM_COUNT: + *static_cast( result ) = CPPProcess::ndiagrams; + break; + case UMAMI_META_HELICITY_COUNT: + *static_cast( result ) = CPPProcess::ncomb; + break; + case UMAMI_META_COLOR_COUNT: + return UMAMI_ERROR_UNSUPPORTED_META; + case UMAMI_META_MASSES: + { + if( g_externalMasses.size() != (size_t)CPPProcess::npar ) return UMAMI_ERROR_UNINITIALIZED_META; + + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + static_cast( result )[ipar] = g_externalMasses[ipar]; + break; + } + default: + return UMAMI_ERROR_UNSUPPORTED_META; + } + return UMAMI_SUCCESS; + } + + UmamiStatus umami_supported_inputs( bool const** supported, int* count ) + { + // MOMENTA, ALPHA_S, FLAVOR_INDEX, RANDOM_COLOR, RANDOM_HELICITY, RANDOM_DIAGRAM, + // HELICITY_INDEX=false, DIAGRAM_INDEX=true, CHANNEL_INDEX=false + static const bool data[UMAMI_INPUT_KEY_COUNT] = { true, true, true, true, true, true, false, true }; + *supported = data; + *count = UMAMI_INPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_required_inputs( bool const** required, int* count ) + { + static const bool data[UMAMI_INPUT_KEY_COUNT] = { true }; // MOMENTA only + *required = data; + *count = UMAMI_INPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_supported_outputs( bool const** supported, int* count ) + { + // MATRIX_ELEMENT, DIAGRAM_AMP2, COLOR_INDEX, HELICITY_INDEX, DIAGRAM_INDEX, GPU_STREAM +#ifdef MGONGPUCPP_GPUIMPL + static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true, true }; +#else + static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true }; +#endif + *supported = data; + *count = UMAMI_OUTPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_initialize( UmamiHandle* handle, char const* param_card_path ) + { + CPPProcess process; + process.initProc( param_card_path ); + + const std::vector& masses = process.getMasses(); + g_externalMasses.assign( masses.begin(), masses.end() ); + + auto instance = new InterfaceInstance(); + *handle = instance; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_set_parameter( + [[maybe_unused]] UmamiHandle handle, + [[maybe_unused]] char const* name, + [[maybe_unused]] double parameter_real, + [[maybe_unused]] double parameter_imag ) + { + return UMAMI_ERROR_NOT_IMPLEMENTED; + } + + UmamiStatus umami_get_parameter( + [[maybe_unused]] UmamiHandle handle, + [[maybe_unused]] char const* name, + [[maybe_unused]] double* parameter_real, + [[maybe_unused]] double* parameter_imag ) + { + return UMAMI_ERROR_NOT_IMPLEMENTED; + } + + UmamiStatus umami_matrix_element( + UmamiHandle handle, + size_t count, + size_t stride, + size_t offset, + size_t input_count, + UmamiInputKey const* input_keys, + void const* const* inputs, + size_t output_count, + UmamiOutputKey const* output_keys, + void* const* outputs ) + { + const double* momenta_in = nullptr; + const double* alpha_s_in = nullptr; + const unsigned int* flavor_indices_in = nullptr; + const double* random_color_in = nullptr; + const double* random_helicity_in = nullptr; + const double* random_diagram_in = nullptr; + [[maybe_unused]] const int* diagram_in = nullptr; // TODO: unused + + for( std::size_t i = 0; i < input_count; ++i ) + { + const void* input = inputs[i]; + switch( input_keys[i] ) + { + case UMAMI_IN_MOMENTA: + momenta_in = static_cast( input ); + break; + case UMAMI_IN_ALPHA_S: + alpha_s_in = static_cast( input ); + break; + case UMAMI_IN_FLAVOR_INDEX: + flavor_indices_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_COLOR: + random_color_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_HELICITY: + random_helicity_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_DIAGRAM: + random_diagram_in = static_cast( input ); + break; + case UMAMI_IN_HELICITY_INDEX: + return UMAMI_ERROR_UNSUPPORTED_INPUT; + case UMAMI_IN_DIAGRAM_INDEX: + diagram_in = static_cast( input ); + break; + default: + return UMAMI_ERROR_UNSUPPORTED_INPUT; + } + } + if( !momenta_in ) return UMAMI_ERROR_MISSING_INPUT; + +#ifdef MGONGPUCPP_GPUIMPL + gpuStream_t gpu_stream = nullptr; +#endif + double* m2_out = nullptr; + double* amp2_out = nullptr; + int* diagram_out = nullptr; + int* color_out = nullptr; + int* helicity_out = nullptr; + for( std::size_t i = 0; i < output_count; ++i ) + { + void* output = outputs[i]; + switch( output_keys[i] ) + { + case UMAMI_OUT_MATRIX_ELEMENT: + m2_out = static_cast( output ); + break; + case UMAMI_OUT_DIAGRAM_AMP2: + amp2_out = static_cast( output ); + break; + case UMAMI_OUT_COLOR_INDEX: + color_out = static_cast( output ); + break; + case UMAMI_OUT_HELICITY_INDEX: + helicity_out = static_cast( output ); + break; + case UMAMI_OUT_DIAGRAM_INDEX: + diagram_out = static_cast( output ); + break; +#ifdef MGONGPUCPP_GPUIMPL + case UMAMI_OUT_GPU_STREAM: + gpu_stream = static_cast( output ); + break; +#endif + default: + return UMAMI_ERROR_UNSUPPORTED_OUTPUT; + } + } + +#ifdef MGONGPUCPP_GPUIMPL + std::size_t n_threads = 256; + std::size_t n_blocks = ( count + n_threads - 1 ) / n_threads; + std::size_t rounded_count = n_blocks * n_threads; + + fptype *momenta, *couplings, *g_s, *helicity_random, *color_random, *diagram_random, *color_jamps; + fptype *matrix_elements, *numerators, *denominators, *ghel_matrix_elements, *ghel_jamps; + int *helicity_index, *color_index; + unsigned int *flavor_indices, *diagram_index; + + std::size_t n_coup = mg5amcGpu::Parameters_dependentCouplings::ndcoup; + std::array, 16> ptrs_and_sizes = {{ + {reinterpret_cast(&momenta), rounded_count * CPPProcess::npar * 4 * sizeof( fptype )}, + {reinterpret_cast(&couplings), rounded_count * n_coup * 2 * sizeof( fptype )}, + {reinterpret_cast(&g_s), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&flavor_indices), rounded_count * sizeof( unsigned int )}, + {reinterpret_cast(&helicity_random), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&color_random), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&diagram_random), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&matrix_elements), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&diagram_index), rounded_count * sizeof( unsigned int )}, + {reinterpret_cast(&color_jamps), rounded_count * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, + {reinterpret_cast(&numerators), rounded_count * CPPProcess::ndiagrams * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&denominators), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&helicity_index), rounded_count * sizeof( int )}, + {reinterpret_cast(&color_index), rounded_count * sizeof( int )}, + {reinterpret_cast(&ghel_matrix_elements), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&ghel_jamps), rounded_count * CPPProcess::ncomb * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, + }}; + std::size_t total_size = 0; + constexpr std::size_t MAX_SIZE = std::max(sizeof(fptype), sizeof(int)); + for (auto [ptr, size] : ptrs_and_sizes) { + std::size_t aligned_size = (size + MAX_SIZE - 1) / MAX_SIZE * MAX_SIZE; + total_size += aligned_size; + } + uint8_t* buffer; + // we can consider caching this between matrix element calls + gpuMallocAsync( &buffer, total_size, gpu_stream ); + std::size_t buf_offset = 0; + for (auto [ptr, size] : ptrs_and_sizes) { + std::size_t aligned_size = (size + 7) / 8 * 8; + *ptr = buffer + buf_offset; + buf_offset += aligned_size; + } + + copy_inputs<<>>( + momenta_in, + random_helicity_in, + random_color_in, + random_diagram_in, + alpha_s_in, + flavor_indices_in, + momenta, + helicity_random, + color_random, + diagram_random, + g_s, + flavor_indices, + count, + stride, + offset ); + computeDependentCouplings<<>>( g_s, couplings ); + checkGpu( gpuPeekAtLastError() ); + + InterfaceInstance* instance = static_cast( handle ); + if( !instance->initialized ) + { + initialize( + momenta, couplings, flavor_indices, matrix_elements, color_jamps, numerators, denominators, rounded_count ); + instance->initialized = true; + } + + sigmaKin( + momenta, + couplings, + flavor_indices, + helicity_random, + color_random, + nullptr, + diagram_random, + matrix_elements, + helicity_index, + color_index, + color_jamps, + numerators, + denominators, + diagram_index, + false, + ghel_matrix_elements, + ghel_jamps, + nullptr, + nullptr, + &gpu_stream, + true, + n_blocks, + n_threads ); + + copy_outputs<<>>( + denominators, + numerators, + matrix_elements, + diagram_index, + color_index, + helicity_index, + m2_out, + amp2_out, + diagram_out, + color_out, + helicity_out, + count, + stride, + offset ); + checkGpu( gpuPeekAtLastError() ); + + gpuFreeAsync( buffer, gpu_stream ); +#else // MGONGPUCPP_GPUIMPL + constexpr std::size_t vector_size = MemoryAccessMomentaBase::neppM; + // need to round to round to double page size for some reason + constexpr std::size_t page_size2 = 2 * vector_size; + std::vector permutation; + std::size_t rounded_count; + + constexpr std::size_t flavor_count = CPPProcess::nmaxflavor; + HostBufferBase flavor_indices( ((count + page_size2 - 1) / page_size2 + flavor_count) * page_size2 ); + bool sort_flavors = vector_size > 1 && flavor_count > 1 && flavor_indices_in; + if ( sort_flavors ) + { + permutation.resize(count); + std::size_t voffset = 0; + std::size_t vector_indices[flavor_count] = {}; + std::size_t vector_counts[flavor_count] = {}; + // determine permutation of inputs such that all entries in a SIMD vector + // have the same flavor index + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + unsigned int flav = flavor_indices_in[i_event + offset]; + auto& vcount = vector_counts[flav]; + auto& vindex = vector_indices[flav]; + if ( vcount == 0 ) + { + vindex = voffset * page_size2; + for ( std::size_t i = 0; i < page_size2; ++i) { + flavor_indices[voffset * page_size2 + i] = flav; + } + voffset += 1; + } + permutation[i_event] = vindex + vcount; + vcount = (vcount + 1) % page_size2; + } + rounded_count = voffset * page_size2; + } else { + rounded_count = ( count + page_size2 - 1 ) / page_size2 * page_size2; + } + + HostBufferBase momenta( rounded_count * CPPProcess::npar * 4 ); + HostBufferBase couplings( rounded_count * mg5amcCpu::Parameters_dependentCouplings::ndcoup * 2 ); + HostBufferBase g_s( rounded_count ); + HostBufferBase helicity_random( rounded_count ); + HostBufferBase color_random( rounded_count ); + HostBufferBase diagram_random( rounded_count ); + HostBufferBase matrix_elements( rounded_count ); + HostBufferBase diagram_index( rounded_count ); + HostBufferBase numerators( rounded_count * CPPProcess::ndiagrams ); + HostBufferBase denominators( rounded_count ); + HostBufferBase helicity_index( rounded_count ); + HostBufferBase color_index( rounded_count ); + if ( sort_flavors ) { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_sorted = permutation[i_event]; + transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_sorted, stride ); + helicity_random[i_sorted] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; + color_random[i_sorted] = random_color_in ? random_color_in[i_event + offset] : 0.5; + diagram_random[i_sorted] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; + g_s[i_sorted] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + } + } else { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_event, stride ); + helicity_random[i_event] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; + color_random[i_event] = random_color_in ? random_color_in[i_event + offset] : 0.5; + diagram_random[i_event] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; + g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; + } + for ( std::size_t i_event = count; i_event < rounded_count; ++i_event ) { + flavor_indices[i_event] = 0; + } + } + computeDependentCouplings( g_s.data(), couplings.data(), rounded_count ); + + InterfaceInstance* instance = static_cast( handle ); + if( !instance->initialized ) + { + initialize( + momenta.data(), + couplings.data(), + flavor_indices.data(), + matrix_elements.data(), + numerators.data(), + denominators.data(), + rounded_count ); + instance->initialized = true; + } + + sigmaKin( + momenta.data(), + couplings.data(), + flavor_indices.data(), + helicity_random.data(), + color_random.data(), + nullptr, + diagram_random.data(), + matrix_elements.data(), + helicity_index.data(), + color_index.data(), + numerators.data(), + denominators.data(), + diagram_index.data(), + false, + rounded_count ); + + if ( sort_flavors ) + { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_sorted = permutation[i_event]; + std::size_t page_size = MemoryAccessMomentaBase::neppM; + std::size_t i_page = i_sorted / page_size; + std::size_t i_vector = i_sorted % page_size; // vector lane + + double denominator = denominators[i_sorted]; + if( m2_out != nullptr ) + { + m2_out[i_event + offset] = matrix_elements[i_sorted]; + } + if( amp2_out != nullptr ) + { + for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; + } + } + if( diagram_out != nullptr ) + { + diagram_out[i_event + offset] = diagram_index[i_sorted] - 1; + } + if( color_out != nullptr ) + { + color_out[i_event + offset] = color_index[i_sorted] - 1; + } + if( helicity_out != nullptr ) + { + helicity_out[i_event + offset] = helicity_index[i_sorted] - 1; + } + } + } else { + std::size_t page_size = MemoryAccessMomentaBase::neppM; + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_page = i_event / page_size; + std::size_t i_vector = i_event % page_size; + + double denominator = denominators[i_event]; + if( m2_out != nullptr ) + { + m2_out[i_event + offset] = matrix_elements[i_event]; + } + if( amp2_out != nullptr ) + { + for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; + } + } + if( diagram_out != nullptr ) + { + diagram_out[i_event + offset] = diagram_index[i_event] - 1; + } + if( color_out != nullptr ) + { + color_out[i_event + offset] = color_index[i_event] - 1; + } + if( helicity_out != nullptr ) + { + helicity_out[i_event + offset] = helicity_index[i_event] - 1; + } + } + } +#endif // MGONGPUCPP_GPUIMPL + return UMAMI_SUCCESS; + } + + UmamiStatus umami_free( UmamiHandle handle ) + { + InterfaceInstance* instance = static_cast( handle ); + delete instance; + return UMAMI_SUCCESS; + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/Bridge.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/Bridge.h new file mode 100644 index 0000000000..8417e3ad15 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/Bridge.h @@ -0,0 +1,633 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Nov 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2021-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef BRIDGE_H +#define BRIDGE_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" // for CPPProcess +#include "CrossSectionKernels.h" // for flagAbnormalMEs +#include "MatrixElementKernels.h" // for MatrixElementKernelHost, MatrixElementKernelDevice +#include "MemoryAccessMomenta.h" // for MemoryAccessMomenta::neppM +#include "MemoryBuffers.h" // for HostBufferMomenta, DeviceBufferMomenta etc + +//#ifdef __HIPCC__ +//#include // see +//https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 #else #include +// // bypass this completely to ease portability on LUMI #803 #endif + +#include // bypass std::filesystem #803 + +#include +#include +#include +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + /** + * A base class for a class whose pointer is passed between Fortran and C++. + * This is not really necessary, but it allows minimal type checks on all such + * pointers. + */ + struct CppObjectInFortran + { + CppObjectInFortran() {} + virtual ~CppObjectInFortran() {} + }; + + //-------------------------------------------------------------------------- + /** + * A templated class for calling the CUDA/C++ matrix element calculations of the + * event generation workflow. The FORTRANFPTYPE template parameter indicates the + * precision of the Fortran momenta from MadEvent (float or double). The + * precision of the matrix element calculation is hardcoded in the fptype + * typedef in CUDA/C++. + * + * The Fortran momenta passed in are in the form of + * DOUBLE PRECISION P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) + * where the dimensions are , , + * . In memory, this is stored in a way that C reads as an array + * P_MULTI[nevtF][nparF][np4F]. The CUDA/C++ momenta are stored as an + * array[npagM][npar][np4][neppM] with nevt=npagM*neppM. The Bridge is + * configured to store nevt==nevtF events in CUDA/C++. It also checks that + * Fortran and C++ parameters match, nparF==npar and np4F==np4. + * + * The cpu/gpu sequences take FORTRANFPTYPE* (not fptype*) momenta/MEs. + * This allows mixing double in MadEvent Fortran with float in CUDA/C++ + * sigmaKin. In the fcheck_sa.f test, Fortran uses double while CUDA/C++ may use + * double or float. In the check_sa "--bridge" test, everything is implemented + * in fptype (double or float). + */ + template + class Bridge final : public CppObjectInFortran + { + public: + /** + * Constructor + * + * @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran array + * loops (VECSIZE_USED <= VECSIZE_MEMMAX) + * @param nparF (NEXTERNAL, nexternal.inc) number of external particles in + * Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + * @param np4F number of momenta components, usually 4, in Fortran arrays + * (KEPT FOR SANITY CHECKS ONLY) + */ + Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F ); + + /** + * Destructor + */ + virtual ~Bridge() {} + + // Delete copy/move constructors and assignment operators + Bridge( const Bridge& ) = delete; + Bridge( Bridge&& ) = delete; + Bridge& operator=( const Bridge& ) = delete; + Bridge& operator=( Bridge&& ) = delete; + +#ifdef MGONGPUCPP_GPUIMPL + /** + * Set the gpublocks and gputhreads for the gpusequence - throws if evnt != + * gpublocks*gputhreads (this is needed for BridgeKernel tests rather than for + * actual production use in Fortran) + * + * @param gpublocks number of gpublocks + * @param gputhreads number of gputhreads + */ + void set_gpugrid( const int gpublocks, const int gputhreads ); + + /** + * Sequence to be executed for the Cuda matrix element calculation + * + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant + * alphas) + * @param iflavorVec the index of the flavor combination + * @param rndhel the pointer to the input random numbers for helicity + * selection + * @param rndcol the pointer to the input random numbers for color selection + * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 + * to n + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void gpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); +#else + /** + * Sequence to be executed for the vectorized CPU matrix element calculation + * + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant + * alphas) + * @param iflavorVec the index of the flavor combination + * @param rndhel the pointer to the input random numbers for helicity + * selection + * @param rndcol the pointer to the input random numbers for color selection + * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 + * to n + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void cpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); +#endif + + // Return the number of good helicities (-1 initially when they have not yet + // been calculated) + int nGoodHel() const { return m_nGoodHel; } + + // Return the total number of helicities (expose cudacpp ncomb in the Bridge + // interface to Fortran) + constexpr int nTotHel() const { return CPPProcess::ncomb; } + + private: + unsigned int m_nevt; // number of events + int m_nGoodHel; // the number of good helicities (-1 initially when they have + // not yet been calculated) + +#ifdef MGONGPUCPP_GPUIMPL + int m_gputhreads; // number of gpu threads (default set from number of + // events, can be modified) + int m_gpublocks; // number of gpu blocks (default set from number of events, + // can be modified) + DeviceBuffer m_devMomentaF; + DeviceBufferMomenta m_devMomentaC; + DeviceBufferGs m_devGs; + DeviceBufferIflavorVec m_devIflavorVec; + DeviceBufferRndNumHelicity m_devRndHel; + DeviceBufferRndNumColor m_devRndCol; + DeviceBufferMatrixElements m_devMEs; + DeviceBufferSelectedHelicity m_devSelHel; + DeviceBufferSelectedColor m_devSelCol; + DeviceBufferChannelIds m_devChannelIds; + PinnedHostBufferIflavorVec m_hstIflavorVec; + PinnedHostBufferGs m_hstGs; + PinnedHostBufferRndNumHelicity m_hstRndHel; + PinnedHostBufferRndNumColor m_hstRndCol; + PinnedHostBufferMatrixElements m_hstMEs; + PinnedHostBufferSelectedHelicity m_hstSelHel; + PinnedHostBufferSelectedColor m_hstSelCol; + PinnedHostBufferChannelIds m_hstChannelIds; + std::unique_ptr m_pmek; + // static constexpr int s_gputhreadsmin = 16; // minimum number of gpu threads + // (TEST VALUE FOR MADEVENT) + static constexpr int s_gputhreadsmin = + 32; // minimum number of gpu threads (DEFAULT) +#else + HostBufferMomenta m_hstMomentaC; + HostBufferGs m_hstGs; + HostBufferIflavorVec m_hstIflavorVec; + HostBufferRndNumHelicity m_hstRndHel; + HostBufferRndNumColor m_hstRndCol; + HostBufferMatrixElements m_hstMEs; + HostBufferSelectedHelicity m_hstSelHel; + HostBufferSelectedColor m_hstSelCol; + HostBufferChannelIds m_hstChannelIds; + std::unique_ptr m_pmek; +#endif + }; + + //-------------------------------------------------------------------------- + // + // Forward declare transposition methods + // + +#ifdef MGONGPUCPP_GPUIMPL + + template + __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); + +#endif // MGONGPUCPP_GPUIMPL + + template + void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); + + template + void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ); + + //-------------------------------------------------------------------------- + // + // Implementations of member functions of class Bridge + // + + template + Bridge::Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F) + : m_nevt( nevtF ), m_nGoodHel( -1 ) +#ifdef MGONGPUCPP_GPUIMPL + , m_gputhreads( 256 ) // default number of gpu threads + , m_gpublocks( m_nevt / m_gputhreads ) // this ensures m_nevt <= m_gpublocks*m_gputhreads + , m_devMomentaF( m_nevt ) + , m_devMomentaC( m_nevt ) + , m_devIflavorVec( m_nevt ) + , m_devGs( m_nevt ) + , m_devRndHel( m_nevt ) + , m_devRndCol( m_nevt ) + , m_devMEs( m_nevt ) + , m_devSelHel( m_nevt ) + , m_devSelCol( m_nevt ) + , m_devChannelIds( m_nevt ) +#else + , m_hstMomentaC( m_nevt ) +#endif + , m_hstGs( m_nevt ) + , m_hstIflavorVec( m_nevt ) + , m_hstRndHel( m_nevt ) + , m_hstRndCol( m_nevt ) + , m_hstMEs( m_nevt ) + , m_hstSelHel( m_nevt ) + , m_hstSelCol( m_nevt ) + , m_hstChannelIds( m_nevt ) + , m_pmek( nullptr ) + { + if( nparF != CPPProcess::npar ) + throw std::runtime_error( "Bridge constructor: npar mismatch" ); + if( np4F != CPPProcess::np4 ) + throw std::runtime_error( "Bridge constructor: np4 mismatch" ); +#ifdef MGONGPUCPP_GPUIMPL + if( ( m_nevt < s_gputhreadsmin ) || ( m_nevt % s_gputhreadsmin != 0 ) ) + throw std::runtime_error( + "Bridge constructor: nevt should be a multiple of " + + std::to_string( s_gputhreadsmin ) ); + while( m_nevt != m_gpublocks * m_gputhreads ) + { + m_gputhreads /= 2; + if( m_gputhreads < s_gputhreadsmin ) + throw std::logic_error( + "Bridge constructor: FIXME! cannot choose gputhreads" ); // this + // should + // never + // happen! + m_gpublocks = m_nevt / m_gputhreads; + } +#ifdef MGONGPUCPP_VERBOSE + std::cout << "WARNING! Instantiate device Bridge (nevt=" << m_nevt + << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads + << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" + << std::endl; +#endif + m_pmek.reset( new MatrixElementKernelDevice( + m_devMomentaC, m_devGs, m_devIflavorVec, m_devRndHel, m_devRndCol, m_devChannelIds, m_devMEs, m_devSelHel, m_devSelCol, m_gpublocks, m_gputhreads) ); +#else +#ifdef MGONGPUCPP_VERBOSE + std::cout << "WARNING! Instantiate host Bridge (nevt=" << m_nevt << ")" + << std::endl; +#endif + m_pmek.reset( new MatrixElementKernelHost( + m_hstMomentaC, m_hstGs, m_hstIflavorVec, m_hstRndHel, m_hstRndCol, m_hstChannelIds, m_hstMEs, m_hstSelHel, m_hstSelCol, m_nevt ) ); +#endif // MGONGPUCPP_GPUIMPL + // Create a process object, read param card and set parameters + // FIXME: the process instance can happily go out of scope because it is only + // needed to read parameters? + // FIXME: the CPPProcess should really be a singleton? what if fbridgecreate + // is called from several Fortran threads? + CPPProcess process( /*verbose=*/false ); + std::string paramCard = + "../Cards/param_card.dat"; // ZW: change default param_card.dat location + // to one dir down + /* +#ifdef __HIPCC__ + if( !std::experimental::filesystem::exists( paramCard ) ) paramCard = "../" + +paramCard; #else if( !std::filesystem::exists( paramCard ) ) paramCard = "../" + +paramCard; #endif + */ + // struct stat dummybuffer; // bypass std::filesystem #803 + // if( !( stat( paramCard.c_str(), &dummyBuffer ) == 0 ) ) paramCard = "../" + + // paramCard; // + auto fileExists = []( std::string& fileName ) + { + struct stat buffer; + return stat( fileName.c_str(), &buffer ) == 0; + }; + size_t paramCardCheck = 2; // ZW: check for paramCard up to 2 directories up + for( size_t k = 0; k < paramCardCheck; ++k ) + { + if( fileExists( paramCard ) ) break; // bypass std::filesystem #803 + paramCard = "../" + paramCard; + } + process.initProc( paramCard ); + } + +#ifdef MGONGPUCPP_GPUIMPL + template + void Bridge::set_gpugrid( const int gpublocks, + const int gputhreads ) + { + if( m_nevt != gpublocks * gputhreads ) + throw std::runtime_error( + "Bridge: gpublocks*gputhreads must equal m_nevt in set_gpugrid" ); + m_gpublocks = gpublocks; + m_gputhreads = gputhreads; +#ifdef MGONGPUCPP_VERBOSE + std::cout << "WARNING! Set grid in Bridge (nevt=" << m_nevt + << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads + << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" + << std::endl; +#endif + m_pmek->setGrid( m_gpublocks, m_gputhreads ); + } +#endif + +#ifdef MGONGPUCPP_GPUIMPL + template + void Bridge::gpu_sequence( const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool goodHelOnly ) + { + constexpr int neppM = MemoryAccessMomenta::neppM; + if constexpr( neppM == 1 && std::is_same_v ) + { + gpuMemcpy( m_devMomentaC.data(), momenta, m_devMomentaC.bytes(), gpuMemcpyHostToDevice ); + } + else + { + gpuMemcpy( m_devMomentaF.data(), momenta, m_devMomentaF.bytes(), gpuMemcpyHostToDevice ); + const int thrPerEvt = + CPPProcess::npar * + CPPProcess::np4; // AV: transpose alg does 1 element per thread (NOT 1 + // event per thread) + // const int thrPerEvt = 1; // AV: try new alg with 1 event per thread... + // this seems slower + gpuLaunchKernel( dev_transposeMomentaF2C, m_gpublocks * thrPerEvt, m_gputhreads, m_devMomentaF.data(), m_devMomentaC.data(), m_nevt ); + } + if constexpr( std::is_same_v ) + { + memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); + } + else + { + std::copy( gs, gs + m_nevt, m_hstGs.data() ); + std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); + std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); + } + const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); + if( useChannelIds ) + memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); + // else ... // no need to initialize m_hstChannel: it is allocated with + // gpuMallocHost and NOT initialized in PinnedHostBufferBase, but it is NOT + // used later on + // initialise iflavorVec + memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); + copyDeviceFromHost( m_devGs, m_hstGs ); + copyDeviceFromHost( m_devRndHel, m_hstRndHel ); + copyDeviceFromHost( m_devRndCol, m_hstRndCol ); + if( useChannelIds ) copyDeviceFromHost( m_devChannelIds, m_hstChannelIds ); + copyDeviceFromHost( m_devIflavorVec, m_hstIflavorVec ); + if( m_nGoodHel < 0 ) + { + m_nGoodHel = m_pmek->computeGoodHelicities(); + if( m_nGoodHel < 0 ) + throw std::runtime_error( + "Bridge gpu_sequence: computeGoodHelicities returned nGoodHel<0" ); + } + if( goodHelOnly ) return; + m_pmek->computeMatrixElements( useChannelIds ); + copyHostFromDevice( m_hstMEs, m_devMEs ); +#ifdef MGONGPUCPP_VERBOSE + flagAbnormalMEs( m_hstMEs.data(), m_nevt ); +#endif + copyHostFromDevice( m_hstSelHel, m_devSelHel ); + copyHostFromDevice( m_hstSelCol, m_devSelCol ); + if constexpr( std::is_same_v ) + { + memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); + memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); + memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); + } + else + { + std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); + std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); + std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); + } + } +#endif + +#ifndef MGONGPUCPP_GPUIMPL + template + void Bridge::cpu_sequence( const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool goodHelOnly ) + { + hst_transposeMomentaF2C( momenta, m_hstMomentaC.data(), m_nevt ); + if constexpr( std::is_same_v ) + { + memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); + memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); + } + else + { + std::copy( gs, gs + m_nevt, m_hstGs.data() ); + std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); + std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); + } + const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); + if( useChannelIds ) + memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); + // else ... // no need to initialize m_hstChannel: it is allocated and default + // initialized in HostBufferBase (and it is not used later on anyway) + // initialise iflavorVec + memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); + if( m_nGoodHel < 0 ) + { + m_nGoodHel = m_pmek->computeGoodHelicities(); + if( m_nGoodHel < 0 ) + throw std::runtime_error( + "Bridge cpu_sequence: computeGoodHelicities returned nGoodHel<0" ); + } + if( goodHelOnly ) return; + m_pmek->computeMatrixElements( useChannelIds ); +#ifdef MGONGPUCPP_VERBOSE + flagAbnormalMEs( m_hstMEs.data(), m_nevt ); +#endif + if constexpr( std::is_same_v ) + { + memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); + memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); + memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); + } + else + { + std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); + std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); + std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); + } + } +#endif + + //-------------------------------------------------------------------------- + // + // Implementations of transposition methods + // - FORTRAN arrays: P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) ==> + // p_multi[nevtF][nparF][np4F] in C++ (AOS) + // - C++ array: momenta[npagM][npar][np4][neppM] with nevt=npagM*neppM (AOSOA) + // + +#ifdef MGONGPUCPP_GPUIMPL + template + __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool oldImplementation = true; // default: use old implementation + if constexpr( oldImplementation ) + { + // SR initial implementation + constexpr int part = CPPProcess::npar; + constexpr int mome = CPPProcess::np4; + constexpr int strd = MemoryAccessMomenta::neppM; + int pos = blockDim.x * blockIdx.x + threadIdx.x; + int arrlen = nevt * part * mome; + if( pos < arrlen ) + { + int page_i = pos / ( strd * mome * part ); + int rest_1 = pos % ( strd * mome * part ); + int part_i = rest_1 / ( strd * mome ); + int rest_2 = rest_1 % ( strd * mome ); + int mome_i = rest_2 / strd; + int strd_i = rest_2 % strd; + int inpos = ( page_i * strd + strd_i ) // event number + * ( part * mome ) // event size (pos of event) + + part_i * mome // particle inside event + + mome_i; // momentum inside particle + out[pos] = in[inpos]; // F2C (Fortran to C) + } + } + else + { + // AV attempt another implementation with 1 event per thread: this seems + // slower... F-style: AOS[nevtF][nparF][np4F] C-style: + // AOSOA[npagM][npar][np4][neppM] with nevt=npagM*neppM + constexpr int npar = CPPProcess::npar; + constexpr int np4 = CPPProcess::np4; + constexpr int neppM = MemoryAccessMomenta::neppM; + assert( nevt % neppM == + 0 ); // number of events is not a multiple of neppM??? + int ievt = blockDim.x * blockIdx.x + threadIdx.x; + int ipagM = ievt / neppM; + int ieppM = ievt % neppM; + for( int ip4 = 0; ip4 < np4; ip4++ ) + for( int ipar = 0; ipar < npar; ipar++ ) + { + int cpos = ipagM * npar * np4 * neppM + ipar * np4 * neppM + + ip4 * neppM + ieppM; + int fpos = ievt * npar * np4 + ipar * np4 + ip4; + out[cpos] = in[fpos]; // F2C (Fortran to C) + } + } + } +#endif + + template + void hst_transposeMomenta( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool oldImplementation = false; // default: use new implementation + if constexpr( oldImplementation ) + { + // SR initial implementation + constexpr unsigned int part = CPPProcess::npar; + constexpr unsigned int mome = CPPProcess::np4; + constexpr unsigned int strd = MemoryAccessMomenta::neppM; + unsigned int arrlen = nevt * part * mome; + for( unsigned int pos = 0; pos < arrlen; ++pos ) + { + unsigned int page_i = pos / ( strd * mome * part ); + unsigned int rest_1 = pos % ( strd * mome * part ); + unsigned int part_i = rest_1 / ( strd * mome ); + unsigned int rest_2 = rest_1 % ( strd * mome ); + unsigned int mome_i = rest_2 / strd; + unsigned int strd_i = rest_2 % strd; + unsigned int inpos = ( page_i * strd + strd_i ) // event number + * ( part * mome ) // event size (pos of event) + + part_i * mome // particle inside event + + mome_i; // momentum inside particle + if constexpr( F2C ) // needs c++17 and cuda >=11.2 (#333) + out[pos] = in[inpos]; // F2C (Fortran to C) + else + out[inpos] = in[pos]; // C2F (C to Fortran) + } + } + else + { + // AV attempt another implementation: this is slightly faster (better c++ + // pipelining?) [NB! this is not a transposition, it is an AOS to AOSOA + // conversion: if neppM=1, a memcpy is enough] F-style: + // AOS[nevtF][nparF][np4F] C-style: AOSOA[npagM][npar][np4][neppM] with + // nevt=npagM*neppM + constexpr unsigned int npar = CPPProcess::npar; + constexpr unsigned int np4 = CPPProcess::np4; + constexpr unsigned int neppM = MemoryAccessMomenta::neppM; + if constexpr( neppM == 1 && std::is_same_v ) + { + memcpy( out, in, nevt * npar * np4 * sizeof( Tin ) ); + } + else + { + const unsigned int npagM = nevt / neppM; + assert( nevt % neppM == + 0 ); // number of events is not a multiple of neppM??? + for( unsigned int ipagM = 0; ipagM < npagM; ipagM++ ) + for( unsigned int ip4 = 0; ip4 < np4; ip4++ ) + for( unsigned int ipar = 0; ipar < npar; ipar++ ) + for( unsigned int ieppM = 0; ieppM < neppM; ieppM++ ) + { + unsigned int ievt = ipagM * neppM + ieppM; + unsigned int cpos = ipagM * npar * np4 * neppM + + ipar * np4 * neppM + ip4 * neppM + ieppM; + unsigned int fpos = ievt * npar * np4 + ipar * np4 + ip4; + if constexpr( F2C ) + out[cpos] = in[fpos]; // F2C (Fortran to C) + else + out[fpos] = in[cpos]; // C2F (C to Fortran) + } + } + } + } + + template + void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool F2C = true; + hst_transposeMomenta( in, out, nevt ); + } + + template + void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ) + { + constexpr bool F2C = false; + hst_transposeMomenta( in, out, nevt ); + } + + //-------------------------------------------------------------------------- +} // namespace mg5amcGpu +#endif // BRIDGE_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.cc new file mode 100644 index 0000000000..65c97616d8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.cc @@ -0,0 +1,164 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "BridgeKernels.h" + +#include "GpuAbstraction.h" +#include "MemoryAccessMomenta.h" + +#include + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) + constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- + + //-------------------------------------------------------------------------- + + BridgeKernelBase::BridgeKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( nevt ) + , m_bridge( nevt, npar, np4 ) + { + if( m_momenta.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: momenta must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: matrixElements must be a host array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +#ifndef MGONGPUCPP_GPUIMPL +namespace mg5amcCpu +{ + + //-------------------------------------------------------------------------- + + BridgeKernelHost::BridgeKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: Gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt) + : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, nevt) + , m_fortranMomenta( nevt ) + { + } + + //-------------------------------------------------------------------------- + + void BridgeKernelHost::transposeInputMomentaC2F() + { + hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); + } + + //-------------------------------------------------------------------------- + + int BridgeKernelHost::computeGoodHelicities() + { + constexpr bool goodHelOnly = true; + constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering + m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + return m_bridge.nGoodHel(); + } + + //-------------------------------------------------------------------------- + + void BridgeKernelHost::computeMatrixElements( const bool useChannelIds ) + { + constexpr bool goodHelOnly = false; + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + + //-------------------------------------------------------------------------- + + BridgeKernelDevice::BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: Gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads) + : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, gpublocks * gputhreads) + , m_fortranMomenta( nevt() ) + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + if( m_gpublocks == 0 ) throw std::runtime_error( "BridgeKernelDevice: gpublocks must be > 0" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "BridgeKernelDevice: gputhreads must be > 0" ); + m_bridge.set_gpugrid( gpublocks, gputhreads ); + } + + //-------------------------------------------------------------------------- + + void BridgeKernelDevice::transposeInputMomentaC2F() + { + hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); + } + + //-------------------------------------------------------------------------- + + int BridgeKernelDevice::computeGoodHelicities() + { + constexpr bool goodHelOnly = true; + constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering + m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + return m_bridge.nGoodHel(); + } + + //-------------------------------------------------------------------------- + + void BridgeKernelDevice::computeMatrixElements( const bool useChannelIds ) + { + constexpr bool goodHelOnly = false; + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.h new file mode 100644 index 0000000000..2d873486a8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.h @@ -0,0 +1,146 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally originally by: A. Valassi (Jan 2022). +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef BRIDGEKERNELS_H +#define BRIDGEKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "Bridge.h" +#include "MatrixElementKernels.h" +#include "MemoryBuffers.h" + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // A Bridge wrapper base class encapsulating matrix element calculations on a CPU host + class BridgeKernelBase : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + BridgeKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~BridgeKernelBase() {} + + // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge + virtual void transposeInputMomentaC2F() = 0; + + protected: + + // The wrapped bridge + Bridge m_bridge; + }; + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + // A Bridge wrapper class encapsulating matrix element calculations on a CPU host + class BridgeKernelHost final : public BridgeKernelBase + { + public: + + // Constructor from existing input and output buffers + BridgeKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~BridgeKernelHost() {} + + // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge + void transposeInputMomentaC2F() override final; + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // The buffer for the input momenta, transposed to Fortran array indexing + HostBufferMomenta m_fortranMomenta; + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A Bridge wrapper class encapsulating matrix element calculations on a GPU device + class BridgeKernelDevice : public BridgeKernelBase + { + public: + + // Constructor from existing input and output buffers + BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads); + + // Destructor + virtual ~BridgeKernelDevice() {} + + // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge + void transposeInputMomentaC2F() override final; + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The buffer for the input momenta, transposed to Fortran array indexing + PinnedHostBufferMomenta m_fortranMomenta; + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + }; +#endif + + //-------------------------------------------------------------------------- +} +#endif // BRIDGEKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc new file mode 100644 index 0000000000..16671b5686 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc @@ -0,0 +1,39 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "CommonRandomNumbers.h" +#include "GpuAbstraction.h" +#include "MemoryBuffers.h" +#include "RandomNumberKernels.h" + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + CommonRandomNumberKernel::CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ) + : RandomNumberKernelBase( rnarray ) + , m_seed( 20211220 ) + { + if( m_rnarray.isOnDevice() ) + throw std::runtime_error( "CommonRandomNumberKernel on host with a device random number array" ); + } + + //-------------------------------------------------------------------------- + + void CommonRandomNumberKernel::generateRnarray() + { + std::vector rnd = CommonRandomNumbers::generate( m_rnarray.size(), m_seed ); // NB: generate as double (HARDCODED) + std::copy( rnd.begin(), rnd.end(), m_rnarray.data() ); // NB: copy may imply a double-to-float conversion + } + + //-------------------------------------------------------------------------- +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc new file mode 100644 index 0000000000..7a3e0e0519 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc @@ -0,0 +1,238 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "CrossSectionKernels.h" + +#include "GpuAbstraction.h" +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessWeights.h" +#include "MemoryBuffers.h" + +#include + +// ****************************************************************************************** +// *** NB: Disabling fast math is essential here, otherwise results are undefined *** +// *** NB: This file CrossSectionKernels.cc IS BUILT WITH -fno-fast-math in the Makefile! *** +// *** NB: Attempts with __attribute__((optimize("-fno-fast-math"))) were unsatisfactory *** +// ****************************************************************************************** + +inline bool +fp_is_nan( const fptype& fp ) +{ + //#pragma clang diagnostic push + //#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) + return std::isnan( fp ); // always false for clang in fast math mode (tautological compare)? + //#pragma clang diagnostic pop +} + +inline bool +fp_is_abnormal( const fptype& fp ) +{ + if( fp_is_nan( fp ) ) return true; + if( fp != fp ) return true; + return false; +} + +inline bool +fp_is_zero( const fptype& fp ) +{ + if( fp == 0 ) return true; + return false; +} + +// See https://en.cppreference.com/w/cpp/numeric/math/FP_categories +inline const char* +fp_show_class( const fptype& fp ) +{ + switch( std::fpclassify( fp ) ) + { + case FP_INFINITE: return "Inf"; + case FP_NAN: return "NaN"; + case FP_NORMAL: return "normal"; + case FP_SUBNORMAL: return "subnormal"; + case FP_ZERO: return "zero"; + default: return "unknown"; + } +} + +inline void +debug_me_is_abnormal( const fptype& me, size_t ievtALL ) +{ + std::cout << "DEBUG[" << ievtALL << "]" + << " ME=" << me + << " fpisabnormal=" << fp_is_abnormal( me ) + << " fpclass=" << fp_show_class( me ) + << " (me==me)=" << ( me == me ) + << " (me==me+1)=" << ( me == me + 1 ) + << " isnan=" << fp_is_nan( me ) + << " isfinite=" << std::isfinite( me ) + << " isnormal=" << std::isnormal( me ) + << " is0=" << ( me == 0 ) + << " is1=" << ( me == 1 ) + << " abs(ME)=" << std::abs( me ) + << " isnan=" << fp_is_nan( std::abs( me ) ) + << std::endl; +} + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ) + { + for( unsigned int ievt = 0; ievt < nevt; ievt++ ) + { + if( fp_is_abnormal( hstMEs[ievt] ) ) + { + std::cout << "WARNING! flagging abnormal ME for ievt=" << ievt << std::endl; + hstMEs[ievt] = std::sqrt( -1. ); + } + } + } + + //-------------------------------------------------------------------------- + + CrossSectionKernelHost::CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t nevt ) + : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) + , NumberOfEvents( nevt ) + { + if( m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: samplingWeights must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: matrixElements must be a host array" ); + if( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with samplingWeights" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelHost::updateEventStatistics( const bool debug ) + { + EventStatistics stats; // new statistics for the new nevt events + // FIRST PASS: COUNT ALL/ABN/ZERO EVENTS, COMPUTE MIN/MAX, COMPUTE REFS AS MEANS OF SIMPLE SUMS + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + const size_t ievtALL = m_iter * nevt() + ievt; + // The following events are abnormal in a run with "-p 2048 256 12 -d" + // - check.exe/commonrand: ME[310744,451171,3007871,3163868,4471038,5473927] with fast math + // - check.exe/curand: ME[578162,1725762,2163579,5407629,5435532,6014690] with fast math + // - gcheck.exe/curand: ME[596016,1446938] with fast math + // Debug NaN/abnormal issues + //if ( ievtALL == 310744 ) // this ME is abnormal both with and without fast math + // debug_me_is_abnormal( me, ievtALL ); + //if ( ievtALL == 5473927 ) // this ME is abnormal only with fast math + // debug_me_is_abnormal( me, ievtALL ); + stats.nevtALL++; + if( fp_is_abnormal( me ) ) + { + if( debug ) // only printed out with "-p -d" (matrixelementALL is not filled without -p) + std::cout << "WARNING! ME[" << ievtALL << "] is NaN/abnormal" << std::endl; + stats.nevtABN++; + continue; + } + if( fp_is_zero( me ) ) stats.nevtZERO++; + stats.minME = std::min( stats.minME, (double)me ); + stats.maxME = std::max( stats.maxME, (double)me ); + stats.minWG = std::min( stats.minWG, (double)wg ); + stats.maxWG = std::max( stats.maxWG, (double)wg ); + stats.sumMEdiff += me; // NB stats.refME is 0 here + stats.sumWGdiff += wg; // NB stats.refWG is 0 here + } + stats.refME = stats.meanME(); // draft ref + stats.refWG = stats.meanWG(); // draft ref + stats.sumMEdiff = 0; + stats.sumWGdiff = 0; + // SECOND PASS: IMPROVE MEANS FROM SUMS OF DIFFS TO PREVIOUS REF, UPDATE REF + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + if( fp_is_abnormal( me ) ) continue; + stats.sumMEdiff += ( me - stats.refME ); + stats.sumWGdiff += ( wg - stats.refWG ); + } + stats.refME = stats.meanME(); // final ref + stats.refWG = stats.meanWG(); // final ref + stats.sumMEdiff = 0; + stats.sumWGdiff = 0; + // THIRD PASS: COMPUTE STDDEV FROM SQUARED SUMS OF DIFFS TO REF + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + if( fp_is_abnormal( me ) ) continue; + stats.sqsMEdiff += std::pow( me - stats.refME, 2 ); + stats.sqsWGdiff += std::pow( wg - stats.refWG, 2 ); + } + // FOURTH PASS: UPDATE THE OVERALL STATS BY ADDING THE NEW STATS + m_stats += stats; + // Increment the iterations counter + m_iter++; + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + + /* + //-------------------------------------------------------------------------- + + CrossSectionKernelDevice::CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t gpublocks, + const size_t gputhreads ) + : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) + , NumberOfEvents( gpublocks*gputhreads ) + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + if ( ! m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: samplingWeights must be a device array" ); + if ( ! m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: matrixElements must be a device array" ); + if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0" ); + if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0" ); + if ( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with samplingWeights" ); + if ( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelDevice::setGrid( const size_t gpublocks, const size_t gputhreads ) + { + if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0 in setGrid" ); + if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0 in setGrid" ); + if ( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch in setGrid" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelDevice::updateEventStatistics( const bool debug ) + { + // Increment the iterations counter + m_iter++; + } + + //-------------------------------------------------------------------------- + */ + +} +#endif + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h new file mode 100644 index 0000000000..d1e29b7999 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h @@ -0,0 +1,139 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef CROSSSECTIONKERNELS_H +#define CROSSSECTIONKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "EventStatistics.h" +#include "MemoryBuffers.h" + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // Helper function for Bridge.h: must be compiled without fast math + // Iterate through all output MEs and replace any NaN/abnormal ones by sqrt(-1) + void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ); + + //-------------------------------------------------------------------------- + + // A base class encapsulating the calculation of event statistics on a CPU host or on a GPU device + class CrossSectionKernelBase //: virtual public ICrossSectionKernel + { + protected: + + // Constructor from existing input and output buffers + CrossSectionKernelBase( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats ) // output: event statistics + : m_samplingWeights( samplingWeights ) + , m_matrixElements( matrixElements ) + , m_stats( stats ) + , m_iter( 0 ) + { + // NB: do not initialise EventStatistics (you may be asked to update an existing result) + } + + public: + + // Destructor + virtual ~CrossSectionKernelBase() {} + + // Update event statistics + virtual void updateEventStatistics( const bool debug = false ) = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The buffer for the sampling weights + const BufferWeights& m_samplingWeights; + + // The buffer for the output matrix elements + const BufferMatrixElements& m_matrixElements; + + // The event statistics + EventStatistics& m_stats; + + // The number of iterations processed so far + size_t m_iter; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating the calculation of event statistics on a CPU host + class CrossSectionKernelHost final : public CrossSectionKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t nevt ); + + // Destructor + virtual ~CrossSectionKernelHost() {} + + // Update event statistics + void updateEventStatistics( const bool debug = false ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + }; + + //-------------------------------------------------------------------------- + + /* +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating the calculation of event statistics on a GPU device + class CrossSectionKernelDevice : public CrossSectionKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t gpublocks, + const size_t gputhreads ); + + // Destructor + virtual ~CrossSectionKernelDevice(){} + + // Reset gpublocks and gputhreads + void setGrid( const size_t gpublocks, const size_t gputhreads ); + + // Update event statistics + void updateEventStatistics( const bool debug=false ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + + }; +#endif + */ + + //-------------------------------------------------------------------------- +} +#endif // CROSSSECTIONKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc new file mode 100644 index 0000000000..0c8e6e6445 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc @@ -0,0 +1,136 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "GpuRuntime.h" +#include "MemoryBuffers.h" +#include "RandomNumberKernels.h" + +#include + +#ifndef MGONGPU_HAS_NO_CURAND /* clang-format off */ +// NB This must come AFTER mgOnGpuConfig.h which contains our definition of __global__ when MGONGPUCPP_GPUIMPL is not defined +#include "curand.h" +#define checkCurand( code ){ assertCurand( code, __FILE__, __LINE__ ); } +inline void assertCurand( curandStatus_t code, const char *file, int line, bool abort = true ) +{ + if ( code != CURAND_STATUS_SUCCESS ) + { + printf( "CurandAssert: %s:%d code=%d\n", file, line, code ); + if ( abort ) assert( code == CURAND_STATUS_SUCCESS ); + } +} +#endif /* clang-format on */ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- +#ifndef MGONGPU_HAS_NO_CURAND + CurandRandomNumberKernel::CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) + : RandomNumberKernelBase( rnarray ) + , m_isOnDevice( onDevice ) + { + if( m_isOnDevice ) + { +#ifdef MGONGPUCPP_GPUIMPL + if( !m_rnarray.isOnDevice() ) + throw std::runtime_error( "CurandRandomNumberKernel on device with a host random number array" ); +#else + throw std::runtime_error( "CurandRandomNumberKernel does not support CurandDevice on CPU host" ); +#endif + } + else + { + if( m_rnarray.isOnDevice() ) + throw std::runtime_error( "CurandRandomNumberKernel on host with a device random number array" ); + } + createGenerator(); + } + + //-------------------------------------------------------------------------- + + CurandRandomNumberKernel::~CurandRandomNumberKernel() + { + destroyGenerator(); + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::seedGenerator( const unsigned int seed ) + { + if( m_isOnDevice ) + { + destroyGenerator(); // workaround for #429 + createGenerator(); // workaround for #429 + } + //printf( "seedGenerator: seed %d\n", seed ); + checkCurand( curandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::createGenerator() + { + // [NB Timings are for GenRnGen host|device (cpp|cuda) generation of 256*32*1 events with nproc=1: rn(0) is host=0.0012s] + const curandRngType_t type = CURAND_RNG_PSEUDO_MTGP32; // 0.00082s | 0.00064s (FOR FAST TESTS) + //const curandRngType_t type = CURAND_RNG_PSEUDO_XORWOW; // 0.049s | 0.0016s + //const curandRngType_t type = CURAND_RNG_PSEUDO_MRG32K3A; // 0.71s | 0.0012s (better but slower, especially in c++) + //const curandRngType_t type = CURAND_RNG_PSEUDO_MT19937; // 21s | 0.021s + //const curandRngType_t type = CURAND_RNG_PSEUDO_PHILOX4_32_10; // 0.024s | 0.00026s (used to segfault?) + if( m_isOnDevice ) + { + checkCurand( curandCreateGenerator( &m_rnGen, type ) ); + } + else + { + checkCurand( curandCreateGeneratorHost( &m_rnGen, type ) ); + } + //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_LEGACY ) ); // fails with code=104 (see #429) + checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_BEST ) ); + //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_DYNAMIC ) ); // fails with code=104 (see #429) + //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_SEEDED ) ); // fails with code=104 (see #429) + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::destroyGenerator() + { + checkCurand( curandDestroyGenerator( m_rnGen ) ); + } + + //-------------------------------------------------------------------------- + + void CurandRandomNumberKernel::generateRnarray() + { +#if defined MGONGPU_FPTYPE_DOUBLE + checkCurand( curandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#elif defined MGONGPU_FPTYPE_FLOAT + checkCurand( curandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#endif + /* + printf( "\nCurandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); + fptype* data = m_rnarray.data(); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) + { + data = new fptype[m_rnarray.size()](); + checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); + } +#endif + for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) + printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) delete[] data; +#endif + */ + } + + //-------------------------------------------------------------------------- +#endif +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h new file mode 100644 index 0000000000..4a1fd8464d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h @@ -0,0 +1,175 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef EventStatistics_H +#define EventStatistics_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" // for npar (meGeVexponent) + +#include +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // The EventStatistics struct is used to accumulate running aggregates of event statistics. + // This will eventually include the process cross section and the process maximum weight: + // one important case of EventStatistics will then be the "gridpack" result set, which is + // the output of the "integration" step and the input to "unweighted event generation" step. + // The current implementation only includes statistics for matrix elements (ME) and sampling weights (WG); + // in first approximation, the process cross section and maximum weight are just the mean ME and maximum ME, + // but eventually the sampling weights WG (e.g. from Rambo) must also be taken into account in the calculation. + // The implementation uses differences to reference values to improve numerical precision. + struct EventStatistics + { + public: + size_t nevtALL; // total number of events used + size_t nevtABN; // number of events used, where ME is abnormal (nevtABN <= nevtALL) + size_t nevtZERO; // number of not-abnormal events used, where ME is zero (nevtZERO <= nevtOK) + double minME; // minimum matrix element + double maxME; // maximum matrix element + double minWG; // minimum sampling weight + double maxWG; // maximum sampling weight + double refME; // "reference" matrix element (normally the current mean) + double refWG; // "reference" sampling weight (normally the current mean) + double sumMEdiff; // sum of diff to ref for matrix element + double sumWGdiff; // sum of diff to ref for sampling weight + double sqsMEdiff; // squared sum of diff to ref for matrix element + double sqsWGdiff; // squared sum of diff to ref for sampling weight + std::string tag; // a text tag for printouts + // Number of events used, where ME is not abnormal + size_t nevtOK() const { return nevtALL - nevtABN; } + // Mean matrix element + // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] + double meanME() const + { + return refME + ( nevtOK() > 0 ? sumMEdiff / nevtOK() : 0 ); + } + // Mean sampling weight + // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] + double meanWG() const + { + return refWG + ( nevtOK() > 0 ? sumWGdiff / nevtOK() : 0 ); + } + // Variance matrix element + // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] + double varME() const { return ( sqsMEdiff - std::pow( sumMEdiff, 2 ) / nevtOK() ) / nevtOK(); } + // Variance sampling weight + // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] + double varWG() const { return ( sqsWGdiff - std::pow( sumWGdiff, 2 ) / nevtOK() ) / nevtOK(); } + // Standard deviation matrix element + double stdME() const { return std::sqrt( varME() ); } + // Standard deviation sampling weight + double stdWG() const { return std::sqrt( varWG() ); } + // Update reference matrix element + void updateRefME( const double newRef ) + { + const double deltaRef = refME - newRef; + sqsMEdiff += deltaRef * ( 2 * sumMEdiff + nevtOK() * deltaRef ); + sumMEdiff += deltaRef * nevtOK(); + refME = newRef; + } + // Update reference sampling weight + void updateRefWG( const double newRef ) + { + const double deltaRef = refWG - newRef; + sqsWGdiff += deltaRef * ( 2 * sumWGdiff + nevtOK() * deltaRef ); + sumWGdiff += deltaRef * nevtOK(); + refWG = newRef; + } + // Constructor + EventStatistics() + : nevtALL( 0 ) + , nevtABN( 0 ) + , nevtZERO( 0 ) + , minME( std::numeric_limits::max() ) + , maxME( std::numeric_limits::lowest() ) + , minWG( std::numeric_limits::max() ) + , maxWG( std::numeric_limits::lowest() ) + , refME( 0 ) + , refWG( 0 ) + , sumMEdiff( 0 ) + , sumWGdiff( 0 ) + , sqsMEdiff( 0 ) + , sqsWGdiff( 0 ) + , tag( "" ) {} + // Combine two EventStatistics +#ifdef __clang__ + // Disable optimizations for this function in HIP (work around FPE crash #1003: originally using #if __HIP_CLANG_ONLY__) + // Disable optimizations for this function in clang tout court (work around FPE crash #1005: now using #ifdef __clang__) + // See https://clang.llvm.org/docs/LanguageExtensions.html#extensions-for-selectively-disabling-optimization + __attribute__( ( optnone ) ) +#endif + EventStatistics& + operator+=( const EventStatistics& stats ) + { + EventStatistics s1 = *this; // temporary copy + EventStatistics s2 = stats; // temporary copy + EventStatistics& sum = *this; + sum.nevtALL = s1.nevtALL + s2.nevtALL; + sum.nevtABN = s1.nevtABN + s2.nevtABN; + sum.nevtZERO = s1.nevtZERO + s2.nevtZERO; + sum.minME = std::min( s1.minME, s2.minME ); + sum.maxME = std::max( s1.maxME, s2.maxME ); + sum.minWG = std::min( s1.minWG, s2.minWG ); + sum.maxWG = std::max( s1.maxWG, s2.maxWG ); + sum.refME = ( s1.meanME() * s1.nevtOK() + s2.meanME() * s2.nevtOK() ) / sum.nevtOK(); // new mean ME + s1.updateRefME( sum.refME ); + s2.updateRefME( sum.refME ); + sum.sumMEdiff = s1.sumMEdiff + s2.sumMEdiff; + sum.sqsMEdiff = s1.sqsMEdiff + s2.sqsMEdiff; + sum.refWG = ( s1.meanWG() * s1.nevtOK() + s2.meanWG() * s2.nevtOK() ) / sum.nevtOK(); // new mean WG + s1.updateRefWG( sum.refWG ); + s2.updateRefWG( sum.refWG ); + sum.sumWGdiff = s1.sumWGdiff + s2.sumWGdiff; + sum.sqsWGdiff = s1.sqsWGdiff + s2.sqsWGdiff; + return sum; + } + // Printout + void printout( std::ostream& out ) const + { + const EventStatistics& s = *this; + constexpr int meGeVexponent = -( 2 * CPPProcess::npar - 8 ); + out << s.tag << "NumMatrixElems(notAbnormal) = " << s.nevtOK() << std::endl + << std::scientific // fixed format: affects all floats (default precision: 6) + << s.tag << "MeanMatrixElemValue = ( " << s.meanME() + << " +- " << s.stdME() / std::sqrt( s.nevtOK() ) << " ) GeV^" << meGeVexponent << std::endl // standard error + << s.tag << "[Min,Max]MatrixElemValue = [ " << s.minME + << " , " << s.maxME << " ] GeV^" << meGeVexponent << std::endl + << s.tag << "StdDevMatrixElemValue = ( " << s.stdME() + << std::string( 16, ' ' ) << " ) GeV^" << meGeVexponent << std::endl + << s.tag << "MeanWeight = ( " << s.meanWG() + << " +- " << s.stdWG() / std::sqrt( s.nevtOK() ) << std::endl // standard error + << s.tag << "[Min,Max]Weight = [ " << s.minWG + << " , " << s.maxWG << " ]" << std::endl + << s.tag << "StdDevWeight = ( " << s.stdWG() + << std::string( 16, ' ' ) << " )" << std::endl + << std::defaultfloat; // default format: affects all floats + } + }; + + //-------------------------------------------------------------------------- + + inline std::ostream& operator<<( std::ostream& out, const EventStatistics& s ) + { + s.printout( out ); + return out; + } + + //-------------------------------------------------------------------------- +} + +#endif // EventStatistics_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuAbstraction.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuAbstraction.h new file mode 100644 index 0000000000..6f709dfe9a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuAbstraction.h @@ -0,0 +1,168 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: J. Teig (Jul 2023) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2020-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MG5AMC_GPUABSTRACTION_H +#define MG5AMC_GPUABSTRACTION_H 1 + +#include "mgOnGpuConfig.h" + +#include + +//-------------------------------------------------------------------------- + +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) + +#ifndef MGONGPU_HAS_NO_BLAS +#include "cublas_v2.h" +#endif + +#define gpuError_t cudaError_t +#define gpuPeekAtLastError cudaPeekAtLastError +#define gpuGetErrorString cudaGetErrorString +#define gpuSuccess cudaSuccess + +#define gpuMallocHost( ptr, size ) checkGpu( cudaMallocHost( ptr, size ) ) +#define gpuMalloc( ptr, size ) checkGpu( cudaMalloc( ptr, size ) ) + +#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( cudaMemcpy( dstData, srcData, srcBytes, func ) ) +#define gpuMemset( data, value, bytes ) checkGpu( cudaMemset( data, value, bytes ) ) +#define gpuMemcpyHostToDevice cudaMemcpyHostToDevice +#define gpuMemcpyDeviceToHost cudaMemcpyDeviceToHost +#define gpuMemcpyDeviceToDevice cudaMemcpyDeviceToDevice +#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( cudaMemcpyToSymbol( type1, type2, size ) ) + +#define gpuFree( ptr ) checkGpu( cudaFree( ptr ) ) +#define gpuFreeHost( ptr ) checkGpu( cudaFreeHost( ptr ) ) + +#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( cudaGetSymbolAddress( devPtr, symbol ) ) + +#define gpuSetDevice cudaSetDevice +#define gpuDeviceSynchronize cudaDeviceSynchronize +#define gpuDeviceReset cudaDeviceReset + +#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) +//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> +#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) +#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) + +#define gpuStream_t cudaStream_t +#define gpuStreamCreate( pStream ) checkGpu( cudaStreamCreate( pStream ) ) +#define gpuStreamDestroy( stream ) checkGpu( cudaStreamDestroy( stream ) ) +#define gpuMallocAsync( ptr, size, stream ) checkGpu( cudaMallocAsync( ptr, size, stream ) ) +#define gpuFreeAsync( ptr, stream ) checkGpu( cudaFreeAsync( ptr, stream ) ) +#define gpuStreamSynchronize( stream ) checkGpu( cudaStreamSynchronize( stream ) ) + +#define gpuBlasStatus_t cublasStatus_t +#define GPUBLAS_STATUS_SUCCESS CUBLAS_STATUS_SUCCESS +#ifndef MGONGPU_HAS_NO_BLAS +#define gpuBlasHandle_t cublasHandle_t +#else +#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds +#endif +#define gpuBlasCreate cublasCreate +#define gpuBlasDestroy cublasDestroy +#define gpuBlasSetStream cublasSetStream + +#define gpuBlasSaxpy cublasSaxpy +#define gpuBlasSdot cublasSdot +#define gpuBlasSgemv cublasSgemv +#define gpuBlasSgemm cublasSgemm +#define gpuBlasSgemmStridedBatched cublasSgemmStridedBatched +#define gpuBlasDaxpy cublasDaxpy +#define gpuBlasDdot cublasDdot +#define gpuBlasDgemv cublasDgemv +#define gpuBlasDgemm cublasDgemm +#define gpuBlasDgemmStridedBatched cublasDgemmStridedBatched +#define GPUBLAS_OP_N CUBLAS_OP_N +#define GPUBLAS_OP_T CUBLAS_OP_T + +//-------------------------------------------------------------------------- + +#elif defined __HIPCC__ + +#ifndef MGONGPU_HAS_NO_BLAS +#include "hipblas/hipblas.h" +#endif + +#define gpuError_t hipError_t +#define gpuPeekAtLastError hipPeekAtLastError +#define gpuGetErrorString hipGetErrorString +#define gpuSuccess hipSuccess + +#define gpuMallocHost( ptr, size ) checkGpu( hipHostMalloc( ptr, size ) ) // HostMalloc better +#define gpuMalloc( ptr, size ) checkGpu( hipMalloc( ptr, size ) ) + +#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( hipMemcpy( dstData, srcData, srcBytes, func ) ) +#define gpuMemset( data, value, bytes ) checkGpu( hipMemset( data, value, bytes ) ) +#define gpuMemcpyHostToDevice hipMemcpyHostToDevice +#define gpuMemcpyDeviceToHost hipMemcpyDeviceToHost +#define gpuMemcpyDeviceToDevice hipMemcpyDeviceToDevice +#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( hipMemcpyToSymbol( type1, type2, size ) ) + +#define gpuFree( ptr ) checkGpu( hipFree( ptr ) ) +#define gpuFreeHost( ptr ) checkGpu( hipHostFree( ptr ) ) + +#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( hipGetSymbolAddress( devPtr, symbol ) ) + +#define gpuSetDevice hipSetDevice +#define gpuDeviceSynchronize hipDeviceSynchronize +#define gpuDeviceReset hipDeviceReset + +#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) +//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> +#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) +#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) + +#define gpuStream_t hipStream_t +#define gpuStreamCreate( pStream ) checkGpu( hipStreamCreate( pStream ) ) +#define gpuStreamDestroy( stream ) checkGpu( hipStreamDestroy( stream ) ) +#define gpuMallocAsync( ptr, size, stream ) checkGpu( hipMallocAsync( ptr, size, stream ) ) +#define gpuFreeAsync( ptr, stream ) checkGpu( hipFreeAsync( ptr, stream ) ) +#define gpuStreamSynchronize( stream ) checkGpu( hipStreamSynchronize( stream ) ) + +#define gpuBlasStatus_t hipblasStatus_t +#define GPUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS +#ifndef MGONGPU_HAS_NO_BLAS +#define gpuBlasHandle_t hipblasHandle_t +#else +#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds +#endif +#define gpuBlasCreate hipblasCreate +#define gpuBlasDestroy hipblasDestroy +#define gpuBlasSetStream hipblasSetStream + +#define gpuBlasSaxpy hipblasSaxpy +#define gpuBlasSdot hipblasSdot +#define gpuBlasSgemv hipblasSgemv +#define gpuBlasSgemm hipblasSgemm +#define gpuBlasSgemmStridedBatched hipblasSgemmStridedBatched +#define gpuBlasDaxpy hipblasDaxpy +#define gpuBlasDdot hipblasDdot +#define gpuBlasDgemv hipblasDgemv +#define gpuBlasDgemm hipblasDgemm +#define gpuBlasDgemmStridedBatched hipblasDgemmStridedBatched +#define GPUBLAS_OP_N HIPBLAS_OP_N +#define GPUBLAS_OP_T HIPBLAS_OP_T + +#endif + +//-------------------------------------------------------------------------- + +#ifdef MGONGPU_FPTYPE2_FLOAT +#define gpuBlasTaxpy gpuBlasSaxpy +#define gpuBlasTdot gpuBlasSdot +#define gpuBlasTgemv gpuBlasSgemv +#define gpuBlasTgemm gpuBlasSgemm +#define gpuBlasTgemmStridedBatched gpuBlasSgemmStridedBatched +#else +#define gpuBlasTaxpy gpuBlasDaxpy +#define gpuBlasTdot gpuBlasDdot +#define gpuBlasTgemv gpuBlasDgemv +#define gpuBlasTgemm gpuBlasDgemm +#define gpuBlasTgemmStridedBatched gpuBlasDgemmStridedBatched +#endif + +#endif // MG5AMC_GPUABSTRACTION_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h new file mode 100644 index 0000000000..2c8ae1a804 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h @@ -0,0 +1,102 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: J. Teig (Jun 2023, based on earlier work by S. Roiser) for the MG5aMC CUDACPP plugin. +// Further modified by: O. Mattelaer, S. Roiser, J. Teig, A. Valassi, Z. Wettersten (2020-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MG5AMC_GPURUNTIME_H +#define MG5AMC_GPURUNTIME_H 1 + +// MG5AMC on GPU uses the CUDA runtime API, not the lower level CUDA driver API +// See https://docs.nvidia.com/cuda/cuda-runtime-api/driver-vs-runtime-api.html#driver-vs-runtime-api + +#include "GpuAbstraction.h" + +#include + +//-------------------------------------------------------------------------- + +// See https://stackoverflow.com/a/14038590 +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ +#define checkGpu( code ) { assertGpu( code, __FILE__, __LINE__ ); } +inline void assertGpu( gpuError_t code, const char* file, int line, bool abort = true ) +{ + if( code != gpuSuccess ) + { + printf( "ERROR! assertGpu: '%s' (%d) in %s:%d\n", gpuGetErrorString( code ), code, file, line ); + if( abort ) assert( code == gpuSuccess ); + } +} +#endif /* clang-format on */ + +//-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ +#ifndef MGONGPU_HAS_NO_BLAS +#define checkGpuBlas( code ){ assertGpuBlas( code, __FILE__, __LINE__ ); } +inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, bool abort = true ) +{ + if ( code != GPUBLAS_STATUS_SUCCESS ) + { + printf( "ERROR! assertGpuBlas: '%d' in %s:%d\n", code, file, line ); + if( abort ) assert( code == GPUBLAS_STATUS_SUCCESS ); + } +} +#endif +#endif /* clang-format on */ + +//-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + // Instantiate a GpuRuntime at the beginnining of the application's main to + // invoke gpuSetDevice(0) in the constructor and book a gpuDeviceReset() call in the destructor + // *** FIXME! This will all need to be designed differently when going to multi-GPU nodes! *** + struct GpuRuntime final + { + GpuRuntime( const bool debug = true ) + : m_debug( debug ) { setUp( m_debug ); } + ~GpuRuntime() { tearDown( m_debug ); } + GpuRuntime( const GpuRuntime& ) = delete; + GpuRuntime( GpuRuntime&& ) = delete; + GpuRuntime& operator=( const GpuRuntime& ) = delete; + GpuRuntime& operator=( GpuRuntime&& ) = delete; + bool m_debug; + + // Set up CUDA application + // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** + // Calling cudaSetDevice on startup is useful to properly book-keep the time spent in CUDA initialization + static void setUp( const bool debug = false ) // ZW: changed debug default to false + { + // ** NB: it is useful to call cudaSetDevice, or cudaFree, to properly book-keep the time spent in CUDA initialization + // ** NB: otherwise, the first CUDA operation (eg a cudaMemcpyToSymbol in CPPProcess ctor) appears to take much longer! + /* + // [We initially added cudaFree(0) to "ease profile analysis" only because it shows up as a big recognizable block!] + // No explicit initialization is needed: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#initialization + // It is not clear what cudaFree(0) does at all: https://stackoverflow.com/questions/69967813/ + if ( debug ) std::cout << "__CudaRuntime: calling cudaFree(0)" << std::endl; + checkCuda( cudaFree( 0 ) ); // SLOW! + */ + // Replace cudaFree(0) by cudaSetDevice(0), even if it is not really needed either + // (but see https://developer.nvidia.com/blog/cuda-pro-tip-always-set-current-device-avoid-multithreading-bugs) + if( debug ) std::cout << "__GpuRuntime: calling GpuSetDevice(0)" << std::endl; + checkGpu( gpuSetDevice( 0 ) ); // SLOW! + } + + // Tear down CUDA application (call cudaDeviceReset) + // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** + // Calling cudaDeviceReset on shutdown is only needed for checking memory leaks in cuda-memcheck + // See https://docs.nvidia.com/cuda/cuda-memcheck/index.html#leak-checking + static void tearDown( const bool debug = false ) // ZW: changed debug default to false + { + if( debug ) std::cout << "__GpuRuntime: calling GpuDeviceReset()" << std::endl; + checkGpu( gpuDeviceReset() ); + } + }; +} +#endif + +//-------------------------------------------------------------------------- + +#endif // MG5AMC_GPURUNTIME_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc new file mode 100644 index 0000000000..b806da2e09 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc @@ -0,0 +1,146 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "mgOnGpuConfig.h" + +#include "GpuRuntime.h" +#include "MemoryBuffers.h" +#include "RandomNumberKernels.h" + +#include + +#ifndef MGONGPU_HAS_NO_HIPRAND /* clang-format off */ +#ifndef __HIP_PLATFORM_AMD__ +#define __HIP_PLATFORM_AMD__ 1 // enable hiprand for AMD (rocrand) +#endif +#include +#define checkHiprand( code ){ assertHiprand( code, __FILE__, __LINE__ ); } +inline void assertHiprand( hiprandStatus_t code, const char *file, int line, bool abort = true ) +{ + if ( code != HIPRAND_STATUS_SUCCESS ) + { + printf( "HiprandAssert: %s:%d code=%d\n", file, line, code ); + if ( abort ) assert( code == HIPRAND_STATUS_SUCCESS ); + } +} +#endif /* clang-format on */ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- +#ifndef MGONGPU_HAS_NO_HIPRAND + HiprandRandomNumberKernel::HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) + : RandomNumberKernelBase( rnarray ) + , m_isOnDevice( onDevice ) + { + if( m_isOnDevice ) + { +#ifdef MGONGPUCPP_GPUIMPL + if( !m_rnarray.isOnDevice() ) + throw std::runtime_error( "HiprandRandomNumberKernel on device with a host random number array" ); +#else + throw std::runtime_error( "HiprandRandomNumberKernel does not support HiprandDevice on CPU host" ); +#endif + } + else + { + if( m_rnarray.isOnDevice() ) + throw std::runtime_error( "HiprandRandomNumberKernel on host with a device random number array" ); + } + createGenerator(); + } + + //-------------------------------------------------------------------------- + + HiprandRandomNumberKernel::~HiprandRandomNumberKernel() + { + destroyGenerator(); + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::seedGenerator( const unsigned int seed ) + { + if( m_isOnDevice ) + { + destroyGenerator(); // workaround for #429 + createGenerator(); // workaround for #429 + } + //printf( "seedGenerator: seed %d\n", seed ); + checkHiprand( hiprandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::createGenerator() + { + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_DEFAULT; + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_XORWOW; + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MRG32K3A; + const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MTGP32; // same as curand; not implemented yet (code=1000) in host code + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MT19937; + //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_PHILOX4_32_10; + if( m_isOnDevice ) + { + checkHiprand( hiprandCreateGenerator( &m_rnGen, type ) ); + } + else + { + // See https://github.com/ROCm/hipRAND/issues/76 + throw std::runtime_error( "HiprandRandomNumberKernel on host is not supported yet (hiprandCreateGeneratorHost is not implemented yet)" ); + //checkHiprand( hiprandCreateGeneratorHost( &m_rnGen, type ) ); // ALWAYS FAILS WITH CODE=1000 + } + // FIXME: hiprand ordering is not implemented yet + // See https://github.com/ROCm/hipRAND/issues/75 + /* + //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_LEGACY ) ); + checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_BEST ) ); + //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_DYNAMIC ) ); + //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_SEEDED ) ); + */ + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::destroyGenerator() + { + checkHiprand( hiprandDestroyGenerator( m_rnGen ) ); + } + + //-------------------------------------------------------------------------- + + void HiprandRandomNumberKernel::generateRnarray() + { +#if defined MGONGPU_FPTYPE_DOUBLE + checkHiprand( hiprandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#elif defined MGONGPU_FPTYPE_FLOAT + checkHiprand( hiprandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); +#endif + /* + printf( "\nHiprandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); + fptype* data = m_rnarray.data(); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) + { + data = new fptype[m_rnarray.size()](); + checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); + } +#endif + for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) + printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); +#ifdef MGONGPUCPP_GPUIMPL + if( m_rnarray.isOnDevice() ) delete[] data; +#endif + */ + } + + //-------------------------------------------------------------------------- +#endif +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h new file mode 100644 index 0000000000..12b7d45adf --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h @@ -0,0 +1,339 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Hageboeck (Dec 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MADGRAPHTEST_H_ +#define MADGRAPHTEST_H_ 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" + +#include + +#include +#include +//#ifdef __HIPCC__ +//#include // see https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 +//#else +//#include // bypass this completely to ease portability on LUMI #803 +//#endif +#include +#include +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +using mg5amcGpu::CPPProcess; +#else +using mg5amcCpu::CPPProcess; +#endif + +namespace +{ + struct ReferenceData + { + std::vector>> momenta; + std::vector MEs; + std::vector ChanIds; + std::vector SelHels; + std::vector SelCols; + }; + + /// Read batches of reference data from a file and store them in a map. + std::map readReferenceData( const std::string& refFileName ) + { + std::cout << "INFO: Opening reference file " << refFileName << std::endl; + std::ifstream referenceFile( refFileName.c_str() ); + EXPECT_TRUE( referenceFile.is_open() ) << refFileName; + std::map referenceData; + unsigned int evtNo; + unsigned int batchNo; + for( std::string line; std::getline( referenceFile, line ); ) + { + std::stringstream lineStr( line ); + if( line.empty() || line[0] == '#' ) + { + continue; + } + else if( line.find( "Event" ) != std::string::npos ) + { + std::string dummy; + lineStr >> dummy >> evtNo >> dummy >> batchNo; + } + else if( line.find( "ME" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].MEs.size() ) + referenceData[batchNo].MEs.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].MEs[evtNo]; + } + else if( line.find( "ChanId" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].ChanIds.size() ) + referenceData[batchNo].ChanIds.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].ChanIds[evtNo]; + referenceData[batchNo].ChanIds[evtNo] = 0; // disable ChanId comparison if multichannel is not supported #976 + } + else if( line.find( "SelHel" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].SelHels.size() ) + referenceData[batchNo].SelHels.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].SelHels[evtNo]; + } + else if( line.find( "SelCol" ) != std::string::npos ) + { + if( evtNo <= referenceData[batchNo].SelCols.size() ) + referenceData[batchNo].SelCols.resize( evtNo + 1 ); + std::string dummy; + lineStr >> dummy >> referenceData[batchNo].SelCols[evtNo]; + } + else + { + unsigned int particleIndex; + lineStr >> particleIndex; + if( evtNo <= referenceData[batchNo].momenta.size() ) + referenceData[batchNo].momenta.resize( evtNo + 1 ); + if( particleIndex <= referenceData[batchNo].momenta[evtNo].size() ) + referenceData[batchNo].momenta[evtNo].resize( particleIndex + 1 ); + auto& fourVec = referenceData[batchNo].momenta[evtNo][particleIndex]; + for( unsigned int i = 0; i < fourVec.size(); ++i ) + { + EXPECT_TRUE( lineStr.good() ); + lineStr >> fourVec[i]; + } + EXPECT_TRUE( lineStr.eof() ); + } + } + return referenceData; + } + +} + +/** + * Test driver providing a common interface for testing different implementations. + * Users need to implement: + * - Functions to retrieve matrix element and 4-momenta. These are used in the tests. + * - Driver functions that run the madgraph workflow. + */ +class TestDriverBase +{ + std::string m_refFileName; +public: + const unsigned int nparticle; + static constexpr unsigned int niter = 2; + static constexpr unsigned int gpublocks = 2; + static constexpr unsigned int gputhreads = 128; + static constexpr unsigned int nevt = gpublocks * gputhreads; + + TestDriverBase( unsigned int npart, const std::string& refFileName ) + : m_refFileName( refFileName ) + , nparticle( npart ) + { + } + TestDriverBase() = delete; + virtual ~TestDriverBase() {} + const std::string& getRefFileName() { return m_refFileName; } + + // ------------------------------------------------ + // Interface for retrieving info from madgraph + // ------------------------------------------------ + virtual fptype getMomentum( std::size_t evtNo, unsigned int particleNo, unsigned int component ) const = 0; + virtual fptype getMatrixElement( std::size_t evtNo ) const = 0; + virtual int getChannelId( std::size_t ievt ) const = 0; + virtual int getSelectedHelicity( std::size_t ievt ) const = 0; + virtual int getSelectedColor( std::size_t ievt ) const = 0; + + // ------------------------------------------------ + // Interface for steering madgraph run + // ------------------------------------------------ + virtual void prepareRandomNumbers( unsigned int iiter ) = 0; + virtual void prepareMomenta( fptype energy ) = 0; + virtual void runSigmaKin( std::size_t iiter ) = 0; + + /// Print the requested event into the stream. If the reference data has enough events, it will be printed as well. + void dumpParticles( std::ostream& stream, std::size_t ievt, unsigned int numParticles, unsigned int nDigit, const ReferenceData& referenceData ) const + { + const auto width = nDigit + 8; + for( unsigned int ipar = 0; ipar < numParticles; ipar++ ) + { + // NB: 'setw' affects only the next field (of any type) + stream << std::scientific // fixed format: affects all floats (default nDigit: 6) + << std::setprecision( nDigit ) + << std::setw( 4 ) << ipar + << std::setw( width ) << getMomentum( ievt, ipar, 0 ) + << std::setw( width ) << getMomentum( ievt, ipar, 1 ) + << std::setw( width ) << getMomentum( ievt, ipar, 2 ) + << std::setw( width ) << getMomentum( ievt, ipar, 3 ) + << "\n"; + if( ievt < referenceData.momenta.size() ) + { + stream << "ref" << ipar; + stream << std::setw( width ) << referenceData.momenta[ievt][ipar][0] + << std::setw( width ) << referenceData.momenta[ievt][ipar][1] + << std::setw( width ) << referenceData.momenta[ievt][ipar][2] + << std::setw( width ) << referenceData.momenta[ievt][ipar][3] + << "\n\n"; + } + stream << std::flush << std::defaultfloat; // default format: affects all floats + } + } +}; + +/** + * Test class that's defining all tests to run with a Madgraph workflow. + */ +class MadgraphTest +{ +public: + MadgraphTest( TestDriverBase& testDriverRef ) + : testDriver( &testDriverRef ) {} + ~MadgraphTest() {} + void CompareMomentaAndME( testing::Test& googleTest ) const; // NB: googleTest is ONLY needed for the HasFailure method... +private: + TestDriverBase* testDriver; // non-owning pointer +}; + +void +MadgraphTest::CompareMomentaAndME( testing::Test& googleTest ) const +{ + const fptype toleranceMomenta = std::is_same::value ? 1.E-10 : 4.E-2; // see #735 +#ifdef __APPLE__ + const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-2; // see #583 +#else + //const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 2.E-3; // fails smeft/hip #843 + const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-3; +#endif + constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) + // Dump events to a new reference file? + const char* dumpEventsC = getenv( "CUDACPP_RUNTEST_DUMPEVENTS" ); + const bool dumpEvents = ( dumpEventsC != 0 ) && ( std::string( dumpEventsC ) != "" ); + const std::string refFileName = testDriver->getRefFileName(); + /* +#ifdef __HIPCC__ + const std::string dumpFileName = std::experimental::filesystem::path( refFileName ).filename(); +#else + const std::string dumpFileName = std::filesystem::path( refFileName ).filename(); +#endif + */ + const std::string dumpFileName = refFileName; // bypass std::filesystem #803 + std::ofstream dumpFile; + if( dumpEvents ) + { + dumpFile.open( dumpFileName, std::ios::trunc ); + } + // Read reference data + std::map referenceData; + if( !dumpEvents ) + { + referenceData = readReferenceData( refFileName ); + } + ASSERT_FALSE( googleTest.HasFailure() ); // It doesn't make any sense to continue if we couldn't read the reference file. + // ************************************** + // *** START MAIN LOOP ON #ITERATIONS *** + // ************************************** + for( unsigned int iiter = 0; iiter < testDriver->niter; ++iiter ) + { + testDriver->prepareRandomNumbers( iiter ); + testDriver->prepareMomenta( energy ); + testDriver->runSigmaKin( iiter ); + // --- Run checks on all events produced in this iteration + for( std::size_t ievt = 0; ievt < testDriver->nevt && !googleTest.HasFailure(); ++ievt ) + { + if( dumpEvents ) + { + ASSERT_TRUE( dumpFile.is_open() ) << dumpFileName; + dumpFile << "Event " << std::setw( 8 ) << ievt << " " + << "Batch " << std::setw( 4 ) << iiter << "\n"; + testDriver->dumpParticles( dumpFile, ievt, testDriver->nparticle, 15, ReferenceData() ); + // Dump matrix element + dumpFile << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) + << testDriver->getMatrixElement( ievt ) << "\n" + << std::defaultfloat; + // Dump channelId + dumpFile << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n"; + // Dump selected helicity and color + dumpFile << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n"; + dumpFile << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" + << std::endl; // leave one line between events + continue; + } + // Check that we have the required reference data + ASSERT_GT( referenceData.size(), iiter ) + << "Don't have enough reference data for iteration " << iiter << ". Ref file:" << refFileName; + ASSERT_GT( referenceData[iiter].MEs.size(), ievt ) + << "Don't have enough reference MEs for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].ChanIds.size(), ievt ) + << "Don't have enough reference ChanIds for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].SelHels.size(), ievt ) + << "Don't have enough reference SelHels for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].SelCols.size(), ievt ) + << "Don't have enough reference SelCols for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GT( referenceData[iiter].momenta.size(), ievt ) + << "Don't have enough reference momenta for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + ASSERT_GE( referenceData[iiter].momenta[ievt].size(), testDriver->nparticle ) + << "Don't have enough reference particles for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; + // This trace will help to understand the event that is being checked. + // It will only be printed in case of failures: + std::stringstream eventTrace; + eventTrace << "In comparing event " << ievt << " from iteration " << iiter << "\n"; + testDriver->dumpParticles( eventTrace, ievt, testDriver->nparticle, 15, referenceData[iiter] ); + eventTrace << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) + << testDriver->getMatrixElement( ievt ) << "\n" + << std::setw( 4 ) << "r.ME" << std::scientific << std::setw( 15 + 8 ) + << referenceData[iiter].MEs[ievt] << std::endl + << std::defaultfloat; + eventTrace << std::setw( 8 ) << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n" + << std::setw( 8 ) << "r.ChanId" << std::setw( 8 ) << referenceData[iiter].ChanIds[ievt] << std::endl; + eventTrace << std::setw( 8 ) << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n" + << std::setw( 8 ) << "r.SelHel" << std::setw( 8 ) << referenceData[iiter].SelHels[ievt] << std::endl; + eventTrace << std::setw( 8 ) << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" + << std::setw( 8 ) << "r.SelCol" << std::setw( 8 ) << referenceData[iiter].SelCols[ievt] << std::endl; + SCOPED_TRACE( eventTrace.str() ); + // Compare Momenta + for( unsigned int ipar = 0; ipar < testDriver->nparticle; ++ipar ) + { + std::stringstream momentumErrors; + for( unsigned int icomp = 0; icomp < CPPProcess::np4; ++icomp ) + { + const fptype pMadg = testDriver->getMomentum( ievt, ipar, icomp ); + const fptype pOrig = referenceData[iiter].momenta[ievt][ipar][icomp]; + //const fptype relDelta = fabs( ( pMadg - pOrig ) / pOrig ); // computing relDelta may lead to FPEs + const fptype delta = fabs( pMadg - pOrig ); + if( delta > toleranceMomenta * fabs( pOrig ) ) // better than "relDelta > toleranceMomenta" + { + momentumErrors << std::setprecision( 15 ) << std::scientific << "\nparticle " << ipar << "\tcomponent " << icomp + << "\n\t madGraph: " << std::setw( 22 ) << pMadg + << "\n\t reference: " << std::setw( 22 ) << pOrig + << "\n\t relative delta exceeds tolerance of " << toleranceMomenta; + } + } + ASSERT_TRUE( momentumErrors.str().empty() ) << momentumErrors.str(); + } + // Compare ME: + EXPECT_NEAR( testDriver->getMatrixElement( ievt ), + referenceData[iiter].MEs[ievt], + toleranceMEs * referenceData[iiter].MEs[ievt] ); + // Compare channelId + EXPECT_EQ( testDriver->getChannelId( ievt ), + referenceData[iiter].ChanIds[ievt] ); + // Compare selected helicity and color + EXPECT_EQ( testDriver->getSelectedHelicity( ievt ), + referenceData[iiter].SelHels[ievt] ); + EXPECT_EQ( testDriver->getSelectedColor( ievt ), + referenceData[iiter].SelCols[ievt] ); + } + } + if( dumpEvents ) + { + std::cout << "Event dump written to " << dumpFileName << std::endl; + } +} + +#endif /* MADGRAPHTEST_H_ */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc new file mode 100644 index 0000000000..872e4795e3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc @@ -0,0 +1,509 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "MatrixElementKernels.h" + +#include "CPPProcess.h" +#include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" + +#include // for fetestexcept +#include +#include + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + MatrixElementKernelBase::MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol ) // output: color selection + : m_momenta( momenta ) + , m_gs( gs ) + , m_iflavorVec( iflavorVec ) + , m_rndhel( rndhel ) + , m_rndcol( rndcol ) + , m_channelIds( channelIds ) + , m_matrixElements( matrixElements ) + , m_selhel( selhel ) + , m_selcol( selcol ) +#ifdef MGONGPU_CHANNELID_DEBUG + , m_nevtProcessedByChannel() + , m_tag() +#endif + { + //std::cout << "DEBUG: MatrixElementKernelBase ctor " << this << std::endl; +#ifdef MGONGPU_CHANNELID_DEBUG + for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + m_nevtProcessedByChannel[channelId] = 0; +#endif + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelBase::~MatrixElementKernelBase() + { + //std::cout << "DEBUG: MatrixElementKernelBase dtor " << this << std::endl; +#ifdef MGONGPU_CHANNELID_DEBUG + MatrixElementKernelBase::dumpNevtProcessedByChannel(); +#endif +#ifdef MGONGPUCPP_VERBOSE + MatrixElementKernelBase::dumpSignallingFPEs(); +#endif + } + + //-------------------------------------------------------------------------- + +#ifdef MGONGPU_CHANNELID_DEBUG + void MatrixElementKernelBase::updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ) + { + if( pHstChannelIds != nullptr ) + { + //std::cout << "DEBUG " << this << ": not nullptr " << nevt << std::endl; + for( unsigned int ievt = 0; ievt < nevt; ievt++ ) + { + const size_t channelId = pHstChannelIds[ievt]; // Fortran indexing + //assert( channelId > 0 ); + //assert( channelId < CPPProcess::ndiagrams ); + m_nevtProcessedByChannel[channelId]++; + } + } + else + { + //std::cout << "DEBUG " << this << ": nullptr " << std::endl; + m_nevtProcessedByChannel[0] += nevt; + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPU_CHANNELID_DEBUG + void MatrixElementKernelBase::dumpNevtProcessedByChannel() + { + size_t nevtProcessed = 0; + for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + nevtProcessed += m_nevtProcessedByChannel[channelId]; + std::ostringstream sstr; + sstr << " {"; + for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + { + if( m_nevtProcessedByChannel[channelId] > 0 ) + { + if( sstr.str() != " {" ) sstr << ","; + if( channelId == 0 ) + sstr << " no-multichannel"; + else + sstr << " " << channelId; + sstr << " : " << m_nevtProcessedByChannel[channelId]; + } + } + sstr << " }"; + std::cout << "DEBUG: MEK " << this; + if( m_tag != "" ) std::cout << " " << m_tag; + std::cout << " processed " << nevtProcessed << " events across " << CPPProcess::ndiagrams << " channels" << sstr.str() << std::endl; + } +#endif + + //-------------------------------------------------------------------------- + + void MatrixElementKernelBase::dumpSignallingFPEs() + { + // New strategy for issue #831: add a final report of FPEs + // Note: normally only underflow will be reported here (inexact is switched off because it would almost always signal; + // divbyzero, invalid and overflow are configured by feenablexcept to send a SIGFPE signal, and are normally fixed in the code) + // Note: this is now called in the individual destructors of MEK classes rather than in that of MatrixElementKernelBase(#837) + std::string fpes; + if( std::fetestexcept( FE_DIVBYZERO ) ) fpes += " FE_DIVBYZERO"; + if( std::fetestexcept( FE_INVALID ) ) fpes += " FE_INVALID"; + if( std::fetestexcept( FE_OVERFLOW ) ) fpes += " FE_OVERFLOW"; + if( std::fetestexcept( FE_UNDERFLOW ) ) fpes += " FE_UNDERFLOW"; + //if( std::fetestexcept( FE_INEXACT ) ) fpes += " FE_INEXACT"; // do not print this out: this would almost always signal! + if( fpes == "" ) + std::cout << "INFO: No Floating Point Exceptions have been reported" << std::endl; + else + std::cerr << "INFO: The following Floating Point Exceptions have been reported:" << fpes << std::endl; + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +#ifndef MGONGPUCPP_GPUIMPL +namespace mg5amcCpu +{ + + //-------------------------------------------------------------------------- + + MatrixElementKernelHost::MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt ) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( nevt ) + , m_couplings( nevt ) + , m_numerators( nevt * CPPProcess::ndiagrams ) + , m_denominators( nevt ) + { + //std::cout << "DEBUG: MatrixElementKernelHost::ctor " << this << std::endl; + if( m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: momenta must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: matrixElements must be a host array" ); + if( m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: channelIds must be a device array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with matrixElements" ); + if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with channelIds" ); + if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with iflavorVec" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MatrixElementKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Fail gently and avoid "Illegal instruction (core dumped)" if the host does not support the SIMD used in the ME calculation + // Note: this prevents a crash on pmpe04 but not on some github CI nodes? + // [NB: SIMD vectorization in mg5amc C++ code is only used in the ME calculation below MatrixElementKernelHost!] + if( !MatrixElementKernelHost::hostSupportsSIMD() ) + throw std::runtime_error( "Host does not support the SIMD implementation of MatrixElementKernelsHost" ); + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelHost::~MatrixElementKernelHost() + { + //std::cout << "DEBUG: MatrixElementKernelBase::dtor " << this << std::endl; + } + + //-------------------------------------------------------------------------- + + int MatrixElementKernelHost::computeGoodHelicities() + { + HostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); + // ... 0d1. Compute good helicity mask on the host + computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); + sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_numerators.data(), m_denominators.data(), hstIsGoodHel.data(), nevt() ); + // ... 0d2. Copy good helicity list to static memory on the host + // [FIXME! REMOVE THIS STATIC THAT BREAKS MULTITHREADING?] + return sigmaKin_setGoodHel( hstIsGoodHel.data() ); + } + + //-------------------------------------------------------------------------- + + void MatrixElementKernelHost::computeMatrixElements( const bool useChannelIds ) + { + computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_numerators.data(), m_denominators.data(), nullptr, true, nevt() ); +#ifdef MGONGPU_CHANNELID_DEBUG + //std::cout << "DEBUG: MatrixElementKernelHost::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; + MatrixElementKernelBase::updateNevtProcessedByChannel( pChannelIds, nevt() ); +#endif + } + + //-------------------------------------------------------------------------- + + // Does this host system support the SIMD used in the matrix element calculation? + bool MatrixElementKernelHost::hostSupportsSIMD( const bool verbose ) + { +#if defined __AVX512VL__ + bool known = true; + bool ok = __builtin_cpu_supports( "avx512vl" ); + const std::string tag = "skylake-avx512 (AVX512VL)"; +#elif defined __AVX2__ + bool known = true; + bool ok = __builtin_cpu_supports( "avx2" ); + const std::string tag = "haswell (AVX2)"; +#elif defined __SSE4_2__ +#ifdef __PPC__ + // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html + bool known = true; + bool ok = __builtin_cpu_supports( "vsx" ); + const std::string tag = "powerpc vsx (128bit as in SSE4.2)"; +#elif defined( __x86_64__ ) || defined( __i386__ ) + bool known = true; + bool ok = __builtin_cpu_supports( "sse4.2" ); + const std::string tag = "nehalem (SSE4.2)"; +#else // AV FIXME! Added by OM for Mac, should identify the correct __xxx__ flag that should be targeted + // DM now we have an explicit NEON target for ARM + bool known = false; // __builtin_cpu_supports is not supported + bool ok = true; // this is just an assumption! + const std::string tag = "simd arch not defined"; +#endif +#elif defined __ARM_NEON // consider using __BUILTIN_CPU_SUPPORTS__ + bool known = false; // __builtin_cpu_supports is not supported + // See https://stackoverflow.com/q/62783908 + // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu + bool ok = true; // this is just an assumption! + const std::string tag = "arm neon (128bit as in SSE4.2)"; +#else + bool known = true; + bool ok = true; + const std::string tag = "none"; +#endif + if( verbose ) + { + if( tag == "none" ) + std::cout << "INFO: The application does not require the host to support any AVX feature" << std::endl; + else if( ok && known ) + std::cout << "INFO: The application is built for " << tag << " and the host supports it" << std::endl; + else if( ok ) + std::cout << "WARNING: The application is built for " << tag << " but it is unknown if the host supports it" << std::endl; + else + std::cout << "ERROR! The application is built for " << tag << " but the host does not support it" << std::endl; + } + return ok; + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +{ + + //-------------------------------------------------------------------------- + + MatrixElementKernelDevice::MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( gpublocks * gputhreads ) + , m_couplings( this->nevt() ) + , m_pHelMEs() + , m_pHelJamps() + , m_pHelNumerators() + , m_pHelDenominators() + , m_colJamp2s( CPPProcess::ncolor * this->nevt() ) +#ifdef MGONGPU_CHANNELID_DEBUG + , m_hstChannelIds( this->nevt() ) +#endif +#ifndef MGONGPU_HAS_NO_BLAS + , m_blasColorSum( false ) + , m_blasTf32Tensor( false ) + , m_pHelBlasTmp() + , m_blasHandle() +#endif + , m_helStreams() + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + //std::cout << "DEBUG: MatrixElementKernelDevice::ctor " << this << std::endl; + if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: momenta must be a device array" ); + if( !m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: matrixElements must be a device array" ); + if( !m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: channelIds must be a device array" ); // FIXME?! + if( !m_iflavorVec.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: iflavorVec must be a device array" ); + if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with matrixElements" ); + if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with channelIds" ); + if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with iflavorVec" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( m_gputhreads % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MatrixElementKernelHost: gputhreads should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Create the "one-helicity" jamp buffer that will be used for helicity filtering + m_pHelJamps.reset( new DeviceBufferSimple( CPPProcess::ncolor * mgOnGpu::nx2 * this->nevt() ) ); + // Create the "one-helicity" numerator and denominator buffers that will be used for helicity filtering + m_pHelNumerators.reset( new DeviceBufferSimple( this->nevt() * CPPProcess::ndiagrams ) ); + m_pHelDenominators.reset( new DeviceBufferSimple( this->nevt() ) ); + // Decide at runtime whether to use BLAS for color sums + // Decide at runtime whether TF32TENSOR math should be used in cuBLAS + static bool first = true; + if( first ) + { + first = false; + // Analyse environment variable CUDACPP_RUNTIME_BLASCOLORSUM + const char* blasEnv = getenv( "CUDACPP_RUNTIME_BLASCOLORSUM" ); + if( blasEnv && std::string( blasEnv ) != "" ) + { +#ifndef MGONGPU_HAS_NO_BLAS + m_blasColorSum = true; // fixme? eventually set default=true and decode "Y" and "N" choices? + std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty: enable BLAS" << std::endl; +#else + throw std::runtime_error( "Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty, but BLAS was disabled at build time" ); +#endif + } + else + { +#ifndef MGONGPU_HAS_NO_BLAS + std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is empty or not set: disable BLAS" << std::endl; +#else + std::cout << "INFO: BLAS was disabled at build time" << std::endl; +#endif + } +#ifndef MGONGPU_HAS_NO_BLAS +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) + // Analyse environment variable CUDACPP_RUNTIME_CUBLASTF32TENSOR + const char* blasEnv2 = getenv( "CUDACPP_RUNTIME_CUBLASTF32TENSOR" ); + if( blasEnv2 && std::string( blasEnv2 ) != "" ) + { + if( m_blasColorSum ) + { +#ifdef MGONGPU_FPTYPE2_FLOAT + m_blasTf32Tensor = true; + std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty: enable CUBLAS_TF32_TENSOR_OP_MATH" << std::endl; +#else + std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but color sums use FP64" << std::endl; +#endif + } + else + std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but BLAS was disabled at runtime" << std::endl; + } +#ifdef MGONGPU_FPTYPE2_FLOAT + else + { + if( m_blasColorSum ) + std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is empty or not set: keep cuBLAS math defaults" << std::endl; + } +#endif +#endif +#endif + } + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelDevice::~MatrixElementKernelDevice() + { + //std::cout << "DEBUG: MatrixElementKernelDevice::dtor " << this << std::endl; +#ifndef MGONGPU_HAS_NO_BLAS + if( m_blasHandle ) gpuBlasDestroy( m_blasHandle ); +#endif + for( int ihel = 0; ihel < CPPProcess::ncomb; ihel++ ) + { + if( m_helStreams[ihel] ) gpuStreamDestroy( m_helStreams[ihel] ); // do not destroy if nullptr + } + } + + //-------------------------------------------------------------------------- + + // FIXME! The relevance of this function should be reassessed (#543 and #902) + void MatrixElementKernelDevice::setGrid( const int /*gpublocks*/, const int /*gputhreads*/ ) + { + if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0 in setGrid" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0 in setGrid" ); + if( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch in setGrid" ); + } + + //-------------------------------------------------------------------------- + + int MatrixElementKernelDevice::computeGoodHelicities() + { + PinnedHostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); + // ... 0d1. Compute good helicity mask (a host variable) on the device + gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); + const int nevt = m_gpublocks * m_gputhreads; + sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_pHelJamps->data(), m_pHelNumerators->data(), m_pHelDenominators->data(), hstIsGoodHel.data(), nevt ); + // ... 0d3. Set good helicity list in host static memory + int nGoodHel = sigmaKin_setGoodHel( hstIsGoodHel.data() ); + assert( nGoodHel > 0 ); // SANITY CHECK: there should be at least one good helicity + // Create one GPU stream for each good helicity + for( int ighel = 0; ighel < nGoodHel; ighel++ ) + gpuStreamCreate( &m_helStreams[ighel] ); +#ifndef MGONGPU_HAS_NO_BLAS + // Create one cuBLAS/hipBLAS handle for each good helicity (attached to the default stream) + if( m_blasColorSum ) + { + checkGpuBlas( gpuBlasCreate( &m_blasHandle ) ); +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) + if( m_blasTf32Tensor ) + checkGpuBlas( cublasSetMathMode( m_blasHandle, CUBLAS_TF32_TENSOR_OP_MATH ) ); // enable TF32 tensor cores +#endif + } +#endif + // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) + m_pHelMEs.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); + // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) + // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) + m_pHelJamps.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); + // ... Create the "many-helicity" super-buffers of nGoodHel numerator and denominator buffers (dynamically allocated) + // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) + m_pHelNumerators.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ndiagrams * nevt ) ); + m_pHelDenominators.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); +#ifndef MGONGPU_HAS_NO_BLAS + // Create the "many-helicity" super-buffers of real/imag ncolor*nevt temporary buffers for cuBLAS/hipBLAS intermediate results in color_sum_blas +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Mixed precision mode: need two fptype2[ncolor*2*nevt] buffers and one fptype2[nevt] buffer per good helicity + if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ( 2 * CPPProcess::ncolor * mgOnGpu::nx2 + 1 ) * nevt ) ); +#else + // Standard single/double precision mode: need one fptype2[ncolor*2*nevt] buffer per good helicity + if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); +#endif +#endif + // Return the number of good helicities + return nGoodHel; + } + + //-------------------------------------------------------------------------- + + void MatrixElementKernelDevice::computeMatrixElements( const bool useChannelIds ) + { + gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); +#ifndef MGONGPU_HAS_NO_BLAS + fptype2* ghelAllBlasTmp = ( m_blasColorSum ? m_pHelBlasTmp->data() : nullptr ); + gpuBlasHandle_t* pBlasHandle = ( m_blasColorSum ? &m_blasHandle : nullptr ); +#else + fptype2* ghelAllBlasTmp = nullptr; + gpuBlasHandle_t* pBlasHandle = nullptr; +#endif + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_colJamp2s.data(), m_pHelNumerators->data(), m_pHelDenominators->data(), nullptr, true, m_pHelMEs->data(), m_pHelJamps->data(), ghelAllBlasTmp, pBlasHandle, m_helStreams, false, m_gpublocks, m_gputhreads ); +#ifdef MGONGPU_CHANNELID_DEBUG + //std::cout << "DEBUG: MatrixElementKernelDevice::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; + copyHostFromDevice( m_hstChannelIds, m_channelIds ); // FIXME?! + const unsigned int* pHstChannelIds = ( useChannelIds ? m_hstChannelIds.data() : nullptr ); + MatrixElementKernelBase::updateNevtProcessedByChannel( pHstChannelIds, nevt() ); +#endif + checkGpu( gpuPeekAtLastError() ); // is this needed? + checkGpu( gpuDeviceSynchronize() ); // probably not needed? but it avoids errors in sigmaKin above from appearing later on in random places... + } + + //-------------------------------------------------------------------------- + +} +#endif + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h new file mode 100644 index 0000000000..01041e43b7 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h @@ -0,0 +1,250 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MATRIXELEMENTKERNELS_H +#define MATRIXELEMENTKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "GpuAbstraction.h" +#include "MemoryBuffers.h" + +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // A base class encapsulating matrix element calculations on a CPU host or on a GPU device + class MatrixElementKernelBase //: virtual public IMatrixElementKernel + { + protected: + + // Constructor from existing input and output buffers + MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol); // output: color selection + + public: + + // Destructor + virtual ~MatrixElementKernelBase(); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + virtual int computeGoodHelicities() = 0; + + // Compute matrix elements + virtual void computeMatrixElements( const bool useChannelIds ) = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + // Dump signalling FPEs (#831 and #837) + static void dumpSignallingFPEs(); + +#ifdef MGONGPU_CHANNELID_DEBUG + // Add a MEK identifier for the channelId debug printout + void setTagForNevtProcessedByChannel( const std::string& tag ) { m_tag = tag; } + + protected: + // Update number of events processed by channel + void updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ); + + // Dump number of events processed by channel + void dumpNevtProcessedByChannel(); +#endif + + protected: + + // The buffer for the input momenta + const BufferMomenta& m_momenta; + + // The buffer for the gs to calculate the alphaS values + const BufferGs& m_gs; + + // The buffer for the flavor indices for the flavor combination + const BufferIflavorVec& m_iflavorVec; + + // The buffer for the random numbers for helicity selection + const BufferRndNumHelicity& m_rndhel; + + // The buffer for the random numbers for color selection + const BufferRndNumColor& m_rndcol; + + // The buffer for the channel ids for single-diagram enhancement + const BufferChannelIds& m_channelIds; + + // The buffer for the output matrix elements + BufferMatrixElements& m_matrixElements; + + // The buffer for the output helicity selection + BufferSelectedHelicity& m_selhel; + + // The buffer for the output color selection + BufferSelectedColor& m_selcol; + +#ifdef MGONGPU_CHANNELID_DEBUG + // The events-per-channel counter for debugging + std::map m_nevtProcessedByChannel; + + // The tag for events-per-channel debugging + std::string m_tag; +#endif + }; + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating matrix element calculations on a CPU host + class MatrixElementKernelHost final : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~MatrixElementKernelHost(); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // Does this host system support the SIMD used in the matrix element calculation? + // [NB: this is private, SIMD vectorization in mg5amc C++ code is currently only used in the ME calculations below MatrixElementKernelHost!] + static bool hostSupportsSIMD( const bool verbose = false ); // ZW: default verbose false + + private: + + // The buffer for the event-by-event couplings that depends on alphas QCD + HostBufferCouplings m_couplings; + + // The buffer for the event-by-event numerators of multichannel factors + HostBufferNumerators m_numerators; + + // The buffer for the event-by-event denominators of multichannel factors + HostBufferDenominators m_denominators; + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating matrix element calculations on a GPU device + class MatrixElementKernelDevice : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t gpublocks, + const size_t gputhreads ); + + // Destructor + virtual ~MatrixElementKernelDevice(); + + // Reset gpublocks and gputhreads + void setGrid( const int gpublocks, const int gputhreads ); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The buffer for the event-by-event couplings that depends on alphas QCD + DeviceBufferCouplings m_couplings; + + // The super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelMEs; + + // The super-buffer of nGoodHel jamp buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelJamps; + + // The super-buffer of nGoodHel numerator buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelNumerators; + + // The super-buffer of nGoodHel denominator buffers (dynamically allocated because nGoodHel is determined at runtime) + std::unique_ptr m_pHelDenominators; + + // The super-buffer of ncolor jamp2 buffers + DeviceBufferSimple m_colJamp2s; + +#ifdef MGONGPU_CHANNELID_DEBUG + // The **host** buffer for the channelId array + // FIXME? MEKD should accept a host buffer as an argument instead of a device buffer, so that a second copy can be avoided? + PinnedHostBufferChannelIds m_hstChannelIds; +#endif + +#ifndef MGONGPU_HAS_NO_BLAS + // Decide at runtime whether to use BLAS for color sums + bool m_blasColorSum; + + // Decide at runtime whether TF32TENSOR math should be used in cuBLAS + bool m_blasTf32Tensor; + + // The super-buffer of nGoodHel cuBLAS/hipBLAS temporary buffers + std::unique_ptr m_pHelBlasTmp; + + // The cuBLAS/hipBLAS handle (a single one for all good helicities) + gpuBlasHandle_t m_blasHandle; +#endif + + // The array of GPU streams (one for each good helicity) + gpuStream_t m_helStreams[CPPProcess::ncomb]; // reserve ncomb streams (but only nGoodHel <= ncomb will be used) + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + }; +#endif + + //-------------------------------------------------------------------------- +} +#endif // MATRIXELEMENTKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h new file mode 100644 index 0000000000..f41e40bf82 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h @@ -0,0 +1,165 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessAmplitudes_H +#define MemoryAccessAmplitudes_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" + +#define MGONGPU_TRIVIAL_AMPLITUDES 1 + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + +#ifndef MGONGPU_TRIVIAL_AMPLITUDES + + // A class describing the internal layout of memory buffers for amplitudes + // This implementation uses an AOSOA[npagA][nx2][neppA] where nevt=npagA*neppA + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessAmplitudesBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the amplitude AOSOA memory buffer layout + static constexpr int neppA = 1; // AOS (just a test...) + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagA = ievt / neppA; // #event "A-page" + const int ieppA = ievt % neppA; // #event in the current event A-page + constexpr int ix2 = 0; + return &( buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA] ); // AOSOA[ipagA][ix2][ieppA] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ix2 ) + { + constexpr int ipagA = 0; + constexpr int ieppA = 0; + return buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA]; // AOSOA[ipagA][ix2][ieppA] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessAmplitudes : public MemoryAccessAmplitudesBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + +#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessAmplitudes + { + public: + +#ifndef MGONGPU_TRIVIAL_AMPLITUDES + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2 = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2Const = + KernelAccessHelper::template kernelAccessFieldConst; + +#else + + static __host__ __device__ inline cxtype_sv* + kernelAccess( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + static __host__ __device__ inline const cxtype_sv* + kernelAccessConst( const fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + +#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessAmplitudes HostAccessAmplitudes; + typedef KernelAccessAmplitudes DeviceAccessAmplitudes; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h new file mode 100644 index 0000000000..331e7f12c3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h @@ -0,0 +1,126 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Dec 2023, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessChannelIds_H +#define MemoryAccessChannelIds_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for channel ids + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessChannelIdsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] + static __host__ __device__ inline unsigned int* + ieventAccessRecord( unsigned int* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline unsigned int& + decodeRecord( unsigned int* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessChannelIds : public MemoryAccessChannelIdsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] + static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessChannelIds + { + public: + + // Expose selected functions from MemoryAccessChannelIds + static constexpr auto ieventAccessRecordConst = MemoryAccessChannelIds::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] + static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] + static __host__ __device__ inline const uint_sv& + kernelAccessConst( const unsigned int* buffer ) + { + const unsigned int& out = kernelAccessConst_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessChannelIds HostAccessChannelIds; + typedef KernelAccessChannelIds DeviceAccessChannelIds; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h new file mode 100644 index 0000000000..fd3fb80c65 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h @@ -0,0 +1,276 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessCouplings_H +#define MemoryAccessCouplings_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM +#include "MemoryBuffers.h" // for HostBufferCouplings::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for couplings + // This implementation uses an AOSOA[npagC][ndcoup][nx2][neppC] "super-buffer" where nevt=npagC*neppC + // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessCouplingsBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the coupling AOSOA memory buffer layout + static constexpr int neppC = MemoryAccessMomentaBase::neppM; // use the same AOSOA striding as for momenta + + // SANITY CHECK: check that neppC is a power of two + static_assert( ispoweroftwo( neppC ), "neppC is not a power of 2" ); + + //-------------------------------------------------------------------------- + // ** NB! A single super-buffer AOSOA[npagC][ndcoup][nx2][neppC] includes data for ndcoup different couplings ** + // ** NB! The ieventAccessRecord and kernelAccess functions refer to the buffer for one individual coupling ** + // ** NB! Use idcoupAccessBuffer to add a fixed offset and locate the buffer for one given individual coupling ** + //-------------------------------------------------------------------------- + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (non-const) ===> fptype* idcoupAccessBuffer( fptype* buffer, const int idcoup ) <===] + // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? + static __host__ __device__ inline fptype* + idcoupAccessBuffer( fptype* buffer, // input "super-buffer" + const int idcoup ) + { + constexpr int ipagC = 0; + constexpr int ieppC = 0; + constexpr int ix2 = 0; + // NB! this effectively adds an offset "idcoup * nx2 * neppC" + return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] + } + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (const) ===> const fptype* idcoupAccessBufferConst( const fptype* buffer, const int idcoup ) <===] + // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? + static __host__ __device__ inline const fptype* + idcoupAccessBufferConst( const fptype* buffer, // input "super-buffer" + const int idcoup ) + { + return idcoupAccessBuffer( const_cast( buffer ), idcoup ); + } + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of couplings that dependent on the running alphas QCD in this specific process + static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagC = ievt / neppC; // #event "C-page" + const int ieppC = ievt %% neppC; // #event in the current event C-page + constexpr int idcoup = 0; + constexpr int ix2 = 0; + return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ix2 ) + { + constexpr int ipagC = 0; + constexpr int ieppC = 0; + // NB! the offset "idcoup * nx2 * neppC" has been added in idcoupAccessBuffer + constexpr int idcoup = 0; + return buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC]; // AOSOA[ipagC][idcoup][ix2][ieppC] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessCouplings : public MemoryAccessCouplingsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessCouplings + { + public: + + // Expose selected functions from MemoryAccessCouplingsBase + static constexpr auto idcoupAccessBuffer = MemoryAccessCouplingsBase::idcoupAccessBuffer; + static constexpr auto idcoupAccessBufferConst = MemoryAccessCouplingsBase::idcoupAccessBufferConst; + + // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. + // For dependent (event-by-event, running-alphas) couplings the value is an AOSOA record [nx2][neppC] + // (real and imaginary SIMD lanes), so consecutive flavor slots are nx2*neppC fptype's apart. + static constexpr int flv_stride = MemoryAccessCouplingsBase::neppC * mgOnGpu::nx2; + + // Expose selected functions from MemoryAccessCouplings + static constexpr auto ieventAccessRecordConst = MemoryAccessCouplings::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2_s = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2Const_s = + KernelAccessHelper::template kernelAccessFieldConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccessIx2( fptype* buffer, + const int ix2 ) + { + fptype& out = kernelAccessIx2_s( buffer, ix2 ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays + constexpr int neppC = MemoryAccessCouplingsBase::neppC; + static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS + static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessIx2Const( const fptype* buffer, + const int ix2 ) + { + return kernelAccessIx2( const_cast( buffer ), ix2 ); + } + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessIx2Const( const fptype* buffer, + const int ix2 ) + { + const fptype& out = kernelAccessIx2Const_s( buffer, ix2 ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays + constexpr int neppC = MemoryAccessCouplingsBase::neppC; + static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS + static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non const, SCALAR OR VECTOR) ===> cxtype_sv_ref kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline cxtype_sv_ref + kernelAccess( fptype* buffer ) + { + /* + fptype_sv& real = kernelAccessIx2( buffer, 0 ); + fptype_sv& imag = kernelAccessIx2( buffer, 1 ); + printf( "C_ACCESS::kernelAccess: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + return cxtype_sv_ref( real, imag ); + */ + return cxtype_sv_ref( kernelAccessIx2( buffer, 0 ), + kernelAccessIx2( buffer, 1 ) ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] + static __host__ __device__ inline cxtype_sv + kernelAccessConst( const fptype* buffer ) + { + /* + const fptype_sv& real = kernelAccessIx2Const( buffer, 0 ); + const fptype_sv& imag = kernelAccessIx2Const( buffer, 1 ); + printf( "C_ACCESS::kernelAccessConst: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + return cxtype_sv( real, imag ); + */ + return cxtype_sv( kernelAccessIx2Const( buffer, 0 ), + kernelAccessIx2Const( buffer, 1 ) ); + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessCouplings HostAccessCouplings; + typedef KernelAccessCouplings DeviceAccessCouplings; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h new file mode 100644 index 0000000000..757de7b6f2 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h @@ -0,0 +1,89 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Apr 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessCouplingsFixed_H +#define MemoryAccessCouplingsFixed_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" +#include "mgOnGpuVectors.h" + +//#include "MemoryAccessHelpers.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for fixed couplings + // This implementation uses a STRUCT[ndcoup][nx2] "super-buffer" layout: in practice, the cIPC global array + // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling + // [If many implementations are used, a suffix _Sv1 should be appended to the class name] + class MemoryAccessCouplingsFixedBase //_Sv1 + { + public: + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (const) ===> const fptype* iicoupAccessBufferConst( const fptype* buffer, const int iicoup ) <===] + static __host__ __device__ inline const fptype* + iicoupAccessBufferConst( const fptype* buffer, // input "super-buffer": in practice, the cIPC global array + const int iicoup ) + { + constexpr int ix2 = 0; + // NB! this effectively adds an offset "iicoup * nx2" + return &( buffer[iicoup * nx2 + ix2] ); // STRUCT[idcoup][ix2] + } + + private: + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessCouplingsFixed + { + public: + + // Expose selected functions from MemoryAccessCouplingsFixedBase + static constexpr auto iicoupAccessBufferConst = MemoryAccessCouplingsFixedBase::iicoupAccessBufferConst; + + // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. + // For fixed (independent) couplings the value is a single scalar complex (real,imag): nx2 fptype's, broadcast across the SIMD vector. + static constexpr int flv_stride = mgOnGpu::nx2; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] + static __host__ __device__ inline const cxtype_sv + kernelAccessConst( const fptype* buffer ) + { + // TRIVIAL ACCESS to fixed-couplings buffers! + //return cxmake( fptype_sv{ buffer[0] }, fptype_sv{ buffer[1] } ); // NO! BUG #339! + const fptype_sv r_sv = fptype_sv{ 0 } + buffer[0]; + const fptype_sv i_sv = fptype_sv{ 0 } + buffer[1]; + return cxmake( r_sv, i_sv ); // ugly but effective + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessCouplingsFixed HostAccessCouplingsFixed; + typedef KernelAccessCouplingsFixed DeviceAccessCouplingsFixed; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h new file mode 100644 index 0000000000..205952e514 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h @@ -0,0 +1,31 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessDenominators_H +#define MemoryAccessDenominators_H 1 + +#include "MemoryAccessGs.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for denominators + // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs + + typedef KernelAccessGs HostAccessDenominators; + typedef KernelAccessGs DeviceAccessDenominators; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h new file mode 100644 index 0000000000..bfcb3e3222 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h @@ -0,0 +1,171 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessGs_H +#define MemoryAccessGs_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for Gs + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessGsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessGs : public MemoryAccessGsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessGs + { + public: + + // Expose selected functions from MemoryAccessGs + static constexpr auto ieventAccessRecord = MemoryAccessGs::ieventAccessRecord; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccess( fptype* buffer ) <===] + static constexpr auto kernelAccess_s = + KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non-const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccess( fptype* buffer ) + { + fptype& out = kernelAccess_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (SCALAR OR VECTOR) ===> fptype_sv* kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv* + kernelAccessP( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + static constexpr auto kernelAccessConst_s = + KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccess( const fptype* buffer ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessConst( const fptype* buffer ) + { + const fptype& out = kernelAccessConst_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessGs HostAccessGs; + typedef KernelAccessGs DeviceAccessGs; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h new file mode 100644 index 0000000000..8b5bd27f4d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h @@ -0,0 +1,158 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessHelpers_H +#define MemoryAccessHelpers_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +//---------------------------------------------------------------------------- + +// A templated helper class that includes the boilerplate code for MemoryAccess classes +template +class MemoryAccessHelper +{ +public: + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = T::ieventAccessRecord; + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline const FT* + ieventAccessRecordConst( const FT* buffer, + const int ievt ) + { + return ieventAccessRecord( const_cast( buffer ), ievt ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + static constexpr auto decodeRecord = T::decodeRecord; + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + decodeRecordConst( const FT* buffer, + Ts... args ) // variadic template + { + return T::decodeRecord( const_cast( buffer ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessField( fptype* buffer, const ievt, Ts... args ) <===] + template + static __host__ __device__ inline FT& + ieventAccessField( FT* buffer, + const int ievt, + Ts... args ) // variadic template + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + return T::decodeRecord( T::ieventAccessRecord( buffer, ievt ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessFieldConst( const fptype* buffer, const ievt, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + ieventAccessFieldConst( const FT* buffer, + const int ievt, + Ts... args ) // variadic template + { + return ieventAccessField( const_cast( buffer ), ievt, args... ); + } +}; + +//---------------------------------------------------------------------------- + +// A templated helper class that includes the boilerplate code for KernelAccess classes +template +class KernelAccessHelper : public MemoryAccessHelper +{ +public: + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non-const) ===> fptype* kernelAccessRecord( fptype* buffer ) <===] + static __host__ __device__ inline FT* + kernelAccessRecord( FT* buffer ) + { + if constexpr( !onDevice ) // requires c++17 also in CUDA (#333) + { + // FIXME #436: clarify that buffer includes all events on device, and only the record for an event subset on host! + // FIXME #436: am I not assuming that the following line is always identical to buffer for all access classes T? + return T::ieventAccessRecord( buffer, 0 ); + } + else + { +#ifdef MGONGPUCPP_GPUIMPL + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid + //printf( "kernelAccessRecord: ievt=%d threadId=%d\n", ievt, threadIdx.x ); + return T::ieventAccessRecord( buffer, ievt ); // NB fptype and fptype_sv coincide for CUDA +#else + throw std::runtime_error( "kernelAccessRecord on device is only implemented in CUDA" ); +#endif + } + } + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const) ===> const fptype* kernelAccessRecordConst( const fptype* buffer ) <===] + static __host__ __device__ inline const FT* + kernelAccessRecordConst( const FT* buffer ) + { + return kernelAccessRecord( const_cast( buffer ) ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessField( fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline FT& + kernelAccessField( FT* buffer, + Ts... args ) // variadic template + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + return T::decodeRecord( kernelAccessRecord( buffer ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessFieldConst( const fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + kernelAccessFieldConst( const FT* buffer, + Ts... args ) // variadic template + { + return kernelAccessField( const_cast( buffer ), args... ); + } + + //-------------------------------------------------------------------------- +}; + +#endif // MemoryAccessHelpers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h new file mode 100644 index 0000000000..ffc74b0f0c --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h @@ -0,0 +1,125 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: D. Massaro (Jan 2026, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessIflavorVec_H +#define MemoryAccessIflavorVec_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for channel ids + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessIflavorVecBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] + static __host__ __device__ inline unsigned int* + ieventAccessRecord( unsigned int* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline unsigned int& + decodeRecord( unsigned int* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessIflavorVec : public MemoryAccessIflavorVecBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] + static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessIflavorVec + { + public: + + // Expose selected functions from MemoryAccessIflavorVec + static constexpr auto ieventAccessRecordConst = MemoryAccessIflavorVec::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] + static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] + static __host__ __device__ inline const uint_sv& + kernelAccessConst( const unsigned int* buffer ) + { + const unsigned int& out = kernelAccessConst_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessIflavorVec HostAccessIflavorVec; + typedef KernelAccessIflavorVec DeviceAccessIflavorVec; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h new file mode 100644 index 0000000000..4236e20602 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h @@ -0,0 +1,147 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessMatrixElements_H +#define MemoryAccessMatrixElements_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for matrix elements + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessMatrixElementsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessMatrixElements : public MemoryAccessMatrixElementsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessMatrixElements + { + public: + + // Expose selected functions from MemoryAccessMatrixElements + static constexpr auto ieventAccessRecord = MemoryAccessMatrixElements::ieventAccessRecord; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccess_s( fptype* buffer ) <===] + static constexpr auto kernelAccess_s = + KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( const fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccess( fptype* buffer ) + { + fptype& out = kernelAccess_s( buffer ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) + static_assert( mg5amcCpu::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast +#endif + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + static constexpr auto kernelAccessConst = + KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessMatrixElements HostAccessMatrixElements; + typedef KernelAccessMatrixElements DeviceAccessMatrixElements; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h new file mode 100644 index 0000000000..3ef4d76fbd --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h @@ -0,0 +1,276 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessMomenta_H +#define MemoryAccessMomenta_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for momenta + // This implementation uses an AOSOA[npagM][npar][np4][neppM] where nevt=npagM*neppM + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessMomentaBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the momenta AOSOA memory buffer layout + // (these are all best kept as a compile-time constants: see issue #23) +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ + // ----------------------------------------------------------------------------------------------- + // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline + // --- This is relevant to ensure coalesced access to momenta in global memory + // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms + // ----------------------------------------------------------------------------------------------- + //static constexpr int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) + static constexpr int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 1.03E9 instead of 1.11E9 in eemumu) +#else + // ----------------------------------------------------------------------------------------------- + // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register + // --- This is relevant to ensure faster access to momenta from C++ memory cache lines + // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded + // --- In practice, neppR, neppM and neppV could now (in principle) all be different + // ----------------------------------------------------------------------------------------------- +#ifdef MGONGPU_CPPSIMD + static constexpr int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance + //static constexpr int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) + //static constexpr int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 4.66E6 instead of 5.09E9 in eemumu) + //static constexpr int neppM = MGONGPU_CPPSIMD*2; // FOR TESTS +#else + static constexpr int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) +#endif +#endif /* clang-format on */ + + // SANITY CHECK: check that neppM is a power of two + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a 4-momentum + static constexpr int np4 = CPPProcess::np4; + + // The number of particles in this physics process + static constexpr int npar = CPPProcess::npar; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagM = ievt / neppM; // #event "M-page" + const int ieppM = ievt % neppM; // #event in the current event M-page + constexpr int ip4 = 0; + constexpr int ipar = 0; + return &( buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM] ); // AOSOA[ipagM][ipar][ip4][ieppM] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ip4, const int ipar" and rename "Field" as "Ip4Ipar"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ip4, + const int ipar ) + { + constexpr int ipagM = 0; + constexpr int ieppM = 0; + return buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessMomenta : public MemoryAccessMomentaBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto decodeRecordIp4Ipar = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto decodeRecordIp4IparConst = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIp4Ipar( fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + static constexpr auto ieventAccessIp4Ipar = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + // DEFAULT VERSION + static constexpr auto ieventAccessIp4IparConst = + MemoryAccessHelper::template ieventAccessFieldConst; + + /* + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + // DEBUG VERSION WITH PRINTOUTS + static __host__ __device__ inline const fptype& + ieventAccessIp4IparConst( const fptype* buffer, + const int ievt, + const int ip4, + const int ipar ) + { + const fptype& out = MemoryAccessHelper::template ieventAccessFieldConst( buffer, ievt, ip4, ipar ); + printf( "ipar=%2d ip4=%2d ievt=%8d out=%8.3f\n", ipar, ip4, ievt, out ); + return out; + } + */ + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessMomenta + { + public: + + // Expose selected functions from MemoryAccessMomenta + static constexpr auto ieventAccessRecordConst = MemoryAccessMomenta::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIp4Ipar( fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto kernelAccessIp4Ipar = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // DEFAULT VERSION + static constexpr auto kernelAccessIp4IparConst_s = + KernelAccessHelper::template kernelAccessFieldConst; + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // DEBUG VERSION WITH PRINTOUTS + static __host__ __device__ inline const fptype& + kernelAccessIp4IparConst_s( const fptype* buffer, + const int ip4, + const int ipar ) + { + const fptype& out = KernelAccessHelper::template kernelAccessFieldConst( buffer, ip4, ipar ); + printf( "ipar=%2d ip4=%2d ievt='kernel' out=%8.3f\n", ipar, ip4, out ); + return out; + } + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> fptype_sv kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // FIXME? Eventually return by const reference and support aligned arrays only? + // FIXME? Currently return by value to support also unaligned and arbitrary arrays + static __host__ __device__ inline fptype_sv + kernelAccessIp4IparConst( const fptype* buffer, + const int ip4, + const int ipar ) + { + const fptype& out = kernelAccessIp4IparConst_s( buffer, ip4, ipar ); +#ifndef MGONGPU_CPPSIMD + return out; +#else + constexpr int neppM = MemoryAccessMomentaBase::neppM; + constexpr bool useContiguousEventsIfPossible = true; // DEFAULT + //constexpr bool useContiguousEventsIfPossible = false; // FOR PERFORMANCE TESTS (treat as arbitrary array even if it is an AOSOA) + // Use c++17 "if constexpr": compile-time branching + if constexpr( useContiguousEventsIfPossible && ( neppM >= neppV ) && ( neppM % neppV == 0 ) ) + { + //constexpr bool skipAlignmentCheck = true; // FASTEST (SEGFAULTS IF MISALIGNED ACCESS, NEEDS A SANITY CHECK ELSEWHERE!) + constexpr bool skipAlignmentCheck = false; // DEFAULT: A BIT SLOWER BUT SAFER [ALLOWS MISALIGNED ACCESS] + if constexpr( skipAlignmentCheck ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! assume aligned AOSOA, skip check" << std::endl; first=false; } // SLOWER (5.06E6) + // FASTEST? (5.09E6 in eemumu 512y) + // This assumes alignment for momenta1d without checking - causes segmentation fault in reinterpret_cast if not aligned! + return mg5amcCpu::fptypevFromAlignedArray( out ); // use reinterpret_cast + } + else if( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ) + { + //static bool first=true; if( first ){ std::cout << "WARNING! aligned AOSOA, reinterpret cast" << std::endl; first=false; } // SLOWER (5.00E6) + // DEFAULT! A tiny bit (<1%) slower because of the alignment check (5.07E6 in eemumu 512y) + // This explicitly checks buffer alignment to avoid segmentation faults in reinterpret_cast + return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWER (4.93E6) + // A bit (1%) slower (5.05E6 in eemumu 512y) + // This does not require buffer alignment, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 + return mg5amcCpu::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) + } + } + else + { + //static bool first=true; if( first ){ std::cout << "WARNING! arbitrary array" << std::endl; first=false; } // SLOWER (5.08E6) + // ?!Used to be much slower, now a tiny bit faster for AOSOA?! (5.11E6 for AOSOA, 4.64E6 for AOS in eemumu 512y) + // This does not even require AOSOA with neppM>=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) + constexpr int ievt0 = 0; // just make it explicit in the code that buffer refers to a given ievt0 and decoderIeppV fetches event ievt0+ieppV + auto decoderIeppv = [buffer, ip4, ipar]( int ieppV ) + -> const fptype& + { return MemoryAccessMomenta::ieventAccessIp4IparConst( buffer, ievt0 + ieppV, ip4, ipar ); }; + return mg5amcCpu::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) + } +#endif + } + + // Is this a HostAccess or DeviceAccess class? + // [this is only needed for a warning printout in rambo.h for nparf==1 #358] + static __host__ __device__ inline constexpr bool + isOnDevice() + { + return onDevice; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessMomenta HostAccessMomenta; + typedef KernelAccessMomenta DeviceAccessMomenta; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h new file mode 100644 index 0000000000..c8d0343b7e --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h @@ -0,0 +1,31 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessNumerators_H +#define MemoryAccessNumerators_H 1 + +#include "MemoryAccessGs.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for numerators + // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs + + typedef KernelAccessGs HostAccessNumerators; + typedef KernelAccessGs DeviceAccessNumerators; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h new file mode 100644 index 0000000000..57f1d02081 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h @@ -0,0 +1,145 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessRandomNumbers_H +#define MemoryAccessRandomNumbers_H 1 + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "MemoryAccessHelpers.h" + +#ifdef MGONGPUCPP_GPUIMPL +using mg5amcGpu::CPPProcess; +#else +using mg5amcCpu::CPPProcess; +#endif + +//---------------------------------------------------------------------------- + +// A class describing the internal layout of memory buffers for random numbers +// This implementation uses an AOSOA[npagR][nparf][np4][neppR] where nevt=npagR*neppR +// [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] +class MemoryAccessRandomNumbersBase //_AOSOAv1 +{ +public: /* clang-format off */ + + // Number of Events Per Page in the random number AOSOA memory buffer layout + // *** NB Different values of neppR lead to different physics results: the *** + // *** same 1d array is generated, but it is interpreted in different ways *** + static constexpr int neppR = 8; // HARDCODED TO GIVE ALWAYS THE SAME PHYSICS RESULTS! + //static constexpr int neppR = 1; // AOS (tests of sectors/requests) + +private: /* clang-format on */ + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a 4-momentum + static constexpr int np4 = CPPProcess::np4; + + // The number of final state particles in this physics process + static constexpr int nparf = CPPProcess::nparf; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagR = ievt / neppR; // #event "R-page" + const int ieppR = ievt % neppR; // #event in the current event R-page + constexpr int ip4 = 0; + constexpr int iparf = 0; + return &( buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR] ); // AOSOA[ipagR][iparf][ip4][ieppR] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ip4, const int iparf" and rename "Field" as "Ip4Iparf"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ip4, + const int iparf ) + { + constexpr int ipagR = 0; + constexpr int ieppR = 0; + return buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR]; // AOSOA[ipagR][iparf][ip4][ieppR] + } +}; + +//---------------------------------------------------------------------------- + +// A class providing access to memory buffers for a given event, based on explicit event numbers +// Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations +class MemoryAccessRandomNumbers : public MemoryAccessRandomNumbersBase +{ +public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto decodeRecordIp4Iparf = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto decodeRecordIp4IparfConst = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIp4Iparf( fptype* buffer, const ievt, const int ipar, const int iparf ) <===] + static constexpr auto ieventAccessIp4Iparf = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparfConst( const fptype* buffer, const ievt, const int ipar, const int iparf ) <===] + static constexpr auto ieventAccessIp4IparfConst = + MemoryAccessHelper::template ieventAccessFieldConst; +}; + +//---------------------------------------------------------------------------- + +// A class providing access to memory buffers for a given event, based on implicit kernel rules +// Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations +template +class KernelAccessRandomNumbers +{ +public: + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIp4Iparf( fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto kernelAccessIp4Iparf = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIp4IparfConst( const fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto kernelAccessIp4IparfConst = + KernelAccessHelper::template kernelAccessFieldConst; +}; + +//---------------------------------------------------------------------------- + +typedef KernelAccessRandomNumbers HostAccessRandomNumbers; +typedef KernelAccessRandomNumbers DeviceAccessRandomNumbers; + +//---------------------------------------------------------------------------- + +#endif // MemoryAccessRandomNumbers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h new file mode 100644 index 0000000000..8563d8f2f7 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h @@ -0,0 +1,138 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessVectors_H +#define MemoryAccessVectors_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#ifndef MGONGPUCPP_GPUIMPL +namespace mg5amcCpu // this is only needed for CPU SIMD vectorization +{ + +#ifdef MGONGPU_CPPSIMD + //-------------------------------------------------------------------------- + + // Cast one non-const fptype_v reference (one vector of neppV fptype values) from one non-const fptype reference (#435), + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned + inline fptype_v& fptypevFromAlignedArray( fptype& ref ) + { + return *reinterpret_cast( &ref ); + } + + inline uint_v& uintvFromAlignedArray( unsigned int& ref ) + { + return *reinterpret_cast( &ref ); + } + + // Cast one const fptype_v reference (one vector of neppV fptype values) from one const fptype reference, + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned + inline const fptype_v& fptypevFromAlignedArray( const fptype& ref ) + { + return *reinterpret_cast( &ref ); + } + + inline const uint_v& uintvFromAlignedArray( const unsigned int& ref ) + { + return *reinterpret_cast( &ref ); + } + + // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", but that the arrays are not aligned + inline fptype_v fptypevFromUnalignedArray( const fptype& ref ) + { +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (2) + *( &ref + 1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (4) + *( &ref + 1 ), + *( &ref + 2 ), + *( &ref + 3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (8) + *( &ref + 1 ), + *( &ref + 2 ), + *( &ref + 3 ), + *( &ref + 4 ), + *( &ref + 5 ), + *( &ref + 6 ), + *( &ref + 7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ *( &ref ), // explicit initialization of all array elements (16) + *( &ref + 1 ), + *( &ref + 2 ), + *( &ref + 3 ), + *( &ref + 4 ), + *( &ref + 5 ), + *( &ref + 6 ), + *( &ref + 7 ), + *( &ref + 8 ), + *( &ref + 9 ), + *( &ref + 10 ), + *( &ref + 11 ), + *( &ref + 12 ), + *( &ref + 13 ), + *( &ref + 14 ), + *( &ref + 15 ) }; +#else +#error Internal error! Unknown MGONGPU_CPPSIMD value +#endif + } + + // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, + // with no a priori assumption on how the input fptype array should be decoded + template + inline fptype_v fptypevFromArbitraryArray( Functor decoderIeppv ) + { +#if MGONGPU_CPPSIMD == 2 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (2) + decoderIeppv( 1 ) }; +#elif MGONGPU_CPPSIMD == 4 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (4) + decoderIeppv( 1 ), + decoderIeppv( 2 ), + decoderIeppv( 3 ) }; +#elif MGONGPU_CPPSIMD == 8 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (8) + decoderIeppv( 1 ), + decoderIeppv( 2 ), + decoderIeppv( 3 ), + decoderIeppv( 4 ), + decoderIeppv( 5 ), + decoderIeppv( 6 ), + decoderIeppv( 7 ) }; +#elif MGONGPU_CPPSIMD == 16 + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (16) + decoderIeppv( 1 ), + decoderIeppv( 2 ), + decoderIeppv( 3 ), + decoderIeppv( 4 ), + decoderIeppv( 5 ), + decoderIeppv( 6 ), + decoderIeppv( 7 ), + decoderIeppv( 8 ), + decoderIeppv( 9 ), + decoderIeppv( 10 ), + decoderIeppv( 11 ), + decoderIeppv( 12 ), + decoderIeppv( 13 ), + decoderIeppv( 14 ), + decoderIeppv( 15 ) }; +#else +#error Internal error! Unknown MGONGPU_CPPSIMD value +#endif + } + + //-------------------------------------------------------------------------- +#endif + +} // end namespace +#endif + +#endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h new file mode 100644 index 0000000000..14a37c7fea --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h @@ -0,0 +1,170 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessWavefunctions_H +#define MemoryAccessWavefunctions_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" + +#define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + +#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + // A class describing the internal layout of memory buffers for wavefunctions + // This implementation uses an AOSOA[npagW][nw6][nx2][neppW] where nevt=npagW*neppW + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessWavefunctionsBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the wavefunction AOSOA memory buffer layout + static constexpr int neppW = 1; // AOS (just a test...) + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a (fermion or vector) wavefunction + static constexpr int nw6 = mgOnGpu::nw6; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagW = ievt / neppW; // #event "W-page" + const int ieppW = ievt % neppW; // #event in the current event W-page + constexpr int iw6 = 0; + constexpr int ix2 = 0; + return &( buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW] ); // AOSOA[ipagW][iw6][ix2][ieppW] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int iw6, const int ix2" and rename "Field" as "Iw6Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int iw6, + const int ix2 ) + { + constexpr int ipagW = 0; + constexpr int ieppW = 0; + return buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW]; // AOSOA[ipagW][iw6][ix2][ieppW] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessWavefunctions : public MemoryAccessWavefunctionsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto decodeRecordIw6Ix2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto decodeRecordIw6Ix2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIw6Ix2( fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] + static constexpr auto ieventAccessIw6Ix2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIw6Ix2Const( const fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] + static constexpr auto ieventAccessIw6Ix2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + +#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessWavefunctions + { + public: + +#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIw6Ix2( fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto kernelAccessIw6Ix2 = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIw6Ix2Const( const fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto kernelAccessIw6Ix2Const = + KernelAccessHelper::template kernelAccessFieldConst; + +#else + + static __host__ __device__ inline cxtype_sv* + kernelAccess( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + static __host__ __device__ inline const cxtype_sv* + kernelAccessConst( const fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + +#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessWavefunctions HostAccessWavefunctions; + typedef KernelAccessWavefunctions DeviceAccessWavefunctions; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h new file mode 100644 index 0000000000..c56527f581 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h @@ -0,0 +1,150 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessWeights_H +#define MemoryAccessWeights_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for weights + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessWeightsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessWeights : public MemoryAccessWeightsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessWeights + { + public: + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] + // FINAL IMPLEMENTATION FOR CUDA 11.4 + static constexpr auto kernelAccess = + KernelAccessHelper::template kernelAccessField<>; + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] + // TEMPORARY HACK FOR CUDA 11.1 + static __host__ __device__ inline fptype& + kernelAccess( fptype* buffer ) + { + return KernelAccessHelper::template kernelAccessField<>( buffer ); + } + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + // FINAL IMPLEMENTATION FOR CUDA 11.4 + static constexpr auto kernelAccessConst = + KernelAccessHelper::template kernelAccessFieldConst<>; + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + // TEMPORARY HACK FOR CUDA 11.1 + static __host__ __device__ inline const fptype& + kernelAccessConst( const fptype* buffer ) + { + return KernelAccessHelper::template kernelAccessFieldConst<>( buffer ); + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessWeights HostAccessWeights; + typedef KernelAccessWeights DeviceAccessWeights; + + //---------------------------------------------------------------------------- + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h new file mode 100644 index 0000000000..d259aa7456 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h @@ -0,0 +1,603 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021, based on earlier work by S. Hageboeck) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryBuffers_H +#define MemoryBuffers_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "CPPProcess.h" +#include "GpuRuntime.h" +#include "Parameters.h" +#include "processConfig.h" + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + namespace MemoryBuffers + { + // Process-independent compile-time constants + static constexpr size_t np4 = CPPProcess::np4; + static constexpr size_t nw6 = CPPProcess::nw6; + static constexpr size_t nx2 = mgOnGpu::nx2; + // Process-dependent compile-time constants + static constexpr size_t nparf = CPPProcess::nparf; + static constexpr size_t npar = CPPProcess::npar; + static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; + static constexpr size_t ncolor = CPPProcess::ncolor; + } + + //-------------------------------------------------------------------------- + + // An abstract interface encapsulating a given number of events + class INumberOfEvents + { + public: + virtual ~INumberOfEvents() {} + virtual size_t nevt() const = 0; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating a given number of events + class NumberOfEvents : virtual public INumberOfEvents + { + public: + NumberOfEvents( const size_t nevt ) + : m_nevt( nevt ) {} + virtual ~NumberOfEvents() {} + virtual size_t nevt() const override { return m_nevt; } + private: + const size_t m_nevt; + }; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer (not necessarily an event buffer) + template + class BufferBase : virtual public INumberOfEvents + { + protected: + BufferBase( const size_t size, const bool onDevice ) + : m_size( size ), m_data( nullptr ), m_isOnDevice( onDevice ) {} + public: + virtual ~BufferBase() {} + T* data() { return m_data; } + const T* data() const { return m_data; } + T& operator[]( const size_t index ) { return m_data[index]; } + const T& operator[]( const size_t index ) const { return m_data[index]; } + size_t size() const { return m_size; } + size_t bytes() const { return m_size * sizeof( T ); } + bool isOnDevice() const { return m_isOnDevice; } + virtual size_t nevt() const override { throw std::runtime_error( "This BufferBase is not an event buffer" ); } + protected: + const size_t m_size; + T* m_data; + const bool m_isOnDevice; + }; + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + constexpr bool HostBufferALIGNED = false; // ismisaligned=false + constexpr bool HostBufferMISALIGNED = true; // ismisaligned=true + + // A class encapsulating a C++ host buffer + template + class HostBufferBase : public BufferBase + { + public: + HostBufferBase( const size_t size ) + : BufferBase( size, false ) + { + if constexpr( !ismisaligned ) + this->m_data = new( std::align_val_t( cppAlign ) ) T[size](); + else + this->m_data = new( std::align_val_t( cppAlign ) ) T[size + 1]() + 1; // TEST MISALIGNMENT! + } + virtual ~HostBufferBase() + { + if constexpr( !ismisaligned ) + ::operator delete[]( this->m_data, std::align_val_t( cppAlign ) ); + else + ::operator delete[]( ( this->m_data ) - 1, std::align_val_t( cppAlign ) ); // TEST MISALIGNMENT! + } + static constexpr bool isaligned() { return !ismisaligned; } + public: + static constexpr size_t cppAlign = mgOnGpu::cppAlign; + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA pinned host buffer + template + class PinnedHostBufferBase : public BufferBase + { + public: + PinnedHostBufferBase( const size_t size ) + : BufferBase( size, false ) + { + gpuMallocHost( &( this->m_data ), this->bytes() ); + } + virtual ~PinnedHostBufferBase() + { + gpuFreeHost( this->m_data ); + } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA device buffer + template + class DeviceBufferBase : public BufferBase + { + public: + DeviceBufferBase( const size_t size ) + : BufferBase( size, true ) + { + gpuMalloc( &( this->m_data ), this->bytes() ); + } + virtual ~DeviceBufferBase() + { + gpuFree( this->m_data ); + } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for a given number of events + template + class HostBuffer : public HostBufferBase, virtual private NumberOfEvents + { + public: + HostBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , HostBufferBase( sizePerEvent * nevt ) + { + //std::cout << "HostBuffer::ctor " << this << " " << nevt << std::endl; + } + virtual ~HostBuffer() + { + //std::cout << "HostBuffer::dtor " << this << std::endl; + } + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA pinned host buffer for a given number of events + template + class PinnedHostBuffer : public PinnedHostBufferBase, virtual private NumberOfEvents + { + public: + PinnedHostBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , PinnedHostBufferBase( sizePerEvent * nevt ) {} + virtual ~PinnedHostBuffer() {} + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a CUDA device buffer for a given number of events + template + class DeviceBuffer : public DeviceBufferBase, virtual protected NumberOfEvents + { + public: + DeviceBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , DeviceBufferBase( sizePerEvent * nevt ) + { + //std::cout << "DeviceBuffer::ctor " << this << " " << nevt << std::endl; + } + virtual ~DeviceBuffer() + { + //std::cout << "DeviceBuffer::dtor " << this << std::endl; + } + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating a simple CUDA device buffer managed on an ad-hoc basis + typedef DeviceBuffer DeviceBufferSimple; + typedef DeviceBuffer DeviceBufferSimple2; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for momenta random numbers + typedef BufferBase BufferRndNumMomenta; + + // The size (number of elements) per event in a memory buffer for momenta random numbers + constexpr size_t sizePerEventRndNumMomenta = MemoryBuffers::np4 * MemoryBuffers::nparf; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for momenta random numbers + typedef HostBuffer HostBufferRndNumMomenta; +#else + // A class encapsulating a CUDA pinned host buffer for momenta random numbers + typedef PinnedHostBuffer PinnedHostBufferRndNumMomenta; + // A class encapsulating a CUDA device buffer for momenta random numbers + typedef DeviceBuffer DeviceBufferRndNumMomenta; +#endif + + //-------------------------------------------------------------------------- + + /* + // A base class encapsulating a memory buffer with ONE fptype per event + typedef BufferBase BufferOneFp; + + // The size (number of elements) per event in a memory buffer with ONE fptype per event + constexpr size_t sizePerEventOneFp = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer with ONE fptype per event + typedef HostBuffer HostBufferOneFp; +#else + // A class encapsulating a CUDA pinned host buffer for gs + typedef PinnedHostBuffer PinnedHostBufferOneFp; + // A class encapsulating a CUDA device buffer for gs + typedef DeviceBuffer DeviceBufferOneFp; +#endif + + // Memory buffers for Gs (related to the event-by-event strength of running coupling constant alphas QCD) + typedef BufferOneFp BufferGs; + typedef HostBufferOneFp HostBufferGs; + typedef PinnedHostBufferOneFp PinnedHostBufferGs; + typedef DeviceBufferOneFp DeviceBufferGs; + */ + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for Gs (related to the event-by-event strength of running coupling constant alphas QCD) + typedef BufferBase BufferGs; + + // The size (number of elements) per event in a memory buffer for Gs + constexpr size_t sizePerEventGs = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for gs + typedef HostBuffer HostBufferGs; +#else + // A class encapsulating a CUDA pinned host buffer for gs + typedef PinnedHostBuffer PinnedHostBufferGs; + // A class encapsulating a CUDA device buffer for gs + typedef DeviceBuffer DeviceBufferGs; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for numerators (of the multichannel single-diagram enhancement factors) + typedef BufferBase BufferNumerators; + + // The size (number of elements) per event in a memory buffer for numerators + // (should be equal to the number of diagrams in the process) + constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for numerators + typedef HostBuffer HostBufferNumerators; +#else + // A class encapsulating a CUDA pinned host buffer for numerators + typedef PinnedHostBuffer PinnedHostBufferNumerators; + // A class encapsulating a CUDA device buffer for numerators + typedef DeviceBuffer DeviceBufferNumerators; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for denominators (of the multichannel single-diagram enhancement factors) + typedef BufferBase BufferDenominators; + + // The size (number of elements) per event in a memory buffer for denominators + constexpr size_t sizePerEventDenominators = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for denominators + typedef HostBuffer HostBufferDenominators; +#else + // A class encapsulating a CUDA pinned host buffer for denominators + typedef PinnedHostBuffer PinnedHostBufferDenominators; + // A class encapsulating a CUDA device buffer for denominators + typedef DeviceBuffer DeviceBufferDenominators; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for couplings that depend on the event-by-event running coupling constant alphas QCD + typedef BufferBase BufferCouplings; + + // The size (number of elements) per event in a memory buffer for random numbers + constexpr size_t sizePerEventCouplings = MemoryBuffers::ndcoup * MemoryBuffers::nx2; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for couplings + typedef HostBuffer HostBufferCouplings; +#else + // A class encapsulating a CUDA pinned host buffer for couplings + typedef PinnedHostBuffer PinnedHostBufferCouplings; + // A class encapsulating a CUDA device buffer for couplings + typedef DeviceBuffer DeviceBufferCouplings; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for momenta + typedef BufferBase BufferMomenta; + + // The size (number of elements) per event in a memory buffer for momenta + constexpr size_t sizePerEventMomenta = MemoryBuffers::np4 * MemoryBuffers::npar; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for momenta + typedef HostBuffer HostBufferMomenta; + //typedef HostBuffer HostBufferMomenta; // TEST MISALIGNMENT! +#else + // A class encapsulating a CUDA pinned host buffer for momenta + typedef PinnedHostBuffer PinnedHostBufferMomenta; + // A class encapsulating a CUDA device buffer for momenta + typedef DeviceBuffer DeviceBufferMomenta; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for sampling weights + typedef BufferBase BufferWeights; + + // The size (number of elements) per event in a memory buffer for sampling weights + constexpr size_t sizePerEventWeights = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for sampling weights + typedef HostBuffer HostBufferWeights; +#else + // A class encapsulating a CUDA pinned host buffer for sampling weights + typedef PinnedHostBuffer PinnedHostBufferWeights; + // A class encapsulating a CUDA device buffer for sampling weights + typedef DeviceBuffer DeviceBufferWeights; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for matrix elements + typedef BufferBase BufferMatrixElements; + + // The size (number of elements) per event in a memory buffer for matrix elements + constexpr size_t sizePerEventMatrixElements = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for matrix elements + typedef HostBuffer HostBufferMatrixElements; +#else + // A class encapsulating a CUDA pinned host buffer for matrix elements + typedef PinnedHostBuffer PinnedHostBufferMatrixElements; + // A class encapsulating a CUDA device buffer for matrix elements + typedef DeviceBuffer DeviceBufferMatrixElements; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for the helicity mask + typedef BufferBase BufferHelicityMask; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for the helicity mask + typedef HostBufferBase HostBufferHelicityMask; +#else + // A class encapsulating a CUDA pinned host buffer for the helicity mask + typedef PinnedHostBufferBase PinnedHostBufferHelicityMask; + // A class encapsulating a CUDA device buffer for the helicity mask + typedef DeviceBufferBase DeviceBufferHelicityMask; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for wavefunctions + typedef BufferBase BufferWavefunctions; + + // The size (number of elements) per event in a memory buffer for wavefunctions + constexpr size_t sizePerEventWavefunctions = MemoryBuffers::nw6 * MemoryBuffers::nx2; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for wavefunctions + typedef HostBuffer HostBufferWavefunctions; +#else + // A class encapsulating a CUDA pinned host buffer for wavefunctions + typedef PinnedHostBuffer PinnedHostBufferWavefunctions; + // A class encapsulating a CUDA device buffer for wavefunctions + typedef DeviceBuffer DeviceBufferWavefunctions; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for helicity random numbers + typedef BufferBase BufferRndNumHelicity; + + // The size (number of elements) per event in a memory buffer for helicity random numbers + constexpr size_t sizePerEventRndNumHelicity = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for helicity random numbers + typedef HostBuffer HostBufferRndNumHelicity; +#else + // A class encapsulating a CUDA pinned host buffer for helicity random numbers + typedef PinnedHostBuffer PinnedHostBufferRndNumHelicity; + // A class encapsulating a CUDA device buffer for helicity random numbers + typedef DeviceBuffer DeviceBufferRndNumHelicity; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for color random numbers + typedef BufferBase BufferRndNumColor; + + // The size (number of elements) per event in a memory buffer for color random numbers + constexpr size_t sizePerEventRndNumColor = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for color random numbers + typedef HostBuffer HostBufferRndNumColor; +#else + // A class encapsulating a CUDA pinned host buffer for color random numbers + typedef PinnedHostBuffer PinnedHostBufferRndNumColor; + // A class encapsulating a CUDA device buffer for color random numbers + typedef DeviceBuffer DeviceBufferRndNumColor; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for channel ids + typedef BufferBase BufferChannelIds; + + // The size (number of elements) per event in a memory buffer for channel ids + constexpr size_t sizePerEventChannelId = 1; + +#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) + // A class encapsulating a C++ host buffer for channel ids + typedef HostBuffer HostBufferChannelIds; +#else + // A class encapsulating a CUDA pinned host buffer for channel ids + typedef PinnedHostBuffer PinnedHostBufferChannelIds; + // A class encapsulating a CUDA device buffer for channel ids + typedef DeviceBuffer DeviceBufferChannelIds; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for channel ids + typedef BufferBase BufferIflavorVec; + + // The size (number of elements) per event in a memory buffer for channel ids + constexpr size_t sizePerEventIflavorVec = 1; + +#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) + // A class encapsulating a C++ host buffer for channel ids + typedef HostBuffer HostBufferIflavorVec; +#else + // A class encapsulating a CUDA pinned host buffer for channel ids + typedef PinnedHostBuffer PinnedHostBufferIflavorVec; + // A class encapsulating a CUDA device buffer for channel ids + typedef DeviceBuffer DeviceBufferIflavorVec; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for helicity selection + typedef BufferBase BufferSelectedHelicity; + + // The size (number of elements) per event in a memory buffer for helicity selection + constexpr size_t sizePerEventSelectedHelicity = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for helicity selection + typedef HostBuffer HostBufferSelectedHelicity; +#else + // A class encapsulating a CUDA pinned host buffer for helicity selection + typedef PinnedHostBuffer PinnedHostBufferSelectedHelicity; + // A class encapsulating a CUDA device buffer for helicity selection + typedef DeviceBuffer DeviceBufferSelectedHelicity; +#endif + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for color selection + typedef BufferBase BufferSelectedColor; + + // The size (number of elements) per event in a memory buffer for color selection + constexpr size_t sizePerEventSelectedColor = 1; + +#ifndef MGONGPUCPP_GPUIMPL + // A class encapsulating a C++ host buffer for color selection + typedef HostBuffer HostBufferSelectedColor; +#else + // A class encapsulating a CUDA pinned host buffer for color selection + typedef PinnedHostBuffer PinnedHostBufferSelectedColor; + // A class encapsulating a CUDA device buffer for color selection + typedef DeviceBuffer DeviceBufferSelectedColor; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // The size (number of elements) per event in a memory buffer for jamps + constexpr size_t sizePerEventJamps = MemoryBuffers::ncolor * MemoryBuffers::nx2; + + // A class encapsulating a CUDA device buffer for color selection + typedef DeviceBuffer DeviceBufferJamps; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + template + void copyDeviceFromHost( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy + { + if( dst.size() != src.size() ) + { + std::ostringstream sstr; + sstr << "Size (#elements) mismatch in copyDeviceFromHost: dst=" << dst.size() << ", src=" << src.size(); + throw std::runtime_error( sstr.str() ); + } + if( dst.bytes() != src.bytes() ) + { + std::ostringstream sstr; + sstr << "Size (#bytes) mismatch in copyDeviceFromHost: dst=" << dst.bytes() << ", src=" << src.bytes(); + throw std::runtime_error( sstr.str() ); + } + // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array + gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyHostToDevice ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + template + void copyHostFromDevice( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy + { + if( dst.size() != src.size() ) + { + std::ostringstream sstr; + sstr << "Size (#elements) mismatch in copyHostFromDevice: dst=" << dst.size() << ", src=" << src.size(); + throw std::runtime_error( sstr.str() ); + } + if( dst.bytes() != src.bytes() ) + { + std::ostringstream sstr; + sstr << "Size (#bytes) mismatch in copyHostFromDevice: dst=" << dst.bytes() << ", src=" << src.bytes(); + throw std::runtime_error( sstr.str() ); + } + // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array + gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyDeviceToHost ); + } +#endif + + //-------------------------------------------------------------------------- +} + +#endif // MemoryBuffers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc new file mode 100644 index 0000000000..d40b1b4174 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc @@ -0,0 +1,246 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "RamboSamplingKernels.h" + +#include "GpuRuntime.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessRandomNumbers.h" +#include "MemoryAccessWeights.h" +#include "MemoryBuffers.h" +#include "rambo.h" // inline classic (massive) RAMBO, ported from standalone_cpp +#include "massless_rambo.h" // inline implementation of massless RAMBO algorithms and kernels + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + MasslessRamboSamplingKernelHost::MasslessRamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t nevt ) + : SamplingKernelBase( energy, rndmom, momenta, weights ) + , NumberOfEvents( nevt ) + { + if( m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: rndmom must be a host array" ); + if( m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: momenta must be a host array" ); + if( m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: weights must be a host array" ); + if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with rndmom" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with weights" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Sanity checks for memory access (random number buffer) + constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout + static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); + if( nevt % neppR != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppR=" << neppR; + throw std::runtime_error( sstr.str() ); + } + } + + //-------------------------------------------------------------------------- + + void + MasslessRamboSamplingKernelHost::getMomentaInitial() + { + constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; + // ** START LOOP ON IEVT ** + for( size_t ievt = 0; ievt < nevt(); ++ievt ) + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); + getMomentaInitial( m_energy, ievtMomenta ); + } + // ** END LOOP ON IEVT ** + } + + //-------------------------------------------------------------------------- + + void + MasslessRamboSamplingKernelHost::getMomentaFinal() + { + constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; + // ** START LOOP ON IEVT ** + for( size_t ievt = 0; ievt < nevt(); ++ievt ) + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + const fptype* ievtRndmom = MemoryAccessRandomNumbers::ieventAccessRecordConst( m_rndmom.data(), ievt ); + fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); + fptype* ievtWeights = MemoryAccessWeights::ieventAccessRecord( m_weights.data(), ievt ); + getMomentaFinal( m_energy, ievtRndmom, ievtMomenta, ievtWeights ); + } + // ** END LOOP ON IEVT ** + } + + //-------------------------------------------------------------------------- + + RamboSamplingKernelHost::RamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED + const std::vector& masses, // input: external-leg masses + const int ninitial, // input: #initial-state particles + const size_t nevt, // input: #events + BufferMomenta& momenta, // output: momenta + BufferWeights& weights ) // output: weights + : SamplingKernelBase( energy, rndmom, momenta, weights ) + , NumberOfEvents( nevt ) + , m_masses( masses.begin(), masses.end() ) + , m_ninitial( ninitial ) + { + if( m_momenta.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: momenta must be a host array" ); + if( m_weights.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: weights must be a host array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with weights" ); + + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "RamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + } + + //-------------------------------------------------------------------------- + + void + RamboSamplingKernelHost::getMomentaInitial() + { + // NOOP + } + + //-------------------------------------------------------------------------- + + void + RamboSamplingKernelHost::getMomentaFinal() + { + const int npar = (int)m_masses.size(); + // ** START LOOP ON IEVT ** + for( size_t ievt = 0; ievt < nevt(); ++ievt ) + { + // Clas. RAMBO returns [E,px,py,pz] vector per ex. particle + // own RNG, intial final once + // For reproducibility betwn fptype = FP32/FP64 generation in FP64 + double wgt = 0.; + const std::vector> point = + rambo::get_momenta( m_ninitial, (double)m_energy, m_masses, wgt ); + for( int ipar = 0; ipar < npar; ++ipar ) + for( int ip4 = 0; ip4 < 4; ++ip4 ) + MemoryAccessMomenta::ieventAccessIp4Ipar( m_momenta.data(), ievt, ip4, ipar ) = (fptype)point[ipar][ip4]; + MemoryAccessWeights::ieventAccess( m_weights.data(), ievt ) = (fptype)wgt; + } + // ** END LOOP ON IEVT ** + } + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + MasslessRamboSamplingKernelDevice::MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t gpublocks, + const size_t gputhreads ) + : SamplingKernelBase( energy, rndmom, momenta, weights ) + , NumberOfEvents( gpublocks * gputhreads ) + , m_gpublocks( gpublocks ) + , m_gputhreads( gputhreads ) + { + if( !m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: rndmom must be a device array" ); + if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: momenta must be a device array" ); + if( !m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: weights must be a device array" ); + if( m_gpublocks == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gpublocks must be > 0" ); + if( m_gputhreads == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gputhreads must be > 0" ); + if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with rndmom" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with momenta" ); + if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with weights" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( m_gputhreads % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelHost: gputhreads should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Sanity checks for memory access (random number buffer) + constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout + static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); + if( m_gputhreads % neppR != 0 ) + { + std::ostringstream sstr; + sstr << "MasslessRamboSamplingKernelDevice: gputhreads should be a multiple of neppR=" << neppR; + throw std::runtime_error( sstr.str() ); + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + getMomentaInitialDevice( const fptype energy, + fptype* momenta ) + { + constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; + return getMomentaInitial( energy, momenta ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + MasslessRamboSamplingKernelDevice::getMomentaInitial() + { + gpuLaunchKernel( getMomentaInitialDevice, m_gpublocks, m_gputhreads, m_energy, m_momenta.data() ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + getMomentaFinalDevice( const fptype energy, + const fptype* rndmom, + fptype* momenta, + fptype* wgts ) + { + constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; + return getMomentaFinal( energy, rndmom, momenta, wgts ); + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + MasslessRamboSamplingKernelDevice::getMomentaFinal() + { + gpuLaunchKernel( getMomentaFinalDevice, m_gpublocks, m_gputhreads, m_energy, m_rndmom.data(), m_momenta.data(), m_weights.data() ); + } +#endif + + //-------------------------------------------------------------------------- +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h new file mode 100644 index 0000000000..6244d55618 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h @@ -0,0 +1,173 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef RAMBOSAMPLINGKERNELS_H +#define RAMBOSAMPLINGKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryBuffers.h" + +#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + // A base class encapsulating phase space sampling on a CPU host or on a GPU device + class SamplingKernelBase //: virtual public ISamplingKernel + { + protected: + + // Constructor from existing input and output buffers + SamplingKernelBase( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights ) // output: weights + : m_energy( energy ) + , m_rndmom( rndmom ) + , m_momenta( momenta ) + , m_weights( weights ) + { + } + + public: + + // Destructor + virtual ~SamplingKernelBase() {} + + // Get momenta of initial state particles + virtual void getMomentaInitial() = 0; + + // Get momenta of final state particles and weights + virtual void getMomentaFinal() = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The energy + const fptype m_energy; + + // The buffer for the input random numbers + const BufferRndNumMomenta& m_rndmom; + + // The buffer for the output momenta + BufferMomenta& m_momenta; + + // The buffer for the output weights + BufferWeights& m_weights; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating RAMBO phase space sampling on a CPU host + class MasslessRamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MasslessRamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t nevt ); + + // Destructor + virtual ~MasslessRamboSamplingKernelHost() {} + + // Get momenta of initial state particles + void getMomentaInitial() override final; + + // Get momenta of final state particles and weights + void getMomentaFinal() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + }; + + //-------------------------------------------------------------------------- + + // Compability port mirroring the massless momenta implementation + // For now own RNG internally (or keep to match Fortran) TODO + // rndmom just for interface (maybe delete later) TODO + class RamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents + { + public: + + RamboSamplingKernelHost( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED + const std::vector& masses, // input: external-leg masses + const int ninitial, // input: #n initial-state particles + const size_t nevt, // input: #n events + BufferMomenta& momenta, // output: momenta + BufferWeights& weights); // output: weights + + virtual ~RamboSamplingKernelHost() {} + + // No-op, kept to match the massless + void getMomentaInitial() override final; + + // All the magic here + void getMomentaFinal() override final; + + bool isOnDevice() const override final { return false; } + + private: + + // The EXTERNAL masses + std::vector m_masses; + + // The number of inital particles + const int m_ninitial; + }; + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + // A class encapsulating RAMBO phase space sampling on a GPU device + class MasslessRamboSamplingKernelDevice final : public SamplingKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy + const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] + BufferMomenta& momenta, // output: momenta + BufferWeights& weights, // output: weights + const size_t gpublocks, + const size_t gputhreads ); + + // Destructor + virtual ~MasslessRamboSamplingKernelDevice() {} + + // Get momenta of initial state particles + void getMomentaInitial() override final; + + // Get momenta of final state particles and weights + void getMomentaFinal() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return true; } + + private: + + // The number of blocks in the GPU grid + size_t m_gpublocks; + + // The number of threads in the GPU grid + size_t m_gputhreads; + }; +#endif + + //-------------------------------------------------------------------------- +} +#endif // RAMBOSAMPLINGKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h new file mode 100644 index 0000000000..0fb880bb36 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h @@ -0,0 +1,192 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef RANDOMNUMBERKERNELS_H +#define RANDOMNUMBERKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryBuffers.h" + +// Forward definition from curand.h (the full header is only needed in CurandRandomKernel.cc) +struct curandGenerator_st; + +// Forward definition from hiprand.h (the full header is only needed in HiprandRandomKernel.cc) +struct rocrand_generator_base_type; +typedef rocrand_generator_base_type hiprandGenerator_st; + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + + /* + // An interface encapsulating random number generation on a CPU host or on a GPU device + class IRandomNumberKernel + { + public: + + // Destructor + virtual ~IRandomNumberKernel(){} + + // Seed the random number generator + virtual void seedGenerator( const unsigned int seed ) = 0; + + // Generate the random number array + virtual void generateRnarray() = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + }; + */ + + //-------------------------------------------------------------------------- + + // A base class encapsulating random number generation on a CPU host or on a GPU device + class RandomNumberKernelBase //: virtual public IRandomNumberKernel + { + + protected: + + // Constructor from an existing output buffer + RandomNumberKernelBase( BufferRndNumMomenta& rnarray ) + : m_rnarray( rnarray ) {} + + public: + + // Destructor + virtual ~RandomNumberKernelBase() {} + + // Seed the random number generator + virtual void seedGenerator( const unsigned int seed ) = 0; + + // Generate the random number array + virtual void generateRnarray() = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The buffer for the output random numbers + BufferRndNumMomenta& m_rnarray; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating common random number generation on a CPU host + class CommonRandomNumberKernel final : public RandomNumberKernelBase + { + public: + + // Constructor from an existing output buffer + CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ); + + // Destructor + ~CommonRandomNumberKernel() {} + + // Seed the random number generator + void seedGenerator( const unsigned int seed ) override final { m_seed = seed; }; + + // Generate the random number array + void generateRnarray() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // The generator seed + unsigned int m_seed; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating CURAND random number generation on a CPU host or on a GPU device + class CurandRandomNumberKernel final : public RandomNumberKernelBase + { + public: + + // Constructor from an existing output buffer + CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); + + // Destructor + ~CurandRandomNumberKernel(); + + // Seed the random number generator + void seedGenerator( const unsigned int seed ) override final; + + // Generate the random number array + void generateRnarray() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return m_isOnDevice; } + + private: + + // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void createGenerator(); + + // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void destroyGenerator(); + + private: + + // Is this a host or device kernel? + const bool m_isOnDevice; + + // The curand generator + // (NB: curand.h defines typedef curandGenerator_t as a pointer to forward-defined 'struct curandGenerator_st') + curandGenerator_st* m_rnGen; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating HIPRAND random number generation on a CPU host or on a GPU device + class HiprandRandomNumberKernel final : public RandomNumberKernelBase + { + public: + + // Constructor from an existing output buffer + HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); + + // Destructor + ~HiprandRandomNumberKernel(); + + // Seed the random number generator + void seedGenerator( const unsigned int seed ) override final; + + // Generate the random number array + void generateRnarray() override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return m_isOnDevice; } + + private: + + // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void createGenerator(); + + // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) + void destroyGenerator(); + + private: + + // Is this a host or device kernel? + const bool m_isOnDevice; + + // The hiprand generator + // (NB: hiprand.h defines typedef hiprandGenerator_t as a pointer to forward-defined 'struct hiprandGenerator_st') + hiprandGenerator_st* m_rnGen; + }; + + //-------------------------------------------------------------------------- +} +#endif // RANDOMNUMBERKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc new file mode 100644 index 0000000000..68e93edb50 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc @@ -0,0 +1,1159 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: O. Mattelaer (Nov 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, D. Massaro, O. Mattelaer, S. Roiser, J. Teig, A. Thete, A. Valassi (2020-2026). +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== +// +// Standalone script for MadGraph7 standalone mode. +// Generates phase-space points with RAMBO and evaluates the matrix element +// through the UMAMI interface (umami.h). +// +// Two run modes: +// * matrix (default): evaluates one phase-space point (generated with the +// classic standalone RAMBO, so identical to the one of +// the Fortran/C++ standalone 'check' drivers at the +// same energy) and prints it together with the matrix +// element of every flavor combination. +// * perf : runs nblocks*nthreads*niter events on a single flavor +// and prints performance counters. +// +//========================================================================== + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "GpuAbstraction.h" +#include "GpuRuntime.h" +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" +#include "RamboSamplingKernels.h" +#include "RandomNumberKernels.h" +#include "epoch_process_id.h" +#include "read_slha.h" +#include "timermap.h" +#include "umami.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STRINGIFY( s ) #s +#define XSTRINGIFY( s ) STRINGIFY( s ) +#define SEP79 79 + +namespace +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + // Fixed physics inputs + fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV + constexpr unsigned long long kSeed = 20200805ULL; // reproducible RAMBO seed + + // Matrix-mode always runs 8 events on a single flavor index. + constexpr unsigned int kMatrixBlocks = 1; + constexpr unsigned int kMatrixThreads = 8; + + // Power of GeV of the matrix-element output; depends only on the number of external legs. + constexpr int kMEGeVExponent = -( 2 * CPPProcess::npar - 8 ); + + bool is_number( const char* s ) + { + const char* t = s; + while( *t != '\0' && isdigit( *t ) ) ++t; + return (int)strlen( s ) == t - s; + } + + // Accepts plain decimal numbers such as "1000" or "1000.0" (used for the + // optional energy argument of matrix mode). + bool is_float( const char* s ) + { + int ndots = 0; + const char* t = s; + while( *t != '\0' && ( isdigit( *t ) || ( *t == '.' && ndots++ == 0 ) ) ) ++t; + return t != s && (int)strlen( s ) == t - s; + } + + enum Mode { MODE_MATRIX, MODE_PERF }; + + enum RamboType { RAMBO_MASSIVE, RAMBO_MASSLESS }; + + // One external-particle list per LHE event, each particle stored as (E, px, py, pz). + using LheEvent = std::array, CPPProcess::npar>; + + bool read_lhe_events( const std::string& path, std::vector& events ) + { + constexpr int npar = CPPProcess::npar; + std::ifstream in( path ); + if( !in ) + { + std::cerr << "ERROR! cannot open LHE file '" << path << "'" << std::endl; + return false; + } + std::string line; + while( std::getline( in, line ) ) + { + if( line.find( "" ) == std::string::npos ) continue; + if( !std::getline( in, line ) ) break; + std::istringstream hdr( line ); + int nptcl = 0; + hdr >> nptcl; + if( nptcl != npar ) + { + std::cerr << "ERROR! LHE event has " << nptcl << " particles, expected " << npar << std::endl; + return false; + } + // particle lines: pdg status mother1 mother2 color1 color2 px py pz E m lifetime spin + LheEvent ev; + int ipar = 0; + while( ipar < npar && std::getline( in, line ) ) + { + if( line.empty() ) continue; + std::istringstream ls( line ); + long pdg; + int status, m1, m2, c1, c2; + double px, py, pz, E; + if( !( ls >> pdg >> status >> m1 >> m2 >> c1 >> c2 >> px >> py >> pz >> E ) ) + { + std::cerr << "ERROR! malformed LHE particle line: " << line << std::endl; + return false; + } + ev[ipar] = { E, px, py, pz }; + ++ipar; + } + if( ipar != npar ) + { + std::cerr << "ERROR! truncated LHE event (got " << ipar << " of " << npar << " particles)" << std::endl; + return false; + } + events.push_back( ev ); + } + if( events.empty() ) + { + std::cerr << "ERROR! no events found in '" << path << "'" << std::endl; + return false; + } + return true; + } + + int usage( const char* argv0, int ret = 1 ) + { + std::cout + << "Usage:\n" + << " " << argv0 << " [matrix] [-v|--verbose] []\n" + << " " << argv0 << " perf [-v|--verbose] [-f|--flavor ] [--rambo-massless]" + << " [-e|--events ] [<#blocksPerGrid> <#threadsPerBlock>] <#iterations>\n" + << " " << argv0 << " -p [opts] (legacy alias for `perf`)\n" + << "\n" + << "Subcommands:\n" + << " matrix (default) Evaluate one phase-space point (classic standalone\n" + << " RAMBO, identical to the Fortran/C++ 'check' drivers,\n" + << " Ecms = , default 1000 GeV) and print it with\n" + << " the matrix element for each flavor combination.\n" + << " With -v also prints backend/fptype/hardcodePARAM header.\n" + << " perf Run #blocks*#threads events over #iterations iterations\n" + << " on a single flavor index, then print performance counters.\n" + << " Always prints inputs + backend/fptype header.\n" + << " With -v also dumps every event's phase-space point and ME.\n" + << "\n" + << "Options:\n" + << " -e|--events (perf only) Read the external momenta from an LHE\n" + << " file instead of generating them with RAMBO. The events are\n" + << " processed in batches of #blocks*#threads; #iterations is\n" + << " ignored (derived from the number of events in the file).\n" + << "\n" + << "perf-mode defaults if positional args are omitted:\n" + << " #blocksPerGrid = 64, #threadsPerBlock = 256, #iterations = 1.\n"; + return ret; + } + + // AOSOA -> UMAMI SoA single-event helper. Layout reminder: + // AOSOA: aosoa[i_page * npar*4*neppM + ipar*4*neppM + ip4*neppM + i_vector] + // UMAMI: soa[ip4 * npar*nevt + ipar*nevt + ievt] + __host__ __device__ inline void + aosoa_to_umami_one( const fptype* aosoa, + double* soa, + std::size_t ievt, + std::size_t nevt ) + { + constexpr int npar = CPPProcess::npar; + for( int ipar = 0; ipar < npar; ++ipar ) + { + for( int ip4 = 0; ip4 < 4; ++ip4 ) + { + soa[(std::size_t)ip4 * npar * nevt + (std::size_t)ipar * nevt + ievt] = + (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, ip4, ipar ); + } + } + } + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + aosoa_to_umami_kernel( const fptype* aosoa, + double* soa, + std::size_t nevt ) + { + std::size_t ievt = blockDim.x * blockIdx.x + threadIdx.x; + if( ievt >= nevt ) return; + aosoa_to_umami_one( aosoa, soa, ievt, nevt ); + } +#endif + + const char* backend_label() + { +#ifdef __CUDACC__ + return "CUDA"; +#elif defined( __HIPCC__ ) + return "HIP"; +#else + return "CPP"; +#endif + } + + const char* fp_label() + { +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + return "MIXED"; +#elif defined MGONGPU_FPTYPE_DOUBLE + return "DOUBLE"; +#elif defined MGONGPU_FPTYPE_FLOAT + return "FLOAT"; +#else + return "UNKNOWN"; +#endif + } + + void print_run_header( std::ostream& os ) + { + os << "Process = " << XSTRINGIFY( MG_EPOCH_PROCESS_ID ) << "_" << backend_label() +#ifdef MGONGPU_HARDCODE_PARAM + << " [hardcodePARAM=1]" << std::endl +#else + << " [hardcodePARAM=0]" << std::endl +#endif + << "FP precision = " << fp_label() << std::endl + << "Random number generation = COMMON RANDOM HOST" << std::endl; + } + + void print_momenta_table( std::ostream& os, const fptype* aosoa, unsigned int ievt ) + { + auto constexpr prec = std::numeric_limits::digits10; + constexpr int npar = CPPProcess::npar; + os << std::string( SEP79, '-' ) << std::endl + << " n E px py pz" << std::endl; + for( int ipar = 0; ipar < npar; ++ipar ) + { + double E = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 0, ipar ); + double px = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 1, ipar ); + double py = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 2, ipar ); + double pz = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 3, ipar ); + os << std::scientific << std::setprecision( prec ) + << std::setw( 4 ) << ipar + 1 + << std::setw( prec + 10 ) << E + << std::setw( prec + 10 ) << px + << std::setw( prec + 10 ) << py + << std::setw( prec + 10 ) << pz + << std::endl + << std::defaultfloat; + } + os << std::string( SEP79, '-' ) << std::endl; + } + + // Run sigmaKin via UMAMI for `nevt` events and copy back the MEs. + // Both the momenta (UMAMI SoA layout) and the per-event flavor buffer must be set + // by the caller. On GPU the buffers are device pointers and `hstMEs` receives the + // host-side copy; on CPU `umamiMEs` is the output buffer. + bool run_umami( + UmamiHandle handle, + unsigned int nevt, + mgOnGpu::TimerMap& timermap, + double& wavetime, +#ifdef MGONGPUCPP_GPUIMPL + const DeviceBufferBase& devUmamiMomenta, + const DeviceBufferBase& devFlv, + DeviceBufferBase& devUmamiMEs, + std::vector& hstMEs +#else + const std::vector& umamiMomenta, + const std::vector& flvVec, + std::vector& umamiMEs +#endif + ) + { + constexpr unsigned int UmamiInKeyNum = 2; + timermap.start( "3a SigmaKin" ); + UmamiInputKey in_keys[UmamiInKeyNum] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX }; + UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; +#ifdef MGONGPUCPP_GPUIMPL + const void* inputs[UmamiInKeyNum] = { devUmamiMomenta.data(), devFlv.data() }; + void* outputs[1] = { devUmamiMEs.data() }; +#else + const void* inputs[UmamiInKeyNum] = { umamiMomenta.data(), flvVec.data() }; + void* outputs[1] = { umamiMEs.data() }; +#endif + UmamiStatus st = umami_matrix_element( + handle, nevt, nevt, 0, UmamiInKeyNum, in_keys, inputs, 1, out_keys, outputs ); + wavetime += timermap.stop(); + if( st != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; + return false; + } + +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "3b CpDTHmes" ); + gpuMemcpy( hstMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); + wavetime += timermap.stop(); +#endif + return true; + } + + // -------------------------------------------------------------------------- + // Classic MadGraph standalone RAMBO (RANMAR generator seeded with the fixed + // values 1802/9373), reproducing the exact phase-space point used by the + // Fortran and C++ standalone 'check' drivers at the same energy, so that + // matrix-mode output can be compared line by line across backends. + // Host-side only; ported from madgraph/iolibs/template_files/rambo.cc. + // -------------------------------------------------------------------------- + namespace classic_rambo + { + struct Random + { + double ranu[98]; + double ranc, rancd, rancm; + int iranmr, jranmr; + + // universal random number generator proposed by Marsaglia and Zaman + double ranmar() + { + double uni = ranu[iranmr] - ranu[jranmr]; + if( uni < 0 ) uni = uni + 1; + ranu[iranmr] = uni; + iranmr = iranmr - 1; + jranmr = jranmr - 1; + if( iranmr == 0 ) iranmr = 97; + if( jranmr == 0 ) jranmr = 97; + ranc = ranc - rancd; + if( ranc < 0 ) ranc = ranc + rancm; + uni = uni - ranc; + if( uni < 0 ) uni = uni + 1; + return uni; + } + + void rmarin( int ij, int kl ) + { + int i = ij / 177 % 177 + 2; + int j = ij % 177 + 2; + int k = ( kl / 169 ) % 178 + 1; + int l = kl % 169; + for( int ii = 1; ii < 98; ii++ ) + { + double s = 0; + double t = .5; + for( int jj = 1; jj < 25; jj++ ) + { + int m = ( ( i * j % 179 ) * k ) % 179; + i = j; + j = k; + k = m; + l = ( 53 * l + 1 ) % 169; + if( ( l * m ) % 64 >= 32 ) s = s + t; + t = .5 * t; + } + ranu[ii] = s; + } + ranc = 362436. / 16777216.; + rancd = 7654321. / 16777216.; + rancm = 16777213. / 16777216.; + iranmr = 97; + jranmr = 33; + } + }; + + inline double rn() + { + static Random rand; + static bool init = true; + if( init ) + { + init = false; + rand.rmarin( 1802, 9373 ); + } + double ran; + while( true ) + { + ran = rand.ranmar(); + if( ran > 1e-16 ) break; + } + return ran; + } + + // RAMBO: democratic multi-particle phase space generator (S.D. Ellis, + // R. Kleiss, W.J. Stirling); weights are logarithmic. + inline std::vector> + rambo( double et, const std::vector& xm, double& wt ) + { + const int n = (int)xm.size(); + std::vector> q( n, std::vector( 4 ) ); + std::vector> p( n, std::vector( 4 ) ); + std::vector z( n ), r( 4 ), b( 3 ), p2( n ), xm2( n ), e( n ), v( n ); + const double acc = 1e-14; + const int itmax = 6; + const double twopi = 8. * atan( 1. ); + const double po2log = log( twopi / 4. ); + + // factorials for the phase-space weight + z[1] = po2log; + for( int k = 2; k < n; k++ ) z[k] = z[k - 1] + po2log - 2. * log( double( k - 1 ) ); + for( int k = 2; k < n; k++ ) z[k] = z[k] - log( double( k ) ); + + if( n < 1 || n > 101 ) + { + std::cout << "Too few or many particles: " << n << std::endl; + exit( -1 ); + } + double xmt = 0.; + int nm = 0; + for( int i = 0; i < n; i++ ) + { + if( xm[i] != 0. ) nm = nm + 1; + xmt = xmt + std::abs( xm[i] ); + } + if( xmt > et ) + { + std::cout << "Too low energy: " << et << " needed " << xmt << std::endl; + exit( -1 ); + } + + // generate n massless momenta in infinite phase space + for( int i = 0; i < n; i++ ) + { + double r1 = rn(); + double c = 2. * r1 - 1.; + double s = sqrt( 1. - c * c ); + double f = twopi * rn(); + r1 = rn(); + double r2 = rn(); + q[i][0] = -log( r1 * r2 ); + q[i][3] = q[i][0] * c; + q[i][2] = q[i][0] * s * cos( f ); + q[i][1] = q[i][0] * s * sin( f ); + } + + // parameters of the conformal transformation + for( int k = 0; k < 4; k++ ) r[k] = 0.; + for( int i = 0; i < n; i++ ) + for( int k = 0; k < 4; k++ ) r[k] = r[k] + q[i][k]; + double rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); + for( int k = 1; k < 4; k++ ) b[k - 1] = -r[k] / rmas; + double g = r[0] / rmas; + double a = 1. / ( 1. + g ); + double x = et / rmas; + + // transform the q's conformally into the p's + for( int i = 0; i < n; i++ ) + { + double bq = b[0] * q[i][1] + b[1] * q[i][2] + b[2] * q[i][3]; + for( int k = 1; k < 4; k++ ) p[i][k] = x * ( q[i][k] + b[k - 1] * ( q[i][0] + a * bq ) ); + p[i][0] = x * ( g * q[i][0] + bq ); + } + + wt = po2log; + if( n != 2 ) wt = ( 2. * n - 4. ) * log( et ) + z[n - 1]; + + // massless case is done + if( nm == 0 ) return p; + + // massive particles: rescale the momenta by a factor x + double xmax = sqrt( 1. - pow( xmt / et, 2 ) ); + for( int i = 0; i < n; i++ ) + { + xm2[i] = pow( xm[i], 2 ); + p2[i] = pow( p[i][0], 2 ); + } + int iter = 0; + x = xmax; + double accu = et * acc; + while( true ) + { + double f0 = -et; + double g0 = 0.; + double x2 = x * x; + for( int i = 0; i < n; i++ ) + { + e[i] = sqrt( xm2[i] + x2 * p2[i] ); + f0 = f0 + e[i]; + g0 = g0 + p2[i] / e[i]; + } + if( std::abs( f0 ) <= accu ) break; + iter = iter + 1; + if( iter > itmax ) + { + std::cout << "Too many iterations without desired accuracy: " << itmax << std::endl; + break; + } + x = x - f0 / ( x * g0 ); + } + for( int i = 0; i < n; i++ ) + { + v[i] = x * p[i][0]; + for( int k = 1; k < 4; k++ ) p[i][k] = x * p[i][k]; + p[i][0] = e[i]; + } + + double wt2 = 1.; + double wt3 = 0.; + for( int i = 0; i < n; i++ ) + { + wt2 = wt2 * v[i] / e[i]; + wt3 = wt3 + pow( v[i], 2 ) / e[i]; + } + double wtm = ( 2. * n - 3. ) * log( x ) + log( wt2 / wt3 * et ); + wt = wt + wtm; + return p; + } + + // Auxiliary function changing convention between MadGraph5_aMC@NLO and + // RAMBO four-momenta (same as get_momenta in the standalone_cpp driver). + inline std::vector> + get_momenta( int ninitial, double energy, const std::vector& masses, double& wgt ) + { + const int nexternal = (int)masses.size(); + const int nfinal = nexternal - ninitial; + const double e2 = pow( energy, 2 ); + const double m1 = masses[0]; + + if( ninitial == 1 ) + { + std::vector> p( 1, std::vector( 4, 0. ) ); + p[0][0] = m1; + std::vector finalmasses( masses.begin() + 1, masses.end() ); + std::vector> p_rambo = rambo( m1, finalmasses, wgt ); + p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); + return p; + } + + if( ninitial != 2 ) + { + std::cout << "Rambo needs 1 or 2 incoming particles" << std::endl; + exit( -1 ); + } + + double etot = energy; + if( nfinal == 1 ) etot = m1; + const double m2 = masses[1]; + const double mom = sqrt( ( pow( e2, 2 ) - 2 * e2 * pow( m1, 2 ) + pow( m1, 4 ) - 2 * e2 * pow( m2, 2 ) - 2 * pow( m1, 2 ) * pow( m2, 2 ) + pow( m2, 4 ) ) / ( 4 * e2 ) ); + const double energy1 = sqrt( pow( mom, 2 ) + pow( m1, 2 ) ); + const double energy2 = sqrt( pow( mom, 2 ) + pow( m2, 2 ) ); + std::vector> p( 2, std::vector( 4, 0. ) ); + p[0][0] = energy1; + p[0][3] = mom; + p[1][0] = energy2; + p[1][3] = -mom; + + if( nfinal == 1 ) + { + p.push_back( std::vector( 4, 0. ) ); + p[2][0] = etot; + wgt = 1; + return p; + } + std::vector finalmasses( masses.begin() + 2, masses.end() ); + std::vector> p_rambo = rambo( etot, finalmasses, wgt ); + p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); + return p; + } + } + + // -------------------------------------------------------------------------- + // matrix mode: same PS point fed to every flavor combination, print event 0. + // The point is generated with the classic standalone RAMBO so it is + // identical to the one of the Fortran/C++ standalone 'check' drivers. + // -------------------------------------------------------------------------- + int run_matrix_mode( bool verbose ) + { + constexpr unsigned int nevt = kMatrixBlocks * kMatrixThreads; + const unsigned int nFlavors = CPPProcess::nmaxflavor; + + mgOnGpu::TimerMap timermap; + +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "00 GpuInit" ); + GpuRuntime gpuRuntime( false ); + + PinnedHostBufferRndNumMomenta hstRndmom( nevt ); + PinnedHostBufferMomenta hstMomenta( nevt ); + PinnedHostBufferWeights hstWeights( nevt ); + DeviceBufferRndNumMomenta devRndmom( nevt ); + DeviceBufferMomenta devMomenta( nevt ); + DeviceBufferWeights devWeights( nevt ); + DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + DeviceBufferBase devUmamiMEs( nevt ); + DeviceBufferBase devFlv( nevt ); + std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + std::vector flvVec( nevt ); + std::vector hstUmamiMEs( nevt ); +#else + HostBufferRndNumMomenta hstRndmom( nevt ); + HostBufferMomenta hstMomenta( nevt ); + HostBufferWeights hstWeights( nevt ); + std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + std::vector umamiMEs( nevt ); + std::vector flvVec( nevt ); +#endif + + UmamiHandle umami_handle = nullptr; + if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_initialize failed" << std::endl; + return 2; + } + + // Generate one shared phase-space point used by every flavor, with the + // classic standalone RAMBO so it matches the Fortran/C++ 'check' drivers. + CPPProcess process; + process.initProc( "../../Cards/param_card.dat" ); + double rambowgt = 0.; + + // Retrieve masses + int npar_meta = 0; + if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + std::vector massesD( npar_meta ); + if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + const std::vector masses( massesD.begin(), massesD.end() ); + + std::vector> point = + classic_rambo::get_momenta( CPPProcess::npari, (double)kEnergy, masses, rambowgt ); + + // alpha_s from the param card so the couplings match the Fortran/C++ + // 'check' drivers (UMAMI otherwise falls back to a hardcoded g_s). + SLHAReader slha( "../../Cards/param_card.dat", false ); + const double alphaS = slha.get_block_entry( "sminputs", 3, 1.180000e-01 ); + std::vector alphasVec( nevt, alphaS ); +#ifdef MGONGPUCPP_GPUIMPL + DeviceBufferBase devAlphaS( nevt ); + gpuMemcpy( devAlphaS.data(), alphasVec.data(), nevt * sizeof( double ), gpuMemcpyHostToDevice ); +#endif + + // Always massive RAMBO + std::unique_ptr prsk( + new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); + prsk->getMomentaInitial(); + prsk->getMomentaFinal(); + + // Fill the UMAMI SoA buffer with nevt copies of the same event: + // soa[ip4 * npar*nevt + ipar*nevt + ievt] + for( int ip4 = 0; ip4 < 4; ++ip4 ) + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + for( unsigned int ievt = 0; ievt < nevt; ++ievt ) + umamiMomenta[(std::size_t)ip4 * CPPProcess::npar * nevt + (std::size_t)ipar * nevt + ievt] = point[ipar][ip4]; +#ifdef MGONGPUCPP_GPUIMPL + gpuMemcpy( devUmamiMomenta.data(), umamiMomenta.data(), umamiMomenta.size() * sizeof( double ), gpuMemcpyHostToDevice ); + // Host only implementation now (copy) + copyDeviceFromHost( devMomenta, hstMomenta ); + gpuLaunchKernel( aosoa_to_umami_kernel, kMatrixBlocks, kMatrixThreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); + checkGpu( gpuPeekAtLastError() ); +#else + for( std::size_t ievt = 0; ievt < nevt; ++ievt ) + aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); +#endif + + if( verbose ) + { + std::cout << std::string( SEP79, '*' ) << std::endl; + print_run_header( std::cout ); + std::cout << std::string( SEP79, '*' ) << std::endl; + } + + std::cout << "Phase space point:" << std::endl + << std::string( SEP79, '-' ) << std::endl + << " n E px py pz" << std::endl; + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + { + std::cout << std::scientific << std::setprecision( 7 ) + << std::setw( 2 ) << ipar + 1 + << std::setw( 16 ) << point[ipar][0] + << std::setw( 16 ) << point[ipar][1] + << std::setw( 16 ) << point[ipar][2] + << std::setw( 16 ) << point[ipar][3] + << std::endl + << std::defaultfloat; + } + std::cout << std::string( SEP79, '-' ) << std::endl; + + for( unsigned int iflav = 0; iflav < nFlavors; ++iflav ) + { + std::fill( flvVec.begin(), flvVec.end(), iflav ); +#ifdef MGONGPUCPP_GPUIMPL + gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); +#endif + timermap.start( "3a SigmaKin" ); + UmamiInputKey in_keys[3] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S }; + UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; +#ifdef MGONGPUCPP_GPUIMPL + const void* inputs[3] = { devUmamiMomenta.data(), devFlv.data(), devAlphaS.data() }; + void* outputs[1] = { devUmamiMEs.data() }; +#else + const void* inputs[3] = { umamiMomenta.data(), flvVec.data(), alphasVec.data() }; + void* outputs[1] = { umamiMEs.data() }; +#endif + UmamiStatus st = umami_matrix_element( + umami_handle, nevt, nevt, 0, 3, in_keys, inputs, 1, out_keys, outputs ); + timermap.stop(); + if( st != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; + umami_free( umami_handle ); + return 3; + } +#ifdef MGONGPUCPP_GPUIMPL + gpuMemcpy( hstUmamiMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); + const double* mes = hstUmamiMEs.data(); +#else + const double* mes = umamiMEs.data(); +#endif + + std::cout << " PDG"; + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + std::cout << std::setw( 12 ) << CPPProcess::flavorPDG( iflav, ipar ); + std::cout << std::endl + << " Matrix element = " << std::scientific << std::setprecision( 16 ) + << mes[0] << " GeV^" << kMEGeVExponent << std::endl + << std::defaultfloat + << std::string( SEP79, '-' ) << std::endl; + } + + umami_free( umami_handle ); + return 0; + } + + // -------------------------------------------------------------------------- + // perf mode: nblocks*nthreads events per iteration on a single flavor. + // -------------------------------------------------------------------------- + int run_perf_mode( bool verbose, + unsigned int gpublocks, + unsigned int gputhreads, + unsigned int niter, + unsigned int flavorID, + RamboType ramboType, + const std::string& lheFile = "" ) + { + const unsigned int nevt = gpublocks * gputhreads; + + // LHE instead of generating. Processed in batches of nevt and + // niter is derived from the number of events read. + std::vector lheEvents; + if( !lheFile.empty() ) + { + if( !read_lhe_events( lheFile, lheEvents ) ) return 2; + niter = (unsigned int)( ( lheEvents.size() + nevt - 1 ) / nevt ); + std::cout << "Reading events from LHE file = " << lheFile + << " (" << lheEvents.size() << " events, " << niter + << " batches of " << nevt << ")" << std::endl; + } + + mgOnGpu::TimerMap timermap; + +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "00 GpuInit" ); + GpuRuntime gpuRuntime( false ); + + PinnedHostBufferRndNumMomenta hstRndmom( nevt ); + PinnedHostBufferMomenta hstMomenta( nevt ); + PinnedHostBufferWeights hstWeights( nevt ); + DeviceBufferRndNumMomenta devRndmom( nevt ); + DeviceBufferMomenta devMomenta( nevt ); + DeviceBufferWeights devWeights( nevt ); + DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + DeviceBufferBase devUmamiMEs( nevt ); + DeviceBufferBase devFlv( nevt ); + std::vector flvVec( nevt, flavorID ); + std::vector hstUmamiMEs( nevt ); + // perf-mode runs a single flavor, so the device-side flavor buffer is filled once. + gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); +#else + HostBufferRndNumMomenta hstRndmom( nevt ); + HostBufferMomenta hstMomenta( nevt ); + HostBufferWeights hstWeights( nevt ); + std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); + std::vector umamiMEs( nevt ); + std::vector flvVec( nevt, flavorID ); +#endif + + std::unique_ptr prnk( + new CommonRandomNumberKernel( hstRndmom ) ); + + UmamiHandle umami_handle = nullptr; + if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_initialize failed" << std::endl; + return 2; + } + + // Retrieve masses + std::vector masses; + if( ramboType != RAMBO_MASSLESS) + { + int npar_meta = 0; + if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + std::vector massesD( npar_meta ); + if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) + { + std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; + umami_free( umami_handle ); + return 2; + } + masses.assign( massesD.begin(), massesD.end() ); + } + + std::unique_ptr prsk; + if( ramboType != RAMBO_MASSLESS ) + { + // Massive host only (copy) + prsk.reset( new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); + } + else + { +#ifdef MGONGPUCPP_GPUIMPL + prsk.reset( new MasslessRamboSamplingKernelDevice( kEnergy, devRndmom, devMomenta, devWeights, gpublocks, gputhreads ) ); +#else + prsk.reset( new MasslessRamboSamplingKernelHost( kEnergy, hstRndmom, hstMomenta, hstWeights, nevt ) ); +#endif + } + + std::unique_ptr genrtimes( new double[niter] ); + std::unique_ptr rambtimes( new double[niter] ); + std::unique_ptr wavetimes( new double[niter] ); + + unsigned int nevtABN = 0; + unsigned int nevtZERO = 0; + double sumME = 0.; + double sumMEsq = 0.; + double minME = std::numeric_limits::infinity(); + double maxME = -std::numeric_limits::infinity(); + unsigned int nevtALL = 0; + + for( unsigned int iiter = 0; iiter < niter; ++iiter ) + { + double genrtime = 0; + double rambtime = 0; + unsigned int nreal = nevt; // number of real (non-padding) events in this batch + if( lheFile.empty() ) + { + timermap.start( "1a GenSeed " ); + prnk->seedGenerator( kSeed + iiter ); + genrtime += timermap.stop(); + timermap.start( "1b GenRnGen" ); + prnk->generateRnarray(); + genrtime += timermap.stop(); +#ifdef MGONGPUCPP_GPUIMPL + if( ramboType == RAMBO_MASSLESS ) + { + timermap.start( "1c CpHTDrnd" ); + copyDeviceFromHost( devRndmom, hstRndmom ); + genrtime += timermap.stop(); + } +#endif + + timermap.start( "2a RamboIni" ); + prsk->getMomentaInitial(); + rambtime += timermap.stop(); + timermap.start( "2b RamboFin" ); + prsk->getMomentaFinal(); + rambtime += timermap.stop(); +#ifdef MGONGPUCPP_GPUIMPL + // Massive host only (copy) + if( ramboType != RAMBO_MASSLESS ) + { + timermap.start( "2c CpHTDmom" ); + copyDeviceFromHost( devMomenta, hstMomenta ); + rambtime += timermap.stop(); + } +#endif + } + else + { + // Fill this batch from the LHE events (AOSOA layout, (E,px,py,pz) per leg). + // padded by repeating its last real event so the SIMD page is valid + // only the nreal real events are counted below. + timermap.start( "2e ReadLHE " ); + const std::size_t base = (std::size_t)iiter * nevt; + nreal = (unsigned int)std::min( nevt, lheEvents.size() - base ); + for( unsigned int ievt = 0; ievt < nevt; ++ievt ) + { + const std::size_t src = base + std::min( ievt, (std::size_t)nreal - 1 ); + const LheEvent& ev = lheEvents[src]; + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + for( int ip4 = 0; ip4 < 4; ++ip4 ) + MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar ) = (fptype)ev[ipar][ip4]; + } + rambtime += timermap.stop(); +#ifdef MGONGPUCPP_GPUIMPL + timermap.start( "2c CpHTDmom" ); + copyDeviceFromHost( devMomenta, hstMomenta ); + rambtime += timermap.stop(); +#endif + } + + timermap.start( "2d Aosoa2U " ); +#ifdef MGONGPUCPP_GPUIMPL + gpuLaunchKernel( aosoa_to_umami_kernel, gpublocks, gputhreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); + checkGpu( gpuPeekAtLastError() ); +#else + for( std::size_t ievt = 0; ievt < nevt; ++ievt ) + aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); +#endif + rambtime += timermap.stop(); + + double wavetime = 0; + if( !run_umami( umami_handle, nevt, timermap, wavetime, +#ifdef MGONGPUCPP_GPUIMPL + devUmamiMomenta, devFlv, devUmamiMEs, hstUmamiMEs +#else + umamiMomenta, flvVec, umamiMEs +#endif + ) ) + { + umami_free( umami_handle ); + return 3; + } + +#ifdef MGONGPUCPP_GPUIMPL + if( verbose ) + { + timermap.start( "3c CpDTHmom" ); + copyHostFromDevice( hstMomenta, devMomenta ); + wavetime += timermap.stop(); + } + const double* mes = hstUmamiMEs.data(); +#else + const double* mes = umamiMEs.data(); +#endif + + timermap.start( "4@ UpdtStat" ); + for( unsigned int ievt = 0; ievt < nreal; ++ievt ) + { + double me = mes[ievt]; + ++nevtALL; + if( !std::isfinite( me ) ) + ++nevtABN; + else if( me == 0. ) + ++nevtZERO; + sumME += me; + sumMEsq += me * me; + if( me < minME ) minME = me; + if( me > maxME ) maxME = me; + } + + genrtimes[iiter] = genrtime; + rambtimes[iiter] = rambtime; + wavetimes[iiter] = wavetime; + + if( verbose ) + { + std::cout << std::string( SEP79, '*' ) << std::endl + << "Iteration #" << iiter + 1 << " of " << niter << std::endl; + for( unsigned int ievt = 0; ievt < nreal; ++ievt ) + { + std::cout << "Event #" << ievt + 1 << std::endl; + print_momenta_table( std::cout, hstMomenta.data(), ievt ); + std::cout << " Matrix element = " << std::scientific << std::setprecision( 16 ) + << mes[ievt] << " GeV^" << kMEGeVExponent << std::endl + << std::defaultfloat + << std::string( SEP79, '-' ) << std::endl; + } + } + } + + double sumgtim = 0, sumrtim = 0, sumwtim = 0; + double minwtim = wavetimes[0], maxwtim = wavetimes[0]; + for( unsigned int i = 0; i < niter; ++i ) + { + sumgtim += genrtimes[i]; + sumrtim += rambtimes[i]; + sumwtim += wavetimes[i]; + minwtim = std::min( minwtim, wavetimes[i] ); + maxwtim = std::max( maxwtim, wavetimes[i] ); + } + double meanwtim = sumwtim / niter; + + unsigned int nevtGood = nevtALL - nevtABN; + double meanME = ( nevtGood > 0 ) ? sumME / nevtGood : 0.; + double varME = ( nevtGood > 0 ) ? sumMEsq / nevtGood - meanME * meanME : 0.; + double stdME = ( varME > 0 ) ? std::sqrt( varME ) : 0.; + + std::cout << std::string( SEP79, '*' ) << std::endl; + print_run_header( std::cout ); + std::cout << "NumBlocksPerGrid = " << gpublocks << std::endl + << "NumThreadsPerBlock = " << gputhreads << std::endl + << "NumIterations = " << niter << std::endl + << "FlavorIndex = " << flavorID << " / " << CPPProcess::nmaxflavor << std::endl + << std::string( SEP79, '-' ) << std::endl + << "NaN/abnormal MEs = " << nevtABN << std::endl + << "Zero MEs = " << nevtZERO << std::endl + << std::string( SEP79, '-' ) << std::endl + << "NumberOfEntries = " << niter << std::endl + << std::scientific + << "TotalTime[Rnd+Rmb+ME] (123) = ( " << sumgtim + sumrtim + sumwtim << " ) sec" << std::endl + << "TotalTime[Rambo+ME] (23) = ( " << sumrtim + sumwtim << " ) sec" << std::endl + << "TotalTime[RndNumGen] (1) = ( " << sumgtim << " ) sec" << std::endl + << "TotalTime[Rambo] (2) = ( " << sumrtim << " ) sec" << std::endl + << "TotalTime[MatrixElems] (3) = ( " << sumwtim << " ) sec" << std::endl + << "MeanTimeInMatrixElems = ( " << meanwtim << " ) sec" << std::endl + << "[Min,Max]TimeInMatrixElems = [ " << minwtim << " , " << maxwtim << " ] sec" << std::endl + << std::string( SEP79, '-' ) << std::endl + << "TotalEventsComputed = " << nevtALL << std::endl + << "EvtsPerSec[Rnd+Rmb+ME](123) = ( " << nevtALL / ( sumgtim + sumrtim + sumwtim ) << " ) sec^-1" << std::endl + << "EvtsPerSec[Rmb+ME] (23) = ( " << nevtALL / ( sumrtim + sumwtim ) << " ) sec^-1" << std::endl + << "EvtsPerSec[MatrixElems] (3) = ( " << nevtALL / sumwtim << " ) sec^-1" << std::endl + << std::defaultfloat + << std::string( SEP79, '*' ) << std::endl + << "MeanMatrixElemValue = ( " << meanME << " +- " << stdME / std::sqrt( (double)std::max( 1u, nevtGood ) ) + << " ) GeV^" << kMEGeVExponent << std::endl + << "[Min,Max]MatrixElemValue = [ " << minME << " , " << maxME << " ] GeV^" << kMEGeVExponent << std::endl + << std::string( SEP79, '*' ) << std::endl; + timermap.dump(); + std::cout << std::string( SEP79, '*' ) << std::endl; + + umami_free( umami_handle ); + return 0; + } +} + +int main( int argc, char** argv ) +{ + + Mode mode = MODE_MATRIX; + RamboType ramboType = RAMBO_MASSIVE; // default + bool ramboTypeSet = false; + bool verbose = false; + unsigned int flavorID = 0; + unsigned int gpublocks = 64; + unsigned int gputhreads = 256; + unsigned int niter = 1; + unsigned int numvec[3] = { 0, 0, 0 }; + int nnum = 0; + std::string lheFile; // -e/--events: read momenta from this LHE file (perf mode only) + + // Optional leading subcommand (no leading dash). + int firstArg = 1; + if( firstArg < argc ) + { + std::string a = argv[firstArg]; + if( a == "matrix" ) { mode = MODE_MATRIX; ++firstArg; } + else if( a == "perf" ) { mode = MODE_PERF; ++firstArg; } + } + + double energyArg = -1.; + + for( int argn = firstArg; argn < argc; ++argn ) + { + std::string arg = argv[argn]; + if( arg == "--verbose" || arg == "-v" ) + verbose = true; + else if( arg == "--performance" || arg == "-p" ) + mode = MODE_PERF; // legacy alias + else if( ( arg == "--flavor" || arg == "-f" ) && argn + 1 < argc && is_number( argv[argn + 1] ) ) + flavorID = strtoul( argv[++argn], nullptr, 0 ); + else if( arg == "--rambo-massless" ) + { + std::string r = argv[++argn]; + ramboType = RAMBO_MASSLESS; + ramboTypeSet = true; + } + else if( ( arg == "--events" || arg == "-e" ) && argn + 1 < argc ) + { + lheFile = argv[++argn]; + mode = MODE_PERF; // reading events from file only makes sense in perf mode + } + else if( is_number( argv[argn] ) && nnum < 3 ) + { + numvec[nnum++] = strtoul( argv[argn], nullptr, 0 ); + if( energyArg < 0 ) energyArg = atof( argv[argn] ); + } + else if( is_float( argv[argn] ) && energyArg < 0 ) + { + // decimal number: only meaningful as the matrix-mode energy + energyArg = atof( argv[argn] ); + } + else + return usage( argv[0] ); + } +//ENERGY CHANGE FOR THE MATRIX MODE +// (default 1000 GeV as for the Fortran/C++ standalone 'check' drivers; +// can be overridden with a single positional argument) + if( mode == MODE_MATRIX ) kEnergy = ( energyArg > 0 ) ? energyArg : 1000.; + + if( mode == MODE_MATRIX ) + { + if( ramboType == RAMBO_MASSLESS ) + { + std::cerr << "ERROR: matrix mode only supports the classic RAMBO (-r c)." << std::endl; + return usage( argv[0] ); + } + if( nnum > 1 ) + { + std::cerr << "WARNING: extra positional args are ignored in matrix mode " + << "(dimensions are fixed at " << kMatrixBlocks << " " << kMatrixThreads << " 1)." + << std::endl; + } + return run_matrix_mode( verbose ); + } + + // perf mode + if( nnum == 3 ) + { + gpublocks = numvec[0]; + gputhreads = numvec[1]; + niter = numvec[2]; + } + else if( nnum == 1 ) + { + niter = numvec[0]; + } + else if( nnum != 0 ) + { + return usage( argv[0] ); + } + if( niter == 0 && lheFile.empty() ) return usage( argv[0] ); // niter is derived from the file in LHE mode + + if( flavorID >= CPPProcess::nmaxflavor ) + { + std::cerr << "ERROR: flavor index " << flavorID + << " is out of range [0, " << CPPProcess::nmaxflavor << ")." << std::endl; + return 1; + } + + return run_perf_mode( verbose, gpublocks, gputhreads, niter, flavorID, ramboType, lheFile ); +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc new file mode 100644 index 0000000000..30c6799932 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc @@ -0,0 +1,434 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "color_sum.h" + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessMatrixElements.h" + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors + + //-------------------------------------------------------------------------- + + // *** COLOR MATRIX BELOW *** +%(color_matrix_lines)s + +#ifdef MGONGPUCPP_GPUIMPL + // The normalized color matrix (divide each column by denom) + template + struct NormalizedColorMatrix + { + constexpr __host__ __device__ NormalizedColorMatrix() + : value() + { + for( int icol = 0; icol < ncolor; icol++ ) + for( int jcol = 0; jcol < ncolor; jcol++ ) + value[icol * ncolor + jcol] = colorMatrix[icol][jcol] / colorDenom[icol]; + } + T value[ncolor * ncolor]; + }; + // The fptype2 version is the default used by kernels (supporting mixed floating point mode also in blas) + static __device__ fptype2 s_pNormalizedColorMatrix2[ncolor * ncolor]; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void createNormalizedColorMatrix() + { + static bool first = true; + if( first ) + { + first = false; + constexpr NormalizedColorMatrix normalizedColorMatrix2; + gpuMemcpyToSymbol( s_pNormalizedColorMatrix2, normalizedColorMatrix2.value, ncolor * ncolor * sizeof( fptype2 ) ); + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + void + color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity + const int ievt0 ) // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) + { + // Pre-compute a constexpr triangular color matrix properly normalized #475 + struct TriangularNormalizedColorMatrix + { + // See https://stackoverflow.com/a/34465458 + __host__ __device__ constexpr TriangularNormalizedColorMatrix() + : value() + { + for( int icol = 0; icol < ncolor; icol++ ) + { + // Diagonal terms + value[icol][icol] = colorMatrix[icol][icol] / colorDenom[icol]; + // Off-diagonal terms + for( int jcol = icol + 1; jcol < ncolor; jcol++ ) + value[icol][jcol] = 2 * colorMatrix[icol][jcol] / colorDenom[icol]; + } + } + fptype2 value[ncolor][ncolor]; + }; + static constexpr auto cf2 = TriangularNormalizedColorMatrix(); + // Use the property that M is a real matrix (see #475): + // we can rewrite the quadratic form (A-iB)(M)(A+iB) as AMA - iBMA + iBMA + BMB = AMA + BMB + // In addition, on C++ use the property that M is symmetric (see #475), + // and also use constexpr to compute "2*" and "/colorDenom[icol]" once and for all at compile time: + // we gain (not a factor 2...) in speed here as we only loop over the up diagonal part of the matrix. + // Strangely, CUDA is slower instead, so keep the old implementation for the moment. + fptype_sv deltaMEs = { 0 }; +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype_sv deltaMEs_next = { 0 }; + // Mixed mode: merge two neppV vectors into one neppV2 vector + fptype2_sv jampR_sv[ncolor]; + fptype2_sv jampI_sv[ncolor]; + for( int icol = 0; icol < ncolor; icol++ ) + { + jampR_sv[icol] = fpvmerge( cxreal( allJamp_sv[icol] ), cxreal( allJamp_sv[ncolor + icol] ) ); + jampI_sv[icol] = fpvmerge( cximag( allJamp_sv[icol] ), cximag( allJamp_sv[ncolor + icol] ) ); + } +#else + const cxtype_sv* jamp_sv = allJamp_sv; +#endif + // Loop over icol + for( int icol = 0; icol < ncolor; icol++ ) + { + // Diagonal terms +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype2_sv& jampRi_sv = jampR_sv[icol]; + fptype2_sv& jampIi_sv = jampI_sv[icol]; +#else + fptype2_sv jampRi_sv = (fptype2_sv)( cxreal( jamp_sv[icol] ) ); + fptype2_sv jampIi_sv = (fptype2_sv)( cximag( jamp_sv[icol] ) ); +#endif + fptype2_sv ztempR_sv = cf2.value[icol][icol] * jampRi_sv; + fptype2_sv ztempI_sv = cf2.value[icol][icol] * jampIi_sv; + // Loop over jcol + for( int jcol = icol + 1; jcol < ncolor; jcol++ ) + { + // Off-diagonal terms +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype2_sv& jampRj_sv = jampR_sv[jcol]; + fptype2_sv& jampIj_sv = jampI_sv[jcol]; +#else + fptype2_sv jampRj_sv = (fptype2_sv)( cxreal( jamp_sv[jcol] ) ); + fptype2_sv jampIj_sv = (fptype2_sv)( cximag( jamp_sv[jcol] ) ); +#endif + ztempR_sv += cf2.value[icol][jcol] * jampRj_sv; + ztempI_sv += cf2.value[icol][jcol] * jampIj_sv; + } + fptype2_sv deltaMEs2 = ( jampRi_sv * ztempR_sv + jampIi_sv * ztempI_sv ); // may underflow #831 +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + deltaMEs += fpvsplit0( deltaMEs2 ); + deltaMEs_next += fpvsplit1( deltaMEs2 ); +#else + deltaMEs += deltaMEs2; +#endif + } + // *** STORE THE RESULTS *** + using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv += deltaMEs; // fix #435 +#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype* MEs_next = E_ACCESS::ieventAccessRecord( allMEs, ievt0 + neppV ); + fptype_sv& MEs_sv_next = E_ACCESS::kernelAccess( MEs_next ); + MEs_sv_next += deltaMEs_next; +#endif + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity + const int nGoodHel, // input: number of good helicities + const int nevtIfAllHelicities ) // input: zero in single-helicity mode, number of events in multi-helicity mode + { + if (nevtIfAllHelicities) { + int ighel = blockIdx.y; + allMEs = allMEs + ighel * nevtIfAllHelicities; // MEs for one specific helicity ighel + allJamps = allJamps + ighel * nevtIfAllHelicities; // Jamps for one specific helicity ighel + } + using J_ACCESS = DeviceAccessJamp; + fptype jampR[ncolor]; + fptype jampI[ncolor]; + for( int icol = 0; icol < ncolor; icol++ ) + { + constexpr int ihel0 = 0; // the input buffer allJamps already points to a specific helicity + cxtype jamp = J_ACCESS::kernelAccessIcolIhelNhelConst( allJamps, icol, ihel0, nGoodHel ); + jampR[icol] = jamp.real(); + jampI[icol] = jamp.imag(); + } + // Loop over icol + fptype deltaMEs = { 0 }; + for( int icol = 0; icol < ncolor; icol++ ) + { + fptype2 ztempR = { 0 }; + fptype2 ztempI = { 0 }; + fptype2 jampRi = jampR[icol]; + fptype2 jampIi = jampI[icol]; + // OLD IMPLEMENTATION (ihel3: symmetric square matrix) - Loop over all jcol + //for( int jcol = 0; jcol < ncolor; jcol++ ) + //{ + // fptype2 jampRj = jampR[jcol]; + // fptype2 jampIj = jampI[jcol]; + // ztempR += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix + // ztempI += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix + //} + // NEW IMPLEMENTATION #475 (ihel3p1: triangular lower diagonal matrix) - Loop over jcol < icol + ztempR += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampRi; // use fptype2 version of color matrix + ztempI += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampIi; // use fptype2 version of color matrix + for( int jcol = 0; jcol < icol; jcol++ ) + { + fptype2 jampRj = jampR[jcol]; + fptype2 jampIj = jampI[jcol]; + ztempR += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix + ztempI += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix + } + deltaMEs += ztempR * jampRi; + deltaMEs += ztempI * jampIi; + } + // *** STORE THE RESULTS *** + using E_ACCESS = DeviceAccessMatrixElements; // non-trivial access: buffer includes all events + // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) + E_ACCESS::kernelAccess( allMEs ) += deltaMEs; // fix #435 + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +#ifndef MGONGPU_HAS_NO_BLAS +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + __global__ void + convertD2F_Jamps( fptype2* allJampsFpt2, // output: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel + const fptype* allJamps, // input: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel + const int nhel ) // input: number of good helicities nGoodHel + { + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + constexpr int ihel = 0; // the input buffer allJamps already points to a specific helicity + // NB! From a functional point of view, any striding will be ok here as long as ncolor*2*nevt elements are all correctly copied! + // NB! Just in case this may be better for performance reasons, however, the same striding as in compute_jamps and cuBLAS is used here + for( int ix2 = 0; ix2 < mgOnGpu::nx2; ix2++ ) + for( int icol = 0; icol < ncolor; icol++ ) + allJampsFpt2[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] = + allJamps[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt]; + } +#endif +#endif +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +#ifndef MGONGPU_HAS_NO_BLAS +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + __global__ void + convertF2D_MEs( fptype* allMEs, // output: allMEs[nevt] for one specific helicity + const fptype2* allMEsFpt2 ) // input: allMEs[nevt] for one specific helicity + { + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + allMEs[ievt] = allMEsFpt2[ievt]; + } +#endif +#endif +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ +#ifndef MGONGPU_HAS_NO_BLAS + void + color_sum_blas( fptype* ghelAllMEs, // output: allMEs super-buffer[nhel][nevt], add |M|^2 separately for each helicity + const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nhel][nevt] for nhel good helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nhel good helicities + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) +#else + gpuStream_t* /*ghelStreams*/, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) +#endif + const int nhel, // input: number of good helicities (nhel == nGoodHel) + const int gpublocks, // input: cuda gpublocks + const int gputhreads ) // input: cuda gputhreads + { + const int nevt = gpublocks * gputhreads; + + // Get the address associated with the normalized color matrix in device memory + static fptype2* devNormColMat = nullptr; + if( !devNormColMat ) gpuGetSymbolAddress( (void**)&devNormColMat, s_pNormalizedColorMatrix2 ); + +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Mixed precision mode: need two fptype2[2*ncolor*nhel*nevt] buffers and one fptype2[nhel*nevt] buffers for the nhel helicities + fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of first fptype2[ncolor*2*nhel*nevt] buffer + fptype2* ghelAllJampsFpt2 = ghelAllBlasTmp + ncolor * mgOnGpu::nx2 * nhel * nevt; // start of second fptype2[ncolor*2*nhel*nevt] buffer + fptype2* ghelAllMEsFpt2 = ghelAllBlasTmp + 2 * ncolor * mgOnGpu::nx2 * nhel * nevt; // start of fptype2[nhel*nevt] buffer + // Convert jamps from double to float + for( int ighel = 0; ighel < nhel; ighel++ ) + { + const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // jamps for a single helicity ihel + fptype2* hAllJampsFpt2 = ghelAllJampsFpt2 + ighel * nevt; // jamps for a single helicity ihel + gpuLaunchKernelStream( convertD2F_Jamps, gpublocks, gputhreads, ghelStreams[ighel], hAllJampsFpt2, hAllJamps, nhel ); + } + // Real and imaginary components + const fptype2* ghelAllJampsReal = ghelAllJampsFpt2; + const fptype2* ghelAllJampsImag = ghelAllJampsFpt2 + ncolor * nhel * nevt; +#else + // Standard single or double precision mode: need one fptype2[ncolor*2*nhel*nevt] buffer + static_assert( std::is_same::value ); + fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of fptype2[ncolor*2*nhel*nevt] buffer + fptype2* ghelAllMEsFpt2 = ghelAllMEs; + // Real and imaginary components + const fptype2* ghelAllJampsReal = ghelAllJamps; // this is not a cast (the two types are identical) + const fptype2* ghelAllJampsImag = ghelAllJamps + ncolor * nhel * nevt; // this is not a cast (the two types are identical) +#endif + // Real and imaginary components + fptype2* ghelAllZtempReal = ghelAllZtempBoth; + fptype2* ghelAllZtempImag = ghelAllZtempBoth + ncolor * nhel * nevt; + + // Note: striding for cuBLAS from DeviceAccessJamp: + // - ghelAllJamps(icol,ihel,ievt).real is ghelAllJamps[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] + // - ghelAllJamps(icol,ihel,ievt).imag is ghelAllJamps[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] + + // Step 1: Compute Ztemp[ncolor][nhel*nevt] = ColorMatrix[ncolor][ncolor] * JampsVector[ncolor][nhel*nevt] for both real and imag + // In this case alpha=1 and beta=0: the operation is Ztemp = alpha * ColorMatrix * JampsVector + beta * Ztemp + fptype2 alpha1 = 1; + fptype2 beta1 = 0; + const int ncolorM = ncolor; + const int nevtN = nhel*nevt; + const int ncolorK = ncolor; + checkGpuBlas( gpuBlasTgemm( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose ColMat + GPUBLAS_OP_T, // transpose JampsV (new1) + ncolorM, nevtN, ncolorK, + &alpha1, + devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK + ghelAllJampsReal, nevtN, // JampsV is nevtN x ncolorK + &beta1, + ghelAllZtempReal, ncolorM ) ); // Ztemp is ncolorM x nevtN + checkGpuBlas( gpuBlasTgemm( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose ColMat + GPUBLAS_OP_T, // transpose JampsV (new1) + ncolorM, nevtN, ncolorK, + &alpha1, + devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK + ghelAllJampsImag, nevtN, // JampsV is nevtN x ncolorK (new1) + &beta1, + ghelAllZtempImag, ncolorM ) ); // Ztemp is ncolorM x nevtN + + // Step 2: For each ievt, compute the dot product of JampsVector[ncolor][ievt] dot tmp[ncolor][ievt] + // In this case alpha=1 and beta=1: the operation is ME = alpha * ( Tmp dot JampsVector ) + beta * ME + // Use cublasSgemmStridedBatched to perform these batched dot products in one call + fptype2 alpha2 = 1; + fptype2 beta2 = 1; + checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose JampsV (new1) + GPUBLAS_OP_N, // do not transpose Tmp + 1, 1, ncolor, // result is 1x1 (dot product) + &alpha2, + ghelAllJampsReal, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column + ghelAllZtempReal, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column + &beta2, + ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) + nevtN ) ); // there are nevtN (nhel*nevt) "batches" + checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, + GPUBLAS_OP_N, // do not transpose JampsV (new1) + GPUBLAS_OP_N, // do not transpose Tmp + 1, 1, ncolor, // result is 1x1 (dot product) + &alpha2, + ghelAllJampsImag, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column (new1) + ghelAllZtempImag, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column + &beta2, + ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) + nevtN ) ); // there are nevt (nhel*nevt) "batches" + +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Convert MEs from float to double + for( int ighel = 0; ighel < nhel; ighel++ ) + { + fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for a single helicity ihel + fptype2* hAllMEsFpt2 = ghelAllMEsFpt2 + ighel * nevt; // MEs for a single helicity ihel + gpuLaunchKernelStream( convertF2D_MEs, gpublocks, gputhreads, ghelStreams[ighel], hAllMEs, hAllMEsFpt2 ); + } +#endif + } +#endif /* clang-format on */ +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) + const int nGoodHel, // input: number of good helicities + const int gpublocks, // input: cuda gpublocks + const int gputhreads, // input: cuda gputhreads + const bool processAllHelicities ) // input: if true, use blockIdx.y to index helicities + { + const int nevt = gpublocks * gputhreads; + // CASE 1: KERNEL + if( !pBlasHandle ) + { + assert( ghelAllBlasTmp == nullptr ); // sanity check for HASBLAS=hasNoBlas or CUDACPP_RUNTIME_BLASCOLORSUM not set + if (processAllHelicities) { + gpuLaunchKernel2D( color_sum_kernel, gpublocks, nGoodHel, gputhreads, ghelStreams[0], ghelAllMEs, ghelAllJamps, nGoodHel, nevt ); + } else { + // Loop over helicities + for( int ighel = 0; ighel < nGoodHel; ighel++ ) + { + fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for one specific helicity ighel + const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // Jamps for one specific helicity ighel + gpuStream_t hStream = ghelStreams[ighel]; + gpuLaunchKernelStream( color_sum_kernel, gpublocks, gputhreads, hStream, hAllMEs, hAllJamps, nGoodHel, 0 ); + } + } + } + // CASE 2: BLAS + else + { +#ifdef MGONGPU_HAS_NO_BLAS + assert( false ); // sanity check: no path to this statement for HASBLAS=hasNoBlas +#else + if (processAllHelicities) { + assert( false ); // BLAS in async mode not supported for now + } else { + checkGpu( gpuDeviceSynchronize() ); // do not start the BLAS color sum for all helicities until the loop over helicities has completed + // Reset the tmp buffer +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( 2 * ncolor * mgOnGpu::nx2 + 1 ) * sizeof( fptype2 ) ); +#else + gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( ncolor * mgOnGpu::nx2 ) * sizeof( fptype2 ) ); +#endif + // Delegate the color sum to BLAS for + color_sum_blas( ghelAllMEs, ghelAllJamps, ghelAllBlasTmp, pBlasHandle, ghelStreams, nGoodHel, gpublocks, gputhreads ); + } +#endif + } + } +#endif + + //-------------------------------------------------------------------------- + +} // end namespace diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h new file mode 100644 index 0000000000..347184c4e1 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h @@ -0,0 +1,105 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef COLOR_SUM_H +#define COLOR_SUM_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#include "CPPProcess.h" +#include "GpuAbstraction.h" + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + class DeviceAccessJamp + { + public: + static __device__ inline cxtype_ref + kernelAccessIcolIhelNhel( fptype* buffer, const int icol, const int ihel, const int nhel ) + { + const int ncolor = CPPProcess::ncolor; // the number of leading colors + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) + //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" + // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) + // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS + //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" + // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) + return cxtype_ref( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], + buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); + } + static __device__ inline const cxtype + kernelAccessIcolIhelNhelConst( const fptype* buffer, const int icol, const int ihel, const int nhel ) + { + const int ncolor = CPPProcess::ncolor; // the number of leading colors + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) + //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" + // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) + // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS + //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" + // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) + return cxtype( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], + buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); + } + }; +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void createNormalizedColorMatrix(); +#endif + + //-------------------------------------------------------------------------- + +#ifndef MGONGPUCPP_GPUIMPL + void + color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity + const int ievt0 ); // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + void + color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) + const int nGoodHel, // input: number of good helicities + const int gpublocks, // input: cuda gpublocks + const int gputhreads, // input: cuda gputhreads + const bool processAllHelicities); // input: if true, use blockIdx.y to index helicities +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL + __global__ void + color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity + const int nGoodHel, // input: number of good helicities + const int nevtIfAllHelicities); // input: zero in single-helicity mode, number of events in multi-helicity mode +#endif + + //-------------------------------------------------------------------------- +} + +#endif // COLOR_SUM_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h new file mode 100644 index 0000000000..027f1aa44e --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h @@ -0,0 +1,68 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: O. Mattelaer, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef COLORAMPS_H +#define COLORAMPS_H 1 + +#include "CPPProcess.h" + + +namespace mgOnGpu +{ + // Summary of numbering and indexing conventions for the relevant concepts (see issue #826 and PR #852) + // - Diagram number (no variable) in [0, N_diagrams-1]: all values are allowed (N_diagrams distinct values) + // It follows the same C-style indexing of MadSpace + // => this number is displayed for information before each block of code in CPPProcess.cc + // - Channel number ("channelId" in C, CHANNEL_ID in F) in [1, N_channels]: not all values are allowed (N_config <= N_channels <= N_diagrams distinct values) + // *** NB channelId is a diagram number: but ALL diagrams > N_channels, and also some < N_channels, do not have an associated SDE config number (#919) *** + // => this number (with F indexing as in ps/pdf output) is passed around as an API argument between cudacpp functions + // Note: the old API passes around a single CHANNEL_ID (and uses CHANNEL_ID=0 to indicate no-multichannel mode, but this is not used in coloramps.h), + // while the new API passes around an array of CHANNEL_ID's (and uses a NULL array pointer to indicate no-multichannel mode) + // - Channel number in C indexing: "channelID - 1" + // => this number (with C indexing) is used as the index of the channel2iconfig array below + // - Config number ("iconfig" in C, ICONFIG in F) in [1, N_config]: all values are allowed (N_config <= N_channels <= N_diagrams distinct values) + // - Config number in C indexing: "iconfig - 1" + // => this number (with C indexing) is used as the index of the icolamp array below + + // The number of channels in the channel2iconfig array below + // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** + constexpr unsigned int nchannels = %(nb_diag)i; +#ifdef MGONGPUCPP_GPUIMPL + static_assert( nchannels <= mg5amcGpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 +#else + static_assert( nchannels <= mg5amcCpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 +#endif + + // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) + // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) + // This array has N_diagrams elements, but only N_config <= N_diagrams valid values (iconfig>0) + // (NB: this array is created on the host in C++ code and on the device in GPU code, but a host copy is also needed in runTest #917) + __device__ constexpr int channel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed +%(channelc2iconfig_lines)s + }; + + // Host copy of the channel2iconfig array (this is needed in runTest #917) +#ifndef MGONGPUCPP_GPUIMPL + constexpr const int* hostChannel2iconfig = channel2iconfig; +#else + constexpr int hostChannel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed +%(channelc2iconfig_lines)s + }; +#endif + + // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) + constexpr unsigned int nconfigSDE = %(nb_channel)s; + + // Map iconfig to the mask of allowed colors (e.g. "colormask = icolamp[iconfig - 1]": input index uses C indexing) + // This array has N_config <= N_diagrams elements + // (NB: this array is created on the host in C++ code and on the device in GPU code) + __device__ constexpr bool icolamp[%(nb_channel)s][%(nb_color)s] = { // note: a trailing comma in the initializer list is allowed +%(is_LC)s + }; + +} + +#endif // COLORAMPS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h new file mode 100644 index 0000000000..32c1bbb963 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h @@ -0,0 +1,335 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Feb 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef constexpr_math_h +#define constexpr_math_h 1 + +#include "mgOnGpuConfig.h" + +#include +#include +#include +#include + +// SWITCH ON/OFF DEBUGGING +#undef CONSTEXPR_MATH_DEBUG // no-debug +//#define CONSTEXPR_MATH_DEBUG 1 // debug + +// FOR DEBUGGING +#ifdef CONSTEXPR_MATH_DEBUG +#define CONSTEXPRMATHFUN inline +#define CONSTEXPRMATHVAR const +#else +#define CONSTEXPRMATHFUN constexpr +#define CONSTEXPRMATHVAR constexpr +#endif + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) + constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) + { + return curr == prev ? curr : sqrtNewtonRaphson( xx, 0.5 * ( curr + xx / curr ), curr ); + } + constexpr long double constexpr_sqrt( const long double xx ) + { + return xx >= 0 // && x < std::numeric_limits::infinity() // avoid -Wtautological-constant-compare warning in fast math + ? sqrtNewtonRaphson( xx, xx, 0 ) + : std::numeric_limits::quiet_NaN(); + } + + // SQRT constants + constexpr long double constexpr_sqrt2 = constexpr_sqrt( 2 ); + + // Constexpr implementation of floor (see https://stackoverflow.com/a/66146159) + constexpr int constexpr_floor( const long double xx ) + { + const int i = static_cast( xx ); + return xx < i ? i - 1 : i; + } + + // Constexpr implementation of pow + constexpr long double constexpr_pow( const long double base, const long double exp, const bool requireExpGe0 = false ) + { + // NB(1): this iterative integer implementation of constexpr_pow requires exponent >= 0 + if( requireExpGe0 ) assert( exp >= 0 ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + if( exp < 0 ) return 1. / constexpr_pow( base, -exp, true ); + // NB(2): this iterative integer implementation of constexpr_pow requires an integer exponent, excexpt for special cases (1/2, 1/4) + if( exp == 0.5 ) return constexpr_sqrt( base ); + if( exp == 0.25 ) return constexpr_sqrt( constexpr_sqrt( base ) ); + const int iexp = constexpr_floor( exp ); + assert( static_cast( iexp ) == exp ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + // Iterative implementation of pow if exp is a non negative integer + return iexp == 0 ? 1 : base * constexpr_pow( base, iexp - 1 ); + } + + // PI constants + // NB1: M_PIl from from cmath is not defined on Mac + // NB2: std::numbers::pi needs c++20 but we are still using c++17 + // NB3: I could use my constexpr_atan(1)*4... but a literal is better? + //constexpr long double constexpr_pi = M_PIl; // pi + //constexpr long double constexpr_pi_by_2 = M_PI_2l; // pi/2 + //constexpr long double constexpr_pi_by_4 = M_PI_4l; // pi/4 + constexpr long double constexpr_pi = 3.141592653589793238462643383279502884L; // same as M_PIl in gcc + constexpr long double constexpr_pi_by_2 = 1.570796326794896619231321691639751442L; // same as M_PI_2l in gcc + constexpr long double constexpr_pi_by_4 = 0.785398163397448309615660845819875721L; // same as M_PI_4l in gcc + static_assert( constexpr_pi_by_4 * 4 == constexpr_pi ); + static_assert( constexpr_pi_by_4 * 2 == constexpr_pi_by_2 ); + static_assert( constexpr_pi_by_2 * 2 == constexpr_pi ); + + // Constexpr implementation of sin for 0= constexpr_pi_by_4 ) std::cout << "sinTaylor xx=" << xx << std::endl; +#endif + assert( xx >= 0 && "The argument of sinTaylor is lower than the expected range [0,pi/4)" ); + assert( xx < constexpr_pi_by_4 && "The argument of sinTaylor is higher than the expected range [0,pi/4)" ); + long double sinx = 0; + int ipow = 1; + long double delta = xx; + while( true ) + { + long double sinxlast = sinx; + sinx += delta; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + if( sinx == sinxlast ) break; + // Next iteration + ipow += 2; + delta *= -xx * xx / ( ipow - 1 ) / ipow; + } +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + return sinx; + } + +#ifdef CONSTEXPR_MATH_DEBUG + // Debug printouts for trig functions + inline void debug_constexpr_trig( const long double xx, size_t call ) + { + CONSTEXPRMATHVAR long double xxminuspi = xx - constexpr_pi; + CONSTEXPRMATHVAR long double twopiminusxx = 2 * constexpr_pi - xx; + std::cout << std::setprecision( 40 ); + std::cout << "constexpr_sin_quad call=" << call << std::endl + << " xx=" << xx << std::endl + << " (xx-pi)=" << xxminuspi << std::endl + << " (2pi-xx)=" << twopiminusxx << std::endl; + std::cout << std::setprecision( 6 ); + if( xx < 0 ) // (-inf, 0) + std::cout << " -- case 1 (xx < 0)" << std::endl; + else if( xx == 0 ) // [0] *** NEW + std::cout << " -- case 2 (xx == 0)" << std::endl; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + std::cout << " -- case 3 (xx < pi/4)" << std::endl; + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + std::cout << " -- case 4 (xx == pi/4)" << std::endl; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + std::cout << " -- case 5 (xx < pi/2)" << std::endl; + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + std::cout << " -- case 6 (xx == pi/2)" << std::endl; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + std::cout << " -- case 7 (xx < 3*pi/4)" << std::endl; + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + std::cout << " -- case 8 (xx == 3*pi/4)" << std::endl; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + std::cout << " -- case 9 (xx < pi)" << std::endl; + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + std::cout << " -- case 10 (xx == pi)" << std::endl; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + std::cout << " -- case 11 (xx < 2*pi)" << std::endl; + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + std::cout << " -- case 12 (xx == 2*pi)" << std::endl; + else // (2*pi, +inf) + std::cout << " -- case 13 (xx > 2*pi)" << std::endl; + } +#endif + + // Mapping to [0,2*pi) range (long double signature) + constexpr long double mapIn0to2Pi( const long double xx ) + { + return xx - constexpr_floor( xx / 2 / constexpr_pi ) * 2 * constexpr_pi; + } + + // Constexpr implementation of cos (long double signature) + CONSTEXPRMATHFUN long double constexpr_cos_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 1; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx ), 2 ) ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return sinTaylor( constexpr_pi_by_2 - xx ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 0; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return -sinTaylor( xx - constexpr_pi_by_2 ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return -1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return -constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi - xx ), 2 ) ); + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + return -1; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return constexpr_cos_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 1; + else // (2*pi, +inf) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of cos (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_cos( const double x ) + { + return constexpr_cos_quad( x ); + } + + // Constexpr implementation of sin (long double signature) + CONSTEXPRMATHFUN long double constexpr_sin_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 0; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return sinTaylor( xx ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi_by_2 - xx ), 2 ) ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 1; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx - constexpr_pi_by_2 ), 2 ) ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return sinTaylor( constexpr_pi - xx ); + else if( xx == constexpr_pi ) // [pi] *** NEW (1st fix #903 infinite recursion) + return 0; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return -constexpr_sin_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 0; + else // (2*pi, +inf) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of sin (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_sin( const double x ) + { + return constexpr_sin_quad( x ); + } + + // Constexpr implementation of tan (long double signature) + CONSTEXPRMATHFUN long double constexpr_tan_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } + if( xx < 0 ) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + else if( xx < 2 * constexpr_pi ) // [0, 2*pi) + return constexpr_sin_quad( xx, assume0to2Pi ) / constexpr_cos_quad( xx, assume0to2Pi ); + else // [8/4*pi, +inf) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of tan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_tan( const double x ) + { + return constexpr_tan_quad( x ); + } + + // Constexpr implementation of atan for -1= -1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + assert( xx < 1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + long double atanx = 0; + int ipow = 1; + long double xpow = xx; + while( true ) + { + long double atanxlast = atanx; + atanx += xpow / ipow; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", xpow=" << xpow << ", atanx=" << atanx << std::endl; // for debugging (not constexpr) +#endif + if( atanx == atanxlast ) break; + // Next iteration + ipow += 2; + xpow *= -xx * xx; + } + return atanx; + } + + // Constexpr implementation of atan (long double signature) + CONSTEXPRMATHFUN long double constexpr_atan_quad( const long double xx ) + { + if( xx > 1 ) + return constexpr_pi_by_2 - atanTaylor( 1 / xx ); + else if( xx == 1 ) + return constexpr_pi_by_4; + else if( xx > -1 ) + return atanTaylor( xx ); + else if( xx == -1 ) + return -constexpr_pi_by_4; + else // if( xx < -1 ) + return -constexpr_pi_by_2 - atanTaylor( 1 / xx ); + } + + // Constexpr implementation of atan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_atan( const double x ) + { + return constexpr_atan_quad( x ); + } +} + +#endif // constexpr_math_h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_hel_amps_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_hel_amps_h.inc new file mode 100644 index 0000000000..8dde0fff4b --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_hel_amps_h.inc @@ -0,0 +1,90 @@ +// Copyright (C) 2010 The ALOHA Development team and Contributors. +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Sep 2010) for the MG5aMC backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== +// This file has been automatically generated for %(output_name)s by +%(info_lines)s +//========================================================================== + +#ifndef HelAmps_%(model_name)s_H +#define HelAmps_%(model_name)s_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#include "Parameters.h" + +#include +//#include +//#include +//#include +//#include + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + + // ALOHA-style object for easy flavor consolidation and non-template API + struct ALOHAOBJ { + + static constexpr int np4 = 4; // dimensions of 4-momenta (E,px,py,pz) + static constexpr int nw6 = %(nwave)d; // dimensions of each wavefunction (notice, this is +1 in case of FD gauge) + fptype_sv * pvec; + fptype * w; + int flv_index; + + __host__ __device__ ALOHAOBJ() {} + __host__ __device__ ALOHAOBJ(fptype_sv * pvec_sv, cxtype_sv * w_sv, int flv = -1) + : pvec(pvec_sv), w(reinterpret_cast(w_sv)), flv_index(flv) {} + }; + + struct FLV_COUPLING_VIEW { + + const int* const partner1; + const int* const partner2; + const fptype* const value; + + __host__ __device__ + FLV_COUPLING_VIEW(const int* p1, const int* p2, const fptype* v) + : partner1(p1), partner2(p2), value(v) {} + }; + + // FSTRIDE is the number of fptype's used to store one flavor slot of the value buffer: + // - independent (fixed) flavored couplings: FSTRIDE = nx2 = 2 (a single scalar complex, broadcast across the SIMD vector) + // - dependent (event-by-event, running-alphas) flavored couplings: FSTRIDE = nx2*neppC (an AOSOA SIMD record) + // It must match C_ACCESS::flv_stride of the access type the consuming vertex routine is instantiated with. + template + class FLV_COUPLING_ARRAY { + + static_assert(SIZE >= 0, "flvCOUPs SIZE must be non-negative"); + static_assert(STRIDE > 0, "flvCOUPs STRIDE must be positive"); + static_assert(FSTRIDE > 0, "flvCOUPs FSTRIDE must be positive"); + const int* const partner1; + const int* const partner2; + const fptype* const value; + + public: + __host__ __device__ + FLV_COUPLING_ARRAY(const int* p1, const int* p2, const fptype* v) + : partner1(p1), partner2(p2), value(v) {} + + __host__ __device__ + FLV_COUPLING_VIEW operator[](const int i) const { + return FLV_COUPLING_VIEW{ + partner1 + i*STRIDE, + partner2 + i*STRIDE, + value + i*FSTRIDE*STRIDE + }; + } + }; +%(function_declarations)s diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_cc.inc b/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_cc.inc new file mode 100644 index 0000000000..4650fc76c8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_cc.inc @@ -0,0 +1,101 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== +// This file has been automatically generated for CUDA/C++ standalone by +%(info_lines)s +//========================================================================== + +#include "Parameters.h" + +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +using namespace mg5amcGpu; +#else +using namespace mg5amcCpu; +#endif + +#ifndef MGONGPU_HARDCODE_PARAM + +// Initialize static instance +Parameters* Parameters::instance = 0; + +// Function to get static instance - only one instance per program +Parameters* +Parameters::getInstance() +{ + if( instance == 0 ) + instance = new Parameters(); + return instance; +} + +void +Parameters::setIndependentParameters( SLHAReader& slha ) +{ + zero = 0; // define "zero" + ZERO = 0; // define "zero" + std::vector indices( 2, 0 ); // prepare a vector for indices + %(set_independent_parameters)s +} + +void +Parameters::setIndependentCouplings() +{ + %(set_independent_couplings)s + %(set_flv_couplings)s +} + +/* +void +Parameters::setDependentParameters() // now computed event-by-event (running alphas #373) +{ + %(set_dependent_parameters)s +} + +void +Parameters::setDependentCouplings() // now computed event-by-event (running alphas #373) +{ + %(set_dependent_couplings)s +} +*/ + +#endif + +// Routines for printing out parameters +void +Parameters::printIndependentParameters() +{ + std::cout << "model parameters independent of event kinematics:" << std::endl; + std::cout << "(Warning: aS in the runcard is ignored because event-by-event Gs are hardcoded or retrieved from Fortran)" << std::endl; + %(print_independent_parameters)s +} + +void +Parameters::printIndependentCouplings() +{ + std::cout << "model couplings independent of event kinematics:" << std::endl; + %(print_independent_couplings)s +} + +/* +void +Parameters::printDependentParameters() // now computed event-by-event (running alphas #373) +{ + std::cout << "model parameters dependent on event kinematics:" << std::endl; + %(print_dependent_parameters)s +} + +void +Parameters::printDependentCouplings() // now computed event-by-event (running alphas #373) +{ + std::cout << "model couplings dependent on event kinematics:" << std::endl; + %(print_dependent_couplings)s +} +*/ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_h.inc new file mode 100644 index 0000000000..c13f8ebe2f --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_h.inc @@ -0,0 +1,264 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. +// ========================================================================== +// This file has been automatically generated for CUDA/C++ standalone by +%(info_lines)s +//========================================================================== + +#ifndef Parameters_H +#define Parameters_H + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" +#include "mgOnGpuVectors.h" + +#include "constexpr_math.h" + +//========================================================================== + +// AV Jan 2024 (PR #625): this ugly #define was the only way I found to avoid creating arrays[nBsm] in CPPProcess.cc if nBsm is 0 +// The problem is that nBsm is determined when generating Parameters.h, which happens after CPPProcess.cc has already been generated +// For simplicity, keep this code hardcoded also for SM processes (a nullptr is needed as in the case nBsm == 0) +%(bsmdefine)s + +#ifndef MGONGPU_HARDCODE_PARAM%(eftwarn0)s + +#include "read_slha.h" + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // Define FLV_COUPLING struct + struct FLV_COUPLING { + static constexpr int max_flavor = %(max_flavor)d; + + int partner1[max_flavor]; + int partner2[max_flavor]; + cxtype* value[max_flavor]; // because it will refer already calculated couplings + + FLV_COUPLING() { + for (int i = 0; i < max_flavor; ++i) { + partner1[i] = -1; + partner2[i] = -1; + value[i] = nullptr; // flavor slots with no coupling stay null so the + // cIPF_value setup (value[j] ? *value[j] : 0) does + // not dereference an uninitialised pointer + } + } + }; + + class Parameters + { + public: + + static Parameters* getInstance(); + + // Define "zero" + double zero, ZERO; + + %(independent_parameters)s + + %(independent_couplings)s + + %(dependent_parameters)s + + %(dependent_couplings)s + + %(flavor_independent_couplings)s + + %(flavor_dependent_couplings)s + + // Set parameters that are unchanged during the run + void setIndependentParameters( SLHAReader& slha ); + + // Set couplings that are unchanged during the run + void setIndependentCouplings(); + + // Set parameters that are changed event by event + //void setDependentParameters(); // now computed event-by-event (running alphas #373) + + // Set couplings that are changed event by event + //void setDependentCouplings(); // now computed event-by-event (running alphas #373) + + // Print parameters that are unchanged during the run + void printIndependentParameters(); + + // Print couplings that are unchanged during the run + void printIndependentCouplings(); + + // Print parameters that are changed event by event + //void printDependentParameters(); // now computed event-by-event (running alphas #373) + + // Print couplings that are changed event by event + //void printDependentCouplings(); // now computed event-by-event (running alphas #373) + + // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; + static constexpr int nBsmIndepParam = %(nbsmip)i; + %(hasbsmip)sdouble mdl_bsmIndepParam[nBsmIndepParam]; + + private: + + static Parameters* instance; + }; + +} // end namespace mg5amcGpu/mg5amcCpu + +#else%(eftwarn1)s + +#include +#include + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // Hardcoded constexpr physics parameters + namespace Parameters // keep the same name rather than HardcodedParameters for simplicity + { + // Model parameters independent of aS + constexpr double zero = 0; + constexpr double ZERO = 0; + %(hardcoded_independent_parameters)s + // Model couplings independent of aS + %(hardcoded_independent_couplings)s + // Model parameters dependent on aS + %(hardcoded_dependent_parameters)s + // Model couplings dependent on aS + %(hardcoded_dependent_couplings)s + // Print parameters that are unchanged during the run + void printIndependentParameters(); + + // Print couplings that are unchanged during the run + void printIndependentCouplings(); + + // Print parameters that are changed event by event + //void printDependentParameters(); // now computed event-by-event (running alphas #373) + + // Print couplings that are changed event by event + //void printDependentCouplings(); // now computed event-by-event (running alphas #373) + + // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; + constexpr int nBsmIndepParam = %(nbsmip)i; + %(hasbsmip)s__device__ constexpr double mdl_bsmIndepParam[nBsmIndepParam] = { %(bsmip)s }; + } + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif + +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + namespace Parameters_dependentCouplings + { + constexpr size_t ndcoup = %(ndcoup)d; // #couplings that vary event by event because they depend on the running alphas QCD +%(idcoup)s + struct DependentCouplings_sv + { +%(dcoupdecl)s + }; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" // e.g. <> +#pragma GCC diagnostic ignored "-Wunused-variable" // e.g. <> +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> +#ifdef MGONGPUCPP_GPUIMPL +#pragma nv_diagnostic push +#pragma nv_diag_suppress 177 // e.g. <> +#endif + __host__ __device__ inline const DependentCouplings_sv computeDependentCouplings_fromG( const fptype_sv& G_sv, const double* bsmIndepParamPtr ) + { +#ifdef MGONGPU_HARDCODE_PARAM + using namespace Parameters; +#else%(eftspecial0)s +#endif + // NB: hardcode cxtype cI(0,1) instead of cxtype (or hardcoded cxsmpl) mdl_complexi (which exists in Parameters) because: + // (1) mdl_complexi is always (0,1); (2) mdl_complexi is undefined in device code; (3) need cxsmpl conversion to cxtype in code below + const cxtype cI( 0., 1. ); + DependentCouplings_sv out; +#if not( defined MGONGPU_CPPSIMD && defined MGONGPU_FPTYPE_FLOAT ) + // Couplings are (scalar, or vector of) doubles, or scalar floats - default implementation + { + const fptype_sv& G = G_sv; + // Model parameters dependent on aS +%(dcoupsetdpar)s + // Model couplings dependent on aS +%(dcoupsetdcoup)s + } +#else + // Couplings are VECTORS OF FLOATS: #439 special handling is needed (variable Gs are vector floats, fixed parameters are scalar doubles) + // Use an explicit loop to avoid <> + // Problems may come e.g. in EFTs from multiplying a vector float (related to aS-dependent G) by a scalar double (aS-independent parameters) + // (NB in pure SM processes this special handling is not needed, but we keep it here for simplicity, see PR #824)%(dcoupoutfptypev2)s + for( int i = 0; i < neppV; i++ ) + { + const fptype& G = G_sv[i]; + // Model parameters dependent on aS +%(dcoupsetdpar2)s + // Model couplings dependent on aS + %(dcoupsetdcoup2)s + }%(dcoupoutdcoup2)s +#endif + return out; + } +#ifdef MGONGPUCPP_GPUIMPL +#pragma GCC diagnostic pop +#pragma nv_diagnostic pop +#endif + } + + //========================================================================== + + namespace Parameters_independentCouplings + { + constexpr size_t nicoup = %(nicoup)d; // #couplings that are fixed for all events because they do not depend on the running alphas QCD +%(iicoup)s + } + + //========================================================================== + +#pragma GCC diagnostic push +#ifndef __clang__ +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> +#endif + // Compute the output couplings (e.g. gc10 and gc11) from the input gs + template + __device__ inline void + G2COUP( const fptype gs[], + fptype couplings[], + const double* bsmIndepParamPtr ) + { + mgDebug( 0, __FUNCTION__ ); + using namespace Parameters_dependentCouplings; + const fptype_sv& gs_sv = G_ACCESS::kernelAccessConst( gs ); + DependentCouplings_sv couplings_sv = computeDependentCouplings_fromG( gs_sv, bsmIndepParamPtr ); +%(dcoupaccessbuffer)s%(dcoupkernelaccess)s%(dcoupcompute)s + mgDebug( 1, __FUNCTION__ ); + return; + } +#pragma GCC diagnostic pop + +} // end namespace mg5amcGpu/mg5amcCpu + +//========================================================================== + +#endif // Parameters_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc new file mode 100644 index 0000000000..05c3b3e0c4 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc @@ -0,0 +1,161 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Oct 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "Bridge.h" +#include "CPPProcess.h" +#include "GpuRuntime.h" + +extern "C" +{ + /** + * The namespace where the Bridge class is taken from. + * + * In the current implementation, two separate shared libraries are created for the GPU/CUDA and CPU/C++ implementations. + * Actually, two shared libraries for GPU and CPU are created for each of the five SIMD implementations on CPUs (none, sse4, avx2, 512y, 512z). + * A single fcreatebridge_ symbol is created in each library with the same name, connected to the appropriate Bridge on CPU or GPU. + * The Fortran MadEvent code is always the same: the choice whether to use a CPU or GPU implementation is done by linking the appropriate library. + * As the names of the two CPU/GPU libraries are the same in the five SIMD implementations, the choice of SIMD is done by setting LD_LIBRARY_PATH. + * + * In a future implementation, a single heterogeneous shared library may be created, with the same interface. + * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. + * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. + */ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + /** + * The floating point precision used in Fortran arrays. + * This is presently hardcoded to double precision (REAL*8). + */ + using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays + //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays + + /** + * Create a Bridge and return its pointer. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param nevtF the pointer to the number of events in the Fortran arrays + * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + */ + void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) + { +#ifdef MGONGPUCPP_GPUIMPL + GpuRuntime::setUp(); +#endif + // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) + // FIXME: disable OMP in Bridge when called from Fortran + *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); + } + + /** + * Delete a Bridge. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + */ + void fbridgedelete_( CppObjectInFortran** ppbridge ) + { + Bridge* pbridge = dynamic_cast*>( *ppbridge ); + if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); + delete pbridge; +#ifdef MGONGPUCPP_GPUIMPL + GpuRuntime::tearDown(); +#endif + } + + /** + * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant alphas) + * @param rndhel the pointer to the input random numbers for helicity selection + * @param rndcol the pointer to the input random numbers for color selection + * @param channelIds the pointer to the input channels i.e. Feynman diagrams to enhance (1 to n: 0 is an invalid value!) + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void fbridgesequence_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ) + { + Bridge* pbridge = dynamic_cast*>( *ppbridge ); + //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); + if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); +#ifdef MGONGPUCPP_GPUIMPL + // Use the device/GPU implementation in the CUDA library + // (there is also a host implementation in this library) + pbridge->gpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); +#else + // Use the host/CPU implementation in the C++ library + // (there is no device implementation in this library) + pbridge->cpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); +#endif + } + + /** + * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++, without multi-channel mode. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param momenta the pointer to the input 4-momenta + * @param gs the pointer to the input Gs (running QCD coupling constant alphas) + * @param rndhel the pointer to the input random numbers for helicity selection + * @param rndcol the pointer to the input random numbers for color selection + * @param mes the pointer to the output matrix elements + * @param selhel the pointer to the output selected helicities + * @param selcol the pointer to the output selected colors + * @param goodHelOnly quit after computing good helicities? + */ + void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ) + { + //printf("fbridgesequence_nomultichannel_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); + fbridgesequence_( ppbridge, momenta, gs, iflavorVec, rndhel, rndcol, nullptr, mes, selhel, selcol, pgoodHelOnly ); + } + + /** + * Retrieve the number of good helicities for helicity filtering in the Bridge. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) + * @param pngoodhel the pointer to the output number of good helicities + * @param pntothel the pointer to the output total number of helicities + */ + void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, + unsigned int* pngoodhel, + unsigned int* pntothel ) + { + Bridge* pbridge = dynamic_cast*>( *ppbridge ); + if( pbridge == 0 ) throw std::runtime_error( "fbridgegetngoodhel_: invalid Bridge address" ); + *pngoodhel = pbridge->nGoodHel(); + *pntothel = pbridge->nTotHel(); + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h new file mode 100644 index 0000000000..e676df0a33 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h @@ -0,0 +1,52 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: Z. Wettersten (Oct 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, A. Thete, A. Valassi (2025). + +#include "Bridge.h" +#include "CPPProcess.h" +#include "GpuRuntime.h" + +#ifndef _FBRIDGE_H_ +#define _FBRIDGE_H_ + +extern "C" +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + using FORTRANFPTYPE = double; + + void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ); + + void fbridgedelete_( CppObjectInFortran** ppbridge ); + + void fbridgesequence_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + const unsigned int* channelIds, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ); + + void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, + const FORTRANFPTYPE* momenta, + const FORTRANFPTYPE* gs, + const unsigned int* iflavorVec, + const FORTRANFPTYPE* rndhel, + const FORTRANFPTYPE* rndcol, + FORTRANFPTYPE* mes, + int* selhel, + int* selcol, + const bool* pgoodHelOnly ); + + void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, unsigned int* pngoodhel, unsigned int* pntothel ); +} +#endif // _FBRIDGE_H_ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc new file mode 100644 index 0000000000..8a4468fb3c --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc @@ -0,0 +1,166 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Feb 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "mgOnGpuConfig.h" + +#include "Bridge.h" +#include "CPPProcess.h" +#include "MemoryBuffers.h" +#include "RamboSamplingKernels.h" +#include "RandomNumberKernels.h" + +//-------------------------------------------------------------------------- + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + template + class Sampler final : public CppObjectInFortran + { + public: + // Constructor + // @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran arrays + // @param nparF (NEXTERNAL, nexternal.inc) number of external particles in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) + // @param np4F number of momenta components, usually 4, in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) + Sampler( int nevtF, int nparF, int np4F ); + // Destructor + virtual ~Sampler() {} + // Delete copy/move constructors and assignment operators + Sampler( const Sampler& ) = delete; + Sampler( Sampler&& ) = delete; + Sampler& operator=( const Sampler& ) = delete; + Sampler& operator=( Sampler&& ) = delete; + // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta + void samplerHostSequence( FORTRANFPTYPE* fortranMomenta ); + private: + const int m_nevt; // The number of events in each iteration + int m_iiter; // The iteration counter (for random number seeding) +#ifndef MGONGPUCPP_GPUIMPL + HostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers + HostBufferMomenta m_hstMomenta; // Memory buffers for momenta + HostBufferWeights m_hstWeights; // Memory buffers for sampling weights +#else + PinnedHostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers + PinnedHostBufferMomenta m_hstMomenta; // Memory buffers for momenta + PinnedHostBufferWeights m_hstWeights; // Memory buffers for sampling weights +#endif + std::unique_ptr m_prnk; // The appropriate RandomNumberKernel + std::unique_ptr m_prsk; // The appropriate SamplingKernel + // HARDCODED DEFAULTS + static constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) + }; + + template + Sampler::Sampler( int nevtF, int nparF, int np4F ) + : m_nevt( nevtF ) + , m_iiter( 0 ) + , m_hstRndmom( nevtF ) + , m_hstMomenta( nevtF ) + , m_hstWeights( nevtF ) + , m_prnk( new CommonRandomNumberKernel( m_hstRndmom ) ) + , m_prsk( new RamboSamplingKernelHost( energy, m_hstRndmom, m_hstMomenta, m_hstWeights, nevtF ) ) + { + if( nparF != CPPProcess::npar ) throw std::runtime_error( "Sampler constructor: npar mismatch" ); + if( np4F != CPPProcess::np4 ) throw std::runtime_error( "Sampler constructor: np4 mismatch" ); + std::cout << "WARNING! Instantiate host Sampler (nevt=" << m_nevt << ")" << std::endl; + } + + // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta + template + void Sampler::samplerHostSequence( FORTRANFPTYPE* fortranMomenta ) + { + std::cout << "Iteration #" << m_iiter + 1 << std::endl; + // === STEP 1 OF 3 + // --- 1a. Seed rnd generator (to get same results on host and device in curand) + // [NB This should not be necessary using the host API: "Generation functions + // can be called multiple times on the same generator to generate successive + // blocks of results. For pseudorandom generators, multiple calls to generation + // functions will yield the same result as a single call with a large size."] + // *** NB! REMEMBER THAT THE FORTRAN SAMPLER ALWAYS USES COMMON RANDOM NUMBERS! *** + constexpr unsigned long long seed = 20200805; + m_prnk->seedGenerator( seed + m_iiter ); + m_iiter++; + // --- 1b. Generate all relevant numbers to build nevt events (i.e. nevt phase space points) on the host + m_prnk->generateRnarray(); + //std::cout << "Got random numbers" << std::endl; + // === STEP 2 OF 3 + // --- 2a. Fill in momenta of initial state particles on the device + m_prsk->getMomentaInitial(); + //std::cout << "Got initial momenta" << std::endl; + // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device + // (i.e. map random numbers to final-state particle momenta for each of nevt events) + m_prsk->getMomentaFinal(); + //std::cout << "Got final momenta" << std::endl; + // --- 2c. TransposeC2F + hst_transposeMomentaC2F( m_hstMomenta.data(), fortranMomenta, m_nevt ); + } +} + +//-------------------------------------------------------------------------- + +extern "C" +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + /** + * The floating point precision used in Fortran arrays. + * This is presently hardcoded to double precision (REAL*8). + */ + using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays + //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays + + /** + * Create a Sampler and return its pointer. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) + * @param nevtF the pointer to the number of events in the Fortran arrays + * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) + */ + void fsamplercreate_( CppObjectInFortran** ppsampler, const int* pnevtF, const int* pnparF, const int* pnp4F ) + { + *ppsampler = new Sampler( *pnevtF, *pnparF, *pnp4F ); + } + + /** + * Delete a Sampler. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) + */ + void fsamplerdelete_( CppObjectInFortran** ppsampler ) + { + Sampler* psampler = dynamic_cast*>( *ppsampler ); + if( psampler == 0 ) throw std::runtime_error( "fsamplerdelete_: invalid Sampler address" ); + delete psampler; + } + + /** + * Execute the matrix-element calculation "sequence" via a Sampler on GPU/CUDA or CUDA/C++. + * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). + * + * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) + * @param momenta the pointer to the input 4-momenta + * @param mes the pointer to the output matrix elements + */ + void fsamplersequence_( CppObjectInFortran** ppsampler, FORTRANFPTYPE* momenta ) + { + Sampler* psampler = dynamic_cast*>( *ppsampler ); + if( psampler == 0 ) throw std::runtime_error( "fsamplersequence_: invalid Sampler address" ); + // Use the host/CPU implementation (there is no device implementation) + psampler->samplerHostSequence( momenta ); + } +} + +//-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h new file mode 100644 index 0000000000..bcd3990bed --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h @@ -0,0 +1,200 @@ +// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. +// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. +//========================================================================== +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Modified originally by: S. Roiser (Feb 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. +//========================================================================== + +#ifndef MASSLESS_RAMBO_H +#define MASSLESS_RAMBO_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +#include "CPPProcess.h" + +#include +#include +#include + +// Simplified rambo version for 2 to N (with N>=2) processes with massless particles +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +namespace massless_rambo { + + constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) + constexpr int npari = CPPProcess::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- + constexpr int nparf = CPPProcess::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- + constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- + + //-------------------------------------------------------------------------- + + // Fill in the momenta of the initial particles + // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] + template + __host__ __device__ void + ramboGetMomentaInitial( const fptype energy, // input: energy + fptype* momenta ) // output: momenta for one event or for a set of events + { + const fptype energy1 = energy / 2; + const fptype energy2 = energy / 2; + const fptype mom = energy / 2; + M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 0 ) = energy1; + M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 0 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 0 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 0 ) = mom; + M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 1 ) = energy2; + M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 1 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 1 ) = 0; + M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 1 ) = -mom; + } + + //-------------------------------------------------------------------------- + + // Fill in the momenta of the final particles using the RAMBO algorithm + // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] + template + __host__ __device__ void + ramboGetMomentaFinal( const fptype energy, // input: energy + const fptype* rndmom, // input: random numbers in [0,1] for one event or for a set of events + fptype* momenta, // output: momenta for one event or for a set of events + fptype* wgts ) // output: weights for one event or for a set of events + { + /**************************************************************************** + * rambo * + * ra(ndom) m(omenta) b(eautifully) o(rganized) * + * * + * a democratic multi-particle phase space generator * + * authors: s.d. ellis, r. kleiss, w.j. stirling * + * this is version 1.0 - written by r. kleiss * + * -- adjusted by hans kuijf, weights are logarithmic (1990-08-20) * + * -- adjusted by madgraph@sheffield_gpu_hackathon team (2020-07-29) * + * * + ****************************************************************************/ + + // output weight + fptype& wt = W_ACCESS::kernelAccess( wgts ); + + // AV special case nparf==1 (issue #358) + if constexpr( nparf == 1 ) + { + static bool first = true; + if( first ) + { +#ifdef MGONGPUCPP_GPUIMPL + if constexpr( M_ACCESS::isOnDevice() ) // avoid + { + const int ievt0 = 0; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid + if( ievt == ievt0 ) + printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); + } + else +#endif + { + printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); + } + first = false; + } + const int iparf = 0; + for( int i4 = 0; i4 < np4; i4++ ) + { + M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = 0; + for( int ipari = 0; ipari < npari; ipari++ ) + { + M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) += M_ACCESS::kernelAccessIp4Ipar( momenta, i4, ipari ); + } + } + wt = 1; + return; + } + + // initialization step: factorials for the phase space weight + const fptype twopi = 8. * atan( 1. ); + const fptype po2log = log( twopi / 4. ); + fptype z[nparf]; + if constexpr( nparf > 1 ) // avoid build warning on clang (related to #358) + z[1] = po2log; + for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = z[kpar - 1] + po2log - 2. * log( fptype( kpar - 1 ) ); + for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = ( z[kpar] - log( fptype( kpar ) ) ); + + // generate n massless momenta in infinite phase space + fptype q[nparf][np4]; + for( int iparf = 0; iparf < nparf; iparf++ ) + { + const fptype r1 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 0, iparf ); + const fptype r2 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 1, iparf ); + const fptype r3 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 2, iparf ); + const fptype r4 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 3, iparf ); + const fptype c = 2. * r1 - 1.; + const fptype s = sqrt( 1. - c * c ); + const fptype f = twopi * r2; + q[iparf][0] = -log( r3 * r4 ); + q[iparf][3] = q[iparf][0] * c; + q[iparf][2] = q[iparf][0] * s * cos( f ); + q[iparf][1] = q[iparf][0] * s * sin( f ); + } + + // calculate the parameters of the conformal transformation + fptype r[np4]; + fptype b[np4 - 1]; + for( int i4 = 0; i4 < np4; i4++ ) r[i4] = 0.; + for( int iparf = 0; iparf < nparf; iparf++ ) + { + for( int i4 = 0; i4 < np4; i4++ ) r[i4] = r[i4] + q[iparf][i4]; + } + const fptype rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); + for( int i4 = 1; i4 < np4; i4++ ) b[i4 - 1] = -r[i4] / rmas; + const fptype g = r[0] / rmas; + const fptype a = 1. / ( 1. + g ); + const fptype x0 = energy / rmas; + + // transform the q's conformally into the p's (i.e. the 'momenta') + for( int iparf = 0; iparf < nparf; iparf++ ) + { + fptype bq = b[0] * q[iparf][1] + b[1] * q[iparf][2] + b[2] * q[iparf][3]; + for( int i4 = 1; i4 < np4; i4++ ) + { + M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = x0 * ( q[iparf][i4] + b[i4 - 1] * ( q[iparf][0] + a * bq ) ); + } + M_ACCESS::kernelAccessIp4Ipar( momenta, 0, iparf + npari ) = x0 * ( g * q[iparf][0] + bq ); + } + + // calculate weight (NB return log of weight) + wt = po2log; + if( nparf != 2 ) wt = ( 2. * nparf - 4. ) * log( energy ) + z[nparf - 1]; + +#ifndef MGONGPUCPP_GPUIMPL + // issue warnings if weight is too small or too large + static int iwarn[5] = { 0, 0, 0, 0, 0 }; + if( wt < -180. ) + { + if( iwarn[0] <= 5 ) std::cout << "Too small wt, risk for underflow: " << wt << std::endl; + iwarn[0] = iwarn[0] + 1; + } + if( wt > 174. ) + { + if( iwarn[1] <= 5 ) std::cout << "Too large wt, risk for overflow: " << wt << std::endl; + iwarn[1] = iwarn[1] + 1; + } +#endif + + // return for weighted massless momenta + // nothing else to do in this event if all particles are massless (nm==0) + + return; + } + + //-------------------------------------------------------------------------- +} +} + +#endif // MASSLESS_RAMBO_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h new file mode 100644 index 0000000000..09c76a6bd5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h @@ -0,0 +1,296 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jul 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCONFIG_H +#define MGONGPUCONFIG_H 1 + +// Is this a GPU (CUDA, HIP) or CPU implementation? +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#define MGONGPUCPP_GPUIMPL cuda +#elif defined __HIPCC__ +#define MGONGPUCPP_GPUIMPL hip +#include "hip/hip_runtime.h" // needed for blockDim, blockIdx, threadIdx: better in mgOnGpuConfig.h than in GpuAbstraction.h +#else +#undef MGONGPUCPP_GPUIMPL +#endif + +// Make sure that __HIP_PLATFORM_NVIDIA__ is undefined +// (__HIP_PLATFORM_AMD__ is defined by hipcc or in HiprandRandomNumberKernel.cc) +#undef __HIP_PLATFORM_NVIDIA__ // disable hiprand for NVidia (curand) + +// ** NB1 Throughputs (e.g. 6.8E8) are events/sec for "./gcheck.exe -p 65536 128 12" +// ** NB2 Baseline on b7g47n0004 fluctuates (probably depends on load on other VMs) + +// Choose if curand is supported for generating random numbers +// For HIP, by default, do not allow curand to be used (hiprand or common random numbers will be used instead) +// For both CUDA and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_CURAND +// (there exist CUDA installations, e.g. using the HPC package, which do not include curand - see PR #784 and #785) +#if defined __HIPCC__ +#define MGONGPU_HAS_NO_CURAND 1 +#else +//#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#undef MGONGPU_HAS_NO_CURAND // default +////#define MGONGPU_HAS_NO_CURAND 1 +//#else +//#undef MGONGPU_HAS_NO_CURAND // default +////#define MGONGPU_HAS_NO_CURAND 1 +//#endif +#endif + +// Choose if hiprand is supported for generating random numbers +// For CUDA, by default, do not allow hiprand to be used (curand or common random numbers will be used instead) +// For both HIP and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_HIPRAND +// (there may exist HIP installations which do not include hiprand?) +#if defined __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#define MGONGPU_HAS_NO_HIPRAND 1 +#else +//#ifdef __HIPCC__ +//#undef MGONGPU_HAS_NO_HIPRAND // default +////#define MGONGPU_HAS_NO_HIPRAND 1 +//#else +//#undef MGONGPU_HAS_NO_HIPRAND // default +////#define MGONGPU_HAS_NO_HIPRAND 1 +//#endif +#endif + +// Choose floating point precision (for everything but color algebra #537) +// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE_FLOAT, nothing happens (issue #167) +#if not defined MGONGPU_FPTYPE_DOUBLE and not defined MGONGPU_FPTYPE_FLOAT +// Floating point precision (CHOOSE ONLY ONE) +#define MGONGPU_FPTYPE_DOUBLE 1 // default +//#define MGONGPU_FPTYPE_FLOAT 1 // 2x faster +#endif + +// Choose floating point precision (for color algebra alone #537) +// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE2_FLOAT, nothing happens (issue #167) +#if not defined MGONGPU_FPTYPE2_DOUBLE and not defined MGONGPU_FPTYPE2_FLOAT +// Floating point precision (CHOOSE ONLY ONE) +#define MGONGPU_FPTYPE2_DOUBLE 1 // default +//#define MGONGPU_FPTYPE2_FLOAT 1 // 2x faster +#endif + +// Choose whether to inline all HelAmps functions +// This optimization can gain almost a factor 4 in C++, similar to -flto (issue #229) +// By default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_INLINE_HELAMPS +//#undef MGONGPU_INLINE_HELAMPS // default +////#define MGONGPU_INLINE_HELAMPS 1 + +// Choose whether to hardcode the cIPD physics parameters rather than reading them from user cards +// This optimization can gain 20%% in CUDA in eemumu (issue #39) +// By default, do not hardcode, but allow this macro to be set from outside with e.g. -DMGONGPU_HARDCODE_PARAM +// ** NB: The option to use hardcoded cIPD physics parameters is supported again even now when alphas is running (#373) +// ** NB: Note however that it now only refers to cIPD parameters (cIPC parameters are always accessed through global memory) +//#undef MGONGPU_HARDCODE_PARAM // default +////#define MGONGPU_HARDCODE_PARAM 1 + +/* clang-format off */ +// Complex type in CUDA: thrust or cucomplex or cxsmpl (CHOOSE ONLY ONE) +// (NB THIS IS MGONGPU_*CU*CXTYPE_xxx) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#define MGONGPU_CUCXTYPE_THRUST 1 // default (~1.15E9/double, ~3.2E9/float) +//#define MGONGPU_CUCXTYPE_CUCOMPLEX 1 // ~10 percent slower (1.03E9/double, ~2.8E9/float) +//#define MGONGPU_CUCXTYPE_CXSMPL 1 // ~10 percent slower (1.00E9/double, ~2.9E9/float) + +// Complex type in HIP: cxsmpl (ONLY ONE OPTION POSSIBLE? #810) +// (NB THIS IS MGONGPU_*HIP*CXTYPE_xxx) +#elif defined __HIPCC__ +#define MGONGPU_HIPCXTYPE_CXSMPL 1 // default for HIP + +// Complex type in C++: std::complex or cxsmpl (CHOOSE ONLY ONE) +// (NB THIS IS MGONGPU_*CPP*CXTYPE_xxx) +#else +//#define MGONGPU_CPPCXTYPE_STDCOMPLEX 1 // ~8 percent slower on float, same on double (5.1E6/double, 9.4E6/float) +#define MGONGPU_CPPCXTYPE_CXSMPL 1 // new default (5.1E6/double, 10.2E6/float) +#endif + +// Choose if cuBLAS and hipBLAS are supported for generating random numbers +// For both CUDA and HIP, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_BLAS +// (there may exist CUDA/HIP installations, e.g. using the HPC package, which do not include cuBLAS/hipBLAS?) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#undef MGONGPU_HAS_NO_BLAS // default +////#define MGONGPU_HAS_NO_BLAS 1 +#elif defined __HIPCC__ +//#undef MGONGPU_HAS_NO_BLAS // default +////#define MGONGPU_HAS_NO_BLAS 1 +#else +#define MGONGPU_HAS_NO_BLAS 1 +#endif + +// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#undef MGONGPU_NSIGHT_DEBUG // default in CUDA +//#define MGONGPU_NSIGHT_DEBUG 1 // CURRENTLY NO LONGER SUPPORTED! +#else +#undef MGONGPU_NSIGHT_DEBUG // only option in HIP or C++ +#endif /* clang-format on */ + +// SANITY CHECKS (floating point precision for everything but color algebra #537) +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE_DOUBLE or defined MGONGPU_FPTYPE_FLOAT +#endif + +// SANITY CHECKS (floating point precision for color algebra alone #537) +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE2_DOUBLE or defined MGONGPU_FPTYPE2_FLOAT +#endif +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You cannot use double precision for color algebra and single precision elsewhere +#endif + +// SANITY CHECKS (CUDA complex number implementation) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CUCOMPLEX +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX for CUDA +#elif defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CXSMPL for CUDA +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX and defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE OF MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL for CUDA +#endif +#endif + +// SANITY CHECKS (C++ complex number implementation) +#ifndef MGONGPUCPP_GPUIMPL +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX and defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL for C++ +#endif +#endif + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu +{ + + // --- Type definitions + + // Floating point type (for everything but color algebra #537): fptype +#if defined MGONGPU_FPTYPE_DOUBLE + typedef double fptype; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE_FLOAT + typedef float fptype; // single precision (4 bytes, fp32) +#endif + + // Floating point type (for color algebra alone #537): fptype2 +#if defined MGONGPU_FPTYPE2_DOUBLE + typedef double fptype2; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE2_FLOAT + typedef float fptype2; // single precision (4 bytes, fp32) +#endif + + // --- Platform-specific software implementation details + + // Maximum number of blocks per grid + // ** NB Some arrays of pointers will be allocated statically to fit all these blocks + // ** (the actual memory for each block will then be allocated dynamically only for existing blocks) + //const int nbpgMAX = 2048; + + // Maximum number of threads per block + //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? + const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" + + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) + // Only needed for C++ code but can be enforced also in NVCC builds of C++ code using CUDA>=11.2 and C++17 (#318, #319, #333) +#ifndef MGONGPUCPP_GPUIMPL + constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) +#endif + +} + +// Expose typedefs and operators outside the namespace +using mgOnGpu::fptype; +using mgOnGpu::fptype2; + +// Undefine ARM_NEON (hack for cppnone on Apple silicon ARM) +#ifdef MGONGPU_NOARMNEON +#undef __ARM_NEON +#endif + +// C++ SIMD vectorization width (this will be used to set neppV) +#ifdef MGONGPUCPP_GPUIMPL // CUDA and HIP implementations have no SIMD +#undef MGONGPU_CPPSIMD +#elif defined __AVX512VL__ && defined MGONGPU_PVW512 // C++ "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 8 +#else +#define MGONGPU_CPPSIMD 16 +#endif +#elif defined __AVX512VL__ // C++ "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif +#elif defined __AVX2__ // C++ "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif +#elif defined __SSE4_2__ // C++ "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [Power9 default] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 +#else +#define MGONGPU_CPPSIMD 4 +#endif +#elif defined __ARM_NEON // C++ "sse4" ARM NEON (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [ARM default] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 +#else +#define MGONGPU_CPPSIMD 4 +#endif +#else // C++ "none" i.e. no SIMD +#undef MGONGPU_CPPSIMD +#endif + +/* clang-format off */ +// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation [NB: CURRENTLY NO LONGER SUPPORTED!] +// Arguments (not used so far): text is __FUNCTION__, code is 0 (start) or 1 (end) +//#if defined __CUDACC__ && defined MGONGPU_NSIGHT_DEBUG // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#define mgDebugDeclare() __shared__ float mgDebugCounter[mgOnGpu::ntpbMAX]; +//#define mgDebugInitialise() { mgDebugCounter[threadIdx.x] = 0; } +//#define mgDebug( code, text ) { mgDebugCounter[threadIdx.x] += 1; } +//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%%f\n", mgDebugCounter[threadIdx.x] ); } +//#else +#define mgDebugDeclare() /*noop*/ +#define mgDebugInitialise() /*noop*/ +#define mgDebug( code, text ) /*noop*/ +#define mgDebugFinalise() /*noop*/ +//#endif /* clang-format on */ + +// Define empty CUDA/HIP declaration specifiers for C++ +#ifndef MGONGPUCPP_GPUIMPL +#define __global__ +#define __host__ +#define __device__ +#endif + +// For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two (https://stackoverflow.com/a/108360) +inline constexpr bool +ispoweroftwo( int n ) +{ + return ( n > 0 ) && !( n & ( n - 1 ) ); +} + +// Compiler version support (#96): require nvcc from CUDA >= 11.2, e.g. to use C++17 (see #333) +#ifdef __NVCC__ +#if( __CUDACC_VER_MAJOR__ < 11 ) || ( __CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ < 2 ) +#error Unsupported CUDA version: please use CUDA >= 11.2 +#endif +#endif + +// Compiler version support (#96): require clang >= 11 +#if defined __clang__ +#if( __clang_major__ < 11 ) +#error Unsupported clang version: please use clang >= 11 +#endif +// Compiler version support (#96): require gcc >= 9.3, e.g. for some OMP issues (see #269) +// [NB skip this check for the gcc toolchain below clang or icx (TEMPORARY? #355)] +#elif defined __GNUC__ +#if( __GNUC__ < 9 ) || ( __GNUC__ == 9 && __GNUC_MINOR__ < 3 ) +#error Unsupported gcc version: please gcc >= 9.3 +#endif +#endif + +#endif // MGONGPUCONFIG_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h new file mode 100644 index 0000000000..d6e1d02e05 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h @@ -0,0 +1,745 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022, based on earlier work by D. Smith) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCXTYPES_H +#define MGONGPUCXTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +#include + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) HEADERS +//========================================================================== + +#include + +// Complex type in cuda: thrust or cucomplex or cxsmpl +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_THRUST +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) +#include +#pragma clang diagnostic pop +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX +#include +#elif not defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL +#endif +// Complex type in HIP: cxsmpl +#elif defined __HIPCC__ +#if not defined MGONGPU_HIPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_HIPCXTYPE_CXSMPL +#endif +#else +// Complex type in c++ or HIP: std::complex or cxsmpl +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX +#include +#elif not defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL +#endif +#endif + +//========================================================================== +// COMPLEX TYPES: INSTRUMENTED CUCOMPLEX CLASS (cucomplex) +//========================================================================== + +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_CUCOMPLEX +namespace mg5amcGpu +{ +#if defined MGONGPU_FPTYPE_DOUBLE + class cucomplex + { + public: + __host__ __device__ cucomplex( const double& r = 0, const double& i = 0 ) + : m_ri( make_cuDoubleComplex( r, i ) ) {} + __host__ __device__ constexpr cucomplex( const cuDoubleComplex& ri ) + : m_ri( ri ) {} + //__host__ __device__ operator cuDoubleComplex&() { return m_ri; } + __host__ __device__ constexpr operator cuDoubleComplex() const { return m_ri; } + __host__ __device__ double real() const { return cuCreal( m_ri ); } + __host__ __device__ double imag() const { return cuCimag( m_ri ); } + inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) + { + m_ri = cuCadd( m_ri, c ); + return *this; + } + inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) + { + m_ri = cuCsub( m_ri, c ); + return *this; + } + private: + cuDoubleComplex m_ri; + }; +#elif defined MGONGPU_FPTYPE_FLOAT + class cucomplex + { + public: + __host__ __device__ cucomplex( const float& r = 0, const float& i = 0 ) + : m_ri( make_cuFloatComplex( r, i ) ) {} + __host__ __device__ constexpr cucomplex( const cuFloatComplex& ri ) + : m_ri( ri ) {} + //__host__ __device__ operator cuFloatComplex&() { return m_ri; } + __host__ __device__ constexpr operator cuFloatComplex() const { return m_ri; } + __host__ __device__ float real() const { return cuCrealf( m_ri ); } + __host__ __device__ float imag() const { return cuCimagf( m_ri ); } + inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) + { + m_ri = cuCaddf( m_ri, c ); + return *this; + } + inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) + { + m_ri = cuCsubf( m_ri, c ); + return *this; + } + private: + cuFloatComplex m_ri; + }; +#endif +} +#endif +#endif + +//========================================================================== +// COMPLEX TYPES: SIMPLE COMPLEX CLASS (cxsmpl) +//========================================================================== + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu /* clang-format off */ +{ + // The number of floating point types in a complex type (real, imaginary) + constexpr int nx2 = 2; + + // --- Type definition (simple complex type derived from cxtype_v) + template + class cxsmpl + { + public: + __host__ __device__ constexpr cxsmpl() : m_real( 0 ), m_imag( 0 ) {} + cxsmpl( const cxsmpl& ) = default; + cxsmpl( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl( const FP& r, const FP& i = 0 ) : m_real( r ), m_imag( i ) {} + __host__ __device__ constexpr cxsmpl( const std::complex& c ) : m_real( c.real() ), m_imag( c.imag() ) {} + cxsmpl& operator=( const cxsmpl& ) = default; + cxsmpl& operator=( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl& operator+=( const cxsmpl& c ) { m_real += c.real(); m_imag += c.imag(); return *this; } + __host__ __device__ constexpr cxsmpl& operator-=( const cxsmpl& c ) { m_real -= c.real(); m_imag -= c.imag(); return *this; } + __host__ __device__ constexpr const FP& real() const { return m_real; } + __host__ __device__ constexpr const FP& imag() const { return m_imag; } + template __host__ __device__ constexpr operator cxsmpl() const { return cxsmpl( m_real, m_imag ); } +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef MGONGPU_CUCXTYPE_THRUST + template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX + __host__ __device__ constexpr operator mg5amcGpu::cucomplex() const { return mg5amcGpu::cucomplex( m_real, m_imag ); } +#endif +#else +#ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX + template __host__ __device__ constexpr operator std::complex() const { return std::complex( m_real, m_imag ); } +#endif +#endif + private: + FP m_real, m_imag; // RI + }; + + template + constexpr // (NB: now valid code? in the past this failed as "a constexpr function cannot have a nonliteral return type mgOnGpu::cxsmpl") + inline __host__ __device__ cxsmpl + conj( const cxsmpl& c ) + { + return cxsmpl( c.real(), -c.imag() ); + } +} /* clang-format on */ + +// Expose the cxsmpl class outside the namespace +using mgOnGpu::cxsmpl; + +// Printout to stream for user defined types +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + template + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxsmpl& c ) + { + //out << std::complex( c.real(), c.imag() ); + out << "(" << c.real() << ", " << c.imag() << ")"; // add a space after the comma + return out; + } + + // Operators for cxsmpl + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl a ) + { + return a; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a ) + { + return cxsmpl( -a.real(), -a.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() + b.real(), a.imag() + b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) + b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() - b.real(), a.imag() - b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) - b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const double& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const double& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const cxsmpl& b ) + { + FP bnorm = b.real() * b.real() + b.imag() * b.imag(); + return cxsmpl( ( a.real() * b.real() + a.imag() * b.imag() ) / bnorm, + ( a.imag() * b.real() - a.real() * b.imag() ) / bnorm ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) / b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const FP& b ) + { + return a + cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const FP& b ) + { + return a - cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const FP& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const FP& b ) + { + return a / cxsmpl( b, 0 ); + } +} + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // --- Type definitions (complex type: cxtype) +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CUCXTYPE_THRUST + typedef thrust::complex cxtype; +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX + typedef cucomplex cxtype; +#else + typedef cxsmpl cxtype; +#endif +#else // c++ +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX + typedef std::complex cxtype; +#else + typedef cxsmpl cxtype; +#endif +#endif + + // SANITY CHECK: memory access may be based on casts of fptype[2] to cxtype (e.g. for wavefunctions) + static_assert( sizeof( cxtype ) == mgOnGpu::nx2 * sizeof( fptype ), "sizeof(cxtype) is not 2*sizeof(fptype)" ); +} + +// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) +// DO NOT expose typedefs and operators outside the namespace +//using mgOnGpu::cxtype; + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL + + //------------------------------ + // CUDA or C++ - using cxsmpl + //------------------------------ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // cxsmpl constructor + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return c.real(); // cxsmpl::real() + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return c.imag(); // cxsmpl::imag() + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( cxsmpl ) + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +#endif // #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL + + //========================================================================== + +#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST // cuda + thrust (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) + + //------------------------------ + // CUDA - using thrust::complex + //------------------------------ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // thrust::complex constructor + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return c.real(); // thrust::complex::real() + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return c.imag(); // thrust::complex::imag() + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( thrust::complex ) + } + + inline __host__ __device__ const cxtype& + cxmake( const cxtype& c ) + { + return c; + } + +#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST + + //========================================================================== + +#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX // cuda + cucomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) + + //------------------------------ + // CUDA - using cuComplex + //------------------------------ + +#if defined MGONGPU_FPTYPE_DOUBLE // cuda + cucomplex + double + + //+++++++++++++++++++++++++ + // cuDoubleComplex ONLY + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return make_cuDoubleComplex( r, i ); + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return cuCreal( c ); // returns by value + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return cuCimag( c ); // returns by value + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const cxtype& b ) + { + return cuCadd( a, b ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const cxtype& b ) + { + return cuCsub( a, b ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const cxtype& b ) + { + return cuCmul( a, b ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const cxtype& b ) + { + return cuCdiv( a, b ); + } + + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + //out << std::complex( cxreal( c ), cximag( c ) ); + out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma + return out; + } + +#elif defined MGONGPU_FPTYPE_FLOAT // cuda + cucomplex + float + + //+++++++++++++++++++++++++ + // cuFloatComplex ONLY + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return make_cuFloatComplex( r, i ); + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return cuCrealf( c ); // returns by value + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return cuCimagf( c ); // returns by value + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const cxtype& b ) + { + return cuCaddf( a, b ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const cxtype& b ) + { + return cuCsubf( a, b ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const cxtype& b ) + { + return cuCmulf( a, b ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const cxtype& b ) + { + return cuCdivf( a, b ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cucomplex (cast double-to-float) + { + return cxmake( (fptype)c.real(), (fptype)c.imag() ); + } + + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + //out << std::complex( cxreal( c ), cximag( c ) ); + out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma + return out; + } + +#endif + + //+++++++++++++++++++++++++ + // cuDoubleComplex OR + // cuFloatComplex + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + operator+( const cxtype a ) + { + return a; + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a ) + { + return cxmake( -cxreal( a ), -cximag( a ) ); + } + + inline __host__ __device__ cxtype + operator+( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) + b; + } + + inline __host__ __device__ cxtype + operator-( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) - b; + } + + inline __host__ __device__ cxtype + operator*( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) * b; + } + + inline __host__ __device__ cxtype + operator/( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) / b; + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const fptype& b ) + { + return a + cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const fptype& b ) + { + return a - cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const fptype& b ) + { + return a * cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const fptype& b ) + { + return a / cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return cxmake( cxreal( c ), -cximag( c ) ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cucomplex (float-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX + + //========================================================================== + +#if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX // c++/hip + stdcomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) + + //------------------------------ + // C++ - using std::complex + //------------------------------ + + inline cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // std::complex constructor + } + + inline fptype + cxreal( const cxtype& c ) + { + return c.real(); // std::complex::real() + } + + inline fptype + cximag( const cxtype& c ) + { + return c.imag(); // std::complex::imag() + } + + inline cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( std::complex ) + } + + inline const cxtype& + cxmake( const cxtype& c ) // std::complex to std::complex (float-to-float or double-to-double) + { + return c; + } + +#if defined MGONGPU_FPTYPE_FLOAT + inline cxtype + cxmake( const std::complex& c ) // std::complex to std::complex (cast double-to-float) + { + return cxmake( (fptype)c.real(), (fptype)c.imag() ); + } +#endif + +#endif // #if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX + + //========================================================================== + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +} // end namespace mg5amcGpu/mg5amcCpu + +//========================================================================== +// COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] + // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined + // It is now always included in the code because it is needed also to access an fptype wavefunction buffer as a cxtype + class cxtype_ref + { + public: + cxtype_ref() = delete; + cxtype_ref( const cxtype_ref& ) = delete; + cxtype_ref( cxtype_ref&& ) = default; // copy const refs + __host__ __device__ cxtype_ref( fptype& r, fptype& i ) + : m_preal( &r ), m_pimag( &i ) {} // copy (create from) const refs + cxtype_ref& operator=( const cxtype_ref& ) = delete; + //__host__ __device__ cxtype_ref& operator=( cxtype_ref&& c ) {...} // REMOVED! Should copy refs or copy values? No longer needed in cxternary + __host__ __device__ cxtype_ref& operator=( const cxtype& c ) + { + *m_preal = cxreal( c ); + *m_pimag = cximag( c ); + return *this; + } // copy (assign) non-const values + __host__ __device__ operator cxtype() const { return cxmake( *m_preal, *m_pimag ); } + private: + fptype* const m_preal; // const pointer to non-const fptype R + fptype* const m_pimag; // const pointer to non-const fptype I + }; + + // Printout to stream for user defined types + inline __host__ __device__ std::ostream& + operator<<( std::ostream& out, const cxtype_ref& c ) + { + out << (cxtype)c; + return out; + } + +} // end namespace mg5amcGpu/mg5amcCpu + +//========================================================================== + +#endif // MGONGPUCXTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h new file mode 100644 index 0000000000..a2489d05d5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h @@ -0,0 +1,102 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUFPTYPES_H +#define MGONGPUFPTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include +#include + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL // cuda +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + //========================================================================== + +#ifdef MGONGPUCPP_GPUIMPL // cuda + + //------------------------------ + // Floating point types - Cuda + //------------------------------ + + /* + inline __host__ __device__ fptype + fpmax( const fptype& a, const fptype& b ) + { + return max( a, b ); + } + + inline __host__ __device__ fptype + fpmin( const fptype& a, const fptype& b ) + { + return min( a, b ); + } + */ + + inline __host__ __device__ const fptype& + fpmax( const fptype& a, const fptype& b ) + { + return ( ( b < a ) ? a : b ); + } + + inline __host__ __device__ const fptype& + fpmin( const fptype& a, const fptype& b ) + { + return ( ( a < b ) ? a : b ); + } + + inline __host__ __device__ fptype + fpsqrt( const fptype& f ) + { +#if defined MGONGPU_FPTYPE_FLOAT + // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__SINGLE.html + return sqrtf( f ); +#else + // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__DOUBLE.html + return sqrt( f ); +#endif + } + +#endif // #ifdef MGONGPUCPP_GPUIMPL + + //========================================================================== + +#ifndef MGONGPUCPP_GPUIMPL + + //------------------------------ + // Floating point types - C++ + //------------------------------ + + inline const fptype& + fpmax( const fptype& a, const fptype& b ) + { + return std::max( a, b ); + } + + inline const fptype& + fpmin( const fptype& a, const fptype& b ) + { + return std::min( a, b ); + } + + inline fptype + fpsqrt( const fptype& f ) + { + return std::sqrt( f ); + } + +#endif // #ifndef MGONGPUCPP_GPUIMPL + + //========================================================================== + +} // end namespace mg5amcGpu/mg5amcCpu + +#endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h new file mode 100644 index 0000000000..74d93f05a3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h @@ -0,0 +1,932 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Nov 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi, Z. Wettersten (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUVECTORS_H +#define MGONGPUVECTORS_H 1 + +#include "mgOnGpuCxtypes.h" +#include "mgOnGpuFptypes.h" + +#include + +//========================================================================== + +//------------------------------ +// Vector types - C++ +//------------------------------ + +#ifdef __clang__ +// If set: return a pair of (fptype&, fptype&) by non-const reference in cxtype_v::operator[] +// This is forbidden in clang ("non-const reference cannot bind to vector element") +// See also https://stackoverflow.com/questions/26554829 +//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // clang test (compilation fails also on clang 12.0, issue #182) +#undef MGONGPU_HAS_CPPCXTYPEV_BRK // clang default +#elif defined __INTEL_COMPILER +//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // icc default? +#undef MGONGPU_HAS_CPPCXTYPEV_BRK // icc test +#else +#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // gcc default +//#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) +#endif + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#ifdef MGONGPU_CPPSIMD + + const int neppV = MGONGPU_CPPSIMD; + + // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) + static_assert( mgOnGpu::cppAlign % ( neppV * sizeof( fptype ) ) == 0 ); + + // SANITY CHECK: check that neppV is a power of two + static_assert( ispoweroftwo( neppV ), "neppV is not a power of 2" ); + + // --- Type definition (using vector compiler extensions: need -march=...) + // For gcc: https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html + // For clang: https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors +#ifdef __clang__ + typedef fptype fptype_v __attribute__( ( ext_vector_type( neppV ) ) ); // RRRR +#else + typedef fptype fptype_v __attribute__( ( vector_size( neppV * sizeof(fptype) ), aligned( neppV * sizeof(fptype) ) ) ); // RRRR +#endif + + // Mixed fptypes #537: float for color algebra and double elsewhere +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int neppV2 = MGONGPU_CPPSIMD * 2; + static_assert( mgOnGpu::cppAlign % ( neppV2 * sizeof( fptype2 ) ) == 0 ); + static_assert( ispoweroftwo( neppV2 ), "neppV2 is not a power of 2" ); +#ifdef __clang__ + typedef fptype2 fptype2_v __attribute__( ( ext_vector_type( neppV2 ) ) ); // RRRRRRRR +#else + typedef fptype2 fptype2_v __attribute__( ( vector_size( neppV2 * sizeof( fptype2 ) ), aligned( neppV2 * sizeof( fptype2 ) ) ) ); // RRRRRRRR +#endif +#else + typedef fptype_v fptype2_v; +#endif + + // --- Type definition (using vector compiler extensions: need -march=...) + class cxtype_v // no need for "class alignas(2*sizeof(fptype_v)) cxtype_v" + { + public: + // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) + // See https://en.cppreference.com/w/c/language/array_initialization#Notes + cxtype_v() + : m_real{ 0 }, m_imag{ 0 } {} // RRRR=0000 IIII=0000 + cxtype_v( const cxtype_v& ) = default; + cxtype_v( cxtype_v&& ) = default; + cxtype_v( const fptype_v& r, const fptype_v& i ) + : m_real( r ), m_imag( i ) {} + cxtype_v( const fptype_v& r ) + : m_real( r ), m_imag{ 0 } {} // IIII=0000 + cxtype_v( const fptype& r ) + : m_real( fptype_v{} + r ), m_imag{ 0 } {} // IIII=0000 + cxtype_v& operator=( const cxtype_v& ) = default; + cxtype_v& operator=( cxtype_v&& ) = default; + cxtype_v& operator+=( const cxtype_v& c ) + { + m_real += c.real(); + m_imag += c.imag(); + return *this; + } + cxtype_v& operator-=( const cxtype_v& c ) + { + m_real -= c.real(); + m_imag -= c.imag(); + return *this; + } +#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK + // NB: THIS IS THE FUNDAMENTAL DIFFERENCE BETWEEN MGONGPU_HAS_CPPCXTYPEV_BRK DEFINED AND NOT DEFINED + // NB: the alternative "clang" implementation is simpler: it simply does not have any bracket operator[] + //cxtype_ref operator[]( size_t i ) const { return cxtype_ref( m_real[i], m_imag[i] ); } // gcc14.2 build fails #1004 + cxtype_ref operator[]( size_t i ) { return cxtype_ref( m_real[i], m_imag[i] ); } + cxtype operator[]( size_t i ) const { return cxtype( m_real[i], m_imag[i] ); } +#endif + const fptype_v& real() const + { + return m_real; + } + const fptype_v& imag() const { return m_imag; } + private: + fptype_v m_real, m_imag; // RRRRIIII + }; + + // --- Type definition (using vector compiler extensions: need -march=...) +#ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors + typedef unsigned int uint_v __attribute__( ( ext_vector_type( neppV ) ) ); +#if defined MGONGPU_FPTYPE_DOUBLE + typedef long int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb +#elif defined MGONGPU_FPTYPE_FLOAT + typedef int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb +#endif +#else // gcc + typedef unsigned int uint_v __attribute__( ( vector_size( neppV * sizeof( unsigned int ) ), aligned( neppV * sizeof( unsigned int ) ) ) ); +#if defined MGONGPU_FPTYPE_DOUBLE + typedef long int bool_v __attribute__( ( vector_size( neppV * sizeof( long int ) ), aligned( neppV * sizeof( long int ) ) ) ); // bbbb +#elif defined MGONGPU_FPTYPE_FLOAT + typedef int bool_v __attribute__( ( vector_size( neppV * sizeof( int ) ), aligned( neppV * sizeof( int ) ) ) ); // bbbb +#endif +#endif + +#else // i.e #ifndef MGONGPU_CPPSIMD (this includes #ifdef MGONGPUCPP_GPUIMPL) + + const int neppV = 1; + +#endif // #ifdef MGONGPU_CPPSIMD +} + +//-------------------------------------------------------------------------- + +// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) +// DO NOT expose typedefs outside the namespace +//using mgOnGpu::neppV; +//#ifdef MGONGPU_CPPSIMD +//using mgOnGpu::fptype_v; +//using mgOnGpu::fptype2_v; +//using mgOnGpu::cxtype_v; +//using mgOnGpu::bool_v; +//#endif + +//========================================================================== + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#ifndef MGONGPUCPP_GPUIMPL + + // Printout to stream for user defined types + +#ifndef MGONGPU_CPPCXTYPE_CXSMPL // operator<< for cxsmpl has already been defined! + inline std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + out << "[" << cxreal( c ) << "," << cximag( c ) << "]"; + //out << cxreal(c) << "+i" << cximag(c); + return out; + } +#endif + + /* +#ifdef MGONGPU_CPPSIMD + inline std::ostream& + operator<<( std::ostream& out, const bool_v& v ) + { + out << "{ " << v[0]; + for ( int i=1; i 0 ) outi = fpsqrt( (fptype)v[i] ); + out[i] = outi; + } + return out; + } + + inline fptype_v + fpsqrt( const fptype_v& v ) + { + // See https://stackoverflow.com/questions/18921049/gcc-vector-extensions-sqrt + fptype_v out = {}; // avoid warning 'out' may be used uninitialized: see #594 + for( int i = 0; i < neppV; i++ ) out[i] = fpsqrt( v[i] ); + return out; + } +#endif + + /* +#ifdef MGONGPU_CPPSIMD + inline fptype_v + fpvmake( const fptype v[neppV] ) + { + fptype_v out = {}; // see #594 + for ( int i=0; i + +#ifdef MGONGPUCPP_GPUIMPL +using namespace mg5amcGpu; +#else +using namespace mg5amcCpu; +#endif + +struct CUDA_CPU_TestBase : public TestDriverBase +{ + static constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static constexpr int np4 = CPPProcess::np4; + static constexpr int npar = CPPProcess::npar; + static_assert( gputhreads % neppM == 0, "ERROR! #threads/block should be a multiple of neppM" ); + static_assert( gputhreads <= mgOnGpu::ntpbMAX, "ERROR! #threads/block should be <= ntpbMAX" ); + CUDA_CPU_TestBase( const std::string& refFileName ) + : TestDriverBase( npar, refFileName ) {} + // Does this test use channelIds? + virtual bool useChannelIds() const = 0; + // Set channelId array (in the same way for CUDA and CPU tests) + static constexpr unsigned int warpSize = 32; // FIXME: add a sanity check in madevent that this is the minimum? (would need to expose this from cudacpp to madevent) + static void setChannelIds( BufferChannelIds& hstChannelIds, std::size_t iiter ) + { + static const char* debugC = getenv( "CUDACPP_RUNTEST_DEBUG" ); + static const bool debug = ( debugC != 0 ) && ( std::string( debugC ) != "" ); + // Fill channelIds for multi-channel tests #896 + // (NB: these are only used if useChannelIds == true) + // TEMPORARY(0): debug multichannel tests with channelId=1 for all events + //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1; + // TEMPORARY(1): debug multichannel tests with channelId=1,2,..,ndiag,1,2,..ndiag,... (every event gets a different channel, no warps) + //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1 + i % CPPProcess::ndiagrams; + // ALMOST FINAL test implementation: 1111222233331111... (every 32-event warp gets a different channel) + // FINAL(?) test implementation: 2222333344442222... (every 32-event warp gets a different channel, skip those without associated iconfig #917) + static_assert( nevt % warpSize == 0, "ERROR! nevt should be a multiple of warpSize" ); + constexpr unsigned int nWarp = nevt / warpSize; + for( unsigned int iWarp = 0; iWarp < nWarp; ++iWarp ) + { + //const unsigned int channelId = 1 + ( iWarp + iiter * nWarp ) % CPPProcess::ndiagrams; // bug #917 + const int iconfig = 1 + ( iWarp + iiter * nWarp ) % mgOnGpu::nconfigSDE; + unsigned int channelId = 0; + //for( unsigned int idiagram = 1; idiagram < CPPProcess::ndiagrams; idiagram++ ) // two bugs #920 and #919 + for( unsigned int idiagram = 0; idiagram < mgOnGpu::nchannels; idiagram++ ) // fix #920 and work around #919 + { + if( mgOnGpu::hostChannel2iconfig[idiagram] == iconfig ) + { + channelId = idiagram + 1; // fix #917 (NB add +1 because channelId uses F indexing) + break; + } + } + assert( channelId > 0 ); // sanity check that the channelId for the given iconfig was found + if( debug ) std::cout << "CUDA_CPU_TestBase::setChannelIds: iWarp=" << iWarp << ", iconfig=" << iconfig << ", channelId=" << channelId << std::endl; + for( unsigned int i = 0; i < warpSize; ++i ) + hstChannelIds[iWarp * warpSize + i] = channelId; + } + } +}; + +#ifndef MGONGPUCPP_GPUIMPL +struct CPUTest : public CUDA_CPU_TestBase +{ + // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) + // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] + CPPProcess process; + HostBufferRndNumMomenta hstRndMom; + HostBufferChannelIds hstChannelIds; + HostBufferMomenta hstMomenta; + HostBufferGs hstGs; + HostBufferRndNumHelicity hstRndHel; + HostBufferRndNumColor hstRndCol; + HostBufferWeights hstWeights; + HostBufferMatrixElements hstMatrixElements; + HostBufferSelectedHelicity hstSelHel; + HostBufferSelectedColor hstSelCol; + HostBufferHelicityMask hstIsGoodHel; + std::unique_ptr pmek; + + // Create a process object + // Read param_card and set parameters + // ** WARNING EVIL EVIL ** + // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. + // Don't remove! + CPUTest( const std::string& refFileName ) + : CUDA_CPU_TestBase( refFileName ) + , process( /*verbose=*/false ) + , hstRndMom( nevt ) + , hstChannelIds( nevt ) + , hstMomenta( nevt ) + , hstGs( nevt ) + , hstRndHel( nevt ) + , hstRndCol( nevt ) + , hstWeights( nevt ) + , hstMatrixElements( nevt ) + , hstSelHel( nevt ) + , hstSelCol( nevt ) + , hstIsGoodHel( CPPProcess::ncomb ) + , pmek( new MatrixElementKernelHost( hstMomenta, hstGs, hstRndHel, hstRndCol, hstChannelIds, hstMatrixElements, hstSelHel, hstSelCol, nevt ) ) + { + // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? + // FIXME: the CPPProcess should really be a singleton? + process.initProc( "../../Cards/param_card.dat" ); + } + + virtual ~CPUTest() {} + + void prepareRandomNumbers( unsigned int iiter ) override + { + // Random numbers for momenta + CommonRandomNumberKernel rnk( hstRndMom ); + rnk.seedGenerator( 1337 + iiter ); + rnk.generateRnarray(); + // Random numbers for helicity and color selection (fix #931) + CommonRandomNumberKernel rnk2( hstRndHel ); + rnk2.seedGenerator( 1338 + iiter ); + rnk2.generateRnarray(); + CommonRandomNumberKernel rnk3( hstRndCol ); + rnk3.seedGenerator( 1339 + iiter ); + rnk3.generateRnarray(); + } + + void prepareMomenta( fptype energy ) override + { + RamboSamplingKernelHost rsk( energy, hstRndMom, hstMomenta, hstWeights, nevt ); + // --- 2a. Fill in momenta of initial state particles on the device + rsk.getMomentaInitial(); + // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device + // (i.e. map random numbers to final-state particle momenta for each of nevt events) + rsk.getMomentaFinal(); + } + + void runSigmaKin( std::size_t iiter ) override + { + constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) + for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; + setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 + if( iiter == 0 ) pmek->computeGoodHelicities(); + pmek->computeMatrixElements( useChannelIds() ); + } + + fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override + { + assert( ipar < npar ); + assert( ip4 < np4 ); + return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); + } + + fptype getMatrixElement( std::size_t ievt ) const override + { + return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); + } + + int getChannelId( std::size_t ievt ) const override + { + return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); + } + + int getSelectedHelicity( std::size_t ievt ) const override + { + //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... + return hstSelHel.data()[ievt]; + } + + int getSelectedColor( std::size_t ievt ) const override + { + //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... + return hstSelCol.data()[ievt]; + } +}; + +// Old test with multi-channel disabled #466 +struct CPUTestNoMultiChannel : public CPUTest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return false; } + + // Constructor + CPUTestNoMultiChannel( const std::string& refFileName ) + : CPUTest( refFileName ) {} // suffix .txt + + // Destructor + virtual ~CPUTestNoMultiChannel() {} +}; + +// New test with multi-channel enabled #896 +struct CPUTestMultiChannel : public CPUTest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return true; } + + // Constructor + CPUTestMultiChannel( const std::string& refFileName ) + : CPUTest( refFileName + "2" ) {} // suffix .txt2 + + // Destructor + virtual ~CPUTestMultiChannel() {} +}; +#endif + +#ifdef MGONGPUCPP_GPUIMPL +struct CUDATest : public CUDA_CPU_TestBase +{ + // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) + // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] + CPPProcess process; + PinnedHostBufferRndNumMomenta hstRndMom; + PinnedHostBufferMomenta hstMomenta; + PinnedHostBufferGs hstGs; + PinnedHostBufferRndNumHelicity hstRndHel; + PinnedHostBufferRndNumColor hstRndCol; + PinnedHostBufferWeights hstWeights; + PinnedHostBufferChannelIds hstChannelIds; + PinnedHostBufferMatrixElements hstMatrixElements; + PinnedHostBufferSelectedHelicity hstSelHel; + PinnedHostBufferSelectedColor hstSelCol; + PinnedHostBufferHelicityMask hstIsGoodHel; + DeviceBufferRndNumMomenta devRndMom; + DeviceBufferChannelIds devChannelIds; + DeviceBufferMomenta devMomenta; + DeviceBufferGs devGs; + DeviceBufferRndNumHelicity devRndHel; + DeviceBufferRndNumColor devRndCol; + DeviceBufferWeights devWeights; + DeviceBufferMatrixElements devMatrixElements; + DeviceBufferSelectedHelicity devSelHel; + DeviceBufferSelectedColor devSelCol; + DeviceBufferHelicityMask devIsGoodHel; + std::unique_ptr pmek; + + // Create a process object + // Read param_card and set parameters + // ** WARNING EVIL EVIL ** + // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. + // Don't remove! + CUDATest( const std::string& refFileName ) + : CUDA_CPU_TestBase( refFileName ) + , process( /*verbose=*/false ) + , hstRndMom( nevt ) + , hstChannelIds( nevt ) + , hstMomenta( nevt ) + , hstGs( nevt ) + , hstRndHel( nevt ) + , hstRndCol( nevt ) + , hstWeights( nevt ) + , hstMatrixElements( nevt ) + , hstSelHel( nevt ) + , hstSelCol( nevt ) + , hstIsGoodHel( CPPProcess::ncomb ) + , devRndMom( nevt ) + , devChannelIds( nevt ) + , devMomenta( nevt ) + , devGs( nevt ) + , devRndHel( nevt ) + , devRndCol( nevt ) + , devWeights( nevt ) + , devMatrixElements( nevt ) + , devSelHel( nevt ) + , devSelCol( nevt ) + , devIsGoodHel( CPPProcess::ncomb ) + , pmek( new MatrixElementKernelDevice( devMomenta, devGs, devRndHel, devRndCol, devChannelIds, devMatrixElements, devSelHel, devSelCol, gpublocks, gputhreads ) ) + { + // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? + // FIXME: the CPPProcess should really be a singleton? + process.initProc( "../../Cards/param_card.dat" ); + } + + virtual ~CUDATest() {} + + void prepareRandomNumbers( unsigned int iiter ) override + { + // Random numbers for momenta + CommonRandomNumberKernel rnk( hstRndMom ); + rnk.seedGenerator( 1337 + iiter ); + rnk.generateRnarray(); + copyDeviceFromHost( devRndMom, hstRndMom ); + // Random numbers for helicity and color selection (fix #931) + CommonRandomNumberKernel rnk2( hstRndHel ); + rnk2.seedGenerator( 1338 + iiter ); + rnk2.generateRnarray(); + copyDeviceFromHost( devRndHel, hstRndHel ); + CommonRandomNumberKernel rnk3( hstRndCol ); + rnk3.seedGenerator( 1339 + iiter ); + rnk3.generateRnarray(); + copyDeviceFromHost( devRndCol, hstRndCol ); + } + + void prepareMomenta( fptype energy ) override + { + RamboSamplingKernelDevice rsk( energy, devRndMom, devMomenta, devWeights, gpublocks, gputhreads ); + // --- 2a. Fill in momenta of initial state particles on the device + rsk.getMomentaInitial(); + // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device + // (i.e. map random numbers to final-state particle momenta for each of nevt events) + rsk.getMomentaFinal(); + // --- 2c. CopyDToH Weights + copyHostFromDevice( hstWeights, devWeights ); + // --- 2d. CopyDToH Momenta + copyHostFromDevice( hstMomenta, devMomenta ); + } + + void runSigmaKin( std::size_t iiter ) override + { + constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) + for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; + copyDeviceFromHost( devGs, hstGs ); // BUG FIX #566 + setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 + copyDeviceFromHost( devChannelIds, hstChannelIds ); + if( iiter == 0 ) pmek->computeGoodHelicities(); + pmek->computeMatrixElements( useChannelIds() ); + copyHostFromDevice( hstMatrixElements, devMatrixElements ); + copyHostFromDevice( hstSelHel, devSelHel ); + copyHostFromDevice( hstSelCol, devSelCol ); + } + + fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override + { + assert( ipar < npar ); + assert( ip4 < np4 ); + return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); + } + + fptype getMatrixElement( std::size_t ievt ) const override + { + return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); + } + + int getChannelId( std::size_t ievt ) const override + { + return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); + } + + int getSelectedHelicity( std::size_t ievt ) const override + { + //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... + return hstSelHel.data()[ievt]; + } + + int getSelectedColor( std::size_t ievt ) const override + { + //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... + return hstSelCol.data()[ievt]; + } +}; + +// Old test with multi-channel disabled #466 +struct CUDATestNoMultiChannel : public CUDATest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return false; } + + // Constructor + CUDATestNoMultiChannel( const std::string& refFileName ) + : CUDATest( refFileName ) {} // suffix .txt + + // Destructor + virtual ~CUDATestNoMultiChannel() {} +}; + +// New test with multi-channel enabled #896 +struct CUDATestMultiChannel : public CUDATest +{ + // Does this test use channelIds? + bool useChannelIds() const override final { return true; } + + // Constructor + CUDATestMultiChannel( const std::string& refFileName ) + : CUDATest( refFileName + "2" ) {} // suffix .txt2 + + // Destructor + virtual ~CUDATestMultiChannel() {} +}; +#endif /* clang-format off */ + +// AV July 2024 much simpler class structure without the presently-unnecessary googletest templates +// This is meant as a workaround to prevent not-understood segfault #907 when adding a second test +// Note: instantiate test2 first and test1 second to ensure that the channelid printout from the dtors comes from test1 first and test2 second +#ifdef MGONGPUCPP_GPUIMPL +// CUDA test drivers +CUDATestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID2( s ) s##_GPU_MULTICHANNEL +CUDATestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID1( s ) s##_GPU_NOMULTICHANNEL +#else +// CPU test drivers +CPUTestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID2( s ) s##_CPU_MULTICHANNEL +CPUTestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); +#define TESTID1( s ) s##_CPU_NOMULTICHANNEL +#endif +// Madgraph tests +MadgraphTest mgTest2( driver2 ); +MadgraphTest mgTest1( driver1 ); +// Instantiate Google test 1 +#define XTESTID1( s ) TESTID1( s ) +TEST( XTESTID1( MG_EPOCH_PROCESS_ID ), compareMomAndME ) +{ +#ifdef MGONGPU_CHANNELID_DEBUG + driver1.pmek->setTagForNevtProcessedByChannel( "(no multichannel)" ); +#endif + mgTest1.CompareMomentaAndME( *this ); +} +// Instantiate Google test 2 +#define XTESTID2( s ) TESTID2( s ) +TEST( XTESTID2( MG_EPOCH_PROCESS_ID ), compareMomAndME ) +{ +#ifdef MGONGPU_CHANNELID_DEBUG + driver2.pmek->setTagForNevtProcessedByChannel( "(channelid array)" ); +#endif + mgTest2.CompareMomentaAndME( *this ); +} +/* clang-format on */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc new file mode 100644 index 0000000000..74c4902dc5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc @@ -0,0 +1,517 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. +// ---------------------------------------------------------------------------- +// Use ./runTest.exe --gtest_filter=*misc to run only testmisc.cc tests +// ---------------------------------------------------------------------------- + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#include "constexpr_math.h" +#include "epoch_process_id.h" +#include "valgrind.h" + +#include + +//#include +//#include // needs C++20... https://stackoverflow.com/a/65347016 +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +#define TESTID( s ) s##_GPU_MISC +#else +#define TESTID( s ) s##_CPU_MISC +#endif + +#define XTESTID( s ) TESTID( s ) + +// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ +#ifdef MGONGPU_CPPSIMD /* clang-format off */ +#define EXPECT_TRUE_sv( cond ) { bool_v mask( cond ); EXPECT_TRUE( maskand( mask ) ); } +#else +#define EXPECT_TRUE_sv( cond ) { EXPECT_TRUE( cond ); } +#endif /* clang-format on */ + + inline const std::string + boolTF( const bool& b ) + { + return ( b ? "T" : "F" ); + } + +#ifdef MGONGPU_CPPSIMD + inline const std::string + boolTF( const bool_v& v ) + { + std::stringstream out; + out << "{ " << ( v[0] ? "T" : "F" ); + for( int i = 1; i < neppV; i++ ) out << ", " << ( v[i] ? "T" : "F" ); + out << " }"; + return out.str(); + } +#endif +} + +TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + + //-------------------------------------------------------------------------- + + EXPECT_TRUE( true ); + + //-------------------------------------------------------------------------- + + // Vector initialization for fptype_sv + { + fptype_sv f{ 0 }; + EXPECT_TRUE_sv( f == 0 ); + } + { + fptype_sv f = fptype_sv{ 0 }; + EXPECT_TRUE_sv( f == 0 ); + } + + // Vector initialization for fptype_sv - demonstrate bug #339 in older cxmake implementation + { + fptype_sv f{ 1 }; + //std::cout << f << std::endl << boolTF( f == 1 ) << std::endl; + //EXPECT_TRUE_sv( f == 1 ); // this fails for vectors! TFFF +#ifndef MGONGPU_CPPSIMD + EXPECT_TRUE_sv( f == 1 ); // this succeds: T +#else + EXPECT_TRUE( ( f == 1 )[0] ); // this succeds: TFFF[0] + EXPECT_TRUE( ( f[0] == 1 ) ); + for( int i = 1; i < neppV; i++ ) + { + EXPECT_TRUE( !( ( f == 1 )[i] ) ); // this succeds: FTTT[i>=1] + EXPECT_TRUE( ( f[i] == 0 ) ); // equals 0, not 1 + } +#endif + } + +#ifdef MGONGPU_CPPSIMD + // Vector initialization for cxtype_sv - demonstrate fix for bug #339 + { + fptype_sv f1 = fptype_v{ 0 } + 1; + EXPECT_TRUE_sv( f1 == 1 ); + cxtype_v c12 = cxmake( f1, 2 ); + //std::cout << c12 << std::endl << boolTF( c12.real() == 1 ) << std::endl << boolTF( c12.imag() == 2 ) << std::endl; + EXPECT_TRUE_sv( c12.real() == 1 ); + EXPECT_TRUE_sv( c12.imag() == 2 ); + cxtype_v c21 = cxmake( 2, f1 ); + //std::cout << c21 << std::endl << boolTF( c21.real() == 2 ) << std::endl << boolTF( c21.imag() == 1 ) << std::endl; + EXPECT_TRUE_sv( c21.real() == 2 ); + EXPECT_TRUE_sv( c21.imag() == 1 ); + } +#endif + + // Vector initialization for cxtype_sv + { + cxtype_sv c = cxzero_sv(); + EXPECT_TRUE_sv( c.real() == 0 ); + EXPECT_TRUE_sv( c.imag() == 0 ); + } + { + cxtype_sv c = cxmake( 1, fptype_sv{ 0 } ); // here was a bug #339 + EXPECT_TRUE_sv( c.real() == 1 ); + EXPECT_TRUE_sv( c.imag() == 0 ); + } + { + cxtype_sv c = cxmake( fptype_sv{ 0 }, 1 ); // here was a bug #339 + EXPECT_TRUE_sv( c.real() == 0 ); + EXPECT_TRUE_sv( c.imag() == 1 ); + } + + // Array initialization for cxtype_sv array (example: jamp_sv in CPPProcess.cc) + { + cxtype_sv array[2] = {}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "= {}" is missing!) + //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; + EXPECT_TRUE_sv( array[0].real() == 0 ); + EXPECT_TRUE_sv( array[0].imag() == 0 ); + EXPECT_TRUE_sv( array[1].real() == 0 ); + EXPECT_TRUE_sv( array[1].imag() == 0 ); + } + + // Alternative array initialization for cxtype_sv array (example: was used for outwf in testxxx.cc) + { + cxtype_sv array[2]{}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "{}" is missing!) + //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; + EXPECT_TRUE_sv( array[0].real() == 0 ); + EXPECT_TRUE_sv( array[0].imag() == 0 ); + EXPECT_TRUE_sv( array[1].real() == 0 ); + EXPECT_TRUE_sv( array[1].imag() == 0 ); + } + + //-------------------------------------------------------------------------- + + // Scalar complex references + { + using namespace mgOnGpu; + // Refs to f1, f2 + fptype f1 = 1; + fptype f2 = 2; + cxtype_ref r12( f1, f2 ); // copy refs + //cxtype_ref r12a( r12 ); //deleted + cxtype_ref r12a( cxtype_ref( f1, f2 ) ); // copy refs + //cxtype_ref r12b = r12; // deleted + cxtype_ref r12b = cxtype_ref( f1, f2 ); // copy refs + EXPECT_TRUE( cxtype( r12 ).real() == 1 ); + EXPECT_TRUE( cxtype( r12 ).imag() == 2 ); + EXPECT_TRUE( cxtype( r12a ).real() == 1 ); + EXPECT_TRUE( cxtype( r12a ).imag() == 2 ); + EXPECT_TRUE( cxtype( r12b ).real() == 1 ); + EXPECT_TRUE( cxtype( r12b ).imag() == 2 ); + // Refs to f1c, f2c + fptype f1c = 0; + fptype f2c = 0; + cxtype_ref r12c( f1c, f2c ); + EXPECT_TRUE( cxtype( r12c ).real() == 0 ); + EXPECT_TRUE( cxtype( r12c ).imag() == 0 ); + //r12c = r12; // deleted + r12c = cxtype( r12 ); // copy values + EXPECT_TRUE( cxtype( r12c ).real() == 1 ); + EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); + // Update f1, f2 + f1 = 10; + f2 = 20; + EXPECT_TRUE( cxtype( r12 ).real() == 10 ); + EXPECT_TRUE( cxtype( r12 ).imag() == 20 ); + EXPECT_TRUE( cxtype( r12a ).real() == 10 ); + EXPECT_TRUE( cxtype( r12a ).imag() == 20 ); + EXPECT_TRUE( cxtype( r12b ).real() == 10 ); + EXPECT_TRUE( cxtype( r12b ).imag() == 20 ); + EXPECT_TRUE( cxtype( r12c ).real() == 1 ); // points to f1c, not to f1 + EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); // points to f2c, not to f2 + } + + // Vector complex references + { + using namespace mgOnGpu; + // Refs to f1, f2 + fptype_sv f1 = fptype_sv{ 0 } + 1; + fptype_sv f2 = fptype_sv{ 0 } + 2; + cxtype_sv_ref r12( f1, f2 ); // copy refs + //cxtype_sv_ref r12a( r12 ); //deleted + cxtype_sv_ref r12a( cxtype_sv_ref( f1, f2 ) ); // copy refs + //cxtype_sv_ref r12b = r12; // deleted + cxtype_sv_ref r12b = cxtype_sv_ref( f1, f2 ); // copy refs + EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 2 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 2 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 2 ); + // Refs to f1c, f2c + fptype_sv f1c = fptype_sv{ 0 }; + fptype_sv f2c = fptype_sv{ 0 }; + cxtype_sv_ref r12c( f1c, f2c ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 0 ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 0 ); + //r12c = r12; // deleted + r12c = cxtype_sv( r12 ); // copy values + EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); + // Update f1, f2 + f1 = fptype_sv{ 0 } + 10; + f2 = fptype_sv{ 0 } + 20; + EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 10 ); + EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 20 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 10 ); + EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 20 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 10 ); + EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 20 ); + EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); // points to f1c, not to f1 + EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); // points to f2c, not to f2 + } + + //-------------------------------------------------------------------------- + + // Boolean vector (mask) times FP vector + /* + // From https://github.com/madgraph5/madgraph4gpu/issues/765#issuecomment-1853672838 + channelids_sv = CHANNEL_ACCESS::kernelAccess( pchannelIds ); // the 4 channels in the SIMD vector + bool_sv mask_sv = ( channelids_sv == 1 ); + numerators_sv += mask_sv * cxabs2( amp_sv[0] ); + if( pchannelIds != nullptr ) denominators_sv += cxabs2( amp_sv[0] ); + */ + { + typedef bool_sv test_int_sv; // defined as scalar_or_vector of long int (FPTYPE=double) or int (FPTYPE=float) + test_int_sv channelids0_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) + test_int_sv channelids1_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) + fptype_sv absamp0_sv{}; // mimic cxabs2( amp_sv[0] ) + fptype_sv absamp1_sv{}; // mimic cxabs2( amp_sv[0] ) +#ifdef MGONGPU_CPPSIMD + for( int i = 0; i < neppV; i++ ) + { + channelids0_sv[i] = i; // 0123 + channelids1_sv[i] = i; // 1234 + absamp0_sv[i] = 10. + i; // 10. 11. 12. 13. + absamp1_sv[i] = 11. + i; // 11. 12. 13. 14. + } +#else + channelids0_sv = 0; + channelids1_sv = 1; + absamp0_sv = 10.; + absamp1_sv = 11.; +#endif + bool_sv mask0_sv = ( channelids0_sv % 2 == 0 ); // even channels 0123 -> TFTF (1010) + bool_sv mask1_sv = ( channelids1_sv % 2 == 0 ); // even channels 1234 -> FTFT (0101) + constexpr fptype_sv fpZERO_sv{}; // 0000 + //fptype_sv numerators0_sv = mask0_sv * absamp0_sv; // invalid operands to binary * ('__vector(4) long int' and '__vector(4) double') + fptype_sv numerators0_sv = fpternary( mask0_sv, absamp0_sv, fpZERO_sv ); // equivalent to "mask0_sv * absamp0_sv" + fptype_sv numerators1_sv = fpternary( mask1_sv, absamp1_sv, fpZERO_sv ); // equivalent to "mask1_sv * absamp1_sv" +#ifdef MGONGPU_CPPSIMD + //std::cout << "numerators0_sv: " << numerators0_sv << std::endl; + //std::cout << "numerators1_sv: " << numerators1_sv << std::endl; + for( int i = 0; i < neppV; i++ ) + { + // Values of numerators0_sv: 10.*1 11.*0 12.*1 13.*0 + if( channelids0_sv[i] % 2 == 0 ) // even channels + EXPECT_TRUE( numerators0_sv[i] == ( 10. + i ) ); + else // odd channels + EXPECT_TRUE( numerators0_sv[i] == 0. ); + // Values of numerators1_sv: 11.*0 12.*1 13.*0 14.*1 + if( channelids1_sv[i] % 2 == 0 ) // even channels + EXPECT_TRUE( numerators1_sv[i] == ( 11. + i ) ); + else // odd channels + EXPECT_TRUE( numerators1_sv[i] == 0. ); + } +#else + // Values of numerators0_sv: 10.*1 + EXPECT_TRUE( numerators0_sv == 10. ); + // Values of numerators1_sv: 11.*0 + EXPECT_TRUE( numerators1_sv == 0. ); +#endif + } + + //-------------------------------------------------------------------------- + + // Test constexpr floor + EXPECT_TRUE( constexpr_floor( 1.5 ) == 1 ); + EXPECT_TRUE( constexpr_floor( 0.5 ) == 0 ); + EXPECT_TRUE( constexpr_floor( -0.5 ) == -1 ); + EXPECT_TRUE( constexpr_floor( -1.5 ) == -2 ); + + // Test constexpr pow + EXPECT_TRUE( constexpr_pow( 10, 0 ) == 1 ); + EXPECT_TRUE( constexpr_pow( 10, 1 ) == 10 ); + EXPECT_TRUE( constexpr_pow( 10, 2 ) == 100 ); + EXPECT_NEAR( constexpr_pow( 10, -1 ), 0.1, 0.1 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10, -1 ) = " << constexpr_pow( 10, -1 ); + EXPECT_NEAR( constexpr_pow( 10, -2 ), 0.01, 0.01 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10, -2 ) = " << constexpr_pow( 10, -2 ); + EXPECT_NEAR( constexpr_pow( 100, 0.5 ), 10, 10 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 100, 0.5 ) = " << constexpr_pow( 100, 0.5 ); + EXPECT_NEAR( constexpr_pow( 100, -0.5 ), 0.1, 0.1 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 100, -0.5 ) = " << constexpr_pow( 100, -0.5 ); + EXPECT_NEAR( constexpr_pow( 10000, 0.25 ), 10, 10 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10000, 0.25 ) = " << constexpr_pow( 10000, 0.25 ); + EXPECT_NEAR( constexpr_pow( 10000, -0.25 ), 0.1, 0.1 * 1E-14 ) + << std::setprecision( 40 ) << "constexpr_pow( 10000, -0.25 ) = " << constexpr_pow( 10000, -0.25 ); + +#ifndef __aarch64__ // TO BE UNDERSTOOD? DISABLE CONSTEXPR_SQRT TESTS ON AARCH64 (#1064) + // Distance from the horizontal or vertical axis (i.e. from 0, pi/2, pi, or 3pi/2) + auto distance4 = []( const long double xx ) + { + const long double xx2 = mapIn0to2Pi( xx ); // in [0,2*pi) + const long double xx3 = xx2 - constexpr_floor( xx2 / constexpr_pi_by_2 ) * constexpr_pi_by_2; // in [0,pi/2) + const long double d0 = xx3; // distance from 0 + const long double d1 = constexpr_pi_by_2 - xx3; // distance from pi/2 + return ( d0 < d1 ? d0 : d1 ); + }; + + // Test constexpr sin, cos, tan - specific, problematic, points + auto testSinCosTanX = []( const long double xx, const double tolerance0, const bool debug = false, const long long istep = -999999999 ) + { + const double x = (double)xx; + const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 + if( debug ) + { + //std::cout << std::setprecision(40) << "testSinCosTanX: xx= " << xx << std::endl; + //std::cout << std::setprecision(40) << " x= " << x << std::endl; + } + //std::cout << std::setprecision(40) << "xx - 3pi/2 " << xx - 3 * constexpr_pi_by_2 << std::endl; + //int width = 46; + //char buf[128]; + //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)xx ); + //std::cout << std::setprecision(40) << "testSinCosTanX: xx=" << buf << std::endl; + //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)x ); + //std::cout << std::setprecision(40) << " x= " << buf << std::endl; + EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::abs( std::sin( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::abs( std::cos( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; +#ifndef __aarch64__ + if( !RUNNING_ON_VALGRIND ) + { + EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::abs( std::tan( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + } + else +#endif + { + // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) + const long double ctanx = constexpr_tan( x ); + const long double taninf = 4E14; // declare tan(x) as "infinity" if above this threshold + if( ctanx > -taninf && ctanx < taninf ) + EXPECT_NEAR( std::tan( x ), ctanx, std::abs( std::tan( x ) * tolerance ) ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + else + { + // Allow tan(x)=-inf if ctan(x)=+inf and viceversa + EXPECT_GT( std::abs( std::tan( x ) ), taninf ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + /* + // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) + if( ctanx > 0 ) + EXPECT_GT( std::tan( x ), taninf ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + else + EXPECT_LT( std::tan( x ), -taninf ) + << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; + */ + } + } + std::cout << std::setprecision( 6 ); // default + }; + testSinCosTanX( constexpr_pi, 1E-3, true ); // from math.h + testSinCosTanX( (long double)3.141592653589793238462643383279502884L, 1E-3, true ); // from math.h + testSinCosTanX( 4.712388980384687897640105802565813064575L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) + testSinCosTanX( 3 * constexpr_pi_by_2 - 1.96e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x!=xx) + testSinCosTanX( 3 * constexpr_pi_by_2 - 1.9601e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) + + // Test constexpr sin, cos, tan - 8 points on (or close to) the boundaries of the 8 sectors of [0,2*pi] + auto testSinCosTan8 = [testSinCosTanX]( const double deltax, const double tolerance ) + { + for( int ioff = -1; ioff < 2; ioff++, ioff++ ) // -1, 1 + { + const bool debug = false; + const int nstep = 8; + for( int istep = 0; istep < nstep + 1; istep++ ) + { + long double x0 = deltax * ioff; + long double x1 = deltax * ioff + 2 * constexpr_pi; + double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) + testSinCosTanX( x, tolerance, debug, istep ); + } + } + }; + + // Use much lower tolerance when testing on the boundaries of the 8 sectors of [0,2*pi] + // Use progressively stricter tolerances as you move away from the boundaries of the 8 sectors of [0,2*pi] + testSinCosTan8( 0, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... + testSinCosTan8( 1E-15, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... + testSinCosTan8( 1E-14, 1E-04 ); // fails with 1E-05 + testSinCosTan8( 1E-12, 1E-06 ); // fails with 1E-07 + testSinCosTan8( 1E-09, 1E-09 ); // fails with 1E-10 + testSinCosTan8( 1E-06, 1E-12 ); // fails with 1E-13 + testSinCosTan8( 1E-03, 1E-14 ); // fails with 1E-16: could use 1E-14 but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) + testSinCosTan8( 1E-02, 1E-14 ); // never fails? could use 1E-99(?) but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) + + // Test constexpr sin, cos, tan - N points almost randomly with a varying tolerance + auto testSinCosTanN = [distance4]( const int nstep, const double x0, const double x1 ) + { + auto toleranceForX = [distance4]( const double x ) + { + const double d4 = distance4( x ); + if( d4 < 1E-14 ) + return 1E-03; // NB: absolute distance limited to 1E-14 anyway even if relative tolerance is 1E-3... + else if( d4 < 1E-13 ) + return 1E-04; + else if( d4 < 1E-12 ) + return 1E-05; + else if( d4 < 1E-11 ) + return 1E-06; + else if( d4 < 1E-10 ) + return 1E-07; + else if( d4 < 1E-09 ) + return 1E-08; + else if( d4 < 1E-08 ) + return 1E-09; + else if( d4 < 1E-07 ) + return 1E-10; + else if( d4 < 1E-06 ) + return 1E-11; + else if( d4 < 1E-05 ) + return 1E-12; + else if( d4 < 1E-04 ) + return 1E-13; + else + return 1E-14; // play it safe even if the agreement might even be better? + }; + for( int istep = 0; istep < nstep + 1; istep++ ) + { + double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) + const double tolerance0 = toleranceForX( x ); + const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 + EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::max( std::abs( std::sin( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::max( std::abs( std::cos( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); +#ifndef __aarch64__ + if( !RUNNING_ON_VALGRIND ) + { + EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + } + else +#endif + { + // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) + const long double ctanx = constexpr_tan( x ); + const long double taninf = 4E14; // declare tan(x) as "infinity if above this threshold + if( ctanx > -taninf && ctanx < taninf ) + EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + else + { + // Allow tan(x)=-inf if ctan(x)=+inf and viceversa + EXPECT_GT( std::abs( std::tan( x ) ), taninf ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + /* + // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) + if( ctanx > 0 ) + EXPECT_GT( std::tan( x ), taninf ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + else + EXPECT_LT( std::tan( x ), -taninf ) + << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); + */ + } + } + } + }; + testSinCosTanN( 100, -4 * constexpr_pi, 6 * constexpr_pi ); // this was failing at 3*pi/2 (now fixed by absolute tolerance 3E-15) + testSinCosTanN( 10000, -constexpr_pi_by_2, 5 * constexpr_pi_by_2 ); + + // Test constexpr atan + { + const double tolerance = 1E-12; + const int nstep = 1000; + for( int istep = 0; istep < nstep + 1; istep++ ) + { + long double x0 = -5, x1 = +5; + double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) + EXPECT_NEAR( std::atan( x ), constexpr_atan( x ), std::abs( std::atan( x ) * tolerance ) ) + << "x=" << x << ", istep=" << istep; + } + } +#endif + //-------------------------------------------------------------------------- +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc new file mode 100644 index 0000000000..4fcf7d26b6 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc @@ -0,0 +1,456 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. +// ---------------------------------------------------------------------------- +// Use ./runTest.exe --gtest_filter=*xxx to run only testxxx.cc tests +// ---------------------------------------------------------------------------- + +#include "mgOnGpuConfig.h" + +#include "CPPProcess.h" +#include "HelAmps_%(model_name)s.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessWavefunctions.h" +#include "MemoryBuffers.h" +#include "epoch_process_id.h" + +#include + +#include +#include +#include // for signal and SIGFPE (see https://stackoverflow.com/a/17473528) +#include +#include +#include +#include +#ifdef MGONGPUCPP_GPUIMPL +#define TESTID( s ) s##_GPU_XXX +#else +#define TESTID( s ) s##_CPU_XXX +#endif + +#define XTESTID( s ) TESTID( s ) + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + std::string fpeHandlerMessage = "unknown"; + int fpeHandlerIevt = -1; + inline void fpeHandlerTestxxx( int /*sig*/ ) + { +#ifdef MGONGPUCPP_GPUIMPL + std::cerr << "Floating Point Exception (GPU): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; +#else + std::cerr << "Floating Point Exception (CPU neppV=" << neppV << "): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; +#endif + exit( 1 ); + } +} + +TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) +{ +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif +#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) + auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); +#endif + constexpr bool dumpEvents = false; // dump the expected output of the test? + constexpr bool testEvents = !dumpEvents; // run the test? + constexpr fptype toleranceXXXs = std::is_same::value ? 1.E-15 : 1.E-5; + // Constant parameters + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + constexpr int np4 = CPPProcess::np4; + const int nevt = 32; // 12 independent tests plus 20 duplicates (need a multiple of 16 for floats '512z') + assert( nevt %% neppM == 0 ); // nevt must be a multiple of neppM + assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV + // Fill in the input momenta +#ifdef MGONGPUCPP_GPUIMPL + mg5amcGpu::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] +#else + mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] +#endif /* clang-format off */ + // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! + const fptype par0[np4 * nevt] = // AOS[nevt][np4] + { + 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) + 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) + 500, 180, 240, 400, // #3 (m=0 pT>0 pz>0) + 500, 180, 240, -400, // #4 (m=0 pT>0 pz<0) + 500, 0, 0, 500, // #5 DUPLICATE == #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, -500, // #6 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) + 500, 300, 400, 0, // #7 DUPLICATE == #2 (m=0 pT>0 pz=0) + 500, 180, 240, 400, // #8 DUPLICATE == #3 (m=0 pT>0 pz>0) + 500, 180, 240, -400, // #9 DUPLICATE == #4 (m=0 pT>0 pz<0) + 500, 0, 0, 500, // #10 DUPLICATE == #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, -500, // #11 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) + 500, 300, 400, 0, // #12 DUPLICATE == #2 (m=0 pT>0 pz=0) + 500, 180, 240, 400, // #13 DUPLICATE == #3 (m=0 pT>0 pz>0) + 500, 180, 240, -400, // #14 DUPLICATE == #4 (m=0 pT>0 pz<0) + 500, 0, 0, 500, // #15 DUPLICATE == #0 (m=0 pT=0 E=pz>0) + 500, 0, 0, 0, // #16 (m=50>0 pT=0 pz=0) + 500, 0, 0, 300, // #17 (m=40>0 pT=0 pz>0) + 500, 0, 0, -300, // #18 (m=40>0 pT=0 pz<0) + 500, 180, 240, 0, // #19 (m=40>0 pT>0 pz=0) + 500, -240, -180, 0, // #20 (m=40>0 pT>0 pz=0) + 500, 180, 192, 144, // #21 (m=40>0 pT>0 pz>0) + 500, 180, 192, -144, // #22 (m=40>0 pT>0 pz<0) + 500, 0, 0, 0, // #23 DUPLICATE == #16 (m=50>0 pT=0 pz=0) + 500, 0, 0, 300, // #24 DUPLICATE == #17 (m=40>0 pT=0 pz>0) + 500, 0, 0, -300, // #25 DUPLICATE == #18 (m=40>0 pT=0 pz<0) + 500, 180, 240, 0, // #26 DUPLICATE == #19 (m=40>0 pT>0 pz=0) + 500, -240, -180, 0, // #27 DUPLICATE == #20 (m=40>0 pT>0 pz=0) + 500, 180, 192, 144, // #28 DUPLICATE == #21 (m=40>0 pT>0 pz>0) + 500, 180, 192, -144, // #29 DUPLICATE == #22 (m=40>0 pT>0 pz<0) + 500, 0, 0, 0, // #30 DUPLICATE == #16 (m=50>0 pT=0 pz=0) + 500, 0, 0, 300 // #31 DUPLICATE == #17 (m=40>0 pT=0 pz>0) + }; /* clang-format on */ + // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) + // See https://en.cppreference.com/w/c/language/array_initialization#Notes + fptype mass0[nevt] = {}; + bool ispzgt0[nevt] = {}; + bool ispzlt0[nevt] = {}; + bool isptgt0[nevt] = {}; + for( int ievt = 0; ievt < nevt; ievt++ ) + { + const fptype p0 = par0[ievt * np4 + 0]; + const fptype p1 = par0[ievt * np4 + 1]; + const fptype p2 = par0[ievt * np4 + 2]; + const fptype p3 = par0[ievt * np4 + 3]; + volatile fptype m2 = fpmax( p0 * p0 - p1 * p1 - p2 * p2 - p3 * p3, 0 ); // see #736 + if( m2 > 0 ) + mass0[ievt] = fpsqrt( (fptype)m2 ); + else + mass0[ievt] = 0; + ispzgt0[ievt] = ( p3 > 0 ); + ispzlt0[ievt] = ( p3 < 0 ); + isptgt0[ievt] = ( p1 != 0 ) || ( p2 != 0 ); + } + const int ipar0 = 0; // use only particle0 for this test + for( int ievt = 0; ievt < nevt; ievt++ ) + { + for( int ip4 = 0; ip4 < np4; ip4++ ) + { + MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA + } + } + // Expected output wavefunctions + std::vector> expwfs; +#include "testxxx_cc_ref.txt" // expwfs.push_back( {...} ); + std::string dumpFileName = "testxxx_cc_ref.txt.new"; + // Compute the output wavefunctions + // Dump new reference file if requested + constexpr int nw6 = CPPProcess::nw6; // dimensions of each wavefunction (HELAS KEK 91-11): e.g. 6 for e+ e- -> mu+ mu- (fermions and vectors) + int itest = 0; // index on the expected output vector + std::ofstream dumpFile; + if( dumpEvents ) + { + dumpFile.open( dumpFileName, std::ios::trunc ); + dumpFile << " // Copyright (C) 2020-2024 CERN and UCLouvain." << std::endl + << " // Licensed under the GNU Lesser General Public License (version 3 or later)." << std::endl + << " // Created by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin." << std::endl + << " // Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin." << std::endl; + } + // Lambda function for dumping wavefunctions + auto dumpwf6 = [&]( std::ostream& out, const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) + { + out << std::setprecision( 15 ) << std::scientific; + out << " expwfs.push_back( {"; + out << " // ---------" << std::endl; + for( int iw6 = 0; iw6 < nw6; iw6++ ) + { +#ifdef MGONGPU_CPPSIMD + const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration +#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK + out << std::setw( 26 ) << cxreal( wf[iw6][ieppV] ) << ", "; + out << std::setw( 22 ) << cximag( wf[iw6][ieppV] ); +#else + out << std::setw( 26 ) << wf[iw6].real()[ieppV] << ", "; + out << std::setw( 22 ) << wf[iw6].imag()[ieppV]; +#endif +#else + out << std::setw( 26 ) << wf[iw6].real(); + out << ", " << std::setw( 22 ) << wf[iw6].imag(); +#endif + if( iw6 < nw6 - 1 ) + out << ", "; + else + out << " } );"; + out << " // itest=" << itest << ": " << xxx << "#" << ievt; + out << " nsp=" << nsp << " mass=" << (int)mass << std::endl; + } + out << std::defaultfloat; + }; + // Lambda function for testing wavefunctions (1) + auto testwf6 = [&]( const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) + { + if( dumpEvents ) dumpwf6( dumpFile, wf, xxx, ievt, nsp, mass ); + if( testEvents ) + { + std::array& expwf = expwfs[itest]; + //std::cout << "Testing " << std::setw(3) << itest << ": " << xxx << " #" << ievt << std::endl; + ////for ( int iw6 = 0; iw60) can be used without triggering FPEs (#701) + // This is done by filling the full SIMD vector with the value of ievt, which was already tested to respect the relevant assumptions + for( int jevt = 0; jevt < nevt; jevt++ ) + for( int ip4 = 0; ip4 < np4; ip4++ ) + MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), jevt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA + } + }; + // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) + // See https://en.cppreference.com/w/c/language/array_initialization#Notes + cxtype_sv outwfI[6] = {}; // last result of ixxxxx (mass==0) + cxtype_sv outwfO[6] = {}; // last result of oxxxxx (mass==0) + cxtype_sv outwf[6] = {}; + cxtype_sv outwf3[6] = {}; // NB: only 3 are filled by sxxxxx, but 6 are compared! + fptype* fp_outwfI = reinterpret_cast( outwfI ); // proof of concept for using fptype* in the interface + fptype* fp_outwfO = reinterpret_cast( outwfO ); // proof of concept for using fptype* in the interface + fptype* fp_outwf = reinterpret_cast( outwf ); // proof of concept for using fptype* in the interface + fptype* fp_outwf3 = reinterpret_cast( outwf3 ); // proof of concept for using fptype* in the interface + const int nhel = 1; + // *** START OF TESTING LOOP + for( auto nsp: { -1, +1 } ) // antifermion/fermion (or initial/final for scalar and vector) + { + for( int ievt = 0; ievt < nevt; ievt++ ) + { +#ifdef MGONGPUCPP_GPUIMPL + using namespace mg5amcGpu; +#else + using namespace mg5amcCpu; +#endif + if( debug ) + { + std::cout << std::endl; + std::cout << "nsp=" << nsp << " ievt=" << ievt << ": "; + for( int ip4 = 0; ip4 < np4; ip4++ ) std::cout << par0[ievt * np4 + ip4] << ", "; + std::cout << std::endl; + } + const int ipagV = ievt / neppV; // #event vector in this iteration + const fptype* ievt0Momenta = MemoryAccessMomenta::ieventAccessRecordConst( hstMomenta.data(), ipagV * neppV ); + // Test ixxxxx - NO ASSUMPTIONS + { + prepareTest( "ixxxxx", ievt ); + const fptype fmass = mass0[ievt]; + ixxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfI, ipar0 ); + testwf6( outwfI, "ixxxxx", ievt, nsp, fmass ); + ixxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfI, ipar0 ); + testwf6( outwfI, "ixxxxx", ievt, nsp, -fmass ); + } + // Test ipzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) + { + prepareTest( "ipzxxx", ievt ); + ipzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfI, "ipzxxx", ievt ); + testwf6( outwf, "ipzxxx", ievt, nsp, 0 ); + } + // Test imzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) + { + prepareTest( "imzxxx", ievt ); + imzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfI, "imzxxx", ievt ); + testwf6( outwf, "imzxxx", ievt, nsp, 0 ); + } + // Test ixzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) + if( mass0[ievt] == 0 && isptgt0[ievt] ) + { + prepareTest( "ixzxxx", ievt ); + ixzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfI, "ixzxxx", ievt ); + testwf6( outwf, "ixzxxx", ievt, nsp, 0 ); + } + // Test vxxxxx - NO ASSUMPTIONS + { + prepareTest( "vxxxxx", ievt ); + const fptype vmass = mass0[ievt]; + vxxxxx( ievt0Momenta, vmass, nhel, nsp, fp_outwf, ipar0 ); + testwf6( outwf, "vxxxxx", ievt, nsp, vmass ); + vxxxxx( ievt0Momenta, -vmass, nhel, nsp, fp_outwf, ipar0 ); + testwf6( outwf, "vxxxxx", ievt, nsp, -vmass ); + } + // Test sxxxxx - NO ASSUMPTIONS + { + prepareTest( "sxxxxx", ievt ); + const fptype smass = mass0[ievt]; + sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass>0") + testwf6( outwf3, "sxxxxx", ievt, nsp, smass ); + sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass<0") + testwf6( outwf3, "sxxxxx", ievt, nsp, -smass ); + } + // Test oxxxxx - NO ASSUMPTIONS + { + prepareTest( "oxxxxx", ievt ); + const fptype fmass = mass0[ievt]; + oxxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfO, ipar0 ); + testwf6( outwfO, "oxxxxx", ievt, nsp, fmass ); + oxxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfO, ipar0 ); + testwf6( outwfO, "oxxxxx", ievt, nsp, -fmass ); + } + // Test opzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) + { + prepareTest( "opzxxx", ievt ); + opzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfO, "opzxxx", ievt ); + testwf6( outwf, "opzxxx", ievt, nsp, 0 ); + } + // Test omzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) + if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) + { + prepareTest( "omzxxx", ievt ); + omzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfO, "omzxxx", ievt ); + testwf6( outwf, "omzxxx", ievt, nsp, 0 ); + } + // Test oxzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) + if( mass0[ievt] == 0 && isptgt0[ievt] ) + { + prepareTest( "oxzxxx", ievt ); + oxzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); + testwf6two( outwf, outwfO, "oxzxxx", ievt ); + testwf6( outwf, "oxzxxx", ievt, nsp, 0 ); + } + } + } + // *** END OF TESTING LOOP + if( dumpEvents ) + { + dumpFile.close(); + std::cout << "INFO: New reference data dumped to file '" << dumpFileName << "'" << std::endl; + } +#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) + signal( SIGFPE, fpeHandlerDefault ); +#endif +} + +//========================================================================== + +// Reset the GPU after ALL tests have gone out of scope +// (This was needed to avoid leaks in profilers, but compute-sanitizer reports no leaks, is it STILL needed?) +// ========= NB: resetting the GPU too early causes segfaults that are very difficult to debug #907 ========= +// Try to use atexit (https://stackoverflow.com/a/14610501) but this still crashes! +// ********* FIXME? avoid CUDA API calls in destructors? (see https://stackoverflow.com/a/16982503) ********* +void +myexit() +{ +#ifdef MGONGPUCPP_GPUIMPL + //checkGpu( gpuDeviceReset() ); // FIXME??? this still crashes! should systematically avoid CUDA calls in all destructors? +#endif +} + +// Main function (see https://google.github.io/googletest/primer.html#writing-the-main-function) +// (NB: the test executables are now separate for C++ and CUDA, therefore main must be included all the time) +// (NB: previously, '#ifndef MGONGPUCPP_GPUIMPL' was ensuring that main was only included once while linking both C++ and CUDA tests) +int +main( int argc, char** argv ) +{ + atexit( myexit ); + testing::InitGoogleTest( &argc, argv ); + int status = RUN_ALL_TESTS(); + return status; +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc new file mode 100644 index 0000000000..d19c93bb9b --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc @@ -0,0 +1,667 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: T. Heimel (Nov 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro (2026). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "umami.h" + +#include "CPPProcess.h" +#include "GpuRuntime.h" +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" + +#include +#include +#include +#include + +#ifdef MGONGPUCPP_GPUIMPL +using namespace mg5amcGpu; +#else +using namespace mg5amcCpu; +#endif + +namespace +{ + + void* initialize_impl( + const fptype* momenta, + const fptype* couplings, + const unsigned int* flavor_indices, + fptype* matrix_elements, +#ifdef MGONGPUCPP_GPUIMPL + fptype* color_jamps, +#endif + fptype* numerators, + fptype* denominators, + std::size_t count ) + { + bool is_good_hel[CPPProcess::ncomb]; + sigmaKin_getGoodHel( + momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, +#ifdef MGONGPUCPP_GPUIMPL + color_jamps, +#endif + is_good_hel, + count ); + sigmaKin_setGoodHel( is_good_hel ); + return nullptr; + } + + void initialize( + const fptype* momenta, + const fptype* couplings, + const unsigned int* flavor_indices, + fptype* matrix_elements, +#ifdef MGONGPUCPP_GPUIMPL + fptype* color_jamps, +#endif + fptype* numerators, + fptype* denominators, + std::size_t count ) + { + // static local initialization is called exactly once in a thread-safe way + static void* dummy = initialize_impl( momenta, couplings, flavor_indices, matrix_elements, +#ifdef MGONGPUCPP_GPUIMPL + color_jamps, +#endif + numerators, + denominators, + count ); + } + +#ifdef MGONGPUCPP_GPUIMPL + __device__ +#endif + void + transpose_momenta( const double* momenta_in, fptype* momenta_out, std::size_t i_event_in, std::size_t i_event_out, std::size_t stride ) + { + std::size_t page_size = MemoryAccessMomentaBase::neppM; + std::size_t i_page = i_event_out / page_size; + std::size_t i_vector = i_event_out % page_size; + + for( std::size_t i_part = 0; i_part < CPPProcess::npar; ++i_part ) + { + for( std::size_t i_mom = 0; i_mom < 4; ++i_mom ) + { + momenta_out[i_page * CPPProcess::npar * 4 * page_size + + i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( CPPProcess::npar * i_mom + i_part ) + i_event_in]; + } + } + } + +#ifdef MGONGPUCPP_GPUIMPL + + __global__ void copy_inputs( + const double* momenta_in, + const double* helicity_random_in, + const double* color_random_in, + const double* diagram_random_in, + const double* alpha_s_in, + const unsigned int* flavor_indices_in, + fptype* momenta, + fptype* helicity_random, + fptype* color_random, + fptype* diagram_random, + fptype* g_s, + unsigned int* flavor_indices, + std::size_t count, + std::size_t stride, + std::size_t offset ) + { + std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; + if( i_event >= count ) return; + + transpose_momenta( &momenta_in[offset], momenta, i_event, i_event, stride ); + diagram_random[i_event] = diagram_random_in ? diagram_random_in[i_event + offset] : 0.5; + helicity_random[i_event] = helicity_random_in ? helicity_random_in[i_event + offset] : 0.5; + color_random[i_event] = color_random_in ? color_random_in[i_event + offset] : 0.5; + g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; + } + + __global__ void copy_outputs( + fptype* denominators, + fptype* numerators, + fptype* matrix_elements, + unsigned int* diagram_index, + int* color_index, + int* helicity_index, + double* m2_out, + double* amp2_out, + int* diagram_out, + int* color_out, + int* helicity_out, + std::size_t count, + std::size_t stride, + std::size_t offset ) + { + std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; + if( i_event >= count ) return; + + if( m2_out ) m2_out[i_event + offset] = matrix_elements[i_event]; + if( amp2_out ) + { + double denominator = denominators[i_event]; + for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_event * CPPProcess::ndiagrams + i_diag] / denominator; + } + } + if( diagram_out ) diagram_out[i_event + offset] = diagram_index[i_event] - 1; + if( color_out ) color_out[i_event + offset] = color_index[i_event] - 1; + if( helicity_out ) helicity_out[i_event + offset] = helicity_index[i_event] - 1; + } + +#endif // MGONGPUCPP_GPUIMPL + + struct InterfaceInstance + { + bool initialized = false; + }; + + std::vector g_externalMasses; + +} + +extern "C" +{ + UmamiStatus umami_get_meta( UmamiMetaKey meta_key, void* result ) + { + switch( meta_key ) + { + case UMAMI_META_DEVICE: + { + UmamiDevice& device = *static_cast( result ); +#ifdef MGONGPUCPP_GPUIMPL +#ifdef __CUDACC__ + device = UMAMI_DEVICE_CUDA; +#elif defined( __HIPCC__ ) + device = UMAMI_DEVICE_HIP; +#endif +#else + device = UMAMI_DEVICE_CPU; +#endif + break; + } + case UMAMI_META_PARTICLE_COUNT: + *static_cast( result ) = CPPProcess::npar; + break; + case UMAMI_META_DIAGRAM_COUNT: + *static_cast( result ) = CPPProcess::ndiagrams; + break; + case UMAMI_META_HELICITY_COUNT: + *static_cast( result ) = CPPProcess::ncomb; + break; + case UMAMI_META_COLOR_COUNT: + return UMAMI_ERROR_UNSUPPORTED_META; + case UMAMI_META_MASSES: + { + if( g_externalMasses.size() != (size_t)CPPProcess::npar ) return UMAMI_ERROR_UNINITIALIZED_META; + + for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + static_cast( result )[ipar] = g_externalMasses[ipar]; + break; + } + default: + return UMAMI_ERROR_UNSUPPORTED_META; + } + return UMAMI_SUCCESS; + } + + UmamiStatus umami_supported_inputs( bool const** supported, int* count ) + { + // MOMENTA, ALPHA_S, FLAVOR_INDEX, RANDOM_COLOR, RANDOM_HELICITY, RANDOM_DIAGRAM, + // HELICITY_INDEX=false, DIAGRAM_INDEX=true, CHANNEL_INDEX=false + static const bool data[UMAMI_INPUT_KEY_COUNT] = { true, true, true, true, true, true, false, true }; + *supported = data; + *count = UMAMI_INPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_required_inputs( bool const** required, int* count ) + { + static const bool data[UMAMI_INPUT_KEY_COUNT] = { true }; // MOMENTA only + *required = data; + *count = UMAMI_INPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_supported_outputs( bool const** supported, int* count ) + { + // MATRIX_ELEMENT, DIAGRAM_AMP2, COLOR_INDEX, HELICITY_INDEX, DIAGRAM_INDEX, GPU_STREAM +#ifdef MGONGPUCPP_GPUIMPL + static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true, true }; +#else + static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true }; +#endif + *supported = data; + *count = UMAMI_OUTPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_initialize( UmamiHandle* handle, char const* param_card_path ) + { + CPPProcess process; + process.initProc( param_card_path ); + + const std::vector& masses = process.getMasses(); + g_externalMasses.assign( masses.begin(), masses.end() ); + + auto instance = new InterfaceInstance(); + *handle = instance; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_set_parameter( + [[maybe_unused]] UmamiHandle handle, + [[maybe_unused]] char const* name, + [[maybe_unused]] double parameter_real, + [[maybe_unused]] double parameter_imag ) + { + return UMAMI_ERROR_NOT_IMPLEMENTED; + } + + UmamiStatus umami_get_parameter( + [[maybe_unused]] UmamiHandle handle, + [[maybe_unused]] char const* name, + [[maybe_unused]] double* parameter_real, + [[maybe_unused]] double* parameter_imag ) + { + return UMAMI_ERROR_NOT_IMPLEMENTED; + } + + UmamiStatus umami_matrix_element( + UmamiHandle handle, + size_t count, + size_t stride, + size_t offset, + size_t input_count, + UmamiInputKey const* input_keys, + void const* const* inputs, + size_t output_count, + UmamiOutputKey const* output_keys, + void* const* outputs ) + { + const double* momenta_in = nullptr; + const double* alpha_s_in = nullptr; + const unsigned int* flavor_indices_in = nullptr; + const double* random_color_in = nullptr; + const double* random_helicity_in = nullptr; + const double* random_diagram_in = nullptr; + [[maybe_unused]] const int* diagram_in = nullptr; // TODO: unused + + for( std::size_t i = 0; i < input_count; ++i ) + { + const void* input = inputs[i]; + switch( input_keys[i] ) + { + case UMAMI_IN_MOMENTA: + momenta_in = static_cast( input ); + break; + case UMAMI_IN_ALPHA_S: + alpha_s_in = static_cast( input ); + break; + case UMAMI_IN_FLAVOR_INDEX: + flavor_indices_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_COLOR: + random_color_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_HELICITY: + random_helicity_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_DIAGRAM: + random_diagram_in = static_cast( input ); + break; + case UMAMI_IN_HELICITY_INDEX: + return UMAMI_ERROR_UNSUPPORTED_INPUT; + case UMAMI_IN_DIAGRAM_INDEX: + diagram_in = static_cast( input ); + break; + default: + return UMAMI_ERROR_UNSUPPORTED_INPUT; + } + } + if( !momenta_in ) return UMAMI_ERROR_MISSING_INPUT; + +#ifdef MGONGPUCPP_GPUIMPL + gpuStream_t gpu_stream = nullptr; +#endif + double* m2_out = nullptr; + double* amp2_out = nullptr; + int* diagram_out = nullptr; + int* color_out = nullptr; + int* helicity_out = nullptr; + for( std::size_t i = 0; i < output_count; ++i ) + { + void* output = outputs[i]; + switch( output_keys[i] ) + { + case UMAMI_OUT_MATRIX_ELEMENT: + m2_out = static_cast( output ); + break; + case UMAMI_OUT_DIAGRAM_AMP2: + amp2_out = static_cast( output ); + break; + case UMAMI_OUT_COLOR_INDEX: + color_out = static_cast( output ); + break; + case UMAMI_OUT_HELICITY_INDEX: + helicity_out = static_cast( output ); + break; + case UMAMI_OUT_DIAGRAM_INDEX: + diagram_out = static_cast( output ); + break; +#ifdef MGONGPUCPP_GPUIMPL + case UMAMI_OUT_GPU_STREAM: + gpu_stream = static_cast( output ); + break; +#endif + default: + return UMAMI_ERROR_UNSUPPORTED_OUTPUT; + } + } + +#ifdef MGONGPUCPP_GPUIMPL + std::size_t n_threads = 256; + std::size_t n_blocks = ( count + n_threads - 1 ) / n_threads; + std::size_t rounded_count = n_blocks * n_threads; + + fptype *momenta, *couplings, *g_s, *helicity_random, *color_random, *diagram_random, *color_jamps; + fptype *matrix_elements, *numerators, *denominators, *ghel_matrix_elements, *ghel_jamps; + int *helicity_index, *color_index; + unsigned int *flavor_indices, *diagram_index; + + std::size_t n_coup = mg5amcGpu::Parameters_dependentCouplings::ndcoup; + std::array, 16> ptrs_and_sizes = {{ + {reinterpret_cast(&momenta), rounded_count * CPPProcess::npar * 4 * sizeof( fptype )}, + {reinterpret_cast(&couplings), rounded_count * n_coup * 2 * sizeof( fptype )}, + {reinterpret_cast(&g_s), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&flavor_indices), rounded_count * sizeof( unsigned int )}, + {reinterpret_cast(&helicity_random), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&color_random), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&diagram_random), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&matrix_elements), rounded_count * sizeof( fptype )}, + {reinterpret_cast(&diagram_index), rounded_count * sizeof( unsigned int )}, + {reinterpret_cast(&color_jamps), rounded_count * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, + {reinterpret_cast(&numerators), rounded_count * CPPProcess::ndiagrams * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&denominators), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&helicity_index), rounded_count * sizeof( int )}, + {reinterpret_cast(&color_index), rounded_count * sizeof( int )}, + {reinterpret_cast(&ghel_matrix_elements), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&ghel_jamps), rounded_count * CPPProcess::ncomb * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, + }}; + std::size_t total_size = 0; + constexpr std::size_t MAX_SIZE = std::max(sizeof(fptype), sizeof(int)); + for (auto [ptr, size] : ptrs_and_sizes) { + std::size_t aligned_size = (size + MAX_SIZE - 1) / MAX_SIZE * MAX_SIZE; + total_size += aligned_size; + } + uint8_t* buffer; + // we can consider caching this between matrix element calls + gpuMallocAsync( &buffer, total_size, gpu_stream ); + std::size_t buf_offset = 0; + for (auto [ptr, size] : ptrs_and_sizes) { + std::size_t aligned_size = (size + 7) / 8 * 8; + *ptr = buffer + buf_offset; + buf_offset += aligned_size; + } + + copy_inputs<<>>( + momenta_in, + random_helicity_in, + random_color_in, + random_diagram_in, + alpha_s_in, + flavor_indices_in, + momenta, + helicity_random, + color_random, + diagram_random, + g_s, + flavor_indices, + count, + stride, + offset ); + computeDependentCouplings<<>>( g_s, couplings ); + checkGpu( gpuPeekAtLastError() ); + + InterfaceInstance* instance = static_cast( handle ); + if( !instance->initialized ) + { + initialize( + momenta, couplings, flavor_indices, matrix_elements, color_jamps, numerators, denominators, rounded_count ); + instance->initialized = true; + } + + sigmaKin( + momenta, + couplings, + flavor_indices, + helicity_random, + color_random, + nullptr, + diagram_random, + matrix_elements, + helicity_index, + color_index, + color_jamps, + numerators, + denominators, + diagram_index, + false, + ghel_matrix_elements, + ghel_jamps, + nullptr, + nullptr, + &gpu_stream, + true, + n_blocks, + n_threads ); + + copy_outputs<<>>( + denominators, + numerators, + matrix_elements, + diagram_index, + color_index, + helicity_index, + m2_out, + amp2_out, + diagram_out, + color_out, + helicity_out, + count, + stride, + offset ); + checkGpu( gpuPeekAtLastError() ); + + gpuFreeAsync( buffer, gpu_stream ); +#else // MGONGPUCPP_GPUIMPL + constexpr std::size_t vector_size = MemoryAccessMomentaBase::neppM; + // need to round to round to double page size for some reason + constexpr std::size_t page_size2 = 2 * vector_size; + std::vector permutation; + std::size_t rounded_count; + + constexpr std::size_t flavor_count = CPPProcess::nmaxflavor; + HostBufferBase flavor_indices( ((count + page_size2 - 1) / page_size2 + flavor_count) * page_size2 ); + bool sort_flavors = vector_size > 1 && flavor_count > 1 && flavor_indices_in; + if ( sort_flavors ) + { + permutation.resize(count); + std::size_t voffset = 0; + std::size_t vector_indices[flavor_count] = {}; + std::size_t vector_counts[flavor_count] = {}; + // determine permutation of inputs such that all entries in a SIMD vector + // have the same flavor index + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + unsigned int flav = flavor_indices_in[i_event + offset]; + auto& vcount = vector_counts[flav]; + auto& vindex = vector_indices[flav]; + if ( vcount == 0 ) + { + vindex = voffset * page_size2; + for ( std::size_t i = 0; i < page_size2; ++i) { + flavor_indices[voffset * page_size2 + i] = flav; + } + voffset += 1; + } + permutation[i_event] = vindex + vcount; + vcount = (vcount + 1) % page_size2; + } + rounded_count = voffset * page_size2; + } else { + rounded_count = ( count + page_size2 - 1 ) / page_size2 * page_size2; + } + + HostBufferBase momenta( rounded_count * CPPProcess::npar * 4 ); + HostBufferBase couplings( rounded_count * mg5amcCpu::Parameters_dependentCouplings::ndcoup * 2 ); + HostBufferBase g_s( rounded_count ); + HostBufferBase helicity_random( rounded_count ); + HostBufferBase color_random( rounded_count ); + HostBufferBase diagram_random( rounded_count ); + HostBufferBase matrix_elements( rounded_count ); + HostBufferBase diagram_index( rounded_count ); + HostBufferBase numerators( rounded_count * CPPProcess::ndiagrams ); + HostBufferBase denominators( rounded_count ); + HostBufferBase helicity_index( rounded_count ); + HostBufferBase color_index( rounded_count ); + if ( sort_flavors ) { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_sorted = permutation[i_event]; + transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_sorted, stride ); + helicity_random[i_sorted] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; + color_random[i_sorted] = random_color_in ? random_color_in[i_event + offset] : 0.5; + diagram_random[i_sorted] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; + g_s[i_sorted] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + } + } else { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_event, stride ); + helicity_random[i_event] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; + color_random[i_event] = random_color_in ? random_color_in[i_event + offset] : 0.5; + diagram_random[i_event] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; + g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; + } + for ( std::size_t i_event = count; i_event < rounded_count; ++i_event ) { + flavor_indices[i_event] = 0; + } + } + computeDependentCouplings( g_s.data(), couplings.data(), rounded_count ); + + InterfaceInstance* instance = static_cast( handle ); + if( !instance->initialized ) + { + initialize( + momenta.data(), + couplings.data(), + flavor_indices.data(), + matrix_elements.data(), + numerators.data(), + denominators.data(), + rounded_count ); + instance->initialized = true; + } + + sigmaKin( + momenta.data(), + couplings.data(), + flavor_indices.data(), + helicity_random.data(), + color_random.data(), + nullptr, + diagram_random.data(), + matrix_elements.data(), + helicity_index.data(), + color_index.data(), + numerators.data(), + denominators.data(), + diagram_index.data(), + false, + rounded_count ); + + if ( sort_flavors ) + { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_sorted = permutation[i_event]; + std::size_t page_size = MemoryAccessMomentaBase::neppM; + std::size_t i_page = i_sorted / page_size; + std::size_t i_vector = i_sorted % page_size; // vector lane + + double denominator = denominators[i_sorted]; + if( m2_out != nullptr ) + { + m2_out[i_event + offset] = matrix_elements[i_sorted]; + } + if( amp2_out != nullptr ) + { + for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; + } + } + if( diagram_out != nullptr ) + { + diagram_out[i_event + offset] = diagram_index[i_sorted] - 1; + } + if( color_out != nullptr ) + { + color_out[i_event + offset] = color_index[i_sorted] - 1; + } + if( helicity_out != nullptr ) + { + helicity_out[i_event + offset] = helicity_index[i_sorted] - 1; + } + } + } else { + std::size_t page_size = MemoryAccessMomentaBase::neppM; + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_page = i_event / page_size; + std::size_t i_vector = i_event % page_size; + + double denominator = denominators[i_event]; + if( m2_out != nullptr ) + { + m2_out[i_event + offset] = matrix_elements[i_event]; + } + if( amp2_out != nullptr ) + { + for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; + } + } + if( diagram_out != nullptr ) + { + diagram_out[i_event + offset] = diagram_index[i_event] - 1; + } + if( color_out != nullptr ) + { + color_out[i_event + offset] = color_index[i_event] - 1; + } + if( helicity_out != nullptr ) + { + helicity_out[i_event + offset] = helicity_index[i_event] - 1; + } + } + } +#endif // MGONGPUCPP_GPUIMPL + return UMAMI_SUCCESS; + } + + UmamiStatus umami_free( UmamiHandle handle ) + { + InterfaceInstance* instance = static_cast( handle ); + delete instance; + return UMAMI_SUCCESS; + } +} diff --git a/madmatrix/output.py b/madmatrix/output.py index 1820b10e96..d6218cd7ea 100644 --- a/madmatrix/output.py +++ b/madmatrix/output.py @@ -113,6 +113,17 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): 'madanalysis5_hadron_card_default.dat', 'rivet_card_default.dat'])} + # Backend split (step 1, not yet wired into the build): mirror + # template_files/madmatrix/backend/{cpu,simd,gpu}/ as a top-level + # backend// dir, sibling of SubProcesses/src/lib. + backend_variants = ('cpu', 'simd', 'gpu') + backend_template_dir = pjoin(madmatrix_templates, 'backend') + for _backend_variant in backend_variants: # plain loop: comprehension wouldn't see the locals above + from_template[pjoin('backend', _backend_variant)] = relative_path_list( + pjoin(backend_template_dir, _backend_variant), + sorted(os.listdir(pjoin(backend_template_dir, _backend_variant)))) + del _backend_variant + to_link_in_P = ['nvtx.h', 'GpuRuntime.h', 'GpuAbstraction.h', 'color_sum.h', 'MemoryAccessHelpers.h', 'MemoryAccessVectors.h', 'MemoryAccessMatrixElements.h', 'MemoryAccessMomenta.h', @@ -131,7 +142,11 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): template_src_make = pjoin(madmatrix_templates, 'madmatrix_src.mk') template_Sub_make = pjoin(madmatrix_templates, 'madmatrix.mk') - dirs_to_create = ['bin', 'src', 'lib', 'Cards', 'SubProcesses'] + dirs_to_create = ['bin', 'src', 'lib', 'Cards', 'SubProcesses', + 'backend', + 'backend/cpu', + 'backend/simd', + 'backend/gpu'] # AV - use a custom UFOModelConverter (model/aloha exporter) create_model_class = model_handling.MadMatrixUFOModelConverter @@ -181,8 +196,21 @@ def generate_subprocess_directory(self, matrix_element, cpp_helas_call_writer, p if cpp_helas_call_writer is not None: cpp_helas_call_writer.use_flavor_mask = self.use_flavor_mask out = super().generate_subprocess_directory(matrix_element, cpp_helas_call_writer, proc_number) + self._link_backend_dirs_in_P(matrix_element) return out + # Symlink the top-level backend//* into this P*'s own backend//. + def _link_backend_dirs_in_P(self, matrix_element): + proc_dir_name = "P%s" % matrix_element.get('processes')[0].shell_string() + dirpath = pjoin(self.dir_path, 'SubProcesses', proc_dir_name) + with misc.chdir(dirpath): + for backend in self.backend_variants: + backend_rel = pjoin('backend', backend) + os.makedirs(backend_rel, exist_ok=True) + src_dir = pjoin('..', '..', 'backend', backend) + for fname in sorted(os.listdir(src_dir)): + files.ln(pjoin(src_dir, fname), starting_dir=backend_rel) + # AV (default from OM's tutorial) - add a debug printout def convert_model(self, model, wanted_lorentz=[], wanted_couplings=[]): if hasattr(model , 'cudacpp_wanted_ordered_couplings'): diff --git a/madmatrix/trex.py b/madmatrix/trex.py index 9809fd01cf..1efeba2849 100644 --- a/madmatrix/trex.py +++ b/madmatrix/trex.py @@ -269,7 +269,8 @@ def generate_subprocess_directory(self, matrix_element, cpp_helas_call_writer, # Create the process .h and .cc files process_exporter_cpp.generate_process_files() for file in self.to_link_in_P: - files.ln('../%s' % file) + files.ln('../%s' % file) + self._link_backend_dirs_in_P(matrix_element) return def export_driver(self): From a35e32d9143aed8e646450f0779909e1ff9c92c3 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 11:30:09 +0200 Subject: [PATCH 02/25] Start of prune delete Bridge. Resolving the not nested #ifdefs. next the .inc in template_files --- .../iolibs/template_files/madmatrix/Bridge.h | 633 -------------- .../template_files/madmatrix/BridgeKernels.cc | 164 ---- .../template_files/madmatrix/BridgeKernels.h | 146 ---- .../madmatrix/backend/cpu/Bridge.h | 633 -------------- .../madmatrix/backend/cpu/BridgeKernels.cc | 164 ---- .../madmatrix/backend/cpu/BridgeKernels.h | 146 ---- .../backend/cpu/CommonRandomNumberKernel.cc | 4 - .../backend/cpu/CrossSectionKernels.cc | 51 -- .../backend/cpu/CrossSectionKernels.h | 4 - .../backend/cpu/CurandRandomNumberKernel.cc | 9 - .../madmatrix/backend/cpu/EventStatistics.h | 4 - .../madmatrix/backend/cpu/GpuRuntime.h | 73 -- .../backend/cpu/HiprandRandomNumberKernel.cc | 9 - .../madmatrix/backend/cpu/MadgraphTest.h | 4 - .../backend/cpu/MatrixElementKernels.cc | 225 ----- .../backend/cpu/MatrixElementKernels.h | 90 -- .../backend/cpu/MemoryAccessAmplitudes.h | 4 - .../backend/cpu/MemoryAccessChannelIds.h | 11 - .../backend/cpu/MemoryAccessCouplings.h | 14 - .../backend/cpu/MemoryAccessCouplingsFixed.h | 4 - .../backend/cpu/MemoryAccessDenominators.h | 4 - .../madmatrix/backend/cpu/MemoryAccessGs.h | 18 - .../backend/cpu/MemoryAccessHelpers.h | 6 - .../backend/cpu/MemoryAccessIflavorVec.h | 11 - .../backend/cpu/MemoryAccessMatrixElements.h | 11 - .../backend/cpu/MemoryAccessMomenta.h | 67 -- .../backend/cpu/MemoryAccessNumerators.h | 4 - .../backend/cpu/MemoryAccessRandomNumbers.h | 4 - .../backend/cpu/MemoryAccessVectors.h | 118 --- .../backend/cpu/MemoryAccessWavefunctions.h | 4 - .../backend/cpu/MemoryAccessWeights.h | 4 - .../madmatrix/backend/cpu/MemoryBuffers.h | 238 ------ .../backend/cpu/RamboSamplingKernels.cc | 78 -- .../backend/cpu/RamboSamplingKernels.h | 39 - .../backend/cpu/RandomNumberKernels.h | 4 - .../madmatrix/backend/cpu/check_sa.cc | 139 --- .../madmatrix/backend/cpu/color_sum.cc | 297 ------- .../madmatrix/backend/cpu/color_sum.h | 64 -- .../madmatrix/backend/cpu/coloramps.h | 10 - .../madmatrix/backend/cpu/constexpr_math.h | 4 - .../madmatrix/backend/cpu/fbridge.cc | 16 - .../madmatrix/backend/cpu/fbridge.h | 4 - .../madmatrix/backend/cpu/fsampler.cc | 14 - .../madmatrix/backend/cpu/massless_rambo.h | 16 - .../madmatrix/backend/cpu/mgOnGpuConfig.h | 10 +- .../madmatrix/backend/cpu/mgOnGpuCxtypes.h | 16 - .../madmatrix/backend/cpu/mgOnGpuFptypes.h | 51 -- .../madmatrix/backend/cpu/mgOnGpuVectors.h | 669 --------------- .../madmatrix/backend/cpu/runTest.cc | 187 ---- .../madmatrix/backend/cpu/testmisc.cc | 80 -- .../madmatrix/backend/cpu/testxxx.cc | 72 -- .../madmatrix/backend/cpu/umami.cc | 220 ----- .../madmatrix/backend/gpu/Bridge.h | 633 -------------- .../madmatrix/backend/gpu/BridgeKernels.cc | 164 ---- .../madmatrix/backend/gpu/BridgeKernels.h | 146 ---- .../backend/gpu/CommonRandomNumberKernel.cc | 4 - .../backend/gpu/CrossSectionKernels.cc | 6 - .../backend/gpu/CrossSectionKernels.h | 4 - .../backend/gpu/CurandRandomNumberKernel.cc | 8 - .../madmatrix/backend/gpu/EventStatistics.h | 4 - .../madmatrix/backend/gpu/GpuRuntime.h | 6 - .../backend/gpu/HiprandRandomNumberKernel.cc | 8 - .../madmatrix/backend/gpu/MadgraphTest.h | 4 - .../backend/gpu/MatrixElementKernels.cc | 143 ---- .../backend/gpu/MatrixElementKernels.h | 54 -- .../backend/gpu/MemoryAccessAmplitudes.h | 4 - .../backend/gpu/MemoryAccessChannelIds.h | 11 - .../backend/gpu/MemoryAccessCouplings.h | 14 - .../backend/gpu/MemoryAccessCouplingsFixed.h | 4 - .../backend/gpu/MemoryAccessDenominators.h | 4 - .../madmatrix/backend/gpu/MemoryAccessGs.h | 18 - .../backend/gpu/MemoryAccessHelpers.h | 4 - .../backend/gpu/MemoryAccessIflavorVec.h | 11 - .../backend/gpu/MemoryAccessMatrixElements.h | 11 - .../backend/gpu/MemoryAccessMomenta.h | 66 -- .../backend/gpu/MemoryAccessNumerators.h | 4 - .../backend/gpu/MemoryAccessRandomNumbers.h | 4 - .../backend/gpu/MemoryAccessVectors.h | 123 --- .../backend/gpu/MemoryAccessWavefunctions.h | 4 - .../backend/gpu/MemoryAccessWeights.h | 4 - .../madmatrix/backend/gpu/MemoryBuffers.h | 149 ---- .../backend/gpu/RamboSamplingKernels.cc | 14 - .../backend/gpu/RamboSamplingKernels.h | 6 - .../backend/gpu/RandomNumberKernels.h | 4 - .../madmatrix/backend/gpu/check_sa.cc | 78 -- .../madmatrix/backend/gpu/color_sum.cc | 111 --- .../madmatrix/backend/gpu/color_sum.h | 18 - .../madmatrix/backend/gpu/coloramps.h | 8 - .../madmatrix/backend/gpu/constexpr_math.h | 4 - .../madmatrix/backend/gpu/fbridge.cc | 14 - .../madmatrix/backend/gpu/fbridge.h | 4 - .../madmatrix/backend/gpu/fsampler.cc | 14 - .../madmatrix/backend/gpu/massless_rambo.h | 20 - .../madmatrix/backend/gpu/mgOnGpuConfig.h | 47 -- .../madmatrix/backend/gpu/mgOnGpuCxtypes.h | 16 - .../madmatrix/backend/gpu/mgOnGpuFptypes.h | 31 - .../madmatrix/backend/gpu/mgOnGpuVectors.h | 799 ------------------ .../madmatrix/backend/gpu/runTest.cc | 153 ---- .../madmatrix/backend/gpu/testmisc.cc | 80 -- .../madmatrix/backend/gpu/testxxx.cc | 71 -- .../madmatrix/backend/gpu/umami.cc | 205 ----- .../madmatrix/backend/simd/Bridge.h | 633 -------------- .../madmatrix/backend/simd/BridgeKernels.cc | 164 ---- .../madmatrix/backend/simd/BridgeKernels.h | 146 ---- .../backend/simd/CommonRandomNumberKernel.cc | 4 - .../backend/simd/CrossSectionKernels.cc | 51 -- .../backend/simd/CrossSectionKernels.h | 4 - .../backend/simd/CurandRandomNumberKernel.cc | 9 - .../madmatrix/backend/simd/EventStatistics.h | 4 - .../madmatrix/backend/simd/GpuRuntime.h | 73 -- .../backend/simd/HiprandRandomNumberKernel.cc | 9 - .../madmatrix/backend/simd/MadgraphTest.h | 4 - .../backend/simd/MatrixElementKernels.cc | 225 ----- .../backend/simd/MatrixElementKernels.h | 90 -- .../backend/simd/MemoryAccessAmplitudes.h | 4 - .../backend/simd/MemoryAccessChannelIds.h | 8 - .../backend/simd/MemoryAccessCouplings.h | 8 - .../backend/simd/MemoryAccessCouplingsFixed.h | 4 - .../backend/simd/MemoryAccessDenominators.h | 4 - .../madmatrix/backend/simd/MemoryAccessGs.h | 12 - .../backend/simd/MemoryAccessHelpers.h | 6 - .../backend/simd/MemoryAccessIflavorVec.h | 8 - .../backend/simd/MemoryAccessMatrixElements.h | 8 - .../backend/simd/MemoryAccessMomenta.h | 23 - .../backend/simd/MemoryAccessNumerators.h | 4 - .../backend/simd/MemoryAccessRandomNumbers.h | 4 - .../backend/simd/MemoryAccessVectors.h | 74 -- .../backend/simd/MemoryAccessWavefunctions.h | 4 - .../backend/simd/MemoryAccessWeights.h | 4 - .../madmatrix/backend/simd/MemoryBuffers.h | 238 ------ .../backend/simd/RamboSamplingKernels.cc | 78 -- .../backend/simd/RamboSamplingKernels.h | 39 - .../backend/simd/RandomNumberKernels.h | 4 - .../madmatrix/backend/simd/check_sa.cc | 139 --- .../madmatrix/backend/simd/color_sum.cc | 297 ------- .../madmatrix/backend/simd/color_sum.h | 64 -- .../madmatrix/backend/simd/coloramps.h | 10 - .../madmatrix/backend/simd/constexpr_math.h | 4 - .../madmatrix/backend/simd/fbridge.cc | 16 - .../madmatrix/backend/simd/fbridge.h | 4 - .../madmatrix/backend/simd/fsampler.cc | 14 - .../madmatrix/backend/simd/massless_rambo.h | 16 - .../madmatrix/backend/simd/mgOnGpuConfig.h | 10 +- .../madmatrix/backend/simd/mgOnGpuCxtypes.h | 16 - .../madmatrix/backend/simd/mgOnGpuFptypes.h | 51 -- .../madmatrix/backend/simd/mgOnGpuVectors.h | 149 ---- .../madmatrix/backend/simd/runTest.cc | 187 ---- .../madmatrix/backend/simd/testmisc.cc | 38 - .../madmatrix/backend/simd/testxxx.cc | 48 -- .../madmatrix/backend/simd/umami.cc | 220 ----- 150 files changed, 2 insertions(+), 11418 deletions(-) delete mode 100644 madgraph/iolibs/template_files/madmatrix/Bridge.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/BridgeKernels.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/BridgeKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/Bridge.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/Bridge.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/Bridge.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.h diff --git a/madgraph/iolibs/template_files/madmatrix/Bridge.h b/madgraph/iolibs/template_files/madmatrix/Bridge.h deleted file mode 100644 index 8417e3ad15..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/Bridge.h +++ /dev/null @@ -1,633 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Nov 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2021-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef BRIDGE_H -#define BRIDGE_H 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" // for CPPProcess -#include "CrossSectionKernels.h" // for flagAbnormalMEs -#include "MatrixElementKernels.h" // for MatrixElementKernelHost, MatrixElementKernelDevice -#include "MemoryAccessMomenta.h" // for MemoryAccessMomenta::neppM -#include "MemoryBuffers.h" // for HostBufferMomenta, DeviceBufferMomenta etc - -//#ifdef __HIPCC__ -//#include // see -//https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 #else #include -// // bypass this completely to ease portability on LUMI #803 #endif - -#include // bypass std::filesystem #803 - -#include -#include -#include -#include -#include -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - /** - * A base class for a class whose pointer is passed between Fortran and C++. - * This is not really necessary, but it allows minimal type checks on all such - * pointers. - */ - struct CppObjectInFortran - { - CppObjectInFortran() {} - virtual ~CppObjectInFortran() {} - }; - - //-------------------------------------------------------------------------- - /** - * A templated class for calling the CUDA/C++ matrix element calculations of the - * event generation workflow. The FORTRANFPTYPE template parameter indicates the - * precision of the Fortran momenta from MadEvent (float or double). The - * precision of the matrix element calculation is hardcoded in the fptype - * typedef in CUDA/C++. - * - * The Fortran momenta passed in are in the form of - * DOUBLE PRECISION P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) - * where the dimensions are , , - * . In memory, this is stored in a way that C reads as an array - * P_MULTI[nevtF][nparF][np4F]. The CUDA/C++ momenta are stored as an - * array[npagM][npar][np4][neppM] with nevt=npagM*neppM. The Bridge is - * configured to store nevt==nevtF events in CUDA/C++. It also checks that - * Fortran and C++ parameters match, nparF==npar and np4F==np4. - * - * The cpu/gpu sequences take FORTRANFPTYPE* (not fptype*) momenta/MEs. - * This allows mixing double in MadEvent Fortran with float in CUDA/C++ - * sigmaKin. In the fcheck_sa.f test, Fortran uses double while CUDA/C++ may use - * double or float. In the check_sa "--bridge" test, everything is implemented - * in fptype (double or float). - */ - template - class Bridge final : public CppObjectInFortran - { - public: - /** - * Constructor - * - * @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran array - * loops (VECSIZE_USED <= VECSIZE_MEMMAX) - * @param nparF (NEXTERNAL, nexternal.inc) number of external particles in - * Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F number of momenta components, usually 4, in Fortran arrays - * (KEPT FOR SANITY CHECKS ONLY) - */ - Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F ); - - /** - * Destructor - */ - virtual ~Bridge() {} - - // Delete copy/move constructors and assignment operators - Bridge( const Bridge& ) = delete; - Bridge( Bridge&& ) = delete; - Bridge& operator=( const Bridge& ) = delete; - Bridge& operator=( Bridge&& ) = delete; - -#ifdef MGONGPUCPP_GPUIMPL - /** - * Set the gpublocks and gputhreads for the gpusequence - throws if evnt != - * gpublocks*gputhreads (this is needed for BridgeKernel tests rather than for - * actual production use in Fortran) - * - * @param gpublocks number of gpublocks - * @param gputhreads number of gputhreads - */ - void set_gpugrid( const int gpublocks, const int gputhreads ); - - /** - * Sequence to be executed for the Cuda matrix element calculation - * - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant - * alphas) - * @param iflavorVec the index of the flavor combination - * @param rndhel the pointer to the input random numbers for helicity - * selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 - * to n - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void gpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); -#else - /** - * Sequence to be executed for the vectorized CPU matrix element calculation - * - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant - * alphas) - * @param iflavorVec the index of the flavor combination - * @param rndhel the pointer to the input random numbers for helicity - * selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 - * to n - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void cpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); -#endif - - // Return the number of good helicities (-1 initially when they have not yet - // been calculated) - int nGoodHel() const { return m_nGoodHel; } - - // Return the total number of helicities (expose cudacpp ncomb in the Bridge - // interface to Fortran) - constexpr int nTotHel() const { return CPPProcess::ncomb; } - - private: - unsigned int m_nevt; // number of events - int m_nGoodHel; // the number of good helicities (-1 initially when they have - // not yet been calculated) - -#ifdef MGONGPUCPP_GPUIMPL - int m_gputhreads; // number of gpu threads (default set from number of - // events, can be modified) - int m_gpublocks; // number of gpu blocks (default set from number of events, - // can be modified) - DeviceBuffer m_devMomentaF; - DeviceBufferMomenta m_devMomentaC; - DeviceBufferGs m_devGs; - DeviceBufferIflavorVec m_devIflavorVec; - DeviceBufferRndNumHelicity m_devRndHel; - DeviceBufferRndNumColor m_devRndCol; - DeviceBufferMatrixElements m_devMEs; - DeviceBufferSelectedHelicity m_devSelHel; - DeviceBufferSelectedColor m_devSelCol; - DeviceBufferChannelIds m_devChannelIds; - PinnedHostBufferIflavorVec m_hstIflavorVec; - PinnedHostBufferGs m_hstGs; - PinnedHostBufferRndNumHelicity m_hstRndHel; - PinnedHostBufferRndNumColor m_hstRndCol; - PinnedHostBufferMatrixElements m_hstMEs; - PinnedHostBufferSelectedHelicity m_hstSelHel; - PinnedHostBufferSelectedColor m_hstSelCol; - PinnedHostBufferChannelIds m_hstChannelIds; - std::unique_ptr m_pmek; - // static constexpr int s_gputhreadsmin = 16; // minimum number of gpu threads - // (TEST VALUE FOR MADEVENT) - static constexpr int s_gputhreadsmin = - 32; // minimum number of gpu threads (DEFAULT) -#else - HostBufferMomenta m_hstMomentaC; - HostBufferGs m_hstGs; - HostBufferIflavorVec m_hstIflavorVec; - HostBufferRndNumHelicity m_hstRndHel; - HostBufferRndNumColor m_hstRndCol; - HostBufferMatrixElements m_hstMEs; - HostBufferSelectedHelicity m_hstSelHel; - HostBufferSelectedColor m_hstSelCol; - HostBufferChannelIds m_hstChannelIds; - std::unique_ptr m_pmek; -#endif - }; - - //-------------------------------------------------------------------------- - // - // Forward declare transposition methods - // - -#ifdef MGONGPUCPP_GPUIMPL - - template - __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); - -#endif // MGONGPUCPP_GPUIMPL - - template - void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); - - template - void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ); - - //-------------------------------------------------------------------------- - // - // Implementations of member functions of class Bridge - // - - template - Bridge::Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F) - : m_nevt( nevtF ), m_nGoodHel( -1 ) -#ifdef MGONGPUCPP_GPUIMPL - , m_gputhreads( 256 ) // default number of gpu threads - , m_gpublocks( m_nevt / m_gputhreads ) // this ensures m_nevt <= m_gpublocks*m_gputhreads - , m_devMomentaF( m_nevt ) - , m_devMomentaC( m_nevt ) - , m_devIflavorVec( m_nevt ) - , m_devGs( m_nevt ) - , m_devRndHel( m_nevt ) - , m_devRndCol( m_nevt ) - , m_devMEs( m_nevt ) - , m_devSelHel( m_nevt ) - , m_devSelCol( m_nevt ) - , m_devChannelIds( m_nevt ) -#else - , m_hstMomentaC( m_nevt ) -#endif - , m_hstGs( m_nevt ) - , m_hstIflavorVec( m_nevt ) - , m_hstRndHel( m_nevt ) - , m_hstRndCol( m_nevt ) - , m_hstMEs( m_nevt ) - , m_hstSelHel( m_nevt ) - , m_hstSelCol( m_nevt ) - , m_hstChannelIds( m_nevt ) - , m_pmek( nullptr ) - { - if( nparF != CPPProcess::npar ) - throw std::runtime_error( "Bridge constructor: npar mismatch" ); - if( np4F != CPPProcess::np4 ) - throw std::runtime_error( "Bridge constructor: np4 mismatch" ); -#ifdef MGONGPUCPP_GPUIMPL - if( ( m_nevt < s_gputhreadsmin ) || ( m_nevt % s_gputhreadsmin != 0 ) ) - throw std::runtime_error( - "Bridge constructor: nevt should be a multiple of " + - std::to_string( s_gputhreadsmin ) ); - while( m_nevt != m_gpublocks * m_gputhreads ) - { - m_gputhreads /= 2; - if( m_gputhreads < s_gputhreadsmin ) - throw std::logic_error( - "Bridge constructor: FIXME! cannot choose gputhreads" ); // this - // should - // never - // happen! - m_gpublocks = m_nevt / m_gputhreads; - } -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Instantiate device Bridge (nevt=" << m_nevt - << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads - << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" - << std::endl; -#endif - m_pmek.reset( new MatrixElementKernelDevice( - m_devMomentaC, m_devGs, m_devIflavorVec, m_devRndHel, m_devRndCol, m_devChannelIds, m_devMEs, m_devSelHel, m_devSelCol, m_gpublocks, m_gputhreads) ); -#else -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Instantiate host Bridge (nevt=" << m_nevt << ")" - << std::endl; -#endif - m_pmek.reset( new MatrixElementKernelHost( - m_hstMomentaC, m_hstGs, m_hstIflavorVec, m_hstRndHel, m_hstRndCol, m_hstChannelIds, m_hstMEs, m_hstSelHel, m_hstSelCol, m_nevt ) ); -#endif // MGONGPUCPP_GPUIMPL - // Create a process object, read param card and set parameters - // FIXME: the process instance can happily go out of scope because it is only - // needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? what if fbridgecreate - // is called from several Fortran threads? - CPPProcess process( /*verbose=*/false ); - std::string paramCard = - "../Cards/param_card.dat"; // ZW: change default param_card.dat location - // to one dir down - /* -#ifdef __HIPCC__ - if( !std::experimental::filesystem::exists( paramCard ) ) paramCard = "../" + -paramCard; #else if( !std::filesystem::exists( paramCard ) ) paramCard = "../" + -paramCard; #endif - */ - // struct stat dummybuffer; // bypass std::filesystem #803 - // if( !( stat( paramCard.c_str(), &dummyBuffer ) == 0 ) ) paramCard = "../" + - // paramCard; // - auto fileExists = []( std::string& fileName ) - { - struct stat buffer; - return stat( fileName.c_str(), &buffer ) == 0; - }; - size_t paramCardCheck = 2; // ZW: check for paramCard up to 2 directories up - for( size_t k = 0; k < paramCardCheck; ++k ) - { - if( fileExists( paramCard ) ) break; // bypass std::filesystem #803 - paramCard = "../" + paramCard; - } - process.initProc( paramCard ); - } - -#ifdef MGONGPUCPP_GPUIMPL - template - void Bridge::set_gpugrid( const int gpublocks, - const int gputhreads ) - { - if( m_nevt != gpublocks * gputhreads ) - throw std::runtime_error( - "Bridge: gpublocks*gputhreads must equal m_nevt in set_gpugrid" ); - m_gpublocks = gpublocks; - m_gputhreads = gputhreads; -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Set grid in Bridge (nevt=" << m_nevt - << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads - << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" - << std::endl; -#endif - m_pmek->setGrid( m_gpublocks, m_gputhreads ); - } -#endif - -#ifdef MGONGPUCPP_GPUIMPL - template - void Bridge::gpu_sequence( const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool goodHelOnly ) - { - constexpr int neppM = MemoryAccessMomenta::neppM; - if constexpr( neppM == 1 && std::is_same_v ) - { - gpuMemcpy( m_devMomentaC.data(), momenta, m_devMomentaC.bytes(), gpuMemcpyHostToDevice ); - } - else - { - gpuMemcpy( m_devMomentaF.data(), momenta, m_devMomentaF.bytes(), gpuMemcpyHostToDevice ); - const int thrPerEvt = - CPPProcess::npar * - CPPProcess::np4; // AV: transpose alg does 1 element per thread (NOT 1 - // event per thread) - // const int thrPerEvt = 1; // AV: try new alg with 1 event per thread... - // this seems slower - gpuLaunchKernel( dev_transposeMomentaF2C, m_gpublocks * thrPerEvt, m_gputhreads, m_devMomentaF.data(), m_devMomentaC.data(), m_nevt ); - } - if constexpr( std::is_same_v ) - { - memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); - } - else - { - std::copy( gs, gs + m_nevt, m_hstGs.data() ); - std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); - std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); - } - const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); - if( useChannelIds ) - memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); - // else ... // no need to initialize m_hstChannel: it is allocated with - // gpuMallocHost and NOT initialized in PinnedHostBufferBase, but it is NOT - // used later on - // initialise iflavorVec - memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); - copyDeviceFromHost( m_devGs, m_hstGs ); - copyDeviceFromHost( m_devRndHel, m_hstRndHel ); - copyDeviceFromHost( m_devRndCol, m_hstRndCol ); - if( useChannelIds ) copyDeviceFromHost( m_devChannelIds, m_hstChannelIds ); - copyDeviceFromHost( m_devIflavorVec, m_hstIflavorVec ); - if( m_nGoodHel < 0 ) - { - m_nGoodHel = m_pmek->computeGoodHelicities(); - if( m_nGoodHel < 0 ) - throw std::runtime_error( - "Bridge gpu_sequence: computeGoodHelicities returned nGoodHel<0" ); - } - if( goodHelOnly ) return; - m_pmek->computeMatrixElements( useChannelIds ); - copyHostFromDevice( m_hstMEs, m_devMEs ); -#ifdef MGONGPUCPP_VERBOSE - flagAbnormalMEs( m_hstMEs.data(), m_nevt ); -#endif - copyHostFromDevice( m_hstSelHel, m_devSelHel ); - copyHostFromDevice( m_hstSelCol, m_devSelCol ); - if constexpr( std::is_same_v ) - { - memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); - memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); - memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); - } - else - { - std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); - std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); - std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); - } - } -#endif - -#ifndef MGONGPUCPP_GPUIMPL - template - void Bridge::cpu_sequence( const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool goodHelOnly ) - { - hst_transposeMomentaF2C( momenta, m_hstMomentaC.data(), m_nevt ); - if constexpr( std::is_same_v ) - { - memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); - } - else - { - std::copy( gs, gs + m_nevt, m_hstGs.data() ); - std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); - std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); - } - const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); - if( useChannelIds ) - memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); - // else ... // no need to initialize m_hstChannel: it is allocated and default - // initialized in HostBufferBase (and it is not used later on anyway) - // initialise iflavorVec - memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); - if( m_nGoodHel < 0 ) - { - m_nGoodHel = m_pmek->computeGoodHelicities(); - if( m_nGoodHel < 0 ) - throw std::runtime_error( - "Bridge cpu_sequence: computeGoodHelicities returned nGoodHel<0" ); - } - if( goodHelOnly ) return; - m_pmek->computeMatrixElements( useChannelIds ); -#ifdef MGONGPUCPP_VERBOSE - flagAbnormalMEs( m_hstMEs.data(), m_nevt ); -#endif - if constexpr( std::is_same_v ) - { - memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); - memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); - memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); - } - else - { - std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); - std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); - std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); - } - } -#endif - - //-------------------------------------------------------------------------- - // - // Implementations of transposition methods - // - FORTRAN arrays: P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) ==> - // p_multi[nevtF][nparF][np4F] in C++ (AOS) - // - C++ array: momenta[npagM][npar][np4][neppM] with nevt=npagM*neppM (AOSOA) - // - -#ifdef MGONGPUCPP_GPUIMPL - template - __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool oldImplementation = true; // default: use old implementation - if constexpr( oldImplementation ) - { - // SR initial implementation - constexpr int part = CPPProcess::npar; - constexpr int mome = CPPProcess::np4; - constexpr int strd = MemoryAccessMomenta::neppM; - int pos = blockDim.x * blockIdx.x + threadIdx.x; - int arrlen = nevt * part * mome; - if( pos < arrlen ) - { - int page_i = pos / ( strd * mome * part ); - int rest_1 = pos % ( strd * mome * part ); - int part_i = rest_1 / ( strd * mome ); - int rest_2 = rest_1 % ( strd * mome ); - int mome_i = rest_2 / strd; - int strd_i = rest_2 % strd; - int inpos = ( page_i * strd + strd_i ) // event number - * ( part * mome ) // event size (pos of event) - + part_i * mome // particle inside event - + mome_i; // momentum inside particle - out[pos] = in[inpos]; // F2C (Fortran to C) - } - } - else - { - // AV attempt another implementation with 1 event per thread: this seems - // slower... F-style: AOS[nevtF][nparF][np4F] C-style: - // AOSOA[npagM][npar][np4][neppM] with nevt=npagM*neppM - constexpr int npar = CPPProcess::npar; - constexpr int np4 = CPPProcess::np4; - constexpr int neppM = MemoryAccessMomenta::neppM; - assert( nevt % neppM == - 0 ); // number of events is not a multiple of neppM??? - int ievt = blockDim.x * blockIdx.x + threadIdx.x; - int ipagM = ievt / neppM; - int ieppM = ievt % neppM; - for( int ip4 = 0; ip4 < np4; ip4++ ) - for( int ipar = 0; ipar < npar; ipar++ ) - { - int cpos = ipagM * npar * np4 * neppM + ipar * np4 * neppM + - ip4 * neppM + ieppM; - int fpos = ievt * npar * np4 + ipar * np4 + ip4; - out[cpos] = in[fpos]; // F2C (Fortran to C) - } - } - } -#endif - - template - void hst_transposeMomenta( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool oldImplementation = false; // default: use new implementation - if constexpr( oldImplementation ) - { - // SR initial implementation - constexpr unsigned int part = CPPProcess::npar; - constexpr unsigned int mome = CPPProcess::np4; - constexpr unsigned int strd = MemoryAccessMomenta::neppM; - unsigned int arrlen = nevt * part * mome; - for( unsigned int pos = 0; pos < arrlen; ++pos ) - { - unsigned int page_i = pos / ( strd * mome * part ); - unsigned int rest_1 = pos % ( strd * mome * part ); - unsigned int part_i = rest_1 / ( strd * mome ); - unsigned int rest_2 = rest_1 % ( strd * mome ); - unsigned int mome_i = rest_2 / strd; - unsigned int strd_i = rest_2 % strd; - unsigned int inpos = ( page_i * strd + strd_i ) // event number - * ( part * mome ) // event size (pos of event) - + part_i * mome // particle inside event - + mome_i; // momentum inside particle - if constexpr( F2C ) // needs c++17 and cuda >=11.2 (#333) - out[pos] = in[inpos]; // F2C (Fortran to C) - else - out[inpos] = in[pos]; // C2F (C to Fortran) - } - } - else - { - // AV attempt another implementation: this is slightly faster (better c++ - // pipelining?) [NB! this is not a transposition, it is an AOS to AOSOA - // conversion: if neppM=1, a memcpy is enough] F-style: - // AOS[nevtF][nparF][np4F] C-style: AOSOA[npagM][npar][np4][neppM] with - // nevt=npagM*neppM - constexpr unsigned int npar = CPPProcess::npar; - constexpr unsigned int np4 = CPPProcess::np4; - constexpr unsigned int neppM = MemoryAccessMomenta::neppM; - if constexpr( neppM == 1 && std::is_same_v ) - { - memcpy( out, in, nevt * npar * np4 * sizeof( Tin ) ); - } - else - { - const unsigned int npagM = nevt / neppM; - assert( nevt % neppM == - 0 ); // number of events is not a multiple of neppM??? - for( unsigned int ipagM = 0; ipagM < npagM; ipagM++ ) - for( unsigned int ip4 = 0; ip4 < np4; ip4++ ) - for( unsigned int ipar = 0; ipar < npar; ipar++ ) - for( unsigned int ieppM = 0; ieppM < neppM; ieppM++ ) - { - unsigned int ievt = ipagM * neppM + ieppM; - unsigned int cpos = ipagM * npar * np4 * neppM + - ipar * np4 * neppM + ip4 * neppM + ieppM; - unsigned int fpos = ievt * npar * np4 + ipar * np4 + ip4; - if constexpr( F2C ) - out[cpos] = in[fpos]; // F2C (Fortran to C) - else - out[fpos] = in[cpos]; // C2F (C to Fortran) - } - } - } - } - - template - void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool F2C = true; - hst_transposeMomenta( in, out, nevt ); - } - - template - void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool F2C = false; - hst_transposeMomenta( in, out, nevt ); - } - - //-------------------------------------------------------------------------- -} // namespace mg5amcGpu -#endif // BRIDGE_H diff --git a/madgraph/iolibs/template_files/madmatrix/BridgeKernels.cc b/madgraph/iolibs/template_files/madmatrix/BridgeKernels.cc deleted file mode 100644 index 65c97616d8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/BridgeKernels.cc +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "BridgeKernels.h" - -#include "GpuAbstraction.h" -#include "MemoryAccessMomenta.h" - -#include - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - - //-------------------------------------------------------------------------- - - BridgeKernelBase::BridgeKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( nevt ) - , m_bridge( nevt, npar, np4 ) - { - if( m_momenta.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: momenta must be a host array" ); - if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: matrixElements must be a host array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with matrixElements" ); - } - - //-------------------------------------------------------------------------- -} - -//============================================================================ - -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu -{ - - //-------------------------------------------------------------------------- - - BridgeKernelHost::BridgeKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: Gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt) - : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, nevt) - , m_fortranMomenta( nevt ) - { - } - - //-------------------------------------------------------------------------- - - void BridgeKernelHost::transposeInputMomentaC2F() - { - hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); - } - - //-------------------------------------------------------------------------- - - int BridgeKernelHost::computeGoodHelicities() - { - constexpr bool goodHelOnly = true; - constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering - m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - return m_bridge.nGoodHel(); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelHost::computeMatrixElements( const bool useChannelIds ) - { - constexpr bool goodHelOnly = false; - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - //-------------------------------------------------------------------------- - - BridgeKernelDevice::BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: Gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads) - : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, gpublocks * gputhreads) - , m_fortranMomenta( nevt() ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if( m_gpublocks == 0 ) throw std::runtime_error( "BridgeKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "BridgeKernelDevice: gputhreads must be > 0" ); - m_bridge.set_gpugrid( gpublocks, gputhreads ); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelDevice::transposeInputMomentaC2F() - { - hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); - } - - //-------------------------------------------------------------------------- - - int BridgeKernelDevice::computeGoodHelicities() - { - constexpr bool goodHelOnly = true; - constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering - m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - return m_bridge.nGoodHel(); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelDevice::computeMatrixElements( const bool useChannelIds ) - { - constexpr bool goodHelOnly = false; - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/BridgeKernels.h b/madgraph/iolibs/template_files/madmatrix/BridgeKernels.h deleted file mode 100644 index 2d873486a8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/BridgeKernels.h +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally originally by: A. Valassi (Jan 2022). -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef BRIDGEKERNELS_H -#define BRIDGEKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "Bridge.h" -#include "MatrixElementKernels.h" -#include "MemoryBuffers.h" - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - // A Bridge wrapper base class encapsulating matrix element calculations on a CPU host - class BridgeKernelBase : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - BridgeKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~BridgeKernelBase() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - virtual void transposeInputMomentaC2F() = 0; - - protected: - - // The wrapped bridge - Bridge m_bridge; - }; - - //-------------------------------------------------------------------------- - -#ifndef MGONGPUCPP_GPUIMPL - // A Bridge wrapper class encapsulating matrix element calculations on a CPU host - class BridgeKernelHost final : public BridgeKernelBase - { - public: - - // Constructor from existing input and output buffers - BridgeKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~BridgeKernelHost() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - void transposeInputMomentaC2F() override final; - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // The buffer for the input momenta, transposed to Fortran array indexing - HostBufferMomenta m_fortranMomenta; - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A Bridge wrapper class encapsulating matrix element calculations on a GPU device - class BridgeKernelDevice : public BridgeKernelBase - { - public: - - // Constructor from existing input and output buffers - BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads); - - // Destructor - virtual ~BridgeKernelDevice() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - void transposeInputMomentaC2F() override final; - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The buffer for the input momenta, transposed to Fortran array indexing - PinnedHostBufferMomenta m_fortranMomenta; - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; -#endif - - //-------------------------------------------------------------------------- -} -#endif // BRIDGEKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/Bridge.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/Bridge.h deleted file mode 100644 index 8417e3ad15..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/Bridge.h +++ /dev/null @@ -1,633 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Nov 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2021-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef BRIDGE_H -#define BRIDGE_H 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" // for CPPProcess -#include "CrossSectionKernels.h" // for flagAbnormalMEs -#include "MatrixElementKernels.h" // for MatrixElementKernelHost, MatrixElementKernelDevice -#include "MemoryAccessMomenta.h" // for MemoryAccessMomenta::neppM -#include "MemoryBuffers.h" // for HostBufferMomenta, DeviceBufferMomenta etc - -//#ifdef __HIPCC__ -//#include // see -//https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 #else #include -// // bypass this completely to ease portability on LUMI #803 #endif - -#include // bypass std::filesystem #803 - -#include -#include -#include -#include -#include -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - /** - * A base class for a class whose pointer is passed between Fortran and C++. - * This is not really necessary, but it allows minimal type checks on all such - * pointers. - */ - struct CppObjectInFortran - { - CppObjectInFortran() {} - virtual ~CppObjectInFortran() {} - }; - - //-------------------------------------------------------------------------- - /** - * A templated class for calling the CUDA/C++ matrix element calculations of the - * event generation workflow. The FORTRANFPTYPE template parameter indicates the - * precision of the Fortran momenta from MadEvent (float or double). The - * precision of the matrix element calculation is hardcoded in the fptype - * typedef in CUDA/C++. - * - * The Fortran momenta passed in are in the form of - * DOUBLE PRECISION P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) - * where the dimensions are , , - * . In memory, this is stored in a way that C reads as an array - * P_MULTI[nevtF][nparF][np4F]. The CUDA/C++ momenta are stored as an - * array[npagM][npar][np4][neppM] with nevt=npagM*neppM. The Bridge is - * configured to store nevt==nevtF events in CUDA/C++. It also checks that - * Fortran and C++ parameters match, nparF==npar and np4F==np4. - * - * The cpu/gpu sequences take FORTRANFPTYPE* (not fptype*) momenta/MEs. - * This allows mixing double in MadEvent Fortran with float in CUDA/C++ - * sigmaKin. In the fcheck_sa.f test, Fortran uses double while CUDA/C++ may use - * double or float. In the check_sa "--bridge" test, everything is implemented - * in fptype (double or float). - */ - template - class Bridge final : public CppObjectInFortran - { - public: - /** - * Constructor - * - * @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran array - * loops (VECSIZE_USED <= VECSIZE_MEMMAX) - * @param nparF (NEXTERNAL, nexternal.inc) number of external particles in - * Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F number of momenta components, usually 4, in Fortran arrays - * (KEPT FOR SANITY CHECKS ONLY) - */ - Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F ); - - /** - * Destructor - */ - virtual ~Bridge() {} - - // Delete copy/move constructors and assignment operators - Bridge( const Bridge& ) = delete; - Bridge( Bridge&& ) = delete; - Bridge& operator=( const Bridge& ) = delete; - Bridge& operator=( Bridge&& ) = delete; - -#ifdef MGONGPUCPP_GPUIMPL - /** - * Set the gpublocks and gputhreads for the gpusequence - throws if evnt != - * gpublocks*gputhreads (this is needed for BridgeKernel tests rather than for - * actual production use in Fortran) - * - * @param gpublocks number of gpublocks - * @param gputhreads number of gputhreads - */ - void set_gpugrid( const int gpublocks, const int gputhreads ); - - /** - * Sequence to be executed for the Cuda matrix element calculation - * - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant - * alphas) - * @param iflavorVec the index of the flavor combination - * @param rndhel the pointer to the input random numbers for helicity - * selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 - * to n - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void gpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); -#else - /** - * Sequence to be executed for the vectorized CPU matrix element calculation - * - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant - * alphas) - * @param iflavorVec the index of the flavor combination - * @param rndhel the pointer to the input random numbers for helicity - * selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 - * to n - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void cpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); -#endif - - // Return the number of good helicities (-1 initially when they have not yet - // been calculated) - int nGoodHel() const { return m_nGoodHel; } - - // Return the total number of helicities (expose cudacpp ncomb in the Bridge - // interface to Fortran) - constexpr int nTotHel() const { return CPPProcess::ncomb; } - - private: - unsigned int m_nevt; // number of events - int m_nGoodHel; // the number of good helicities (-1 initially when they have - // not yet been calculated) - -#ifdef MGONGPUCPP_GPUIMPL - int m_gputhreads; // number of gpu threads (default set from number of - // events, can be modified) - int m_gpublocks; // number of gpu blocks (default set from number of events, - // can be modified) - DeviceBuffer m_devMomentaF; - DeviceBufferMomenta m_devMomentaC; - DeviceBufferGs m_devGs; - DeviceBufferIflavorVec m_devIflavorVec; - DeviceBufferRndNumHelicity m_devRndHel; - DeviceBufferRndNumColor m_devRndCol; - DeviceBufferMatrixElements m_devMEs; - DeviceBufferSelectedHelicity m_devSelHel; - DeviceBufferSelectedColor m_devSelCol; - DeviceBufferChannelIds m_devChannelIds; - PinnedHostBufferIflavorVec m_hstIflavorVec; - PinnedHostBufferGs m_hstGs; - PinnedHostBufferRndNumHelicity m_hstRndHel; - PinnedHostBufferRndNumColor m_hstRndCol; - PinnedHostBufferMatrixElements m_hstMEs; - PinnedHostBufferSelectedHelicity m_hstSelHel; - PinnedHostBufferSelectedColor m_hstSelCol; - PinnedHostBufferChannelIds m_hstChannelIds; - std::unique_ptr m_pmek; - // static constexpr int s_gputhreadsmin = 16; // minimum number of gpu threads - // (TEST VALUE FOR MADEVENT) - static constexpr int s_gputhreadsmin = - 32; // minimum number of gpu threads (DEFAULT) -#else - HostBufferMomenta m_hstMomentaC; - HostBufferGs m_hstGs; - HostBufferIflavorVec m_hstIflavorVec; - HostBufferRndNumHelicity m_hstRndHel; - HostBufferRndNumColor m_hstRndCol; - HostBufferMatrixElements m_hstMEs; - HostBufferSelectedHelicity m_hstSelHel; - HostBufferSelectedColor m_hstSelCol; - HostBufferChannelIds m_hstChannelIds; - std::unique_ptr m_pmek; -#endif - }; - - //-------------------------------------------------------------------------- - // - // Forward declare transposition methods - // - -#ifdef MGONGPUCPP_GPUIMPL - - template - __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); - -#endif // MGONGPUCPP_GPUIMPL - - template - void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); - - template - void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ); - - //-------------------------------------------------------------------------- - // - // Implementations of member functions of class Bridge - // - - template - Bridge::Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F) - : m_nevt( nevtF ), m_nGoodHel( -1 ) -#ifdef MGONGPUCPP_GPUIMPL - , m_gputhreads( 256 ) // default number of gpu threads - , m_gpublocks( m_nevt / m_gputhreads ) // this ensures m_nevt <= m_gpublocks*m_gputhreads - , m_devMomentaF( m_nevt ) - , m_devMomentaC( m_nevt ) - , m_devIflavorVec( m_nevt ) - , m_devGs( m_nevt ) - , m_devRndHel( m_nevt ) - , m_devRndCol( m_nevt ) - , m_devMEs( m_nevt ) - , m_devSelHel( m_nevt ) - , m_devSelCol( m_nevt ) - , m_devChannelIds( m_nevt ) -#else - , m_hstMomentaC( m_nevt ) -#endif - , m_hstGs( m_nevt ) - , m_hstIflavorVec( m_nevt ) - , m_hstRndHel( m_nevt ) - , m_hstRndCol( m_nevt ) - , m_hstMEs( m_nevt ) - , m_hstSelHel( m_nevt ) - , m_hstSelCol( m_nevt ) - , m_hstChannelIds( m_nevt ) - , m_pmek( nullptr ) - { - if( nparF != CPPProcess::npar ) - throw std::runtime_error( "Bridge constructor: npar mismatch" ); - if( np4F != CPPProcess::np4 ) - throw std::runtime_error( "Bridge constructor: np4 mismatch" ); -#ifdef MGONGPUCPP_GPUIMPL - if( ( m_nevt < s_gputhreadsmin ) || ( m_nevt % s_gputhreadsmin != 0 ) ) - throw std::runtime_error( - "Bridge constructor: nevt should be a multiple of " + - std::to_string( s_gputhreadsmin ) ); - while( m_nevt != m_gpublocks * m_gputhreads ) - { - m_gputhreads /= 2; - if( m_gputhreads < s_gputhreadsmin ) - throw std::logic_error( - "Bridge constructor: FIXME! cannot choose gputhreads" ); // this - // should - // never - // happen! - m_gpublocks = m_nevt / m_gputhreads; - } -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Instantiate device Bridge (nevt=" << m_nevt - << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads - << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" - << std::endl; -#endif - m_pmek.reset( new MatrixElementKernelDevice( - m_devMomentaC, m_devGs, m_devIflavorVec, m_devRndHel, m_devRndCol, m_devChannelIds, m_devMEs, m_devSelHel, m_devSelCol, m_gpublocks, m_gputhreads) ); -#else -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Instantiate host Bridge (nevt=" << m_nevt << ")" - << std::endl; -#endif - m_pmek.reset( new MatrixElementKernelHost( - m_hstMomentaC, m_hstGs, m_hstIflavorVec, m_hstRndHel, m_hstRndCol, m_hstChannelIds, m_hstMEs, m_hstSelHel, m_hstSelCol, m_nevt ) ); -#endif // MGONGPUCPP_GPUIMPL - // Create a process object, read param card and set parameters - // FIXME: the process instance can happily go out of scope because it is only - // needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? what if fbridgecreate - // is called from several Fortran threads? - CPPProcess process( /*verbose=*/false ); - std::string paramCard = - "../Cards/param_card.dat"; // ZW: change default param_card.dat location - // to one dir down - /* -#ifdef __HIPCC__ - if( !std::experimental::filesystem::exists( paramCard ) ) paramCard = "../" + -paramCard; #else if( !std::filesystem::exists( paramCard ) ) paramCard = "../" + -paramCard; #endif - */ - // struct stat dummybuffer; // bypass std::filesystem #803 - // if( !( stat( paramCard.c_str(), &dummyBuffer ) == 0 ) ) paramCard = "../" + - // paramCard; // - auto fileExists = []( std::string& fileName ) - { - struct stat buffer; - return stat( fileName.c_str(), &buffer ) == 0; - }; - size_t paramCardCheck = 2; // ZW: check for paramCard up to 2 directories up - for( size_t k = 0; k < paramCardCheck; ++k ) - { - if( fileExists( paramCard ) ) break; // bypass std::filesystem #803 - paramCard = "../" + paramCard; - } - process.initProc( paramCard ); - } - -#ifdef MGONGPUCPP_GPUIMPL - template - void Bridge::set_gpugrid( const int gpublocks, - const int gputhreads ) - { - if( m_nevt != gpublocks * gputhreads ) - throw std::runtime_error( - "Bridge: gpublocks*gputhreads must equal m_nevt in set_gpugrid" ); - m_gpublocks = gpublocks; - m_gputhreads = gputhreads; -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Set grid in Bridge (nevt=" << m_nevt - << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads - << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" - << std::endl; -#endif - m_pmek->setGrid( m_gpublocks, m_gputhreads ); - } -#endif - -#ifdef MGONGPUCPP_GPUIMPL - template - void Bridge::gpu_sequence( const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool goodHelOnly ) - { - constexpr int neppM = MemoryAccessMomenta::neppM; - if constexpr( neppM == 1 && std::is_same_v ) - { - gpuMemcpy( m_devMomentaC.data(), momenta, m_devMomentaC.bytes(), gpuMemcpyHostToDevice ); - } - else - { - gpuMemcpy( m_devMomentaF.data(), momenta, m_devMomentaF.bytes(), gpuMemcpyHostToDevice ); - const int thrPerEvt = - CPPProcess::npar * - CPPProcess::np4; // AV: transpose alg does 1 element per thread (NOT 1 - // event per thread) - // const int thrPerEvt = 1; // AV: try new alg with 1 event per thread... - // this seems slower - gpuLaunchKernel( dev_transposeMomentaF2C, m_gpublocks * thrPerEvt, m_gputhreads, m_devMomentaF.data(), m_devMomentaC.data(), m_nevt ); - } - if constexpr( std::is_same_v ) - { - memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); - } - else - { - std::copy( gs, gs + m_nevt, m_hstGs.data() ); - std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); - std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); - } - const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); - if( useChannelIds ) - memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); - // else ... // no need to initialize m_hstChannel: it is allocated with - // gpuMallocHost and NOT initialized in PinnedHostBufferBase, but it is NOT - // used later on - // initialise iflavorVec - memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); - copyDeviceFromHost( m_devGs, m_hstGs ); - copyDeviceFromHost( m_devRndHel, m_hstRndHel ); - copyDeviceFromHost( m_devRndCol, m_hstRndCol ); - if( useChannelIds ) copyDeviceFromHost( m_devChannelIds, m_hstChannelIds ); - copyDeviceFromHost( m_devIflavorVec, m_hstIflavorVec ); - if( m_nGoodHel < 0 ) - { - m_nGoodHel = m_pmek->computeGoodHelicities(); - if( m_nGoodHel < 0 ) - throw std::runtime_error( - "Bridge gpu_sequence: computeGoodHelicities returned nGoodHel<0" ); - } - if( goodHelOnly ) return; - m_pmek->computeMatrixElements( useChannelIds ); - copyHostFromDevice( m_hstMEs, m_devMEs ); -#ifdef MGONGPUCPP_VERBOSE - flagAbnormalMEs( m_hstMEs.data(), m_nevt ); -#endif - copyHostFromDevice( m_hstSelHel, m_devSelHel ); - copyHostFromDevice( m_hstSelCol, m_devSelCol ); - if constexpr( std::is_same_v ) - { - memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); - memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); - memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); - } - else - { - std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); - std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); - std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); - } - } -#endif - -#ifndef MGONGPUCPP_GPUIMPL - template - void Bridge::cpu_sequence( const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool goodHelOnly ) - { - hst_transposeMomentaF2C( momenta, m_hstMomentaC.data(), m_nevt ); - if constexpr( std::is_same_v ) - { - memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); - } - else - { - std::copy( gs, gs + m_nevt, m_hstGs.data() ); - std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); - std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); - } - const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); - if( useChannelIds ) - memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); - // else ... // no need to initialize m_hstChannel: it is allocated and default - // initialized in HostBufferBase (and it is not used later on anyway) - // initialise iflavorVec - memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); - if( m_nGoodHel < 0 ) - { - m_nGoodHel = m_pmek->computeGoodHelicities(); - if( m_nGoodHel < 0 ) - throw std::runtime_error( - "Bridge cpu_sequence: computeGoodHelicities returned nGoodHel<0" ); - } - if( goodHelOnly ) return; - m_pmek->computeMatrixElements( useChannelIds ); -#ifdef MGONGPUCPP_VERBOSE - flagAbnormalMEs( m_hstMEs.data(), m_nevt ); -#endif - if constexpr( std::is_same_v ) - { - memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); - memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); - memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); - } - else - { - std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); - std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); - std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); - } - } -#endif - - //-------------------------------------------------------------------------- - // - // Implementations of transposition methods - // - FORTRAN arrays: P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) ==> - // p_multi[nevtF][nparF][np4F] in C++ (AOS) - // - C++ array: momenta[npagM][npar][np4][neppM] with nevt=npagM*neppM (AOSOA) - // - -#ifdef MGONGPUCPP_GPUIMPL - template - __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool oldImplementation = true; // default: use old implementation - if constexpr( oldImplementation ) - { - // SR initial implementation - constexpr int part = CPPProcess::npar; - constexpr int mome = CPPProcess::np4; - constexpr int strd = MemoryAccessMomenta::neppM; - int pos = blockDim.x * blockIdx.x + threadIdx.x; - int arrlen = nevt * part * mome; - if( pos < arrlen ) - { - int page_i = pos / ( strd * mome * part ); - int rest_1 = pos % ( strd * mome * part ); - int part_i = rest_1 / ( strd * mome ); - int rest_2 = rest_1 % ( strd * mome ); - int mome_i = rest_2 / strd; - int strd_i = rest_2 % strd; - int inpos = ( page_i * strd + strd_i ) // event number - * ( part * mome ) // event size (pos of event) - + part_i * mome // particle inside event - + mome_i; // momentum inside particle - out[pos] = in[inpos]; // F2C (Fortran to C) - } - } - else - { - // AV attempt another implementation with 1 event per thread: this seems - // slower... F-style: AOS[nevtF][nparF][np4F] C-style: - // AOSOA[npagM][npar][np4][neppM] with nevt=npagM*neppM - constexpr int npar = CPPProcess::npar; - constexpr int np4 = CPPProcess::np4; - constexpr int neppM = MemoryAccessMomenta::neppM; - assert( nevt % neppM == - 0 ); // number of events is not a multiple of neppM??? - int ievt = blockDim.x * blockIdx.x + threadIdx.x; - int ipagM = ievt / neppM; - int ieppM = ievt % neppM; - for( int ip4 = 0; ip4 < np4; ip4++ ) - for( int ipar = 0; ipar < npar; ipar++ ) - { - int cpos = ipagM * npar * np4 * neppM + ipar * np4 * neppM + - ip4 * neppM + ieppM; - int fpos = ievt * npar * np4 + ipar * np4 + ip4; - out[cpos] = in[fpos]; // F2C (Fortran to C) - } - } - } -#endif - - template - void hst_transposeMomenta( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool oldImplementation = false; // default: use new implementation - if constexpr( oldImplementation ) - { - // SR initial implementation - constexpr unsigned int part = CPPProcess::npar; - constexpr unsigned int mome = CPPProcess::np4; - constexpr unsigned int strd = MemoryAccessMomenta::neppM; - unsigned int arrlen = nevt * part * mome; - for( unsigned int pos = 0; pos < arrlen; ++pos ) - { - unsigned int page_i = pos / ( strd * mome * part ); - unsigned int rest_1 = pos % ( strd * mome * part ); - unsigned int part_i = rest_1 / ( strd * mome ); - unsigned int rest_2 = rest_1 % ( strd * mome ); - unsigned int mome_i = rest_2 / strd; - unsigned int strd_i = rest_2 % strd; - unsigned int inpos = ( page_i * strd + strd_i ) // event number - * ( part * mome ) // event size (pos of event) - + part_i * mome // particle inside event - + mome_i; // momentum inside particle - if constexpr( F2C ) // needs c++17 and cuda >=11.2 (#333) - out[pos] = in[inpos]; // F2C (Fortran to C) - else - out[inpos] = in[pos]; // C2F (C to Fortran) - } - } - else - { - // AV attempt another implementation: this is slightly faster (better c++ - // pipelining?) [NB! this is not a transposition, it is an AOS to AOSOA - // conversion: if neppM=1, a memcpy is enough] F-style: - // AOS[nevtF][nparF][np4F] C-style: AOSOA[npagM][npar][np4][neppM] with - // nevt=npagM*neppM - constexpr unsigned int npar = CPPProcess::npar; - constexpr unsigned int np4 = CPPProcess::np4; - constexpr unsigned int neppM = MemoryAccessMomenta::neppM; - if constexpr( neppM == 1 && std::is_same_v ) - { - memcpy( out, in, nevt * npar * np4 * sizeof( Tin ) ); - } - else - { - const unsigned int npagM = nevt / neppM; - assert( nevt % neppM == - 0 ); // number of events is not a multiple of neppM??? - for( unsigned int ipagM = 0; ipagM < npagM; ipagM++ ) - for( unsigned int ip4 = 0; ip4 < np4; ip4++ ) - for( unsigned int ipar = 0; ipar < npar; ipar++ ) - for( unsigned int ieppM = 0; ieppM < neppM; ieppM++ ) - { - unsigned int ievt = ipagM * neppM + ieppM; - unsigned int cpos = ipagM * npar * np4 * neppM + - ipar * np4 * neppM + ip4 * neppM + ieppM; - unsigned int fpos = ievt * npar * np4 + ipar * np4 + ip4; - if constexpr( F2C ) - out[cpos] = in[fpos]; // F2C (Fortran to C) - else - out[fpos] = in[cpos]; // C2F (C to Fortran) - } - } - } - } - - template - void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool F2C = true; - hst_transposeMomenta( in, out, nevt ); - } - - template - void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool F2C = false; - hst_transposeMomenta( in, out, nevt ); - } - - //-------------------------------------------------------------------------- -} // namespace mg5amcGpu -#endif // BRIDGE_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.cc deleted file mode 100644 index 65c97616d8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.cc +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "BridgeKernels.h" - -#include "GpuAbstraction.h" -#include "MemoryAccessMomenta.h" - -#include - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - - //-------------------------------------------------------------------------- - - BridgeKernelBase::BridgeKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( nevt ) - , m_bridge( nevt, npar, np4 ) - { - if( m_momenta.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: momenta must be a host array" ); - if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: matrixElements must be a host array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with matrixElements" ); - } - - //-------------------------------------------------------------------------- -} - -//============================================================================ - -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu -{ - - //-------------------------------------------------------------------------- - - BridgeKernelHost::BridgeKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: Gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt) - : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, nevt) - , m_fortranMomenta( nevt ) - { - } - - //-------------------------------------------------------------------------- - - void BridgeKernelHost::transposeInputMomentaC2F() - { - hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); - } - - //-------------------------------------------------------------------------- - - int BridgeKernelHost::computeGoodHelicities() - { - constexpr bool goodHelOnly = true; - constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering - m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - return m_bridge.nGoodHel(); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelHost::computeMatrixElements( const bool useChannelIds ) - { - constexpr bool goodHelOnly = false; - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - //-------------------------------------------------------------------------- - - BridgeKernelDevice::BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: Gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads) - : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, gpublocks * gputhreads) - , m_fortranMomenta( nevt() ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if( m_gpublocks == 0 ) throw std::runtime_error( "BridgeKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "BridgeKernelDevice: gputhreads must be > 0" ); - m_bridge.set_gpugrid( gpublocks, gputhreads ); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelDevice::transposeInputMomentaC2F() - { - hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); - } - - //-------------------------------------------------------------------------- - - int BridgeKernelDevice::computeGoodHelicities() - { - constexpr bool goodHelOnly = true; - constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering - m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - return m_bridge.nGoodHel(); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelDevice::computeMatrixElements( const bool useChannelIds ) - { - constexpr bool goodHelOnly = false; - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.h deleted file mode 100644 index 2d873486a8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/BridgeKernels.h +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally originally by: A. Valassi (Jan 2022). -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef BRIDGEKERNELS_H -#define BRIDGEKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "Bridge.h" -#include "MatrixElementKernels.h" -#include "MemoryBuffers.h" - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - // A Bridge wrapper base class encapsulating matrix element calculations on a CPU host - class BridgeKernelBase : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - BridgeKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~BridgeKernelBase() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - virtual void transposeInputMomentaC2F() = 0; - - protected: - - // The wrapped bridge - Bridge m_bridge; - }; - - //-------------------------------------------------------------------------- - -#ifndef MGONGPUCPP_GPUIMPL - // A Bridge wrapper class encapsulating matrix element calculations on a CPU host - class BridgeKernelHost final : public BridgeKernelBase - { - public: - - // Constructor from existing input and output buffers - BridgeKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~BridgeKernelHost() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - void transposeInputMomentaC2F() override final; - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // The buffer for the input momenta, transposed to Fortran array indexing - HostBufferMomenta m_fortranMomenta; - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A Bridge wrapper class encapsulating matrix element calculations on a GPU device - class BridgeKernelDevice : public BridgeKernelBase - { - public: - - // Constructor from existing input and output buffers - BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads); - - // Destructor - virtual ~BridgeKernelDevice() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - void transposeInputMomentaC2F() override final; - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The buffer for the input momenta, transposed to Fortran array indexing - PinnedHostBufferMomenta m_fortranMomenta; - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; -#endif - - //-------------------------------------------------------------------------- -} -#endif // BRIDGEKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc index 16671b5686..14c1ceeaf5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc @@ -11,11 +11,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc index 7a3e0e0519..861bca8940 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc @@ -79,11 +79,7 @@ debug_me_is_abnormal( const fptype& me, size_t ievtALL ) //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -187,52 +183,5 @@ namespace mg5amcCpu //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - /* - //-------------------------------------------------------------------------- - - CrossSectionKernelDevice::CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats, // output: event statistics - const size_t gpublocks, - const size_t gputhreads ) - : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) - , NumberOfEvents( gpublocks*gputhreads ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if ( ! m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: samplingWeights must be a device array" ); - if ( ! m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: matrixElements must be a device array" ); - if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0" ); - if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0" ); - if ( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with samplingWeights" ); - if ( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with matrixElements" ); - } - - //-------------------------------------------------------------------------- - - void CrossSectionKernelDevice::setGrid( const size_t gpublocks, const size_t gputhreads ) - { - if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0 in setGrid" ); - if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0 in setGrid" ); - if ( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch in setGrid" ); - } - - //-------------------------------------------------------------------------- - - void CrossSectionKernelDevice::updateEventStatistics( const bool debug ) - { - // Increment the iterations counter - m_iter++; - } - - //-------------------------------------------------------------------------- - */ - -} -#endif //============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h index d1e29b7999..9fb3c3fce2 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h @@ -14,11 +14,7 @@ //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc index 0c8e6e6445..24ee78772b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc @@ -24,11 +24,7 @@ inline void assertCurand( curandStatus_t code, const char *file, int line, bool } #endif /* clang-format on */ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_CURAND @@ -38,12 +34,7 @@ namespace mg5amcCpu { if( m_isOnDevice ) { -#ifdef MGONGPUCPP_GPUIMPL - if( !m_rnarray.isOnDevice() ) - throw std::runtime_error( "CurandRandomNumberKernel on device with a host random number array" ); -#else throw std::runtime_error( "CurandRandomNumberKernel does not support CurandDevice on CPU host" ); -#endif } else { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h index 4a1fd8464d..831d477bb6 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h @@ -17,11 +17,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h index 2c8ae1a804..7366509821 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h @@ -17,85 +17,12 @@ //-------------------------------------------------------------------------- // See https://stackoverflow.com/a/14038590 -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ -#define checkGpu( code ) { assertGpu( code, __FILE__, __LINE__ ); } -inline void assertGpu( gpuError_t code, const char* file, int line, bool abort = true ) -{ - if( code != gpuSuccess ) - { - printf( "ERROR! assertGpu: '%s' (%d) in %s:%d\n", gpuGetErrorString( code ), code, file, line ); - if( abort ) assert( code == gpuSuccess ); - } -} -#endif /* clang-format on */ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ -#ifndef MGONGPU_HAS_NO_BLAS -#define checkGpuBlas( code ){ assertGpuBlas( code, __FILE__, __LINE__ ); } -inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, bool abort = true ) -{ - if ( code != GPUBLAS_STATUS_SUCCESS ) - { - printf( "ERROR! assertGpuBlas: '%d' in %s:%d\n", code, file, line ); - if( abort ) assert( code == GPUBLAS_STATUS_SUCCESS ); - } -} -#endif -#endif /* clang-format on */ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - // Instantiate a GpuRuntime at the beginnining of the application's main to - // invoke gpuSetDevice(0) in the constructor and book a gpuDeviceReset() call in the destructor - // *** FIXME! This will all need to be designed differently when going to multi-GPU nodes! *** - struct GpuRuntime final - { - GpuRuntime( const bool debug = true ) - : m_debug( debug ) { setUp( m_debug ); } - ~GpuRuntime() { tearDown( m_debug ); } - GpuRuntime( const GpuRuntime& ) = delete; - GpuRuntime( GpuRuntime&& ) = delete; - GpuRuntime& operator=( const GpuRuntime& ) = delete; - GpuRuntime& operator=( GpuRuntime&& ) = delete; - bool m_debug; - - // Set up CUDA application - // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** - // Calling cudaSetDevice on startup is useful to properly book-keep the time spent in CUDA initialization - static void setUp( const bool debug = false ) // ZW: changed debug default to false - { - // ** NB: it is useful to call cudaSetDevice, or cudaFree, to properly book-keep the time spent in CUDA initialization - // ** NB: otherwise, the first CUDA operation (eg a cudaMemcpyToSymbol in CPPProcess ctor) appears to take much longer! - /* - // [We initially added cudaFree(0) to "ease profile analysis" only because it shows up as a big recognizable block!] - // No explicit initialization is needed: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#initialization - // It is not clear what cudaFree(0) does at all: https://stackoverflow.com/questions/69967813/ - if ( debug ) std::cout << "__CudaRuntime: calling cudaFree(0)" << std::endl; - checkCuda( cudaFree( 0 ) ); // SLOW! - */ - // Replace cudaFree(0) by cudaSetDevice(0), even if it is not really needed either - // (but see https://developer.nvidia.com/blog/cuda-pro-tip-always-set-current-device-avoid-multithreading-bugs) - if( debug ) std::cout << "__GpuRuntime: calling GpuSetDevice(0)" << std::endl; - checkGpu( gpuSetDevice( 0 ) ); // SLOW! - } - - // Tear down CUDA application (call cudaDeviceReset) - // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** - // Calling cudaDeviceReset on shutdown is only needed for checking memory leaks in cuda-memcheck - // See https://docs.nvidia.com/cuda/cuda-memcheck/index.html#leak-checking - static void tearDown( const bool debug = false ) // ZW: changed debug default to false - { - if( debug ) std::cout << "__GpuRuntime: calling GpuDeviceReset()" << std::endl; - checkGpu( gpuDeviceReset() ); - } - }; -} -#endif //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc index b806da2e09..2fd64a2dce 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc @@ -28,11 +28,7 @@ inline void assertHiprand( hiprandStatus_t code, const char *file, int line, boo } #endif /* clang-format on */ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_HIPRAND @@ -42,12 +38,7 @@ namespace mg5amcCpu { if( m_isOnDevice ) { -#ifdef MGONGPUCPP_GPUIMPL - if( !m_rnarray.isOnDevice() ) - throw std::runtime_error( "HiprandRandomNumberKernel on device with a host random number array" ); -#else throw std::runtime_error( "HiprandRandomNumberKernel does not support HiprandDevice on CPU host" ); -#endif } else { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h index 12b7d45adf..641917d8ff 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h @@ -27,11 +27,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -using mg5amcGpu::CPPProcess; -#else using mg5amcCpu::CPPProcess; -#endif namespace { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc index 872e4795e3..fb19139f2d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc @@ -17,11 +17,7 @@ //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -146,7 +142,6 @@ namespace mg5amcCpu //============================================================================ -#ifndef MGONGPUCPP_GPUIMPL namespace mg5amcCpu { @@ -282,228 +277,8 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- } -#endif //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - //-------------------------------------------------------------------------- - - MatrixElementKernelDevice::MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( gpublocks * gputhreads ) - , m_couplings( this->nevt() ) - , m_pHelMEs() - , m_pHelJamps() - , m_pHelNumerators() - , m_pHelDenominators() - , m_colJamp2s( CPPProcess::ncolor * this->nevt() ) -#ifdef MGONGPU_CHANNELID_DEBUG - , m_hstChannelIds( this->nevt() ) -#endif -#ifndef MGONGPU_HAS_NO_BLAS - , m_blasColorSum( false ) - , m_blasTf32Tensor( false ) - , m_pHelBlasTmp() - , m_blasHandle() -#endif - , m_helStreams() - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - //std::cout << "DEBUG: MatrixElementKernelDevice::ctor " << this << std::endl; - if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: momenta must be a device array" ); - if( !m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: matrixElements must be a device array" ); - if( !m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: channelIds must be a device array" ); // FIXME?! - if( !m_iflavorVec.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: iflavorVec must be a device array" ); - if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with matrixElements" ); - if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with channelIds" ); - if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with iflavorVec" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( m_gputhreads % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MatrixElementKernelHost: gputhreads should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Create the "one-helicity" jamp buffer that will be used for helicity filtering - m_pHelJamps.reset( new DeviceBufferSimple( CPPProcess::ncolor * mgOnGpu::nx2 * this->nevt() ) ); - // Create the "one-helicity" numerator and denominator buffers that will be used for helicity filtering - m_pHelNumerators.reset( new DeviceBufferSimple( this->nevt() * CPPProcess::ndiagrams ) ); - m_pHelDenominators.reset( new DeviceBufferSimple( this->nevt() ) ); - // Decide at runtime whether to use BLAS for color sums - // Decide at runtime whether TF32TENSOR math should be used in cuBLAS - static bool first = true; - if( first ) - { - first = false; - // Analyse environment variable CUDACPP_RUNTIME_BLASCOLORSUM - const char* blasEnv = getenv( "CUDACPP_RUNTIME_BLASCOLORSUM" ); - if( blasEnv && std::string( blasEnv ) != "" ) - { -#ifndef MGONGPU_HAS_NO_BLAS - m_blasColorSum = true; // fixme? eventually set default=true and decode "Y" and "N" choices? - std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty: enable BLAS" << std::endl; -#else - throw std::runtime_error( "Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty, but BLAS was disabled at build time" ); -#endif - } - else - { -#ifndef MGONGPU_HAS_NO_BLAS - std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is empty or not set: disable BLAS" << std::endl; -#else - std::cout << "INFO: BLAS was disabled at build time" << std::endl; -#endif - } -#ifndef MGONGPU_HAS_NO_BLAS -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) - // Analyse environment variable CUDACPP_RUNTIME_CUBLASTF32TENSOR - const char* blasEnv2 = getenv( "CUDACPP_RUNTIME_CUBLASTF32TENSOR" ); - if( blasEnv2 && std::string( blasEnv2 ) != "" ) - { - if( m_blasColorSum ) - { -#ifdef MGONGPU_FPTYPE2_FLOAT - m_blasTf32Tensor = true; - std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty: enable CUBLAS_TF32_TENSOR_OP_MATH" << std::endl; -#else - std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but color sums use FP64" << std::endl; -#endif - } - else - std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but BLAS was disabled at runtime" << std::endl; - } -#ifdef MGONGPU_FPTYPE2_FLOAT - else - { - if( m_blasColorSum ) - std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is empty or not set: keep cuBLAS math defaults" << std::endl; - } -#endif -#endif -#endif - } - } - - //-------------------------------------------------------------------------- - - MatrixElementKernelDevice::~MatrixElementKernelDevice() - { - //std::cout << "DEBUG: MatrixElementKernelDevice::dtor " << this << std::endl; -#ifndef MGONGPU_HAS_NO_BLAS - if( m_blasHandle ) gpuBlasDestroy( m_blasHandle ); -#endif - for( int ihel = 0; ihel < CPPProcess::ncomb; ihel++ ) - { - if( m_helStreams[ihel] ) gpuStreamDestroy( m_helStreams[ihel] ); // do not destroy if nullptr - } - } - - //-------------------------------------------------------------------------- - - // FIXME! The relevance of this function should be reassessed (#543 and #902) - void MatrixElementKernelDevice::setGrid( const int /*gpublocks*/, const int /*gputhreads*/ ) - { - if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0 in setGrid" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0 in setGrid" ); - if( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch in setGrid" ); - } - - //-------------------------------------------------------------------------- - - int MatrixElementKernelDevice::computeGoodHelicities() - { - PinnedHostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); - // ... 0d1. Compute good helicity mask (a host variable) on the device - gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); - const int nevt = m_gpublocks * m_gputhreads; - sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_pHelJamps->data(), m_pHelNumerators->data(), m_pHelDenominators->data(), hstIsGoodHel.data(), nevt ); - // ... 0d3. Set good helicity list in host static memory - int nGoodHel = sigmaKin_setGoodHel( hstIsGoodHel.data() ); - assert( nGoodHel > 0 ); // SANITY CHECK: there should be at least one good helicity - // Create one GPU stream for each good helicity - for( int ighel = 0; ighel < nGoodHel; ighel++ ) - gpuStreamCreate( &m_helStreams[ighel] ); -#ifndef MGONGPU_HAS_NO_BLAS - // Create one cuBLAS/hipBLAS handle for each good helicity (attached to the default stream) - if( m_blasColorSum ) - { - checkGpuBlas( gpuBlasCreate( &m_blasHandle ) ); -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) - if( m_blasTf32Tensor ) - checkGpuBlas( cublasSetMathMode( m_blasHandle, CUBLAS_TF32_TENSOR_OP_MATH ) ); // enable TF32 tensor cores -#endif - } -#endif - // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) - m_pHelMEs.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); - // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) - // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) - m_pHelJamps.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); - // ... Create the "many-helicity" super-buffers of nGoodHel numerator and denominator buffers (dynamically allocated) - // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) - m_pHelNumerators.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ndiagrams * nevt ) ); - m_pHelDenominators.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); -#ifndef MGONGPU_HAS_NO_BLAS - // Create the "many-helicity" super-buffers of real/imag ncolor*nevt temporary buffers for cuBLAS/hipBLAS intermediate results in color_sum_blas -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Mixed precision mode: need two fptype2[ncolor*2*nevt] buffers and one fptype2[nevt] buffer per good helicity - if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ( 2 * CPPProcess::ncolor * mgOnGpu::nx2 + 1 ) * nevt ) ); -#else - // Standard single/double precision mode: need one fptype2[ncolor*2*nevt] buffer per good helicity - if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); -#endif -#endif - // Return the number of good helicities - return nGoodHel; - } - - //-------------------------------------------------------------------------- - - void MatrixElementKernelDevice::computeMatrixElements( const bool useChannelIds ) - { - gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); -#ifndef MGONGPU_HAS_NO_BLAS - fptype2* ghelAllBlasTmp = ( m_blasColorSum ? m_pHelBlasTmp->data() : nullptr ); - gpuBlasHandle_t* pBlasHandle = ( m_blasColorSum ? &m_blasHandle : nullptr ); -#else - fptype2* ghelAllBlasTmp = nullptr; - gpuBlasHandle_t* pBlasHandle = nullptr; -#endif - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_colJamp2s.data(), m_pHelNumerators->data(), m_pHelDenominators->data(), nullptr, true, m_pHelMEs->data(), m_pHelJamps->data(), ghelAllBlasTmp, pBlasHandle, m_helStreams, false, m_gpublocks, m_gputhreads ); -#ifdef MGONGPU_CHANNELID_DEBUG - //std::cout << "DEBUG: MatrixElementKernelDevice::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; - copyHostFromDevice( m_hstChannelIds, m_channelIds ); // FIXME?! - const unsigned int* pHstChannelIds = ( useChannelIds ? m_hstChannelIds.data() : nullptr ); - MatrixElementKernelBase::updateNevtProcessedByChannel( pHstChannelIds, nevt() ); -#endif - checkGpu( gpuPeekAtLastError() ); // is this needed? - checkGpu( gpuDeviceSynchronize() ); // probably not needed? but it avoids errors in sigmaKin above from appearing later on in random places... - } - - //-------------------------------------------------------------------------- - -} -#endif //============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h index 01041e43b7..9cfb1c6247 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h @@ -16,11 +16,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -109,7 +105,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating matrix element calculations on a CPU host class MatrixElementKernelHost final : public MatrixElementKernelBase, public NumberOfEvents { @@ -156,94 +151,9 @@ namespace mg5amcCpu // The buffer for the event-by-event denominators of multichannel factors HostBufferDenominators m_denominators; }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating matrix element calculations on a GPU device - class MatrixElementKernelDevice : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~MatrixElementKernelDevice(); - - // Reset gpublocks and gputhreads - void setGrid( const int gpublocks, const int gputhreads ); - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The buffer for the event-by-event couplings that depends on alphas QCD - DeviceBufferCouplings m_couplings; - - // The super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelMEs; - - // The super-buffer of nGoodHel jamp buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelJamps; - - // The super-buffer of nGoodHel numerator buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelNumerators; - - // The super-buffer of nGoodHel denominator buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelDenominators; - - // The super-buffer of ncolor jamp2 buffers - DeviceBufferSimple m_colJamp2s; - -#ifdef MGONGPU_CHANNELID_DEBUG - // The **host** buffer for the channelId array - // FIXME? MEKD should accept a host buffer as an argument instead of a device buffer, so that a second copy can be avoided? - PinnedHostBufferChannelIds m_hstChannelIds; -#endif - -#ifndef MGONGPU_HAS_NO_BLAS - // Decide at runtime whether to use BLAS for color sums - bool m_blasColorSum; - - // Decide at runtime whether TF32TENSOR math should be used in cuBLAS - bool m_blasTf32Tensor; - - // The super-buffer of nGoodHel cuBLAS/hipBLAS temporary buffers - std::unique_ptr m_pHelBlasTmp; - - // The cuBLAS/hipBLAS handle (a single one for all good helicities) - gpuBlasHandle_t m_blasHandle; -#endif - - // The array of GPU streams (one for each good helicity) - gpuStream_t m_helStreams[CPPProcess::ncomb]; // reserve ncomb streams (but only nGoodHel <= ncomb will be used) - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h index f41e40bf82..9cd0e794c0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h @@ -16,11 +16,7 @@ #define MGONGPU_TRIVIAL_AMPLITUDES 1 // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h index 331e7f12c3..1963d3dfff 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h @@ -14,11 +14,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -103,14 +99,7 @@ namespace mg5amcCpu kernelAccessConst( const unsigned int* buffer ) { const unsigned int& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } }; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h index fd3fb80c65..844ee5a5ed 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h @@ -16,11 +16,7 @@ #include "MemoryBuffers.h" // for HostBufferCouplings::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -189,17 +185,7 @@ namespace mg5amcCpu const int ix2 ) { fptype& out = kernelAccessIx2_s( buffer, ix2 ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays - constexpr int neppC = MemoryAccessCouplingsBase::neppC; - static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h index 757de7b6f2..fd1e1abadf 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h @@ -15,11 +15,7 @@ //#include "MemoryAccessHelpers.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h index 205952e514..01611fe92d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h @@ -10,11 +10,7 @@ #include "MemoryAccessGs.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h index bfcb3e3222..9faa088a45 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h @@ -14,11 +14,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -119,14 +115,7 @@ namespace mg5amcCpu kernelAccess( fptype* buffer ) { fptype& out = kernelAccess_s( buffer ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) @@ -148,14 +137,7 @@ namespace mg5amcCpu kernelAccessConst( const fptype* buffer ) { const fptype& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } }; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h index 8b5bd27f4d..525196d770 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h @@ -106,13 +106,7 @@ class KernelAccessHelper : public MemoryAccessHelper } else { -#ifdef MGONGPUCPP_GPUIMPL - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid - //printf( "kernelAccessRecord: ievt=%d threadId=%d\n", ievt, threadIdx.x ); - return T::ieventAccessRecord( buffer, ievt ); // NB fptype and fptype_sv coincide for CUDA -#else throw std::runtime_error( "kernelAccessRecord on device is only implemented in CUDA" ); -#endif } } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h index ffc74b0f0c..1903b43ed4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h @@ -13,11 +13,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -102,14 +98,7 @@ namespace mg5amcCpu kernelAccessConst( const unsigned int* buffer ) { const unsigned int& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } }; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h index 4236e20602..7f9cd4c11c 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h @@ -14,11 +14,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -119,14 +115,7 @@ namespace mg5amcCpu kernelAccess( fptype* buffer ) { fptype& out = kernelAccess_s( buffer ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h index 3ef4d76fbd..a9e428999c 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h @@ -14,11 +14,7 @@ #include "MemoryAccessVectors.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -31,32 +27,13 @@ namespace mg5amcCpu // Number of Events Per Page in the momenta AOSOA memory buffer layout // (these are all best kept as a compile-time constants: see issue #23) -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - // ----------------------------------------------------------------------------------------------- - // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline - // --- This is relevant to ensure coalesced access to momenta in global memory - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------------- - //static constexpr int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) - static constexpr int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) - //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 1.03E9 instead of 1.11E9 in eemumu) -#else // ----------------------------------------------------------------------------------------------- // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register // --- This is relevant to ensure faster access to momenta from C++ memory cache lines // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded // --- In practice, neppR, neppM and neppV could now (in principle) all be different // ----------------------------------------------------------------------------------------------- -#ifdef MGONGPU_CPPSIMD - static constexpr int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance - //static constexpr int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) - //static constexpr int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) - //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 4.66E6 instead of 5.09E9 in eemumu) - //static constexpr int neppM = MGONGPU_CPPSIMD*2; // FOR TESTS -#else static constexpr int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) -#endif -#endif /* clang-format on */ // SANITY CHECK: check that neppM is a power of two static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); @@ -208,51 +185,7 @@ namespace mg5amcCpu const int ipar ) { const fptype& out = kernelAccessIp4IparConst_s( buffer, ip4, ipar ); -#ifndef MGONGPU_CPPSIMD return out; -#else - constexpr int neppM = MemoryAccessMomentaBase::neppM; - constexpr bool useContiguousEventsIfPossible = true; // DEFAULT - //constexpr bool useContiguousEventsIfPossible = false; // FOR PERFORMANCE TESTS (treat as arbitrary array even if it is an AOSOA) - // Use c++17 "if constexpr": compile-time branching - if constexpr( useContiguousEventsIfPossible && ( neppM >= neppV ) && ( neppM % neppV == 0 ) ) - { - //constexpr bool skipAlignmentCheck = true; // FASTEST (SEGFAULTS IF MISALIGNED ACCESS, NEEDS A SANITY CHECK ELSEWHERE!) - constexpr bool skipAlignmentCheck = false; // DEFAULT: A BIT SLOWER BUT SAFER [ALLOWS MISALIGNED ACCESS] - if constexpr( skipAlignmentCheck ) - { - //static bool first=true; if( first ){ std::cout << "WARNING! assume aligned AOSOA, skip check" << std::endl; first=false; } // SLOWER (5.06E6) - // FASTEST? (5.09E6 in eemumu 512y) - // This assumes alignment for momenta1d without checking - causes segmentation fault in reinterpret_cast if not aligned! - return mg5amcCpu::fptypevFromAlignedArray( out ); // use reinterpret_cast - } - else if( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ) - { - //static bool first=true; if( first ){ std::cout << "WARNING! aligned AOSOA, reinterpret cast" << std::endl; first=false; } // SLOWER (5.00E6) - // DEFAULT! A tiny bit (<1%) slower because of the alignment check (5.07E6 in eemumu 512y) - // This explicitly checks buffer alignment to avoid segmentation faults in reinterpret_cast - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast - } - else - { - //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWER (4.93E6) - // A bit (1%) slower (5.05E6 in eemumu 512y) - // This does not require buffer alignment, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 - return mg5amcCpu::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) - } - } - else - { - //static bool first=true; if( first ){ std::cout << "WARNING! arbitrary array" << std::endl; first=false; } // SLOWER (5.08E6) - // ?!Used to be much slower, now a tiny bit faster for AOSOA?! (5.11E6 for AOSOA, 4.64E6 for AOS in eemumu 512y) - // This does not even require AOSOA with neppM>=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) - constexpr int ievt0 = 0; // just make it explicit in the code that buffer refers to a given ievt0 and decoderIeppV fetches event ievt0+ieppV - auto decoderIeppv = [buffer, ip4, ipar]( int ieppV ) - -> const fptype& - { return MemoryAccessMomenta::ieventAccessIp4IparConst( buffer, ievt0 + ieppV, ip4, ipar ); }; - return mg5amcCpu::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) - } -#endif } // Is this a HostAccess or DeviceAccess class? diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h index c8d0343b7e..42356061e7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h @@ -10,11 +10,7 @@ #include "MemoryAccessGs.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h index 57f1d02081..ebdffe9c84 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h @@ -12,11 +12,7 @@ #include "CPPProcess.h" #include "MemoryAccessHelpers.h" -#ifdef MGONGPUCPP_GPUIMPL -using mg5amcGpu::CPPProcess; -#else using mg5amcCpu::CPPProcess; -#endif //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h index 8563d8f2f7..db9addfb21 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h @@ -11,128 +11,10 @@ #include "mgOnGpuVectors.h" -#ifndef MGONGPUCPP_GPUIMPL namespace mg5amcCpu // this is only needed for CPU SIMD vectorization { -#ifdef MGONGPU_CPPSIMD - //-------------------------------------------------------------------------- - - // Cast one non-const fptype_v reference (one vector of neppV fptype values) from one non-const fptype reference (#435), - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned - inline fptype_v& fptypevFromAlignedArray( fptype& ref ) - { - return *reinterpret_cast( &ref ); - } - - inline uint_v& uintvFromAlignedArray( unsigned int& ref ) - { - return *reinterpret_cast( &ref ); - } - - // Cast one const fptype_v reference (one vector of neppV fptype values) from one const fptype reference, - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned - inline const fptype_v& fptypevFromAlignedArray( const fptype& ref ) - { - return *reinterpret_cast( &ref ); - } - - inline const uint_v& uintvFromAlignedArray( const unsigned int& ref ) - { - return *reinterpret_cast( &ref ); - } - - // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", but that the arrays are not aligned - inline fptype_v fptypevFromUnalignedArray( const fptype& ref ) - { -#if MGONGPU_CPPSIMD == 2 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (2) - *( &ref + 1 ) }; -#elif MGONGPU_CPPSIMD == 4 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (4) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ) }; -#elif MGONGPU_CPPSIMD == 8 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (8) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ), - *( &ref + 4 ), - *( &ref + 5 ), - *( &ref + 6 ), - *( &ref + 7 ) }; -#elif MGONGPU_CPPSIMD == 16 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (16) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ), - *( &ref + 4 ), - *( &ref + 5 ), - *( &ref + 6 ), - *( &ref + 7 ), - *( &ref + 8 ), - *( &ref + 9 ), - *( &ref + 10 ), - *( &ref + 11 ), - *( &ref + 12 ), - *( &ref + 13 ), - *( &ref + 14 ), - *( &ref + 15 ) }; -#else -#error Internal error! Unknown MGONGPU_CPPSIMD value -#endif - } - - // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, - // with no a priori assumption on how the input fptype array should be decoded - template - inline fptype_v fptypevFromArbitraryArray( Functor decoderIeppv ) - { -#if MGONGPU_CPPSIMD == 2 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (2) - decoderIeppv( 1 ) }; -#elif MGONGPU_CPPSIMD == 4 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (4) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ) }; -#elif MGONGPU_CPPSIMD == 8 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (8) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ), - decoderIeppv( 4 ), - decoderIeppv( 5 ), - decoderIeppv( 6 ), - decoderIeppv( 7 ) }; -#elif MGONGPU_CPPSIMD == 16 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (16) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ), - decoderIeppv( 4 ), - decoderIeppv( 5 ), - decoderIeppv( 6 ), - decoderIeppv( 7 ), - decoderIeppv( 8 ), - decoderIeppv( 9 ), - decoderIeppv( 10 ), - decoderIeppv( 11 ), - decoderIeppv( 12 ), - decoderIeppv( 13 ), - decoderIeppv( 14 ), - decoderIeppv( 15 ) }; -#else -#error Internal error! Unknown MGONGPU_CPPSIMD value -#endif - } - - //-------------------------------------------------------------------------- -#endif } // end namespace -#endif #endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h index 14a37c7fea..672a66bc09 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h @@ -16,11 +16,7 @@ #define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h index c56527f581..0a82fe49d1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h @@ -12,11 +12,7 @@ #include "MemoryAccessHelpers.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h index d259aa7456..c229fc0a56 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h @@ -18,11 +18,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -90,7 +86,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL constexpr bool HostBufferALIGNED = false; // ismisaligned=false constexpr bool HostBufferMISALIGNED = true; // ismisaligned=true @@ -118,51 +113,15 @@ namespace mg5amcCpu public: static constexpr size_t cppAlign = mgOnGpu::cppAlign; }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA pinned host buffer - template - class PinnedHostBufferBase : public BufferBase - { - public: - PinnedHostBufferBase( const size_t size ) - : BufferBase( size, false ) - { - gpuMallocHost( &( this->m_data ), this->bytes() ); - } - virtual ~PinnedHostBufferBase() - { - gpuFreeHost( this->m_data ); - } - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA device buffer - template - class DeviceBufferBase : public BufferBase - { - public: - DeviceBufferBase( const size_t size ) - : BufferBase( size, true ) - { - gpuMalloc( &( this->m_data ), this->bytes() ); - } - virtual ~DeviceBufferBase() - { - gpuFree( this->m_data ); - } - }; -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for a given number of events template class HostBuffer : public HostBufferBase, virtual private NumberOfEvents @@ -180,53 +139,15 @@ namespace mg5amcCpu } virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA pinned host buffer for a given number of events - template - class PinnedHostBuffer : public PinnedHostBufferBase, virtual private NumberOfEvents - { - public: - PinnedHostBuffer( const size_t nevt ) - : NumberOfEvents( nevt ) - , PinnedHostBufferBase( sizePerEvent * nevt ) {} - virtual ~PinnedHostBuffer() {} - virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA device buffer for a given number of events - template - class DeviceBuffer : public DeviceBufferBase, virtual protected NumberOfEvents - { - public: - DeviceBuffer( const size_t nevt ) - : NumberOfEvents( nevt ) - , DeviceBufferBase( sizePerEvent * nevt ) - { - //std::cout << "DeviceBuffer::ctor " << this << " " << nevt << std::endl; - } - virtual ~DeviceBuffer() - { - //std::cout << "DeviceBuffer::dtor " << this << std::endl; - } - virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a simple CUDA device buffer managed on an ad-hoc basis - typedef DeviceBuffer DeviceBufferSimple; - typedef DeviceBuffer DeviceBufferSimple2; -#endif //-------------------------------------------------------------------------- @@ -236,15 +157,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for momenta random numbers constexpr size_t sizePerEventRndNumMomenta = MemoryBuffers::np4 * MemoryBuffers::nparf; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for momenta random numbers typedef HostBuffer HostBufferRndNumMomenta; -#else - // A class encapsulating a CUDA pinned host buffer for momenta random numbers - typedef PinnedHostBuffer PinnedHostBufferRndNumMomenta; - // A class encapsulating a CUDA device buffer for momenta random numbers - typedef DeviceBuffer DeviceBufferRndNumMomenta; -#endif //-------------------------------------------------------------------------- @@ -280,15 +194,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for Gs constexpr size_t sizePerEventGs = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for gs typedef HostBuffer HostBufferGs; -#else - // A class encapsulating a CUDA pinned host buffer for gs - typedef PinnedHostBuffer PinnedHostBufferGs; - // A class encapsulating a CUDA device buffer for gs - typedef DeviceBuffer DeviceBufferGs; -#endif //-------------------------------------------------------------------------- @@ -299,15 +206,8 @@ namespace mg5amcCpu // (should be equal to the number of diagrams in the process) constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for numerators typedef HostBuffer HostBufferNumerators; -#else - // A class encapsulating a CUDA pinned host buffer for numerators - typedef PinnedHostBuffer PinnedHostBufferNumerators; - // A class encapsulating a CUDA device buffer for numerators - typedef DeviceBuffer DeviceBufferNumerators; -#endif //-------------------------------------------------------------------------- @@ -317,15 +217,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for denominators constexpr size_t sizePerEventDenominators = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for denominators typedef HostBuffer HostBufferDenominators; -#else - // A class encapsulating a CUDA pinned host buffer for denominators - typedef PinnedHostBuffer PinnedHostBufferDenominators; - // A class encapsulating a CUDA device buffer for denominators - typedef DeviceBuffer DeviceBufferDenominators; -#endif //-------------------------------------------------------------------------- @@ -335,15 +228,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for random numbers constexpr size_t sizePerEventCouplings = MemoryBuffers::ndcoup * MemoryBuffers::nx2; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for couplings typedef HostBuffer HostBufferCouplings; -#else - // A class encapsulating a CUDA pinned host buffer for couplings - typedef PinnedHostBuffer PinnedHostBufferCouplings; - // A class encapsulating a CUDA device buffer for couplings - typedef DeviceBuffer DeviceBufferCouplings; -#endif //-------------------------------------------------------------------------- @@ -353,16 +239,9 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for momenta constexpr size_t sizePerEventMomenta = MemoryBuffers::np4 * MemoryBuffers::npar; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for momenta typedef HostBuffer HostBufferMomenta; //typedef HostBuffer HostBufferMomenta; // TEST MISALIGNMENT! -#else - // A class encapsulating a CUDA pinned host buffer for momenta - typedef PinnedHostBuffer PinnedHostBufferMomenta; - // A class encapsulating a CUDA device buffer for momenta - typedef DeviceBuffer DeviceBufferMomenta; -#endif //-------------------------------------------------------------------------- @@ -372,15 +251,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for sampling weights constexpr size_t sizePerEventWeights = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for sampling weights typedef HostBuffer HostBufferWeights; -#else - // A class encapsulating a CUDA pinned host buffer for sampling weights - typedef PinnedHostBuffer PinnedHostBufferWeights; - // A class encapsulating a CUDA device buffer for sampling weights - typedef DeviceBuffer DeviceBufferWeights; -#endif //-------------------------------------------------------------------------- @@ -390,30 +262,16 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for matrix elements constexpr size_t sizePerEventMatrixElements = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for matrix elements typedef HostBuffer HostBufferMatrixElements; -#else - // A class encapsulating a CUDA pinned host buffer for matrix elements - typedef PinnedHostBuffer PinnedHostBufferMatrixElements; - // A class encapsulating a CUDA device buffer for matrix elements - typedef DeviceBuffer DeviceBufferMatrixElements; -#endif //-------------------------------------------------------------------------- // A base class encapsulating a memory buffer for the helicity mask typedef BufferBase BufferHelicityMask; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for the helicity mask typedef HostBufferBase HostBufferHelicityMask; -#else - // A class encapsulating a CUDA pinned host buffer for the helicity mask - typedef PinnedHostBufferBase PinnedHostBufferHelicityMask; - // A class encapsulating a CUDA device buffer for the helicity mask - typedef DeviceBufferBase DeviceBufferHelicityMask; -#endif //-------------------------------------------------------------------------- @@ -423,15 +281,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for wavefunctions constexpr size_t sizePerEventWavefunctions = MemoryBuffers::nw6 * MemoryBuffers::nx2; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for wavefunctions typedef HostBuffer HostBufferWavefunctions; -#else - // A class encapsulating a CUDA pinned host buffer for wavefunctions - typedef PinnedHostBuffer PinnedHostBufferWavefunctions; - // A class encapsulating a CUDA device buffer for wavefunctions - typedef DeviceBuffer DeviceBufferWavefunctions; -#endif //-------------------------------------------------------------------------- @@ -441,15 +292,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for helicity random numbers constexpr size_t sizePerEventRndNumHelicity = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for helicity random numbers typedef HostBuffer HostBufferRndNumHelicity; -#else - // A class encapsulating a CUDA pinned host buffer for helicity random numbers - typedef PinnedHostBuffer PinnedHostBufferRndNumHelicity; - // A class encapsulating a CUDA device buffer for helicity random numbers - typedef DeviceBuffer DeviceBufferRndNumHelicity; -#endif //-------------------------------------------------------------------------- @@ -459,15 +303,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for color random numbers constexpr size_t sizePerEventRndNumColor = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for color random numbers typedef HostBuffer HostBufferRndNumColor; -#else - // A class encapsulating a CUDA pinned host buffer for color random numbers - typedef PinnedHostBuffer PinnedHostBufferRndNumColor; - // A class encapsulating a CUDA device buffer for color random numbers - typedef DeviceBuffer DeviceBufferRndNumColor; -#endif //-------------------------------------------------------------------------- @@ -477,15 +314,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for channel ids constexpr size_t sizePerEventChannelId = 1; -#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) // A class encapsulating a C++ host buffer for channel ids typedef HostBuffer HostBufferChannelIds; -#else - // A class encapsulating a CUDA pinned host buffer for channel ids - typedef PinnedHostBuffer PinnedHostBufferChannelIds; - // A class encapsulating a CUDA device buffer for channel ids - typedef DeviceBuffer DeviceBufferChannelIds; -#endif //-------------------------------------------------------------------------- @@ -495,15 +325,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for channel ids constexpr size_t sizePerEventIflavorVec = 1; -#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) // A class encapsulating a C++ host buffer for channel ids typedef HostBuffer HostBufferIflavorVec; -#else - // A class encapsulating a CUDA pinned host buffer for channel ids - typedef PinnedHostBuffer PinnedHostBufferIflavorVec; - // A class encapsulating a CUDA device buffer for channel ids - typedef DeviceBuffer DeviceBufferIflavorVec; -#endif //-------------------------------------------------------------------------- @@ -513,15 +336,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for helicity selection constexpr size_t sizePerEventSelectedHelicity = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for helicity selection typedef HostBuffer HostBufferSelectedHelicity; -#else - // A class encapsulating a CUDA pinned host buffer for helicity selection - typedef PinnedHostBuffer PinnedHostBufferSelectedHelicity; - // A class encapsulating a CUDA device buffer for helicity selection - typedef DeviceBuffer DeviceBufferSelectedHelicity; -#endif //-------------------------------------------------------------------------- @@ -531,71 +347,17 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for color selection constexpr size_t sizePerEventSelectedColor = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for color selection typedef HostBuffer HostBufferSelectedColor; -#else - // A class encapsulating a CUDA pinned host buffer for color selection - typedef PinnedHostBuffer PinnedHostBufferSelectedColor; - // A class encapsulating a CUDA device buffer for color selection - typedef DeviceBuffer DeviceBufferSelectedColor; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // The size (number of elements) per event in a memory buffer for jamps - constexpr size_t sizePerEventJamps = MemoryBuffers::ncolor * MemoryBuffers::nx2; - - // A class encapsulating a CUDA device buffer for color selection - typedef DeviceBuffer DeviceBufferJamps; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - template - void copyDeviceFromHost( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy - { - if( dst.size() != src.size() ) - { - std::ostringstream sstr; - sstr << "Size (#elements) mismatch in copyDeviceFromHost: dst=" << dst.size() << ", src=" << src.size(); - throw std::runtime_error( sstr.str() ); - } - if( dst.bytes() != src.bytes() ) - { - std::ostringstream sstr; - sstr << "Size (#bytes) mismatch in copyDeviceFromHost: dst=" << dst.bytes() << ", src=" << src.bytes(); - throw std::runtime_error( sstr.str() ); - } - // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array - gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyHostToDevice ); - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - template - void copyHostFromDevice( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy - { - if( dst.size() != src.size() ) - { - std::ostringstream sstr; - sstr << "Size (#elements) mismatch in copyHostFromDevice: dst=" << dst.size() << ", src=" << src.size(); - throw std::runtime_error( sstr.str() ); - } - if( dst.bytes() != src.bytes() ) - { - std::ostringstream sstr; - sstr << "Size (#bytes) mismatch in copyHostFromDevice: dst=" << dst.bytes() << ", src=" << src.bytes(); - throw std::runtime_error( sstr.str() ); - } - // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array - gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyDeviceToHost ); - } -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc index d40b1b4174..e1e17d6247 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc @@ -16,11 +16,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -155,92 +151,18 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - MasslessRamboSamplingKernelDevice::MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t gpublocks, - const size_t gputhreads ) - : SamplingKernelBase( energy, rndmom, momenta, weights ) - , NumberOfEvents( gpublocks * gputhreads ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if( !m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: rndmom must be a device array" ); - if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: momenta must be a device array" ); - if( !m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: weights must be a device array" ); - if( m_gpublocks == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gputhreads must be > 0" ); - if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with rndmom" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with momenta" ); - if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with weights" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( m_gputhreads % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelHost: gputhreads should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Sanity checks for memory access (random number buffer) - constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout - static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); - if( m_gputhreads % neppR != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelDevice: gputhreads should be a multiple of neppR=" << neppR; - throw std::runtime_error( sstr.str() ); - } - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - getMomentaInitialDevice( const fptype energy, - fptype* momenta ) - { - constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; - return getMomentaInitial( energy, momenta ); - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void - MasslessRamboSamplingKernelDevice::getMomentaInitial() - { - gpuLaunchKernel( getMomentaInitialDevice, m_gpublocks, m_gputhreads, m_energy, m_momenta.data() ); - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - getMomentaFinalDevice( const fptype energy, - const fptype* rndmom, - fptype* momenta, - fptype* wgts ) - { - constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; - return getMomentaFinal( energy, rndmom, momenta, wgts ); - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void - MasslessRamboSamplingKernelDevice::getMomentaFinal() - { - gpuLaunchKernel( getMomentaFinalDevice, m_gpublocks, m_gputhreads, m_energy, m_rndmom.data(), m_momenta.data(), m_weights.data() ); - } -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h index 6244d55618..e0aa2aea7f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h @@ -13,11 +13,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -132,41 +128,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating RAMBO phase space sampling on a GPU device - class MasslessRamboSamplingKernelDevice final : public SamplingKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~MasslessRamboSamplingKernelDevice() {} - - // Get momenta of initial state particles - void getMomentaInitial() override final; - - // Get momenta of final state particles and weights - void getMomentaFinal() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h index 0fb880bb36..480824a7e5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h @@ -18,11 +18,7 @@ struct curandGenerator_st; struct rocrand_generator_base_type; typedef rocrand_generator_base_type hiprandGenerator_st; -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc index 68e93edb50..ebce23a97b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc @@ -56,11 +56,7 @@ namespace { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif // Fixed physics inputs fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV @@ -203,17 +199,6 @@ namespace } } -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - aosoa_to_umami_kernel( const fptype* aosoa, - double* soa, - std::size_t nevt ) - { - std::size_t ievt = blockDim.x * blockIdx.x + threadIdx.x; - if( ievt >= nevt ) return; - aosoa_to_umami_one( aosoa, soa, ievt, nevt ); - } -#endif const char* backend_label() { @@ -284,29 +269,17 @@ namespace unsigned int nevt, mgOnGpu::TimerMap& timermap, double& wavetime, -#ifdef MGONGPUCPP_GPUIMPL - const DeviceBufferBase& devUmamiMomenta, - const DeviceBufferBase& devFlv, - DeviceBufferBase& devUmamiMEs, - std::vector& hstMEs -#else const std::vector& umamiMomenta, const std::vector& flvVec, std::vector& umamiMEs -#endif ) { constexpr unsigned int UmamiInKeyNum = 2; timermap.start( "3a SigmaKin" ); UmamiInputKey in_keys[UmamiInKeyNum] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX }; UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; -#ifdef MGONGPUCPP_GPUIMPL - const void* inputs[UmamiInKeyNum] = { devUmamiMomenta.data(), devFlv.data() }; - void* outputs[1] = { devUmamiMEs.data() }; -#else const void* inputs[UmamiInKeyNum] = { umamiMomenta.data(), flvVec.data() }; void* outputs[1] = { umamiMEs.data() }; -#endif UmamiStatus st = umami_matrix_element( handle, nevt, nevt, 0, UmamiInKeyNum, in_keys, inputs, 1, out_keys, outputs ); wavetime += timermap.stop(); @@ -316,11 +289,6 @@ namespace return false; } -#ifdef MGONGPUCPP_GPUIMPL - timermap.start( "3b CpDTHmes" ); - gpuMemcpy( hstMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); - wavetime += timermap.stop(); -#endif return true; } @@ -593,30 +561,12 @@ namespace mgOnGpu::TimerMap timermap; -#ifdef MGONGPUCPP_GPUIMPL - timermap.start( "00 GpuInit" ); - GpuRuntime gpuRuntime( false ); - - PinnedHostBufferRndNumMomenta hstRndmom( nevt ); - PinnedHostBufferMomenta hstMomenta( nevt ); - PinnedHostBufferWeights hstWeights( nevt ); - DeviceBufferRndNumMomenta devRndmom( nevt ); - DeviceBufferMomenta devMomenta( nevt ); - DeviceBufferWeights devWeights( nevt ); - DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - DeviceBufferBase devUmamiMEs( nevt ); - DeviceBufferBase devFlv( nevt ); - std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - std::vector flvVec( nevt ); - std::vector hstUmamiMEs( nevt ); -#else HostBufferRndNumMomenta hstRndmom( nevt ); HostBufferMomenta hstMomenta( nevt ); HostBufferWeights hstWeights( nevt ); std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); std::vector umamiMEs( nevt ); std::vector flvVec( nevt ); -#endif UmamiHandle umami_handle = nullptr; if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) @@ -656,10 +606,6 @@ namespace SLHAReader slha( "../../Cards/param_card.dat", false ); const double alphaS = slha.get_block_entry( "sminputs", 3, 1.180000e-01 ); std::vector alphasVec( nevt, alphaS ); -#ifdef MGONGPUCPP_GPUIMPL - DeviceBufferBase devAlphaS( nevt ); - gpuMemcpy( devAlphaS.data(), alphasVec.data(), nevt * sizeof( double ), gpuMemcpyHostToDevice ); -#endif // Always massive RAMBO std::unique_ptr prsk( @@ -673,16 +619,8 @@ namespace for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) for( unsigned int ievt = 0; ievt < nevt; ++ievt ) umamiMomenta[(std::size_t)ip4 * CPPProcess::npar * nevt + (std::size_t)ipar * nevt + ievt] = point[ipar][ip4]; -#ifdef MGONGPUCPP_GPUIMPL - gpuMemcpy( devUmamiMomenta.data(), umamiMomenta.data(), umamiMomenta.size() * sizeof( double ), gpuMemcpyHostToDevice ); - // Host only implementation now (copy) - copyDeviceFromHost( devMomenta, hstMomenta ); - gpuLaunchKernel( aosoa_to_umami_kernel, kMatrixBlocks, kMatrixThreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); - checkGpu( gpuPeekAtLastError() ); -#else for( std::size_t ievt = 0; ievt < nevt; ++ievt ) aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); -#endif if( verbose ) { @@ -710,19 +648,11 @@ namespace for( unsigned int iflav = 0; iflav < nFlavors; ++iflav ) { std::fill( flvVec.begin(), flvVec.end(), iflav ); -#ifdef MGONGPUCPP_GPUIMPL - gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); -#endif timermap.start( "3a SigmaKin" ); UmamiInputKey in_keys[3] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S }; UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; -#ifdef MGONGPUCPP_GPUIMPL - const void* inputs[3] = { devUmamiMomenta.data(), devFlv.data(), devAlphaS.data() }; - void* outputs[1] = { devUmamiMEs.data() }; -#else const void* inputs[3] = { umamiMomenta.data(), flvVec.data(), alphasVec.data() }; void* outputs[1] = { umamiMEs.data() }; -#endif UmamiStatus st = umami_matrix_element( umami_handle, nevt, nevt, 0, 3, in_keys, inputs, 1, out_keys, outputs ); timermap.stop(); @@ -732,12 +662,7 @@ namespace umami_free( umami_handle ); return 3; } -#ifdef MGONGPUCPP_GPUIMPL - gpuMemcpy( hstUmamiMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); - const double* mes = hstUmamiMEs.data(); -#else const double* mes = umamiMEs.data(); -#endif std::cout << " PDG"; for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) @@ -780,31 +705,12 @@ namespace mgOnGpu::TimerMap timermap; -#ifdef MGONGPUCPP_GPUIMPL - timermap.start( "00 GpuInit" ); - GpuRuntime gpuRuntime( false ); - - PinnedHostBufferRndNumMomenta hstRndmom( nevt ); - PinnedHostBufferMomenta hstMomenta( nevt ); - PinnedHostBufferWeights hstWeights( nevt ); - DeviceBufferRndNumMomenta devRndmom( nevt ); - DeviceBufferMomenta devMomenta( nevt ); - DeviceBufferWeights devWeights( nevt ); - DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - DeviceBufferBase devUmamiMEs( nevt ); - DeviceBufferBase devFlv( nevt ); - std::vector flvVec( nevt, flavorID ); - std::vector hstUmamiMEs( nevt ); - // perf-mode runs a single flavor, so the device-side flavor buffer is filled once. - gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); -#else HostBufferRndNumMomenta hstRndmom( nevt ); HostBufferMomenta hstMomenta( nevt ); HostBufferWeights hstWeights( nevt ); std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); std::vector umamiMEs( nevt ); std::vector flvVec( nevt, flavorID ); -#endif std::unique_ptr prnk( new CommonRandomNumberKernel( hstRndmom ) ); @@ -845,11 +751,7 @@ namespace } else { -#ifdef MGONGPUCPP_GPUIMPL - prsk.reset( new MasslessRamboSamplingKernelDevice( kEnergy, devRndmom, devMomenta, devWeights, gpublocks, gputhreads ) ); -#else prsk.reset( new MasslessRamboSamplingKernelHost( kEnergy, hstRndmom, hstMomenta, hstWeights, nevt ) ); -#endif } std::unique_ptr genrtimes( new double[niter] ); @@ -877,14 +779,6 @@ namespace timermap.start( "1b GenRnGen" ); prnk->generateRnarray(); genrtime += timermap.stop(); -#ifdef MGONGPUCPP_GPUIMPL - if( ramboType == RAMBO_MASSLESS ) - { - timermap.start( "1c CpHTDrnd" ); - copyDeviceFromHost( devRndmom, hstRndmom ); - genrtime += timermap.stop(); - } -#endif timermap.start( "2a RamboIni" ); prsk->getMomentaInitial(); @@ -892,15 +786,6 @@ namespace timermap.start( "2b RamboFin" ); prsk->getMomentaFinal(); rambtime += timermap.stop(); -#ifdef MGONGPUCPP_GPUIMPL - // Massive host only (copy) - if( ramboType != RAMBO_MASSLESS ) - { - timermap.start( "2c CpHTDmom" ); - copyDeviceFromHost( devMomenta, hstMomenta ); - rambtime += timermap.stop(); - } -#endif } else { @@ -919,47 +804,23 @@ namespace MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar ) = (fptype)ev[ipar][ip4]; } rambtime += timermap.stop(); -#ifdef MGONGPUCPP_GPUIMPL - timermap.start( "2c CpHTDmom" ); - copyDeviceFromHost( devMomenta, hstMomenta ); - rambtime += timermap.stop(); -#endif } timermap.start( "2d Aosoa2U " ); -#ifdef MGONGPUCPP_GPUIMPL - gpuLaunchKernel( aosoa_to_umami_kernel, gpublocks, gputhreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); - checkGpu( gpuPeekAtLastError() ); -#else for( std::size_t ievt = 0; ievt < nevt; ++ievt ) aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); -#endif rambtime += timermap.stop(); double wavetime = 0; if( !run_umami( umami_handle, nevt, timermap, wavetime, -#ifdef MGONGPUCPP_GPUIMPL - devUmamiMomenta, devFlv, devUmamiMEs, hstUmamiMEs -#else umamiMomenta, flvVec, umamiMEs -#endif ) ) { umami_free( umami_handle ); return 3; } -#ifdef MGONGPUCPP_GPUIMPL - if( verbose ) - { - timermap.start( "3c CpDTHmom" ); - copyHostFromDevice( hstMomenta, devMomenta ); - wavetime += timermap.stop(); - } - const double* mes = hstUmamiMEs.data(); -#else const double* mes = umamiMEs.data(); -#endif timermap.start( "4@ UpdtStat" ); for( unsigned int ievt = 0; ievt < nreal; ++ievt ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc index 30c6799932..2cbe2e38cc 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc @@ -10,11 +10,7 @@ #include "MemoryAccessMatrixElements.h" -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors @@ -23,42 +19,12 @@ namespace mg5amcCpu // *** COLOR MATRIX BELOW *** %(color_matrix_lines)s -#ifdef MGONGPUCPP_GPUIMPL - // The normalized color matrix (divide each column by denom) - template - struct NormalizedColorMatrix - { - constexpr __host__ __device__ NormalizedColorMatrix() - : value() - { - for( int icol = 0; icol < ncolor; icol++ ) - for( int jcol = 0; jcol < ncolor; jcol++ ) - value[icol * ncolor + jcol] = colorMatrix[icol][jcol] / colorDenom[icol]; - } - T value[ncolor * ncolor]; - }; - // The fptype2 version is the default used by kernels (supporting mixed floating point mode also in blas) - static __device__ fptype2 s_pNormalizedColorMatrix2[ncolor * ncolor]; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void createNormalizedColorMatrix() - { - static bool first = true; - if( first ) - { - first = false; - constexpr NormalizedColorMatrix normalizedColorMatrix2; - gpuMemcpyToSymbol( s_pNormalizedColorMatrix2, normalizedColorMatrix2.value, ncolor * ncolor * sizeof( fptype2 ) ); - } - } -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL void color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity @@ -150,284 +116,21 @@ namespace mg5amcCpu MEs_sv_next += deltaMEs_next; #endif } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity - const int nGoodHel, // input: number of good helicities - const int nevtIfAllHelicities ) // input: zero in single-helicity mode, number of events in multi-helicity mode - { - if (nevtIfAllHelicities) { - int ighel = blockIdx.y; - allMEs = allMEs + ighel * nevtIfAllHelicities; // MEs for one specific helicity ighel - allJamps = allJamps + ighel * nevtIfAllHelicities; // Jamps for one specific helicity ighel - } - using J_ACCESS = DeviceAccessJamp; - fptype jampR[ncolor]; - fptype jampI[ncolor]; - for( int icol = 0; icol < ncolor; icol++ ) - { - constexpr int ihel0 = 0; // the input buffer allJamps already points to a specific helicity - cxtype jamp = J_ACCESS::kernelAccessIcolIhelNhelConst( allJamps, icol, ihel0, nGoodHel ); - jampR[icol] = jamp.real(); - jampI[icol] = jamp.imag(); - } - // Loop over icol - fptype deltaMEs = { 0 }; - for( int icol = 0; icol < ncolor; icol++ ) - { - fptype2 ztempR = { 0 }; - fptype2 ztempI = { 0 }; - fptype2 jampRi = jampR[icol]; - fptype2 jampIi = jampI[icol]; - // OLD IMPLEMENTATION (ihel3: symmetric square matrix) - Loop over all jcol - //for( int jcol = 0; jcol < ncolor; jcol++ ) - //{ - // fptype2 jampRj = jampR[jcol]; - // fptype2 jampIj = jampI[jcol]; - // ztempR += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix - // ztempI += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix - //} - // NEW IMPLEMENTATION #475 (ihel3p1: triangular lower diagonal matrix) - Loop over jcol < icol - ztempR += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampRi; // use fptype2 version of color matrix - ztempI += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampIi; // use fptype2 version of color matrix - for( int jcol = 0; jcol < icol; jcol++ ) - { - fptype2 jampRj = jampR[jcol]; - fptype2 jampIj = jampI[jcol]; - ztempR += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix - ztempI += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix - } - deltaMEs += ztempR * jampRi; - deltaMEs += ztempI * jampIi; - } - // *** STORE THE RESULTS *** - using E_ACCESS = DeviceAccessMatrixElements; // non-trivial access: buffer includes all events - // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) - E_ACCESS::kernelAccess( allMEs ) += deltaMEs; // fix #435 - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -#ifndef MGONGPU_HAS_NO_BLAS -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - __global__ void - convertD2F_Jamps( fptype2* allJampsFpt2, // output: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel - const fptype* allJamps, // input: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel - const int nhel ) // input: number of good helicities nGoodHel - { - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - constexpr int ihel = 0; // the input buffer allJamps already points to a specific helicity - // NB! From a functional point of view, any striding will be ok here as long as ncolor*2*nevt elements are all correctly copied! - // NB! Just in case this may be better for performance reasons, however, the same striding as in compute_jamps and cuBLAS is used here - for( int ix2 = 0; ix2 < mgOnGpu::nx2; ix2++ ) - for( int icol = 0; icol < ncolor; icol++ ) - allJampsFpt2[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] = - allJamps[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt]; - } -#endif -#endif -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -#ifndef MGONGPU_HAS_NO_BLAS -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - __global__ void - convertF2D_MEs( fptype* allMEs, // output: allMEs[nevt] for one specific helicity - const fptype2* allMEsFpt2 ) // input: allMEs[nevt] for one specific helicity - { - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - allMEs[ievt] = allMEsFpt2[ievt]; - } -#endif -#endif -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ -#ifndef MGONGPU_HAS_NO_BLAS - void - color_sum_blas( fptype* ghelAllMEs, // output: allMEs super-buffer[nhel][nevt], add |M|^2 separately for each helicity - const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nhel][nevt] for nhel good helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nhel good helicities - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) -#else - gpuStream_t* /*ghelStreams*/, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) -#endif - const int nhel, // input: number of good helicities (nhel == nGoodHel) - const int gpublocks, // input: cuda gpublocks - const int gputhreads ) // input: cuda gputhreads - { - const int nevt = gpublocks * gputhreads; - - // Get the address associated with the normalized color matrix in device memory - static fptype2* devNormColMat = nullptr; - if( !devNormColMat ) gpuGetSymbolAddress( (void**)&devNormColMat, s_pNormalizedColorMatrix2 ); - -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Mixed precision mode: need two fptype2[2*ncolor*nhel*nevt] buffers and one fptype2[nhel*nevt] buffers for the nhel helicities - fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of first fptype2[ncolor*2*nhel*nevt] buffer - fptype2* ghelAllJampsFpt2 = ghelAllBlasTmp + ncolor * mgOnGpu::nx2 * nhel * nevt; // start of second fptype2[ncolor*2*nhel*nevt] buffer - fptype2* ghelAllMEsFpt2 = ghelAllBlasTmp + 2 * ncolor * mgOnGpu::nx2 * nhel * nevt; // start of fptype2[nhel*nevt] buffer - // Convert jamps from double to float - for( int ighel = 0; ighel < nhel; ighel++ ) - { - const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // jamps for a single helicity ihel - fptype2* hAllJampsFpt2 = ghelAllJampsFpt2 + ighel * nevt; // jamps for a single helicity ihel - gpuLaunchKernelStream( convertD2F_Jamps, gpublocks, gputhreads, ghelStreams[ighel], hAllJampsFpt2, hAllJamps, nhel ); - } - // Real and imaginary components - const fptype2* ghelAllJampsReal = ghelAllJampsFpt2; - const fptype2* ghelAllJampsImag = ghelAllJampsFpt2 + ncolor * nhel * nevt; -#else - // Standard single or double precision mode: need one fptype2[ncolor*2*nhel*nevt] buffer - static_assert( std::is_same::value ); - fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of fptype2[ncolor*2*nhel*nevt] buffer - fptype2* ghelAllMEsFpt2 = ghelAllMEs; - // Real and imaginary components - const fptype2* ghelAllJampsReal = ghelAllJamps; // this is not a cast (the two types are identical) - const fptype2* ghelAllJampsImag = ghelAllJamps + ncolor * nhel * nevt; // this is not a cast (the two types are identical) -#endif - // Real and imaginary components - fptype2* ghelAllZtempReal = ghelAllZtempBoth; - fptype2* ghelAllZtempImag = ghelAllZtempBoth + ncolor * nhel * nevt; - - // Note: striding for cuBLAS from DeviceAccessJamp: - // - ghelAllJamps(icol,ihel,ievt).real is ghelAllJamps[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] - // - ghelAllJamps(icol,ihel,ievt).imag is ghelAllJamps[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] - - // Step 1: Compute Ztemp[ncolor][nhel*nevt] = ColorMatrix[ncolor][ncolor] * JampsVector[ncolor][nhel*nevt] for both real and imag - // In this case alpha=1 and beta=0: the operation is Ztemp = alpha * ColorMatrix * JampsVector + beta * Ztemp - fptype2 alpha1 = 1; - fptype2 beta1 = 0; - const int ncolorM = ncolor; - const int nevtN = nhel*nevt; - const int ncolorK = ncolor; - checkGpuBlas( gpuBlasTgemm( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose ColMat - GPUBLAS_OP_T, // transpose JampsV (new1) - ncolorM, nevtN, ncolorK, - &alpha1, - devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK - ghelAllJampsReal, nevtN, // JampsV is nevtN x ncolorK - &beta1, - ghelAllZtempReal, ncolorM ) ); // Ztemp is ncolorM x nevtN - checkGpuBlas( gpuBlasTgemm( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose ColMat - GPUBLAS_OP_T, // transpose JampsV (new1) - ncolorM, nevtN, ncolorK, - &alpha1, - devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK - ghelAllJampsImag, nevtN, // JampsV is nevtN x ncolorK (new1) - &beta1, - ghelAllZtempImag, ncolorM ) ); // Ztemp is ncolorM x nevtN - - // Step 2: For each ievt, compute the dot product of JampsVector[ncolor][ievt] dot tmp[ncolor][ievt] - // In this case alpha=1 and beta=1: the operation is ME = alpha * ( Tmp dot JampsVector ) + beta * ME - // Use cublasSgemmStridedBatched to perform these batched dot products in one call - fptype2 alpha2 = 1; - fptype2 beta2 = 1; - checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose JampsV (new1) - GPUBLAS_OP_N, // do not transpose Tmp - 1, 1, ncolor, // result is 1x1 (dot product) - &alpha2, - ghelAllJampsReal, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column - ghelAllZtempReal, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column - &beta2, - ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) - nevtN ) ); // there are nevtN (nhel*nevt) "batches" - checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose JampsV (new1) - GPUBLAS_OP_N, // do not transpose Tmp - 1, 1, ncolor, // result is 1x1 (dot product) - &alpha2, - ghelAllJampsImag, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column (new1) - ghelAllZtempImag, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column - &beta2, - ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) - nevtN ) ); // there are nevt (nhel*nevt) "batches" - -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Convert MEs from float to double - for( int ighel = 0; ighel < nhel; ighel++ ) - { - fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for a single helicity ihel - fptype2* hAllMEsFpt2 = ghelAllMEsFpt2 + ighel * nevt; // MEs for a single helicity ihel - gpuLaunchKernelStream( convertF2D_MEs, gpublocks, gputhreads, ghelStreams[ighel], hAllMEs, hAllMEsFpt2 ); - } -#endif - } -#endif /* clang-format on */ -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void - color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) - const int nGoodHel, // input: number of good helicities - const int gpublocks, // input: cuda gpublocks - const int gputhreads, // input: cuda gputhreads - const bool processAllHelicities ) // input: if true, use blockIdx.y to index helicities - { - const int nevt = gpublocks * gputhreads; - // CASE 1: KERNEL - if( !pBlasHandle ) - { - assert( ghelAllBlasTmp == nullptr ); // sanity check for HASBLAS=hasNoBlas or CUDACPP_RUNTIME_BLASCOLORSUM not set - if (processAllHelicities) { - gpuLaunchKernel2D( color_sum_kernel, gpublocks, nGoodHel, gputhreads, ghelStreams[0], ghelAllMEs, ghelAllJamps, nGoodHel, nevt ); - } else { - // Loop over helicities - for( int ighel = 0; ighel < nGoodHel; ighel++ ) - { - fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for one specific helicity ighel - const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // Jamps for one specific helicity ighel - gpuStream_t hStream = ghelStreams[ighel]; - gpuLaunchKernelStream( color_sum_kernel, gpublocks, gputhreads, hStream, hAllMEs, hAllJamps, nGoodHel, 0 ); - } - } - } - // CASE 2: BLAS - else - { -#ifdef MGONGPU_HAS_NO_BLAS - assert( false ); // sanity check: no path to this statement for HASBLAS=hasNoBlas -#else - if (processAllHelicities) { - assert( false ); // BLAS in async mode not supported for now - } else { - checkGpu( gpuDeviceSynchronize() ); // do not start the BLAS color sum for all helicities until the loop over helicities has completed - // Reset the tmp buffer -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( 2 * ncolor * mgOnGpu::nx2 + 1 ) * sizeof( fptype2 ) ); -#else - gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( ncolor * mgOnGpu::nx2 ) * sizeof( fptype2 ) ); -#endif - // Delegate the color sum to BLAS for - color_sum_blas( ghelAllMEs, ghelAllJamps, ghelAllBlasTmp, pBlasHandle, ghelStreams, nGoodHel, gpublocks, gputhreads ); - } -#endif - } - } -#endif //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h index 347184c4e1..2e043fa128 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h @@ -14,90 +14,26 @@ #include "CPPProcess.h" #include "GpuAbstraction.h" -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - class DeviceAccessJamp - { - public: - static __device__ inline cxtype_ref - kernelAccessIcolIhelNhel( fptype* buffer, const int icol, const int ihel, const int nhel ) - { - const int ncolor = CPPProcess::ncolor; // the number of leading colors - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) - //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" - // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) - // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS - //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" - // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) - return cxtype_ref( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], - buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); - } - static __device__ inline const cxtype - kernelAccessIcolIhelNhelConst( const fptype* buffer, const int icol, const int ihel, const int nhel ) - { - const int ncolor = CPPProcess::ncolor; // the number of leading colors - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) - //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" - // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) - // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS - //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" - // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) - return cxtype( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], - buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); - } - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void createNormalizedColorMatrix(); -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL void color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity const int ievt0 ); // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void - color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) - const int nGoodHel, // input: number of good helicities - const int gpublocks, // input: cuda gpublocks - const int gputhreads, // input: cuda gputhreads - const bool processAllHelicities); // input: if true, use blockIdx.y to index helicities -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity - const int nGoodHel, // input: number of good helicities - const int nevtIfAllHelicities); // input: zero in single-helicity mode, number of events in multi-helicity mode -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h index 027f1aa44e..8ff67e63ff 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h @@ -30,11 +30,7 @@ namespace mgOnGpu // The number of channels in the channel2iconfig array below // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** constexpr unsigned int nchannels = %(nb_diag)i; -#ifdef MGONGPUCPP_GPUIMPL - static_assert( nchannels <= mg5amcGpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 -#else static_assert( nchannels <= mg5amcCpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 -#endif // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) @@ -45,13 +41,7 @@ namespace mgOnGpu }; // Host copy of the channel2iconfig array (this is needed in runTest #917) -#ifndef MGONGPUCPP_GPUIMPL constexpr const int* hostChannel2iconfig = channel2iconfig; -#else - constexpr int hostChannel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed -%(channelc2iconfig_lines)s - }; -#endif // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) constexpr unsigned int nconfigSDE = %(nb_channel)s; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h index 32c1bbb963..356f034c92 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h @@ -28,11 +28,7 @@ #endif // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc index 05c3b3e0c4..d0b13c937f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc @@ -23,11 +23,7 @@ extern "C" * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. */ -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif /** * The floating point precision used in Fortran arrays. @@ -47,9 +43,6 @@ extern "C" */ void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) { -#ifdef MGONGPUCPP_GPUIMPL - GpuRuntime::setUp(); -#endif // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) // FIXME: disable OMP in Bridge when called from Fortran *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); @@ -66,9 +59,6 @@ extern "C" Bridge* pbridge = dynamic_cast*>( *ppbridge ); if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); delete pbridge; -#ifdef MGONGPUCPP_GPUIMPL - GpuRuntime::tearDown(); -#endif } /** @@ -101,15 +91,9 @@ extern "C" Bridge* pbridge = dynamic_cast*>( *ppbridge ); //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); -#ifdef MGONGPUCPP_GPUIMPL - // Use the device/GPU implementation in the CUDA library - // (there is also a host implementation in this library) - pbridge->gpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); -#else // Use the host/CPU implementation in the C++ library // (there is no device implementation in this library) pbridge->cpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); -#endif } /** diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h index e676df0a33..4815fdb027 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h @@ -12,11 +12,7 @@ extern "C" { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif using FORTRANFPTYPE = double; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc index 8a4468fb3c..4048cce912 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc @@ -14,11 +14,7 @@ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { template class Sampler final : public CppObjectInFortran @@ -41,15 +37,9 @@ namespace mg5amcCpu private: const int m_nevt; // The number of events in each iteration int m_iiter; // The iteration counter (for random number seeding) -#ifndef MGONGPUCPP_GPUIMPL HostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers HostBufferMomenta m_hstMomenta; // Memory buffers for momenta HostBufferWeights m_hstWeights; // Memory buffers for sampling weights -#else - PinnedHostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers - PinnedHostBufferMomenta m_hstMomenta; // Memory buffers for momenta - PinnedHostBufferWeights m_hstWeights; // Memory buffers for sampling weights -#endif std::unique_ptr m_prnk; // The appropriate RandomNumberKernel std::unique_ptr m_prsk; // The appropriate SamplingKernel // HARDCODED DEFAULTS @@ -106,11 +96,7 @@ namespace mg5amcCpu extern "C" { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif /** * The floating point precision used in Fortran arrays. diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h index bcd3990bed..2b7aaafc19 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h @@ -22,11 +22,7 @@ #include // Simplified rambo version for 2 to N (with N>=2) processes with massless particles -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { namespace massless_rambo { @@ -89,16 +85,6 @@ namespace massless_rambo { static bool first = true; if( first ) { -#ifdef MGONGPUCPP_GPUIMPL - if constexpr( M_ACCESS::isOnDevice() ) // avoid - { - const int ievt0 = 0; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid - if( ievt == ievt0 ) - printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); - } - else -#endif { printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); } @@ -172,7 +158,6 @@ namespace massless_rambo { wt = po2log; if( nparf != 2 ) wt = ( 2. * nparf - 4. ) * log( energy ) + z[nparf - 1]; -#ifndef MGONGPUCPP_GPUIMPL // issue warnings if weight is too small or too large static int iwarn[5] = { 0, 0, 0, 0, 0 }; if( wt < -180. ) @@ -185,7 +170,6 @@ namespace massless_rambo { if( iwarn[1] <= 5 ) std::cout << "Too large wt, risk for overflow: " << wt << std::endl; iwarn[1] = iwarn[1] + 1; } -#endif // return for weighted massless momenta // nothing else to do in this event if all particles are massless (nm==0) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h index 09c76a6bd5..53b41f3b80 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h @@ -152,11 +152,9 @@ #endif // SANITY CHECKS (C++ complex number implementation) -#ifndef MGONGPUCPP_GPUIMPL #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX and defined MGONGPU_CPPCXTYPE_CXSMPL #error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL for C++ #endif -#endif // NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) namespace mgOnGpu @@ -192,9 +190,7 @@ namespace mgOnGpu // Alignment requirement for using reinterpret_cast with SIMD vectorized code // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) // Only needed for C++ code but can be enforced also in NVCC builds of C++ code using CUDA>=11.2 and C++17 (#318, #319, #333) -#ifndef MGONGPUCPP_GPUIMPL constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) -#endif } @@ -208,9 +204,7 @@ using mgOnGpu::fptype2; #endif // C++ SIMD vectorization width (this will be used to set neppV) -#ifdef MGONGPUCPP_GPUIMPL // CUDA and HIP implementations have no SIMD -#undef MGONGPU_CPPSIMD -#elif defined __AVX512VL__ && defined MGONGPU_PVW512 // C++ "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#if defined __AVX512VL__ && defined MGONGPU_PVW512 // C++ "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 8 #else @@ -260,11 +254,9 @@ using mgOnGpu::fptype2; //#endif /* clang-format on */ // Define empty CUDA/HIP declaration specifiers for C++ -#ifndef MGONGPUCPP_GPUIMPL #define __global__ #define __host__ #define __device__ -#endif // For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two (https://stackoverflow.com/a/108360) inline constexpr bool diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h index d6e1d02e05..900a496b7d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h @@ -163,11 +163,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { template inline __host__ std::ostream& @@ -297,11 +293,7 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { // --- Type definitions (complex type: cxtype) #ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) @@ -333,11 +325,7 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL @@ -699,11 +687,7 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h index a2489d05d5..fa465e7658 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h @@ -13,63 +13,13 @@ #include // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // cuda -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //========================================================================== -#ifdef MGONGPUCPP_GPUIMPL // cuda - - //------------------------------ - // Floating point types - Cuda - //------------------------------ - - /* - inline __host__ __device__ fptype - fpmax( const fptype& a, const fptype& b ) - { - return max( a, b ); - } - - inline __host__ __device__ fptype - fpmin( const fptype& a, const fptype& b ) - { - return min( a, b ); - } - */ - - inline __host__ __device__ const fptype& - fpmax( const fptype& a, const fptype& b ) - { - return ( ( b < a ) ? a : b ); - } - - inline __host__ __device__ const fptype& - fpmin( const fptype& a, const fptype& b ) - { - return ( ( a < b ) ? a : b ); - } - - inline __host__ __device__ fptype - fpsqrt( const fptype& f ) - { -#if defined MGONGPU_FPTYPE_FLOAT - // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__SINGLE.html - return sqrtf( f ); -#else - // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__DOUBLE.html - return sqrt( f ); -#endif - } - -#endif // #ifdef MGONGPUCPP_GPUIMPL //========================================================================== -#ifndef MGONGPUCPP_GPUIMPL //------------------------------ // Floating point types - C++ @@ -93,7 +43,6 @@ namespace mg5amcCpu return std::sqrt( f ); } -#endif // #ifndef MGONGPUCPP_GPUIMPL //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h index 74d93f05a3..c464174d5f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h @@ -33,113 +33,11 @@ #endif // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { -#ifdef MGONGPU_CPPSIMD - - const int neppV = MGONGPU_CPPSIMD; - - // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) - static_assert( mgOnGpu::cppAlign % ( neppV * sizeof( fptype ) ) == 0 ); - - // SANITY CHECK: check that neppV is a power of two - static_assert( ispoweroftwo( neppV ), "neppV is not a power of 2" ); - - // --- Type definition (using vector compiler extensions: need -march=...) - // For gcc: https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html - // For clang: https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors -#ifdef __clang__ - typedef fptype fptype_v __attribute__( ( ext_vector_type( neppV ) ) ); // RRRR -#else - typedef fptype fptype_v __attribute__( ( vector_size( neppV * sizeof(fptype) ), aligned( neppV * sizeof(fptype) ) ) ); // RRRR -#endif - - // Mixed fptypes #537: float for color algebra and double elsewhere -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - const int neppV2 = MGONGPU_CPPSIMD * 2; - static_assert( mgOnGpu::cppAlign % ( neppV2 * sizeof( fptype2 ) ) == 0 ); - static_assert( ispoweroftwo( neppV2 ), "neppV2 is not a power of 2" ); -#ifdef __clang__ - typedef fptype2 fptype2_v __attribute__( ( ext_vector_type( neppV2 ) ) ); // RRRRRRRR -#else - typedef fptype2 fptype2_v __attribute__( ( vector_size( neppV2 * sizeof( fptype2 ) ), aligned( neppV2 * sizeof( fptype2 ) ) ) ); // RRRRRRRR -#endif -#else - typedef fptype_v fptype2_v; -#endif - - // --- Type definition (using vector compiler extensions: need -march=...) - class cxtype_v // no need for "class alignas(2*sizeof(fptype_v)) cxtype_v" - { - public: - // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) - // See https://en.cppreference.com/w/c/language/array_initialization#Notes - cxtype_v() - : m_real{ 0 }, m_imag{ 0 } {} // RRRR=0000 IIII=0000 - cxtype_v( const cxtype_v& ) = default; - cxtype_v( cxtype_v&& ) = default; - cxtype_v( const fptype_v& r, const fptype_v& i ) - : m_real( r ), m_imag( i ) {} - cxtype_v( const fptype_v& r ) - : m_real( r ), m_imag{ 0 } {} // IIII=0000 - cxtype_v( const fptype& r ) - : m_real( fptype_v{} + r ), m_imag{ 0 } {} // IIII=0000 - cxtype_v& operator=( const cxtype_v& ) = default; - cxtype_v& operator=( cxtype_v&& ) = default; - cxtype_v& operator+=( const cxtype_v& c ) - { - m_real += c.real(); - m_imag += c.imag(); - return *this; - } - cxtype_v& operator-=( const cxtype_v& c ) - { - m_real -= c.real(); - m_imag -= c.imag(); - return *this; - } -#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK - // NB: THIS IS THE FUNDAMENTAL DIFFERENCE BETWEEN MGONGPU_HAS_CPPCXTYPEV_BRK DEFINED AND NOT DEFINED - // NB: the alternative "clang" implementation is simpler: it simply does not have any bracket operator[] - //cxtype_ref operator[]( size_t i ) const { return cxtype_ref( m_real[i], m_imag[i] ); } // gcc14.2 build fails #1004 - cxtype_ref operator[]( size_t i ) { return cxtype_ref( m_real[i], m_imag[i] ); } - cxtype operator[]( size_t i ) const { return cxtype( m_real[i], m_imag[i] ); } -#endif - const fptype_v& real() const - { - return m_real; - } - const fptype_v& imag() const { return m_imag; } - private: - fptype_v m_real, m_imag; // RRRRIIII - }; - - // --- Type definition (using vector compiler extensions: need -march=...) -#ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors - typedef unsigned int uint_v __attribute__( ( ext_vector_type( neppV ) ) ); -#if defined MGONGPU_FPTYPE_DOUBLE - typedef long int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb -#elif defined MGONGPU_FPTYPE_FLOAT - typedef int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb -#endif -#else // gcc - typedef unsigned int uint_v __attribute__( ( vector_size( neppV * sizeof( unsigned int ) ), aligned( neppV * sizeof( unsigned int ) ) ) ); -#if defined MGONGPU_FPTYPE_DOUBLE - typedef long int bool_v __attribute__( ( vector_size( neppV * sizeof( long int ) ), aligned( neppV * sizeof( long int ) ) ) ); // bbbb -#elif defined MGONGPU_FPTYPE_FLOAT - typedef int bool_v __attribute__( ( vector_size( neppV * sizeof( int ) ), aligned( neppV * sizeof( int ) ) ) ); // bbbb -#endif -#endif - -#else // i.e #ifndef MGONGPU_CPPSIMD (this includes #ifdef MGONGPUCPP_GPUIMPL) const int neppV = 1; -#endif // #ifdef MGONGPU_CPPSIMD } //-------------------------------------------------------------------------- @@ -157,13 +55,8 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { -#ifndef MGONGPUCPP_GPUIMPL // Printout to stream for user defined types @@ -190,16 +83,6 @@ namespace mg5amcCpu #endif */ -#ifdef MGONGPU_CPPSIMD - inline std::ostream& - operator<<( std::ostream& out, const fptype_v& v ) - { - out << "{ " << v[0]; - for( int i = 1; i < neppV; i++ ) out << ", " << v[i]; - out << " }"; - return out; - } -#endif #if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT inline std::ostream& @@ -212,32 +95,7 @@ namespace mg5amcCpu } #endif -#ifdef MGONGPU_CPPSIMD - inline std::ostream& - operator<<( std::ostream& out, const cxtype_v& v ) - { -#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK - out << "{ " << v[0]; - for( int i = 1; i < neppV; i++ ) out << ", " << v[i]; -#else - out << "{ " << cxmake( v.real()[0], v.imag()[0] ); - for( int i = 1; i < neppV; i++ ) out << ", " << cxmake( v.real()[i], v.imag()[i] ); -#endif - out << " }"; - return out; - } -#endif -#ifdef MGONGPU_CPPSIMD - inline std::ostream& - operator<<( std::ostream& out, const uint_v& v ) - { - out << "{ " << v[0]; - for( int i = 1; i < neppV; i++ ) out << ", " << v[i]; - out << " }"; - return out; - } -#endif //-------------------------------------------------------------------------- @@ -261,30 +119,6 @@ namespace mg5amcCpu // Functions and operators for fptype_v -#ifdef MGONGPU_CPPSIMD - inline fptype_v - fpsqrt( const volatile fptype_v& v ) // volatile fixes #736 - { - // See https://stackoverflow.com/questions/18921049/gcc-vector-extensions-sqrt - fptype_v out = {}; // avoid warning 'out' may be used uninitialized: see #594 - for( int i = 0; i < neppV; i++ ) - { - volatile fptype outi = 0; // volatile fixes #736 - if( v[i] > 0 ) outi = fpsqrt( (fptype)v[i] ); - out[i] = outi; - } - return out; - } - - inline fptype_v - fpsqrt( const fptype_v& v ) - { - // See https://stackoverflow.com/questions/18921049/gcc-vector-extensions-sqrt - fptype_v out = {}; // avoid warning 'out' may be used uninitialized: see #594 - for( int i = 0; i < neppV; i++ ) out[i] = fpsqrt( v[i] ); - return out; - } -#endif /* #ifdef MGONGPU_CPPSIMD @@ -302,347 +136,11 @@ namespace mg5amcCpu // Functions and operators for cxtype_v -#ifdef MGONGPU_CPPSIMD - - /* - inline cxtype_v - cxvmake( const cxtype c ) - { - cxtype_v out; - for ( int i=0; i -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif struct CUDA_CPU_TestBase : public TestDriverBase { @@ -78,7 +74,6 @@ struct CUDA_CPU_TestBase : public TestDriverBase } }; -#ifndef MGONGPUCPP_GPUIMPL struct CPUTest : public CUDA_CPU_TestBase { // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) @@ -216,198 +211,16 @@ struct CPUTestMultiChannel : public CPUTest // Destructor virtual ~CPUTestMultiChannel() {} }; -#endif - -#ifdef MGONGPUCPP_GPUIMPL -struct CUDATest : public CUDA_CPU_TestBase -{ - // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) - // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] - CPPProcess process; - PinnedHostBufferRndNumMomenta hstRndMom; - PinnedHostBufferMomenta hstMomenta; - PinnedHostBufferGs hstGs; - PinnedHostBufferRndNumHelicity hstRndHel; - PinnedHostBufferRndNumColor hstRndCol; - PinnedHostBufferWeights hstWeights; - PinnedHostBufferChannelIds hstChannelIds; - PinnedHostBufferMatrixElements hstMatrixElements; - PinnedHostBufferSelectedHelicity hstSelHel; - PinnedHostBufferSelectedColor hstSelCol; - PinnedHostBufferHelicityMask hstIsGoodHel; - DeviceBufferRndNumMomenta devRndMom; - DeviceBufferChannelIds devChannelIds; - DeviceBufferMomenta devMomenta; - DeviceBufferGs devGs; - DeviceBufferRndNumHelicity devRndHel; - DeviceBufferRndNumColor devRndCol; - DeviceBufferWeights devWeights; - DeviceBufferMatrixElements devMatrixElements; - DeviceBufferSelectedHelicity devSelHel; - DeviceBufferSelectedColor devSelCol; - DeviceBufferHelicityMask devIsGoodHel; - std::unique_ptr pmek; - - // Create a process object - // Read param_card and set parameters - // ** WARNING EVIL EVIL ** - // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. - // Don't remove! - CUDATest( const std::string& refFileName ) - : CUDA_CPU_TestBase( refFileName ) - , process( /*verbose=*/false ) - , hstRndMom( nevt ) - , hstChannelIds( nevt ) - , hstMomenta( nevt ) - , hstGs( nevt ) - , hstRndHel( nevt ) - , hstRndCol( nevt ) - , hstWeights( nevt ) - , hstMatrixElements( nevt ) - , hstSelHel( nevt ) - , hstSelCol( nevt ) - , hstIsGoodHel( CPPProcess::ncomb ) - , devRndMom( nevt ) - , devChannelIds( nevt ) - , devMomenta( nevt ) - , devGs( nevt ) - , devRndHel( nevt ) - , devRndCol( nevt ) - , devWeights( nevt ) - , devMatrixElements( nevt ) - , devSelHel( nevt ) - , devSelCol( nevt ) - , devIsGoodHel( CPPProcess::ncomb ) - , pmek( new MatrixElementKernelDevice( devMomenta, devGs, devRndHel, devRndCol, devChannelIds, devMatrixElements, devSelHel, devSelCol, gpublocks, gputhreads ) ) - { - // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? - process.initProc( "../../Cards/param_card.dat" ); - } - - virtual ~CUDATest() {} - - void prepareRandomNumbers( unsigned int iiter ) override - { - // Random numbers for momenta - CommonRandomNumberKernel rnk( hstRndMom ); - rnk.seedGenerator( 1337 + iiter ); - rnk.generateRnarray(); - copyDeviceFromHost( devRndMom, hstRndMom ); - // Random numbers for helicity and color selection (fix #931) - CommonRandomNumberKernel rnk2( hstRndHel ); - rnk2.seedGenerator( 1338 + iiter ); - rnk2.generateRnarray(); - copyDeviceFromHost( devRndHel, hstRndHel ); - CommonRandomNumberKernel rnk3( hstRndCol ); - rnk3.seedGenerator( 1339 + iiter ); - rnk3.generateRnarray(); - copyDeviceFromHost( devRndCol, hstRndCol ); - } - - void prepareMomenta( fptype energy ) override - { - RamboSamplingKernelDevice rsk( energy, devRndMom, devMomenta, devWeights, gpublocks, gputhreads ); - // --- 2a. Fill in momenta of initial state particles on the device - rsk.getMomentaInitial(); - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - rsk.getMomentaFinal(); - // --- 2c. CopyDToH Weights - copyHostFromDevice( hstWeights, devWeights ); - // --- 2d. CopyDToH Momenta - copyHostFromDevice( hstMomenta, devMomenta ); - } - - void runSigmaKin( std::size_t iiter ) override - { - constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) - for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; - copyDeviceFromHost( devGs, hstGs ); // BUG FIX #566 - setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 - copyDeviceFromHost( devChannelIds, hstChannelIds ); - if( iiter == 0 ) pmek->computeGoodHelicities(); - pmek->computeMatrixElements( useChannelIds() ); - copyHostFromDevice( hstMatrixElements, devMatrixElements ); - copyHostFromDevice( hstSelHel, devSelHel ); - copyHostFromDevice( hstSelCol, devSelCol ); - } - - fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override - { - assert( ipar < npar ); - assert( ip4 < np4 ); - return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); - } - - fptype getMatrixElement( std::size_t ievt ) const override - { - return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); - } - - int getChannelId( std::size_t ievt ) const override - { - return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); - } - - int getSelectedHelicity( std::size_t ievt ) const override - { - //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... - return hstSelHel.data()[ievt]; - } - - int getSelectedColor( std::size_t ievt ) const override - { - //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... - return hstSelCol.data()[ievt]; - } -}; -// Old test with multi-channel disabled #466 -struct CUDATestNoMultiChannel : public CUDATest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return false; } - - // Constructor - CUDATestNoMultiChannel( const std::string& refFileName ) - : CUDATest( refFileName ) {} // suffix .txt - - // Destructor - virtual ~CUDATestNoMultiChannel() {} -}; - -// New test with multi-channel enabled #896 -struct CUDATestMultiChannel : public CUDATest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return true; } - - // Constructor - CUDATestMultiChannel( const std::string& refFileName ) - : CUDATest( refFileName + "2" ) {} // suffix .txt2 - - // Destructor - virtual ~CUDATestMultiChannel() {} -}; -#endif /* clang-format off */ // AV July 2024 much simpler class structure without the presently-unnecessary googletest templates // This is meant as a workaround to prevent not-understood segfault #907 when adding a second test // Note: instantiate test2 first and test1 second to ensure that the channelid printout from the dtors comes from test1 first and test2 second -#ifdef MGONGPUCPP_GPUIMPL -// CUDA test drivers -CUDATestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID2( s ) s##_GPU_MULTICHANNEL -CUDATestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID1( s ) s##_GPU_NOMULTICHANNEL -#else // CPU test drivers CPUTestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); #define TESTID2( s ) s##_CPU_MULTICHANNEL CPUTestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); #define TESTID1( s ) s##_CPU_NOMULTICHANNEL -#endif // Madgraph tests MadgraphTest mgTest2( driver2 ); MadgraphTest mgTest1( driver1 ); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc index 74c4902dc5..24ccfb208d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc @@ -23,26 +23,14 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -#define TESTID( s ) s##_GPU_MISC -#else #define TESTID( s ) s##_CPU_MISC -#endif #define XTESTID( s ) TESTID( s ) // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { -#ifdef MGONGPU_CPPSIMD /* clang-format off */ -#define EXPECT_TRUE_sv( cond ) { bool_v mask( cond ); EXPECT_TRUE( maskand( mask ) ); } -#else #define EXPECT_TRUE_sv( cond ) { EXPECT_TRUE( cond ); } -#endif /* clang-format on */ inline const std::string boolTF( const bool& b ) @@ -50,26 +38,11 @@ namespace mg5amcCpu return ( b ? "T" : "F" ); } -#ifdef MGONGPU_CPPSIMD - inline const std::string - boolTF( const bool_v& v ) - { - std::stringstream out; - out << "{ " << ( v[0] ? "T" : "F" ); - for( int i = 1; i < neppV; i++ ) out << ", " << ( v[i] ? "T" : "F" ); - out << " }"; - return out.str(); - } -#endif } TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif //-------------------------------------------------------------------------- @@ -92,34 +65,9 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) fptype_sv f{ 1 }; //std::cout << f << std::endl << boolTF( f == 1 ) << std::endl; //EXPECT_TRUE_sv( f == 1 ); // this fails for vectors! TFFF -#ifndef MGONGPU_CPPSIMD EXPECT_TRUE_sv( f == 1 ); // this succeds: T -#else - EXPECT_TRUE( ( f == 1 )[0] ); // this succeds: TFFF[0] - EXPECT_TRUE( ( f[0] == 1 ) ); - for( int i = 1; i < neppV; i++ ) - { - EXPECT_TRUE( !( ( f == 1 )[i] ) ); // this succeds: FTTT[i>=1] - EXPECT_TRUE( ( f[i] == 0 ) ); // equals 0, not 1 - } -#endif } -#ifdef MGONGPU_CPPSIMD - // Vector initialization for cxtype_sv - demonstrate fix for bug #339 - { - fptype_sv f1 = fptype_v{ 0 } + 1; - EXPECT_TRUE_sv( f1 == 1 ); - cxtype_v c12 = cxmake( f1, 2 ); - //std::cout << c12 << std::endl << boolTF( c12.real() == 1 ) << std::endl << boolTF( c12.imag() == 2 ) << std::endl; - EXPECT_TRUE_sv( c12.real() == 1 ); - EXPECT_TRUE_sv( c12.imag() == 2 ); - cxtype_v c21 = cxmake( 2, f1 ); - //std::cout << c21 << std::endl << boolTF( c21.real() == 2 ) << std::endl << boolTF( c21.imag() == 1 ) << std::endl; - EXPECT_TRUE_sv( c21.real() == 2 ); - EXPECT_TRUE_sv( c21.imag() == 1 ); - } -#endif // Vector initialization for cxtype_sv { @@ -256,48 +204,20 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) test_int_sv channelids1_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) fptype_sv absamp0_sv{}; // mimic cxabs2( amp_sv[0] ) fptype_sv absamp1_sv{}; // mimic cxabs2( amp_sv[0] ) -#ifdef MGONGPU_CPPSIMD - for( int i = 0; i < neppV; i++ ) - { - channelids0_sv[i] = i; // 0123 - channelids1_sv[i] = i; // 1234 - absamp0_sv[i] = 10. + i; // 10. 11. 12. 13. - absamp1_sv[i] = 11. + i; // 11. 12. 13. 14. - } -#else channelids0_sv = 0; channelids1_sv = 1; absamp0_sv = 10.; absamp1_sv = 11.; -#endif bool_sv mask0_sv = ( channelids0_sv % 2 == 0 ); // even channels 0123 -> TFTF (1010) bool_sv mask1_sv = ( channelids1_sv % 2 == 0 ); // even channels 1234 -> FTFT (0101) constexpr fptype_sv fpZERO_sv{}; // 0000 //fptype_sv numerators0_sv = mask0_sv * absamp0_sv; // invalid operands to binary * ('__vector(4) long int' and '__vector(4) double') fptype_sv numerators0_sv = fpternary( mask0_sv, absamp0_sv, fpZERO_sv ); // equivalent to "mask0_sv * absamp0_sv" fptype_sv numerators1_sv = fpternary( mask1_sv, absamp1_sv, fpZERO_sv ); // equivalent to "mask1_sv * absamp1_sv" -#ifdef MGONGPU_CPPSIMD - //std::cout << "numerators0_sv: " << numerators0_sv << std::endl; - //std::cout << "numerators1_sv: " << numerators1_sv << std::endl; - for( int i = 0; i < neppV; i++ ) - { - // Values of numerators0_sv: 10.*1 11.*0 12.*1 13.*0 - if( channelids0_sv[i] % 2 == 0 ) // even channels - EXPECT_TRUE( numerators0_sv[i] == ( 10. + i ) ); - else // odd channels - EXPECT_TRUE( numerators0_sv[i] == 0. ); - // Values of numerators1_sv: 11.*0 12.*1 13.*0 14.*1 - if( channelids1_sv[i] % 2 == 0 ) // even channels - EXPECT_TRUE( numerators1_sv[i] == ( 11. + i ) ); - else // odd channels - EXPECT_TRUE( numerators1_sv[i] == 0. ); - } -#else // Values of numerators0_sv: 10.*1 EXPECT_TRUE( numerators0_sv == 10. ); // Values of numerators1_sv: 11.*0 EXPECT_TRUE( numerators1_sv == 0. ); -#endif } //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc index 4fcf7d26b6..46786a40aa 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc @@ -25,40 +25,24 @@ #include #include #include -#ifdef MGONGPUCPP_GPUIMPL -#define TESTID( s ) s##_GPU_XXX -#else #define TESTID( s ) s##_CPU_XXX -#endif #define XTESTID( s ) TESTID( s ) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { std::string fpeHandlerMessage = "unknown"; int fpeHandlerIevt = -1; inline void fpeHandlerTestxxx( int /*sig*/ ) { -#ifdef MGONGPUCPP_GPUIMPL - std::cerr << "Floating Point Exception (GPU): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; -#else std::cerr << "Floating Point Exception (CPU neppV=" << neppV << "): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; -#endif exit( 1 ); } } TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif #ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); #endif @@ -72,11 +56,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt %% neppM == 0 ); // nevt must be a multiple of neppM assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV // Fill in the input momenta -#ifdef MGONGPUCPP_GPUIMPL - mg5amcGpu::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] -#else mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] -#endif /* clang-format off */ // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! const fptype par0[np4 * nevt] = // AOS[nevt][np4] { @@ -167,19 +147,8 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) out << " // ---------" << std::endl; for( int iw6 = 0; iw6 < nw6; iw6++ ) { -#ifdef MGONGPU_CPPSIMD - const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration -#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK - out << std::setw( 26 ) << cxreal( wf[iw6][ieppV] ) << ", "; - out << std::setw( 22 ) << cximag( wf[iw6][ieppV] ); -#else - out << std::setw( 26 ) << wf[iw6].real()[ieppV] << ", "; - out << std::setw( 22 ) << wf[iw6].imag()[ieppV]; -#endif -#else out << std::setw( 26 ) << wf[iw6].real(); out << ", " << std::setw( 22 ) << wf[iw6].imag(); -#endif if( iw6 < nw6 - 1 ) out << ", "; else @@ -207,25 +176,10 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) const fptype expImag = expwf[iw6 * 2 + 1]; if( true ) { -#ifdef MGONGPU_CPPSIMD - const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration -#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK - EXPECT_NEAR( cxreal( wf[iw6][ieppV] ), expReal, std::abs( expReal * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt; - EXPECT_NEAR( cximag( wf[iw6][ieppV] ), expImag, std::abs( expImag * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt; -#else - EXPECT_NEAR( wf[iw6].real()[ieppV], expReal, std::abs( expReal * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt; - EXPECT_NEAR( wf[iw6].imag()[ieppV], expImag, std::abs( expImag * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt; -#endif -#else EXPECT_NEAR( cxreal( wf[iw6] ), expReal, std::abs( expReal * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt; EXPECT_NEAR( cximag( wf[iw6] ), expImag, std::abs( expImag * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt; -#endif } } } @@ -246,31 +200,12 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { if( true ) { -#ifdef MGONGPU_CPPSIMD - const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration -#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK - const fptype expReal = cxreal( expwf[iw6][ieppV] ); - const fptype expImag = cximag( expwf[iw6][ieppV] ); - EXPECT_NEAR( cxreal( wf[iw6][ieppV] ), expReal, std::abs( expReal * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; - EXPECT_NEAR( cximag( wf[iw6][ieppV] ), expImag, std::abs( expImag * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; -#else - const fptype expReal = expwf[iw6].real()[ieppV]; - const fptype expImag = expwf[iw6].imag()[ieppV]; - EXPECT_NEAR( wf[iw6].real()[ieppV], expReal, std::abs( expReal * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; - EXPECT_NEAR( wf[iw6].imag()[ieppV], expImag, std::abs( expImag * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; -#endif -#else const fptype expReal = cxreal( expwf[iw6] ); const fptype expImag = cximag( expwf[iw6] ); EXPECT_NEAR( cxreal( wf[iw6] ), expReal, std::abs( expReal * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; EXPECT_NEAR( cximag( wf[iw6] ), expImag, std::abs( expImag * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; -#endif } } } @@ -317,11 +252,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { for( int ievt = 0; ievt < nevt; ievt++ ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif if( debug ) { std::cout << std::endl; @@ -438,9 +369,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) void myexit() { -#ifdef MGONGPUCPP_GPUIMPL - //checkGpu( gpuDeviceReset() ); // FIXME??? this still crashes! should systematically avoid CUDA calls in all destructors? -#endif } // Main function (see https://google.github.io/googletest/primer.html#writing-the-main-function) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc index d19c93bb9b..0cfb07ed26 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc @@ -16,11 +16,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif namespace { @@ -30,9 +26,6 @@ namespace const fptype* couplings, const unsigned int* flavor_indices, fptype* matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL - fptype* color_jamps, -#endif fptype* numerators, fptype* denominators, std::size_t count ) @@ -40,9 +33,6 @@ namespace bool is_good_hel[CPPProcess::ncomb]; sigmaKin_getGoodHel( momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, -#ifdef MGONGPUCPP_GPUIMPL - color_jamps, -#endif is_good_hel, count ); sigmaKin_setGoodHel( is_good_hel ); @@ -54,26 +44,17 @@ namespace const fptype* couplings, const unsigned int* flavor_indices, fptype* matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL - fptype* color_jamps, -#endif fptype* numerators, fptype* denominators, std::size_t count ) { // static local initialization is called exactly once in a thread-safe way static void* dummy = initialize_impl( momenta, couplings, flavor_indices, matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL - color_jamps, -#endif numerators, denominators, count ); } -#ifdef MGONGPUCPP_GPUIMPL - __device__ -#endif void transpose_momenta( const double* momenta_in, fptype* momenta_out, std::size_t i_event_in, std::size_t i_event_out, std::size_t stride ) { @@ -91,70 +72,6 @@ namespace } } -#ifdef MGONGPUCPP_GPUIMPL - - __global__ void copy_inputs( - const double* momenta_in, - const double* helicity_random_in, - const double* color_random_in, - const double* diagram_random_in, - const double* alpha_s_in, - const unsigned int* flavor_indices_in, - fptype* momenta, - fptype* helicity_random, - fptype* color_random, - fptype* diagram_random, - fptype* g_s, - unsigned int* flavor_indices, - std::size_t count, - std::size_t stride, - std::size_t offset ) - { - std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; - if( i_event >= count ) return; - - transpose_momenta( &momenta_in[offset], momenta, i_event, i_event, stride ); - diagram_random[i_event] = diagram_random_in ? diagram_random_in[i_event + offset] : 0.5; - helicity_random[i_event] = helicity_random_in ? helicity_random_in[i_event + offset] : 0.5; - color_random[i_event] = color_random_in ? color_random_in[i_event + offset] : 0.5; - g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; - flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; - } - - __global__ void copy_outputs( - fptype* denominators, - fptype* numerators, - fptype* matrix_elements, - unsigned int* diagram_index, - int* color_index, - int* helicity_index, - double* m2_out, - double* amp2_out, - int* diagram_out, - int* color_out, - int* helicity_out, - std::size_t count, - std::size_t stride, - std::size_t offset ) - { - std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; - if( i_event >= count ) return; - - if( m2_out ) m2_out[i_event + offset] = matrix_elements[i_event]; - if( amp2_out ) - { - double denominator = denominators[i_event]; - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) - { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_event * CPPProcess::ndiagrams + i_diag] / denominator; - } - } - if( diagram_out ) diagram_out[i_event + offset] = diagram_index[i_event] - 1; - if( color_out ) color_out[i_event + offset] = color_index[i_event] - 1; - if( helicity_out ) helicity_out[i_event + offset] = helicity_index[i_event] - 1; - } - -#endif // MGONGPUCPP_GPUIMPL struct InterfaceInstance { @@ -174,15 +91,7 @@ extern "C" case UMAMI_META_DEVICE: { UmamiDevice& device = *static_cast( result ); -#ifdef MGONGPUCPP_GPUIMPL -#ifdef __CUDACC__ - device = UMAMI_DEVICE_CUDA; -#elif defined( __HIPCC__ ) - device = UMAMI_DEVICE_HIP; -#endif -#else device = UMAMI_DEVICE_CPU; -#endif break; } case UMAMI_META_PARTICLE_COUNT: @@ -231,11 +140,7 @@ extern "C" UmamiStatus umami_supported_outputs( bool const** supported, int* count ) { // MATRIX_ELEMENT, DIAGRAM_AMP2, COLOR_INDEX, HELICITY_INDEX, DIAGRAM_INDEX, GPU_STREAM -#ifdef MGONGPUCPP_GPUIMPL - static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true, true }; -#else static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true }; -#endif *supported = data; *count = UMAMI_OUTPUT_KEY_COUNT; return UMAMI_SUCCESS; @@ -326,9 +231,6 @@ extern "C" } if( !momenta_in ) return UMAMI_ERROR_MISSING_INPUT; -#ifdef MGONGPUCPP_GPUIMPL - gpuStream_t gpu_stream = nullptr; -#endif double* m2_out = nullptr; double* amp2_out = nullptr; int* diagram_out = nullptr; @@ -354,132 +256,11 @@ extern "C" case UMAMI_OUT_DIAGRAM_INDEX: diagram_out = static_cast( output ); break; -#ifdef MGONGPUCPP_GPUIMPL - case UMAMI_OUT_GPU_STREAM: - gpu_stream = static_cast( output ); - break; -#endif default: return UMAMI_ERROR_UNSUPPORTED_OUTPUT; } } -#ifdef MGONGPUCPP_GPUIMPL - std::size_t n_threads = 256; - std::size_t n_blocks = ( count + n_threads - 1 ) / n_threads; - std::size_t rounded_count = n_blocks * n_threads; - - fptype *momenta, *couplings, *g_s, *helicity_random, *color_random, *diagram_random, *color_jamps; - fptype *matrix_elements, *numerators, *denominators, *ghel_matrix_elements, *ghel_jamps; - int *helicity_index, *color_index; - unsigned int *flavor_indices, *diagram_index; - - std::size_t n_coup = mg5amcGpu::Parameters_dependentCouplings::ndcoup; - std::array, 16> ptrs_and_sizes = {{ - {reinterpret_cast(&momenta), rounded_count * CPPProcess::npar * 4 * sizeof( fptype )}, - {reinterpret_cast(&couplings), rounded_count * n_coup * 2 * sizeof( fptype )}, - {reinterpret_cast(&g_s), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&flavor_indices), rounded_count * sizeof( unsigned int )}, - {reinterpret_cast(&helicity_random), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&color_random), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&diagram_random), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&matrix_elements), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&diagram_index), rounded_count * sizeof( unsigned int )}, - {reinterpret_cast(&color_jamps), rounded_count * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, - {reinterpret_cast(&numerators), rounded_count * CPPProcess::ndiagrams * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&denominators), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&helicity_index), rounded_count * sizeof( int )}, - {reinterpret_cast(&color_index), rounded_count * sizeof( int )}, - {reinterpret_cast(&ghel_matrix_elements), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&ghel_jamps), rounded_count * CPPProcess::ncomb * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, - }}; - std::size_t total_size = 0; - constexpr std::size_t MAX_SIZE = std::max(sizeof(fptype), sizeof(int)); - for (auto [ptr, size] : ptrs_and_sizes) { - std::size_t aligned_size = (size + MAX_SIZE - 1) / MAX_SIZE * MAX_SIZE; - total_size += aligned_size; - } - uint8_t* buffer; - // we can consider caching this between matrix element calls - gpuMallocAsync( &buffer, total_size, gpu_stream ); - std::size_t buf_offset = 0; - for (auto [ptr, size] : ptrs_and_sizes) { - std::size_t aligned_size = (size + 7) / 8 * 8; - *ptr = buffer + buf_offset; - buf_offset += aligned_size; - } - - copy_inputs<<>>( - momenta_in, - random_helicity_in, - random_color_in, - random_diagram_in, - alpha_s_in, - flavor_indices_in, - momenta, - helicity_random, - color_random, - diagram_random, - g_s, - flavor_indices, - count, - stride, - offset ); - computeDependentCouplings<<>>( g_s, couplings ); - checkGpu( gpuPeekAtLastError() ); - - InterfaceInstance* instance = static_cast( handle ); - if( !instance->initialized ) - { - initialize( - momenta, couplings, flavor_indices, matrix_elements, color_jamps, numerators, denominators, rounded_count ); - instance->initialized = true; - } - - sigmaKin( - momenta, - couplings, - flavor_indices, - helicity_random, - color_random, - nullptr, - diagram_random, - matrix_elements, - helicity_index, - color_index, - color_jamps, - numerators, - denominators, - diagram_index, - false, - ghel_matrix_elements, - ghel_jamps, - nullptr, - nullptr, - &gpu_stream, - true, - n_blocks, - n_threads ); - - copy_outputs<<>>( - denominators, - numerators, - matrix_elements, - diagram_index, - color_index, - helicity_index, - m2_out, - amp2_out, - diagram_out, - color_out, - helicity_out, - count, - stride, - offset ); - checkGpu( gpuPeekAtLastError() ); - - gpuFreeAsync( buffer, gpu_stream ); -#else // MGONGPUCPP_GPUIMPL constexpr std::size_t vector_size = MemoryAccessMomentaBase::neppM; // need to round to round to double page size for some reason constexpr std::size_t page_size2 = 2 * vector_size; @@ -654,7 +435,6 @@ extern "C" } } } -#endif // MGONGPUCPP_GPUIMPL return UMAMI_SUCCESS; } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/Bridge.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/Bridge.h deleted file mode 100644 index 8417e3ad15..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/Bridge.h +++ /dev/null @@ -1,633 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Nov 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2021-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef BRIDGE_H -#define BRIDGE_H 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" // for CPPProcess -#include "CrossSectionKernels.h" // for flagAbnormalMEs -#include "MatrixElementKernels.h" // for MatrixElementKernelHost, MatrixElementKernelDevice -#include "MemoryAccessMomenta.h" // for MemoryAccessMomenta::neppM -#include "MemoryBuffers.h" // for HostBufferMomenta, DeviceBufferMomenta etc - -//#ifdef __HIPCC__ -//#include // see -//https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 #else #include -// // bypass this completely to ease portability on LUMI #803 #endif - -#include // bypass std::filesystem #803 - -#include -#include -#include -#include -#include -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - /** - * A base class for a class whose pointer is passed between Fortran and C++. - * This is not really necessary, but it allows minimal type checks on all such - * pointers. - */ - struct CppObjectInFortran - { - CppObjectInFortran() {} - virtual ~CppObjectInFortran() {} - }; - - //-------------------------------------------------------------------------- - /** - * A templated class for calling the CUDA/C++ matrix element calculations of the - * event generation workflow. The FORTRANFPTYPE template parameter indicates the - * precision of the Fortran momenta from MadEvent (float or double). The - * precision of the matrix element calculation is hardcoded in the fptype - * typedef in CUDA/C++. - * - * The Fortran momenta passed in are in the form of - * DOUBLE PRECISION P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) - * where the dimensions are , , - * . In memory, this is stored in a way that C reads as an array - * P_MULTI[nevtF][nparF][np4F]. The CUDA/C++ momenta are stored as an - * array[npagM][npar][np4][neppM] with nevt=npagM*neppM. The Bridge is - * configured to store nevt==nevtF events in CUDA/C++. It also checks that - * Fortran and C++ parameters match, nparF==npar and np4F==np4. - * - * The cpu/gpu sequences take FORTRANFPTYPE* (not fptype*) momenta/MEs. - * This allows mixing double in MadEvent Fortran with float in CUDA/C++ - * sigmaKin. In the fcheck_sa.f test, Fortran uses double while CUDA/C++ may use - * double or float. In the check_sa "--bridge" test, everything is implemented - * in fptype (double or float). - */ - template - class Bridge final : public CppObjectInFortran - { - public: - /** - * Constructor - * - * @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran array - * loops (VECSIZE_USED <= VECSIZE_MEMMAX) - * @param nparF (NEXTERNAL, nexternal.inc) number of external particles in - * Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F number of momenta components, usually 4, in Fortran arrays - * (KEPT FOR SANITY CHECKS ONLY) - */ - Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F ); - - /** - * Destructor - */ - virtual ~Bridge() {} - - // Delete copy/move constructors and assignment operators - Bridge( const Bridge& ) = delete; - Bridge( Bridge&& ) = delete; - Bridge& operator=( const Bridge& ) = delete; - Bridge& operator=( Bridge&& ) = delete; - -#ifdef MGONGPUCPP_GPUIMPL - /** - * Set the gpublocks and gputhreads for the gpusequence - throws if evnt != - * gpublocks*gputhreads (this is needed for BridgeKernel tests rather than for - * actual production use in Fortran) - * - * @param gpublocks number of gpublocks - * @param gputhreads number of gputhreads - */ - void set_gpugrid( const int gpublocks, const int gputhreads ); - - /** - * Sequence to be executed for the Cuda matrix element calculation - * - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant - * alphas) - * @param iflavorVec the index of the flavor combination - * @param rndhel the pointer to the input random numbers for helicity - * selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 - * to n - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void gpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); -#else - /** - * Sequence to be executed for the vectorized CPU matrix element calculation - * - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant - * alphas) - * @param iflavorVec the index of the flavor combination - * @param rndhel the pointer to the input random numbers for helicity - * selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 - * to n - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void cpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); -#endif - - // Return the number of good helicities (-1 initially when they have not yet - // been calculated) - int nGoodHel() const { return m_nGoodHel; } - - // Return the total number of helicities (expose cudacpp ncomb in the Bridge - // interface to Fortran) - constexpr int nTotHel() const { return CPPProcess::ncomb; } - - private: - unsigned int m_nevt; // number of events - int m_nGoodHel; // the number of good helicities (-1 initially when they have - // not yet been calculated) - -#ifdef MGONGPUCPP_GPUIMPL - int m_gputhreads; // number of gpu threads (default set from number of - // events, can be modified) - int m_gpublocks; // number of gpu blocks (default set from number of events, - // can be modified) - DeviceBuffer m_devMomentaF; - DeviceBufferMomenta m_devMomentaC; - DeviceBufferGs m_devGs; - DeviceBufferIflavorVec m_devIflavorVec; - DeviceBufferRndNumHelicity m_devRndHel; - DeviceBufferRndNumColor m_devRndCol; - DeviceBufferMatrixElements m_devMEs; - DeviceBufferSelectedHelicity m_devSelHel; - DeviceBufferSelectedColor m_devSelCol; - DeviceBufferChannelIds m_devChannelIds; - PinnedHostBufferIflavorVec m_hstIflavorVec; - PinnedHostBufferGs m_hstGs; - PinnedHostBufferRndNumHelicity m_hstRndHel; - PinnedHostBufferRndNumColor m_hstRndCol; - PinnedHostBufferMatrixElements m_hstMEs; - PinnedHostBufferSelectedHelicity m_hstSelHel; - PinnedHostBufferSelectedColor m_hstSelCol; - PinnedHostBufferChannelIds m_hstChannelIds; - std::unique_ptr m_pmek; - // static constexpr int s_gputhreadsmin = 16; // minimum number of gpu threads - // (TEST VALUE FOR MADEVENT) - static constexpr int s_gputhreadsmin = - 32; // minimum number of gpu threads (DEFAULT) -#else - HostBufferMomenta m_hstMomentaC; - HostBufferGs m_hstGs; - HostBufferIflavorVec m_hstIflavorVec; - HostBufferRndNumHelicity m_hstRndHel; - HostBufferRndNumColor m_hstRndCol; - HostBufferMatrixElements m_hstMEs; - HostBufferSelectedHelicity m_hstSelHel; - HostBufferSelectedColor m_hstSelCol; - HostBufferChannelIds m_hstChannelIds; - std::unique_ptr m_pmek; -#endif - }; - - //-------------------------------------------------------------------------- - // - // Forward declare transposition methods - // - -#ifdef MGONGPUCPP_GPUIMPL - - template - __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); - -#endif // MGONGPUCPP_GPUIMPL - - template - void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); - - template - void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ); - - //-------------------------------------------------------------------------- - // - // Implementations of member functions of class Bridge - // - - template - Bridge::Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F) - : m_nevt( nevtF ), m_nGoodHel( -1 ) -#ifdef MGONGPUCPP_GPUIMPL - , m_gputhreads( 256 ) // default number of gpu threads - , m_gpublocks( m_nevt / m_gputhreads ) // this ensures m_nevt <= m_gpublocks*m_gputhreads - , m_devMomentaF( m_nevt ) - , m_devMomentaC( m_nevt ) - , m_devIflavorVec( m_nevt ) - , m_devGs( m_nevt ) - , m_devRndHel( m_nevt ) - , m_devRndCol( m_nevt ) - , m_devMEs( m_nevt ) - , m_devSelHel( m_nevt ) - , m_devSelCol( m_nevt ) - , m_devChannelIds( m_nevt ) -#else - , m_hstMomentaC( m_nevt ) -#endif - , m_hstGs( m_nevt ) - , m_hstIflavorVec( m_nevt ) - , m_hstRndHel( m_nevt ) - , m_hstRndCol( m_nevt ) - , m_hstMEs( m_nevt ) - , m_hstSelHel( m_nevt ) - , m_hstSelCol( m_nevt ) - , m_hstChannelIds( m_nevt ) - , m_pmek( nullptr ) - { - if( nparF != CPPProcess::npar ) - throw std::runtime_error( "Bridge constructor: npar mismatch" ); - if( np4F != CPPProcess::np4 ) - throw std::runtime_error( "Bridge constructor: np4 mismatch" ); -#ifdef MGONGPUCPP_GPUIMPL - if( ( m_nevt < s_gputhreadsmin ) || ( m_nevt % s_gputhreadsmin != 0 ) ) - throw std::runtime_error( - "Bridge constructor: nevt should be a multiple of " + - std::to_string( s_gputhreadsmin ) ); - while( m_nevt != m_gpublocks * m_gputhreads ) - { - m_gputhreads /= 2; - if( m_gputhreads < s_gputhreadsmin ) - throw std::logic_error( - "Bridge constructor: FIXME! cannot choose gputhreads" ); // this - // should - // never - // happen! - m_gpublocks = m_nevt / m_gputhreads; - } -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Instantiate device Bridge (nevt=" << m_nevt - << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads - << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" - << std::endl; -#endif - m_pmek.reset( new MatrixElementKernelDevice( - m_devMomentaC, m_devGs, m_devIflavorVec, m_devRndHel, m_devRndCol, m_devChannelIds, m_devMEs, m_devSelHel, m_devSelCol, m_gpublocks, m_gputhreads) ); -#else -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Instantiate host Bridge (nevt=" << m_nevt << ")" - << std::endl; -#endif - m_pmek.reset( new MatrixElementKernelHost( - m_hstMomentaC, m_hstGs, m_hstIflavorVec, m_hstRndHel, m_hstRndCol, m_hstChannelIds, m_hstMEs, m_hstSelHel, m_hstSelCol, m_nevt ) ); -#endif // MGONGPUCPP_GPUIMPL - // Create a process object, read param card and set parameters - // FIXME: the process instance can happily go out of scope because it is only - // needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? what if fbridgecreate - // is called from several Fortran threads? - CPPProcess process( /*verbose=*/false ); - std::string paramCard = - "../Cards/param_card.dat"; // ZW: change default param_card.dat location - // to one dir down - /* -#ifdef __HIPCC__ - if( !std::experimental::filesystem::exists( paramCard ) ) paramCard = "../" + -paramCard; #else if( !std::filesystem::exists( paramCard ) ) paramCard = "../" + -paramCard; #endif - */ - // struct stat dummybuffer; // bypass std::filesystem #803 - // if( !( stat( paramCard.c_str(), &dummyBuffer ) == 0 ) ) paramCard = "../" + - // paramCard; // - auto fileExists = []( std::string& fileName ) - { - struct stat buffer; - return stat( fileName.c_str(), &buffer ) == 0; - }; - size_t paramCardCheck = 2; // ZW: check for paramCard up to 2 directories up - for( size_t k = 0; k < paramCardCheck; ++k ) - { - if( fileExists( paramCard ) ) break; // bypass std::filesystem #803 - paramCard = "../" + paramCard; - } - process.initProc( paramCard ); - } - -#ifdef MGONGPUCPP_GPUIMPL - template - void Bridge::set_gpugrid( const int gpublocks, - const int gputhreads ) - { - if( m_nevt != gpublocks * gputhreads ) - throw std::runtime_error( - "Bridge: gpublocks*gputhreads must equal m_nevt in set_gpugrid" ); - m_gpublocks = gpublocks; - m_gputhreads = gputhreads; -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Set grid in Bridge (nevt=" << m_nevt - << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads - << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" - << std::endl; -#endif - m_pmek->setGrid( m_gpublocks, m_gputhreads ); - } -#endif - -#ifdef MGONGPUCPP_GPUIMPL - template - void Bridge::gpu_sequence( const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool goodHelOnly ) - { - constexpr int neppM = MemoryAccessMomenta::neppM; - if constexpr( neppM == 1 && std::is_same_v ) - { - gpuMemcpy( m_devMomentaC.data(), momenta, m_devMomentaC.bytes(), gpuMemcpyHostToDevice ); - } - else - { - gpuMemcpy( m_devMomentaF.data(), momenta, m_devMomentaF.bytes(), gpuMemcpyHostToDevice ); - const int thrPerEvt = - CPPProcess::npar * - CPPProcess::np4; // AV: transpose alg does 1 element per thread (NOT 1 - // event per thread) - // const int thrPerEvt = 1; // AV: try new alg with 1 event per thread... - // this seems slower - gpuLaunchKernel( dev_transposeMomentaF2C, m_gpublocks * thrPerEvt, m_gputhreads, m_devMomentaF.data(), m_devMomentaC.data(), m_nevt ); - } - if constexpr( std::is_same_v ) - { - memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); - } - else - { - std::copy( gs, gs + m_nevt, m_hstGs.data() ); - std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); - std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); - } - const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); - if( useChannelIds ) - memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); - // else ... // no need to initialize m_hstChannel: it is allocated with - // gpuMallocHost and NOT initialized in PinnedHostBufferBase, but it is NOT - // used later on - // initialise iflavorVec - memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); - copyDeviceFromHost( m_devGs, m_hstGs ); - copyDeviceFromHost( m_devRndHel, m_hstRndHel ); - copyDeviceFromHost( m_devRndCol, m_hstRndCol ); - if( useChannelIds ) copyDeviceFromHost( m_devChannelIds, m_hstChannelIds ); - copyDeviceFromHost( m_devIflavorVec, m_hstIflavorVec ); - if( m_nGoodHel < 0 ) - { - m_nGoodHel = m_pmek->computeGoodHelicities(); - if( m_nGoodHel < 0 ) - throw std::runtime_error( - "Bridge gpu_sequence: computeGoodHelicities returned nGoodHel<0" ); - } - if( goodHelOnly ) return; - m_pmek->computeMatrixElements( useChannelIds ); - copyHostFromDevice( m_hstMEs, m_devMEs ); -#ifdef MGONGPUCPP_VERBOSE - flagAbnormalMEs( m_hstMEs.data(), m_nevt ); -#endif - copyHostFromDevice( m_hstSelHel, m_devSelHel ); - copyHostFromDevice( m_hstSelCol, m_devSelCol ); - if constexpr( std::is_same_v ) - { - memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); - memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); - memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); - } - else - { - std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); - std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); - std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); - } - } -#endif - -#ifndef MGONGPUCPP_GPUIMPL - template - void Bridge::cpu_sequence( const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool goodHelOnly ) - { - hst_transposeMomentaF2C( momenta, m_hstMomentaC.data(), m_nevt ); - if constexpr( std::is_same_v ) - { - memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); - } - else - { - std::copy( gs, gs + m_nevt, m_hstGs.data() ); - std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); - std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); - } - const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); - if( useChannelIds ) - memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); - // else ... // no need to initialize m_hstChannel: it is allocated and default - // initialized in HostBufferBase (and it is not used later on anyway) - // initialise iflavorVec - memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); - if( m_nGoodHel < 0 ) - { - m_nGoodHel = m_pmek->computeGoodHelicities(); - if( m_nGoodHel < 0 ) - throw std::runtime_error( - "Bridge cpu_sequence: computeGoodHelicities returned nGoodHel<0" ); - } - if( goodHelOnly ) return; - m_pmek->computeMatrixElements( useChannelIds ); -#ifdef MGONGPUCPP_VERBOSE - flagAbnormalMEs( m_hstMEs.data(), m_nevt ); -#endif - if constexpr( std::is_same_v ) - { - memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); - memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); - memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); - } - else - { - std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); - std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); - std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); - } - } -#endif - - //-------------------------------------------------------------------------- - // - // Implementations of transposition methods - // - FORTRAN arrays: P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) ==> - // p_multi[nevtF][nparF][np4F] in C++ (AOS) - // - C++ array: momenta[npagM][npar][np4][neppM] with nevt=npagM*neppM (AOSOA) - // - -#ifdef MGONGPUCPP_GPUIMPL - template - __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool oldImplementation = true; // default: use old implementation - if constexpr( oldImplementation ) - { - // SR initial implementation - constexpr int part = CPPProcess::npar; - constexpr int mome = CPPProcess::np4; - constexpr int strd = MemoryAccessMomenta::neppM; - int pos = blockDim.x * blockIdx.x + threadIdx.x; - int arrlen = nevt * part * mome; - if( pos < arrlen ) - { - int page_i = pos / ( strd * mome * part ); - int rest_1 = pos % ( strd * mome * part ); - int part_i = rest_1 / ( strd * mome ); - int rest_2 = rest_1 % ( strd * mome ); - int mome_i = rest_2 / strd; - int strd_i = rest_2 % strd; - int inpos = ( page_i * strd + strd_i ) // event number - * ( part * mome ) // event size (pos of event) - + part_i * mome // particle inside event - + mome_i; // momentum inside particle - out[pos] = in[inpos]; // F2C (Fortran to C) - } - } - else - { - // AV attempt another implementation with 1 event per thread: this seems - // slower... F-style: AOS[nevtF][nparF][np4F] C-style: - // AOSOA[npagM][npar][np4][neppM] with nevt=npagM*neppM - constexpr int npar = CPPProcess::npar; - constexpr int np4 = CPPProcess::np4; - constexpr int neppM = MemoryAccessMomenta::neppM; - assert( nevt % neppM == - 0 ); // number of events is not a multiple of neppM??? - int ievt = blockDim.x * blockIdx.x + threadIdx.x; - int ipagM = ievt / neppM; - int ieppM = ievt % neppM; - for( int ip4 = 0; ip4 < np4; ip4++ ) - for( int ipar = 0; ipar < npar; ipar++ ) - { - int cpos = ipagM * npar * np4 * neppM + ipar * np4 * neppM + - ip4 * neppM + ieppM; - int fpos = ievt * npar * np4 + ipar * np4 + ip4; - out[cpos] = in[fpos]; // F2C (Fortran to C) - } - } - } -#endif - - template - void hst_transposeMomenta( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool oldImplementation = false; // default: use new implementation - if constexpr( oldImplementation ) - { - // SR initial implementation - constexpr unsigned int part = CPPProcess::npar; - constexpr unsigned int mome = CPPProcess::np4; - constexpr unsigned int strd = MemoryAccessMomenta::neppM; - unsigned int arrlen = nevt * part * mome; - for( unsigned int pos = 0; pos < arrlen; ++pos ) - { - unsigned int page_i = pos / ( strd * mome * part ); - unsigned int rest_1 = pos % ( strd * mome * part ); - unsigned int part_i = rest_1 / ( strd * mome ); - unsigned int rest_2 = rest_1 % ( strd * mome ); - unsigned int mome_i = rest_2 / strd; - unsigned int strd_i = rest_2 % strd; - unsigned int inpos = ( page_i * strd + strd_i ) // event number - * ( part * mome ) // event size (pos of event) - + part_i * mome // particle inside event - + mome_i; // momentum inside particle - if constexpr( F2C ) // needs c++17 and cuda >=11.2 (#333) - out[pos] = in[inpos]; // F2C (Fortran to C) - else - out[inpos] = in[pos]; // C2F (C to Fortran) - } - } - else - { - // AV attempt another implementation: this is slightly faster (better c++ - // pipelining?) [NB! this is not a transposition, it is an AOS to AOSOA - // conversion: if neppM=1, a memcpy is enough] F-style: - // AOS[nevtF][nparF][np4F] C-style: AOSOA[npagM][npar][np4][neppM] with - // nevt=npagM*neppM - constexpr unsigned int npar = CPPProcess::npar; - constexpr unsigned int np4 = CPPProcess::np4; - constexpr unsigned int neppM = MemoryAccessMomenta::neppM; - if constexpr( neppM == 1 && std::is_same_v ) - { - memcpy( out, in, nevt * npar * np4 * sizeof( Tin ) ); - } - else - { - const unsigned int npagM = nevt / neppM; - assert( nevt % neppM == - 0 ); // number of events is not a multiple of neppM??? - for( unsigned int ipagM = 0; ipagM < npagM; ipagM++ ) - for( unsigned int ip4 = 0; ip4 < np4; ip4++ ) - for( unsigned int ipar = 0; ipar < npar; ipar++ ) - for( unsigned int ieppM = 0; ieppM < neppM; ieppM++ ) - { - unsigned int ievt = ipagM * neppM + ieppM; - unsigned int cpos = ipagM * npar * np4 * neppM + - ipar * np4 * neppM + ip4 * neppM + ieppM; - unsigned int fpos = ievt * npar * np4 + ipar * np4 + ip4; - if constexpr( F2C ) - out[cpos] = in[fpos]; // F2C (Fortran to C) - else - out[fpos] = in[cpos]; // C2F (C to Fortran) - } - } - } - } - - template - void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool F2C = true; - hst_transposeMomenta( in, out, nevt ); - } - - template - void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool F2C = false; - hst_transposeMomenta( in, out, nevt ); - } - - //-------------------------------------------------------------------------- -} // namespace mg5amcGpu -#endif // BRIDGE_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.cc deleted file mode 100644 index 65c97616d8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.cc +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "BridgeKernels.h" - -#include "GpuAbstraction.h" -#include "MemoryAccessMomenta.h" - -#include - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - - //-------------------------------------------------------------------------- - - BridgeKernelBase::BridgeKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( nevt ) - , m_bridge( nevt, npar, np4 ) - { - if( m_momenta.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: momenta must be a host array" ); - if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: matrixElements must be a host array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with matrixElements" ); - } - - //-------------------------------------------------------------------------- -} - -//============================================================================ - -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu -{ - - //-------------------------------------------------------------------------- - - BridgeKernelHost::BridgeKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: Gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt) - : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, nevt) - , m_fortranMomenta( nevt ) - { - } - - //-------------------------------------------------------------------------- - - void BridgeKernelHost::transposeInputMomentaC2F() - { - hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); - } - - //-------------------------------------------------------------------------- - - int BridgeKernelHost::computeGoodHelicities() - { - constexpr bool goodHelOnly = true; - constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering - m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - return m_bridge.nGoodHel(); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelHost::computeMatrixElements( const bool useChannelIds ) - { - constexpr bool goodHelOnly = false; - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - //-------------------------------------------------------------------------- - - BridgeKernelDevice::BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: Gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads) - : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, gpublocks * gputhreads) - , m_fortranMomenta( nevt() ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if( m_gpublocks == 0 ) throw std::runtime_error( "BridgeKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "BridgeKernelDevice: gputhreads must be > 0" ); - m_bridge.set_gpugrid( gpublocks, gputhreads ); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelDevice::transposeInputMomentaC2F() - { - hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); - } - - //-------------------------------------------------------------------------- - - int BridgeKernelDevice::computeGoodHelicities() - { - constexpr bool goodHelOnly = true; - constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering - m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - return m_bridge.nGoodHel(); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelDevice::computeMatrixElements( const bool useChannelIds ) - { - constexpr bool goodHelOnly = false; - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.h deleted file mode 100644 index 2d873486a8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/BridgeKernels.h +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally originally by: A. Valassi (Jan 2022). -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef BRIDGEKERNELS_H -#define BRIDGEKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "Bridge.h" -#include "MatrixElementKernels.h" -#include "MemoryBuffers.h" - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - // A Bridge wrapper base class encapsulating matrix element calculations on a CPU host - class BridgeKernelBase : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - BridgeKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~BridgeKernelBase() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - virtual void transposeInputMomentaC2F() = 0; - - protected: - - // The wrapped bridge - Bridge m_bridge; - }; - - //-------------------------------------------------------------------------- - -#ifndef MGONGPUCPP_GPUIMPL - // A Bridge wrapper class encapsulating matrix element calculations on a CPU host - class BridgeKernelHost final : public BridgeKernelBase - { - public: - - // Constructor from existing input and output buffers - BridgeKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~BridgeKernelHost() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - void transposeInputMomentaC2F() override final; - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // The buffer for the input momenta, transposed to Fortran array indexing - HostBufferMomenta m_fortranMomenta; - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A Bridge wrapper class encapsulating matrix element calculations on a GPU device - class BridgeKernelDevice : public BridgeKernelBase - { - public: - - // Constructor from existing input and output buffers - BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads); - - // Destructor - virtual ~BridgeKernelDevice() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - void transposeInputMomentaC2F() override final; - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The buffer for the input momenta, transposed to Fortran array indexing - PinnedHostBufferMomenta m_fortranMomenta; - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; -#endif - - //-------------------------------------------------------------------------- -} -#endif // BRIDGEKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc index 16671b5686..4c72479501 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc @@ -11,11 +11,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc index 7a3e0e0519..53430fe8d4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc @@ -79,11 +79,7 @@ debug_me_is_abnormal( const fptype& me, size_t ievtALL ) //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -187,7 +183,6 @@ namespace mg5amcCpu //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu { @@ -233,6 +228,5 @@ namespace mg5amcGpu */ } -#endif //============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h index d1e29b7999..9f2a179e8b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h @@ -14,11 +14,7 @@ //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc index 0c8e6e6445..442c483981 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc @@ -24,11 +24,7 @@ inline void assertCurand( curandStatus_t code, const char *file, int line, bool } #endif /* clang-format on */ -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_CURAND @@ -38,12 +34,8 @@ namespace mg5amcCpu { if( m_isOnDevice ) { -#ifdef MGONGPUCPP_GPUIMPL if( !m_rnarray.isOnDevice() ) throw std::runtime_error( "CurandRandomNumberKernel on device with a host random number array" ); -#else - throw std::runtime_error( "CurandRandomNumberKernel does not support CurandDevice on CPU host" ); -#endif } else { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h index 4a1fd8464d..9e334c5c47 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h @@ -17,11 +17,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h index 2c8ae1a804..ed9887e045 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h @@ -17,7 +17,6 @@ //-------------------------------------------------------------------------- // See https://stackoverflow.com/a/14038590 -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ #define checkGpu( code ) { assertGpu( code, __FILE__, __LINE__ ); } inline void assertGpu( gpuError_t code, const char* file, int line, bool abort = true ) { @@ -27,11 +26,9 @@ inline void assertGpu( gpuError_t code, const char* file, int line, bool abort = if( abort ) assert( code == gpuSuccess ); } } -#endif /* clang-format on */ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ #ifndef MGONGPU_HAS_NO_BLAS #define checkGpuBlas( code ){ assertGpuBlas( code, __FILE__, __LINE__ ); } inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, bool abort = true ) @@ -43,11 +40,9 @@ inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, boo } } #endif -#endif /* clang-format on */ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu { // Instantiate a GpuRuntime at the beginnining of the application's main to @@ -95,7 +90,6 @@ namespace mg5amcGpu } }; } -#endif //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc index b806da2e09..54287cb57d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc @@ -28,11 +28,7 @@ inline void assertHiprand( hiprandStatus_t code, const char *file, int line, boo } #endif /* clang-format on */ -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_HIPRAND @@ -42,12 +38,8 @@ namespace mg5amcCpu { if( m_isOnDevice ) { -#ifdef MGONGPUCPP_GPUIMPL if( !m_rnarray.isOnDevice() ) throw std::runtime_error( "HiprandRandomNumberKernel on device with a host random number array" ); -#else - throw std::runtime_error( "HiprandRandomNumberKernel does not support HiprandDevice on CPU host" ); -#endif } else { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h index 12b7d45adf..f50ee1a0dc 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h @@ -27,11 +27,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL using mg5amcGpu::CPPProcess; -#else -using mg5amcCpu::CPPProcess; -#endif namespace { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc index 872e4795e3..e6aaa2f763 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc @@ -17,11 +17,7 @@ //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -146,147 +142,9 @@ namespace mg5amcCpu //============================================================================ -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu -{ - - //-------------------------------------------------------------------------- - - MatrixElementKernelHost::MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt ) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( nevt ) - , m_couplings( nevt ) - , m_numerators( nevt * CPPProcess::ndiagrams ) - , m_denominators( nevt ) - { - //std::cout << "DEBUG: MatrixElementKernelHost::ctor " << this << std::endl; - if( m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: momenta must be a host array" ); - if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: matrixElements must be a host array" ); - if( m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: channelIds must be a device array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with matrixElements" ); - if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with channelIds" ); - if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with iflavorVec" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( nevt % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MatrixElementKernelHost: nevt should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Fail gently and avoid "Illegal instruction (core dumped)" if the host does not support the SIMD used in the ME calculation - // Note: this prevents a crash on pmpe04 but not on some github CI nodes? - // [NB: SIMD vectorization in mg5amc C++ code is only used in the ME calculation below MatrixElementKernelHost!] - if( !MatrixElementKernelHost::hostSupportsSIMD() ) - throw std::runtime_error( "Host does not support the SIMD implementation of MatrixElementKernelsHost" ); - } - - //-------------------------------------------------------------------------- - - MatrixElementKernelHost::~MatrixElementKernelHost() - { - //std::cout << "DEBUG: MatrixElementKernelBase::dtor " << this << std::endl; - } - - //-------------------------------------------------------------------------- - - int MatrixElementKernelHost::computeGoodHelicities() - { - HostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); - // ... 0d1. Compute good helicity mask on the host - computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); - sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_numerators.data(), m_denominators.data(), hstIsGoodHel.data(), nevt() ); - // ... 0d2. Copy good helicity list to static memory on the host - // [FIXME! REMOVE THIS STATIC THAT BREAKS MULTITHREADING?] - return sigmaKin_setGoodHel( hstIsGoodHel.data() ); - } - - //-------------------------------------------------------------------------- - - void MatrixElementKernelHost::computeMatrixElements( const bool useChannelIds ) - { - computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_numerators.data(), m_denominators.data(), nullptr, true, nevt() ); -#ifdef MGONGPU_CHANNELID_DEBUG - //std::cout << "DEBUG: MatrixElementKernelHost::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; - MatrixElementKernelBase::updateNevtProcessedByChannel( pChannelIds, nevt() ); -#endif - } - - //-------------------------------------------------------------------------- - - // Does this host system support the SIMD used in the matrix element calculation? - bool MatrixElementKernelHost::hostSupportsSIMD( const bool verbose ) - { -#if defined __AVX512VL__ - bool known = true; - bool ok = __builtin_cpu_supports( "avx512vl" ); - const std::string tag = "skylake-avx512 (AVX512VL)"; -#elif defined __AVX2__ - bool known = true; - bool ok = __builtin_cpu_supports( "avx2" ); - const std::string tag = "haswell (AVX2)"; -#elif defined __SSE4_2__ -#ifdef __PPC__ - // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html - bool known = true; - bool ok = __builtin_cpu_supports( "vsx" ); - const std::string tag = "powerpc vsx (128bit as in SSE4.2)"; -#elif defined( __x86_64__ ) || defined( __i386__ ) - bool known = true; - bool ok = __builtin_cpu_supports( "sse4.2" ); - const std::string tag = "nehalem (SSE4.2)"; -#else // AV FIXME! Added by OM for Mac, should identify the correct __xxx__ flag that should be targeted - // DM now we have an explicit NEON target for ARM - bool known = false; // __builtin_cpu_supports is not supported - bool ok = true; // this is just an assumption! - const std::string tag = "simd arch not defined"; -#endif -#elif defined __ARM_NEON // consider using __BUILTIN_CPU_SUPPORTS__ - bool known = false; // __builtin_cpu_supports is not supported - // See https://stackoverflow.com/q/62783908 - // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu - bool ok = true; // this is just an assumption! - const std::string tag = "arm neon (128bit as in SSE4.2)"; -#else - bool known = true; - bool ok = true; - const std::string tag = "none"; -#endif - if( verbose ) - { - if( tag == "none" ) - std::cout << "INFO: The application does not require the host to support any AVX feature" << std::endl; - else if( ok && known ) - std::cout << "INFO: The application is built for " << tag << " and the host supports it" << std::endl; - else if( ok ) - std::cout << "WARNING: The application is built for " << tag << " but it is unknown if the host supports it" << std::endl; - else - std::cout << "ERROR! The application is built for " << tag << " but the host does not support it" << std::endl; - } - return ok; - } - - //-------------------------------------------------------------------------- - -} -#endif //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu { @@ -504,6 +362,5 @@ namespace mg5amcGpu //-------------------------------------------------------------------------- } -#endif //============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h index 01041e43b7..5e30723a3f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h @@ -16,11 +16,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -109,58 +105,9 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating matrix element calculations on a CPU host - class MatrixElementKernelHost final : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~MatrixElementKernelHost(); - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // Does this host system support the SIMD used in the matrix element calculation? - // [NB: this is private, SIMD vectorization in mg5amc C++ code is currently only used in the ME calculations below MatrixElementKernelHost!] - static bool hostSupportsSIMD( const bool verbose = false ); // ZW: default verbose false - - private: - - // The buffer for the event-by-event couplings that depends on alphas QCD - HostBufferCouplings m_couplings; - - // The buffer for the event-by-event numerators of multichannel factors - HostBufferNumerators m_numerators; - - // The buffer for the event-by-event denominators of multichannel factors - HostBufferDenominators m_denominators; - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating matrix element calculations on a GPU device class MatrixElementKernelDevice : public MatrixElementKernelBase, public NumberOfEvents { @@ -243,7 +190,6 @@ namespace mg5amcCpu // The number of threads in the GPU grid size_t m_gputhreads; }; -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h index f41e40bf82..d0b9d95642 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h @@ -16,11 +16,7 @@ #define MGONGPU_TRIVIAL_AMPLITUDES 1 // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h index 331e7f12c3..95ab36a00a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h @@ -14,11 +14,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -103,14 +99,7 @@ namespace mg5amcCpu kernelAccessConst( const unsigned int* buffer ) { const unsigned int& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } }; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h index fd3fb80c65..a5e1fcc262 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h @@ -16,11 +16,7 @@ #include "MemoryBuffers.h" // for HostBufferCouplings::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -189,17 +185,7 @@ namespace mg5amcCpu const int ix2 ) { fptype& out = kernelAccessIx2_s( buffer, ix2 ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays - constexpr int neppC = MemoryAccessCouplingsBase::neppC; - static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h index 757de7b6f2..43dff43510 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h @@ -15,11 +15,7 @@ //#include "MemoryAccessHelpers.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h index 205952e514..b7aa50c871 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h @@ -10,11 +10,7 @@ #include "MemoryAccessGs.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h index bfcb3e3222..faf362d745 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h @@ -14,11 +14,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -119,14 +115,7 @@ namespace mg5amcCpu kernelAccess( fptype* buffer ) { fptype& out = kernelAccess_s( buffer ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) @@ -148,14 +137,7 @@ namespace mg5amcCpu kernelAccessConst( const fptype* buffer ) { const fptype& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } }; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h index 8b5bd27f4d..b42bc349a2 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h @@ -106,13 +106,9 @@ class KernelAccessHelper : public MemoryAccessHelper } else { -#ifdef MGONGPUCPP_GPUIMPL const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid //printf( "kernelAccessRecord: ievt=%d threadId=%d\n", ievt, threadIdx.x ); return T::ieventAccessRecord( buffer, ievt ); // NB fptype and fptype_sv coincide for CUDA -#else - throw std::runtime_error( "kernelAccessRecord on device is only implemented in CUDA" ); -#endif } } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h index ffc74b0f0c..dafb4b4fcb 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h @@ -13,11 +13,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -102,14 +98,7 @@ namespace mg5amcCpu kernelAccessConst( const unsigned int* buffer ) { const unsigned int& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } }; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h index 4236e20602..17a1f33460 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h @@ -14,11 +14,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -119,14 +115,7 @@ namespace mg5amcCpu kernelAccess( fptype* buffer ) { fptype& out = kernelAccess_s( buffer ); -#ifndef MGONGPU_CPPSIMD return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h index 3ef4d76fbd..6301587740 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h @@ -14,11 +14,7 @@ #include "MemoryAccessVectors.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -31,7 +27,6 @@ namespace mg5amcCpu // Number of Events Per Page in the momenta AOSOA memory buffer layout // (these are all best kept as a compile-time constants: see issue #23) -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ // ----------------------------------------------------------------------------------------------- // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline // --- This is relevant to ensure coalesced access to momenta in global memory @@ -40,23 +35,6 @@ namespace mg5amcCpu //static constexpr int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) static constexpr int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 1.03E9 instead of 1.11E9 in eemumu) -#else - // ----------------------------------------------------------------------------------------------- - // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register - // --- This is relevant to ensure faster access to momenta from C++ memory cache lines - // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded - // --- In practice, neppR, neppM and neppV could now (in principle) all be different - // ----------------------------------------------------------------------------------------------- -#ifdef MGONGPU_CPPSIMD - static constexpr int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance - //static constexpr int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) - //static constexpr int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) - //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 4.66E6 instead of 5.09E9 in eemumu) - //static constexpr int neppM = MGONGPU_CPPSIMD*2; // FOR TESTS -#else - static constexpr int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) -#endif -#endif /* clang-format on */ // SANITY CHECK: check that neppM is a power of two static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); @@ -208,51 +186,7 @@ namespace mg5amcCpu const int ipar ) { const fptype& out = kernelAccessIp4IparConst_s( buffer, ip4, ipar ); -#ifndef MGONGPU_CPPSIMD return out; -#else - constexpr int neppM = MemoryAccessMomentaBase::neppM; - constexpr bool useContiguousEventsIfPossible = true; // DEFAULT - //constexpr bool useContiguousEventsIfPossible = false; // FOR PERFORMANCE TESTS (treat as arbitrary array even if it is an AOSOA) - // Use c++17 "if constexpr": compile-time branching - if constexpr( useContiguousEventsIfPossible && ( neppM >= neppV ) && ( neppM % neppV == 0 ) ) - { - //constexpr bool skipAlignmentCheck = true; // FASTEST (SEGFAULTS IF MISALIGNED ACCESS, NEEDS A SANITY CHECK ELSEWHERE!) - constexpr bool skipAlignmentCheck = false; // DEFAULT: A BIT SLOWER BUT SAFER [ALLOWS MISALIGNED ACCESS] - if constexpr( skipAlignmentCheck ) - { - //static bool first=true; if( first ){ std::cout << "WARNING! assume aligned AOSOA, skip check" << std::endl; first=false; } // SLOWER (5.06E6) - // FASTEST? (5.09E6 in eemumu 512y) - // This assumes alignment for momenta1d without checking - causes segmentation fault in reinterpret_cast if not aligned! - return mg5amcCpu::fptypevFromAlignedArray( out ); // use reinterpret_cast - } - else if( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ) - { - //static bool first=true; if( first ){ std::cout << "WARNING! aligned AOSOA, reinterpret cast" << std::endl; first=false; } // SLOWER (5.00E6) - // DEFAULT! A tiny bit (<1%) slower because of the alignment check (5.07E6 in eemumu 512y) - // This explicitly checks buffer alignment to avoid segmentation faults in reinterpret_cast - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast - } - else - { - //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWER (4.93E6) - // A bit (1%) slower (5.05E6 in eemumu 512y) - // This does not require buffer alignment, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 - return mg5amcCpu::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) - } - } - else - { - //static bool first=true; if( first ){ std::cout << "WARNING! arbitrary array" << std::endl; first=false; } // SLOWER (5.08E6) - // ?!Used to be much slower, now a tiny bit faster for AOSOA?! (5.11E6 for AOSOA, 4.64E6 for AOS in eemumu 512y) - // This does not even require AOSOA with neppM>=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) - constexpr int ievt0 = 0; // just make it explicit in the code that buffer refers to a given ievt0 and decoderIeppV fetches event ievt0+ieppV - auto decoderIeppv = [buffer, ip4, ipar]( int ieppV ) - -> const fptype& - { return MemoryAccessMomenta::ieventAccessIp4IparConst( buffer, ievt0 + ieppV, ip4, ipar ); }; - return mg5amcCpu::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) - } -#endif } // Is this a HostAccess or DeviceAccess class? diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h index c8d0343b7e..e69d9e82fc 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h @@ -10,11 +10,7 @@ #include "MemoryAccessGs.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h index 57f1d02081..d553211648 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h @@ -12,11 +12,7 @@ #include "CPPProcess.h" #include "MemoryAccessHelpers.h" -#ifdef MGONGPUCPP_GPUIMPL using mg5amcGpu::CPPProcess; -#else -using mg5amcCpu::CPPProcess; -#endif //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h index 8563d8f2f7..4db1875156 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h @@ -11,128 +11,5 @@ #include "mgOnGpuVectors.h" -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu // this is only needed for CPU SIMD vectorization -{ - -#ifdef MGONGPU_CPPSIMD - //-------------------------------------------------------------------------- - - // Cast one non-const fptype_v reference (one vector of neppV fptype values) from one non-const fptype reference (#435), - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned - inline fptype_v& fptypevFromAlignedArray( fptype& ref ) - { - return *reinterpret_cast( &ref ); - } - - inline uint_v& uintvFromAlignedArray( unsigned int& ref ) - { - return *reinterpret_cast( &ref ); - } - - // Cast one const fptype_v reference (one vector of neppV fptype values) from one const fptype reference, - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned - inline const fptype_v& fptypevFromAlignedArray( const fptype& ref ) - { - return *reinterpret_cast( &ref ); - } - - inline const uint_v& uintvFromAlignedArray( const unsigned int& ref ) - { - return *reinterpret_cast( &ref ); - } - - // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", but that the arrays are not aligned - inline fptype_v fptypevFromUnalignedArray( const fptype& ref ) - { -#if MGONGPU_CPPSIMD == 2 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (2) - *( &ref + 1 ) }; -#elif MGONGPU_CPPSIMD == 4 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (4) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ) }; -#elif MGONGPU_CPPSIMD == 8 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (8) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ), - *( &ref + 4 ), - *( &ref + 5 ), - *( &ref + 6 ), - *( &ref + 7 ) }; -#elif MGONGPU_CPPSIMD == 16 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (16) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ), - *( &ref + 4 ), - *( &ref + 5 ), - *( &ref + 6 ), - *( &ref + 7 ), - *( &ref + 8 ), - *( &ref + 9 ), - *( &ref + 10 ), - *( &ref + 11 ), - *( &ref + 12 ), - *( &ref + 13 ), - *( &ref + 14 ), - *( &ref + 15 ) }; -#else -#error Internal error! Unknown MGONGPU_CPPSIMD value -#endif - } - - // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, - // with no a priori assumption on how the input fptype array should be decoded - template - inline fptype_v fptypevFromArbitraryArray( Functor decoderIeppv ) - { -#if MGONGPU_CPPSIMD == 2 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (2) - decoderIeppv( 1 ) }; -#elif MGONGPU_CPPSIMD == 4 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (4) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ) }; -#elif MGONGPU_CPPSIMD == 8 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (8) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ), - decoderIeppv( 4 ), - decoderIeppv( 5 ), - decoderIeppv( 6 ), - decoderIeppv( 7 ) }; -#elif MGONGPU_CPPSIMD == 16 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (16) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ), - decoderIeppv( 4 ), - decoderIeppv( 5 ), - decoderIeppv( 6 ), - decoderIeppv( 7 ), - decoderIeppv( 8 ), - decoderIeppv( 9 ), - decoderIeppv( 10 ), - decoderIeppv( 11 ), - decoderIeppv( 12 ), - decoderIeppv( 13 ), - decoderIeppv( 14 ), - decoderIeppv( 15 ) }; -#else -#error Internal error! Unknown MGONGPU_CPPSIMD value -#endif - } - - //-------------------------------------------------------------------------- -#endif - -} // end namespace -#endif #endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h index 14a37c7fea..f023a5ecc0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h @@ -16,11 +16,7 @@ #define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h index c56527f581..fd36d9be67 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h @@ -12,11 +12,7 @@ #include "MemoryAccessHelpers.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h index d259aa7456..d27c3ebbbc 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h @@ -18,11 +18,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -90,39 +86,9 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL - constexpr bool HostBufferALIGNED = false; // ismisaligned=false - constexpr bool HostBufferMISALIGNED = true; // ismisaligned=true - - // A class encapsulating a C++ host buffer - template - class HostBufferBase : public BufferBase - { - public: - HostBufferBase( const size_t size ) - : BufferBase( size, false ) - { - if constexpr( !ismisaligned ) - this->m_data = new( std::align_val_t( cppAlign ) ) T[size](); - else - this->m_data = new( std::align_val_t( cppAlign ) ) T[size + 1]() + 1; // TEST MISALIGNMENT! - } - virtual ~HostBufferBase() - { - if constexpr( !ismisaligned ) - ::operator delete[]( this->m_data, std::align_val_t( cppAlign ) ); - else - ::operator delete[]( ( this->m_data ) - 1, std::align_val_t( cppAlign ) ); // TEST MISALIGNMENT! - } - static constexpr bool isaligned() { return !ismisaligned; } - public: - static constexpr size_t cppAlign = mgOnGpu::cppAlign; - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating a CUDA pinned host buffer template class PinnedHostBufferBase : public BufferBase @@ -138,11 +104,9 @@ namespace mg5amcCpu gpuFreeHost( this->m_data ); } }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating a CUDA device buffer template class DeviceBufferBase : public BufferBase @@ -158,33 +122,12 @@ namespace mg5amcCpu gpuFree( this->m_data ); } }; -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for a given number of events - template - class HostBuffer : public HostBufferBase, virtual private NumberOfEvents - { - public: - HostBuffer( const size_t nevt ) - : NumberOfEvents( nevt ) - , HostBufferBase( sizePerEvent * nevt ) - { - //std::cout << "HostBuffer::ctor " << this << " " << nevt << std::endl; - } - virtual ~HostBuffer() - { - //std::cout << "HostBuffer::dtor " << this << std::endl; - } - virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating a CUDA pinned host buffer for a given number of events template class PinnedHostBuffer : public PinnedHostBufferBase, virtual private NumberOfEvents @@ -196,11 +139,9 @@ namespace mg5amcCpu virtual ~PinnedHostBuffer() {} virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating a CUDA device buffer for a given number of events template class DeviceBuffer : public DeviceBufferBase, virtual protected NumberOfEvents @@ -218,15 +159,12 @@ namespace mg5amcCpu } virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating a simple CUDA device buffer managed on an ad-hoc basis typedef DeviceBuffer DeviceBufferSimple; typedef DeviceBuffer DeviceBufferSimple2; -#endif //-------------------------------------------------------------------------- @@ -236,15 +174,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for momenta random numbers constexpr size_t sizePerEventRndNumMomenta = MemoryBuffers::np4 * MemoryBuffers::nparf; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for momenta random numbers - typedef HostBuffer HostBufferRndNumMomenta; -#else // A class encapsulating a CUDA pinned host buffer for momenta random numbers typedef PinnedHostBuffer PinnedHostBufferRndNumMomenta; // A class encapsulating a CUDA device buffer for momenta random numbers typedef DeviceBuffer DeviceBufferRndNumMomenta; -#endif //-------------------------------------------------------------------------- @@ -280,15 +213,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for Gs constexpr size_t sizePerEventGs = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for gs - typedef HostBuffer HostBufferGs; -#else // A class encapsulating a CUDA pinned host buffer for gs typedef PinnedHostBuffer PinnedHostBufferGs; // A class encapsulating a CUDA device buffer for gs typedef DeviceBuffer DeviceBufferGs; -#endif //-------------------------------------------------------------------------- @@ -299,15 +227,10 @@ namespace mg5amcCpu // (should be equal to the number of diagrams in the process) constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for numerators - typedef HostBuffer HostBufferNumerators; -#else // A class encapsulating a CUDA pinned host buffer for numerators typedef PinnedHostBuffer PinnedHostBufferNumerators; // A class encapsulating a CUDA device buffer for numerators typedef DeviceBuffer DeviceBufferNumerators; -#endif //-------------------------------------------------------------------------- @@ -317,15 +240,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for denominators constexpr size_t sizePerEventDenominators = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for denominators - typedef HostBuffer HostBufferDenominators; -#else // A class encapsulating a CUDA pinned host buffer for denominators typedef PinnedHostBuffer PinnedHostBufferDenominators; // A class encapsulating a CUDA device buffer for denominators typedef DeviceBuffer DeviceBufferDenominators; -#endif //-------------------------------------------------------------------------- @@ -335,15 +253,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for random numbers constexpr size_t sizePerEventCouplings = MemoryBuffers::ndcoup * MemoryBuffers::nx2; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for couplings - typedef HostBuffer HostBufferCouplings; -#else // A class encapsulating a CUDA pinned host buffer for couplings typedef PinnedHostBuffer PinnedHostBufferCouplings; // A class encapsulating a CUDA device buffer for couplings typedef DeviceBuffer DeviceBufferCouplings; -#endif //-------------------------------------------------------------------------- @@ -353,16 +266,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for momenta constexpr size_t sizePerEventMomenta = MemoryBuffers::np4 * MemoryBuffers::npar; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for momenta - typedef HostBuffer HostBufferMomenta; - //typedef HostBuffer HostBufferMomenta; // TEST MISALIGNMENT! -#else // A class encapsulating a CUDA pinned host buffer for momenta typedef PinnedHostBuffer PinnedHostBufferMomenta; // A class encapsulating a CUDA device buffer for momenta typedef DeviceBuffer DeviceBufferMomenta; -#endif //-------------------------------------------------------------------------- @@ -372,15 +279,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for sampling weights constexpr size_t sizePerEventWeights = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for sampling weights - typedef HostBuffer HostBufferWeights; -#else // A class encapsulating a CUDA pinned host buffer for sampling weights typedef PinnedHostBuffer PinnedHostBufferWeights; // A class encapsulating a CUDA device buffer for sampling weights typedef DeviceBuffer DeviceBufferWeights; -#endif //-------------------------------------------------------------------------- @@ -390,30 +292,20 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for matrix elements constexpr size_t sizePerEventMatrixElements = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for matrix elements - typedef HostBuffer HostBufferMatrixElements; -#else // A class encapsulating a CUDA pinned host buffer for matrix elements typedef PinnedHostBuffer PinnedHostBufferMatrixElements; // A class encapsulating a CUDA device buffer for matrix elements typedef DeviceBuffer DeviceBufferMatrixElements; -#endif //-------------------------------------------------------------------------- // A base class encapsulating a memory buffer for the helicity mask typedef BufferBase BufferHelicityMask; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for the helicity mask - typedef HostBufferBase HostBufferHelicityMask; -#else // A class encapsulating a CUDA pinned host buffer for the helicity mask typedef PinnedHostBufferBase PinnedHostBufferHelicityMask; // A class encapsulating a CUDA device buffer for the helicity mask typedef DeviceBufferBase DeviceBufferHelicityMask; -#endif //-------------------------------------------------------------------------- @@ -423,15 +315,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for wavefunctions constexpr size_t sizePerEventWavefunctions = MemoryBuffers::nw6 * MemoryBuffers::nx2; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for wavefunctions - typedef HostBuffer HostBufferWavefunctions; -#else // A class encapsulating a CUDA pinned host buffer for wavefunctions typedef PinnedHostBuffer PinnedHostBufferWavefunctions; // A class encapsulating a CUDA device buffer for wavefunctions typedef DeviceBuffer DeviceBufferWavefunctions; -#endif //-------------------------------------------------------------------------- @@ -441,15 +328,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for helicity random numbers constexpr size_t sizePerEventRndNumHelicity = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for helicity random numbers - typedef HostBuffer HostBufferRndNumHelicity; -#else // A class encapsulating a CUDA pinned host buffer for helicity random numbers typedef PinnedHostBuffer PinnedHostBufferRndNumHelicity; // A class encapsulating a CUDA device buffer for helicity random numbers typedef DeviceBuffer DeviceBufferRndNumHelicity; -#endif //-------------------------------------------------------------------------- @@ -459,15 +341,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for color random numbers constexpr size_t sizePerEventRndNumColor = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for color random numbers - typedef HostBuffer HostBufferRndNumColor; -#else // A class encapsulating a CUDA pinned host buffer for color random numbers typedef PinnedHostBuffer PinnedHostBufferRndNumColor; // A class encapsulating a CUDA device buffer for color random numbers typedef DeviceBuffer DeviceBufferRndNumColor; -#endif //-------------------------------------------------------------------------- @@ -477,15 +354,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for channel ids constexpr size_t sizePerEventChannelId = 1; -#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) - // A class encapsulating a C++ host buffer for channel ids - typedef HostBuffer HostBufferChannelIds; -#else // A class encapsulating a CUDA pinned host buffer for channel ids typedef PinnedHostBuffer PinnedHostBufferChannelIds; // A class encapsulating a CUDA device buffer for channel ids typedef DeviceBuffer DeviceBufferChannelIds; -#endif //-------------------------------------------------------------------------- @@ -495,15 +367,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for channel ids constexpr size_t sizePerEventIflavorVec = 1; -#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) - // A class encapsulating a C++ host buffer for channel ids - typedef HostBuffer HostBufferIflavorVec; -#else // A class encapsulating a CUDA pinned host buffer for channel ids typedef PinnedHostBuffer PinnedHostBufferIflavorVec; // A class encapsulating a CUDA device buffer for channel ids typedef DeviceBuffer DeviceBufferIflavorVec; -#endif //-------------------------------------------------------------------------- @@ -513,15 +380,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for helicity selection constexpr size_t sizePerEventSelectedHelicity = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for helicity selection - typedef HostBuffer HostBufferSelectedHelicity; -#else // A class encapsulating a CUDA pinned host buffer for helicity selection typedef PinnedHostBuffer PinnedHostBufferSelectedHelicity; // A class encapsulating a CUDA device buffer for helicity selection typedef DeviceBuffer DeviceBufferSelectedHelicity; -#endif //-------------------------------------------------------------------------- @@ -531,29 +393,21 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for color selection constexpr size_t sizePerEventSelectedColor = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for color selection - typedef HostBuffer HostBufferSelectedColor; -#else // A class encapsulating a CUDA pinned host buffer for color selection typedef PinnedHostBuffer PinnedHostBufferSelectedColor; // A class encapsulating a CUDA device buffer for color selection typedef DeviceBuffer DeviceBufferSelectedColor; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // The size (number of elements) per event in a memory buffer for jamps constexpr size_t sizePerEventJamps = MemoryBuffers::ncolor * MemoryBuffers::nx2; // A class encapsulating a CUDA device buffer for color selection typedef DeviceBuffer DeviceBufferJamps; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL template void copyDeviceFromHost( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy { @@ -572,11 +426,9 @@ namespace mg5amcCpu // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyHostToDevice ); } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL template void copyHostFromDevice( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy { @@ -595,7 +447,6 @@ namespace mg5amcCpu // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyDeviceToHost ); } -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc index d40b1b4174..ae42513fd6 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc @@ -16,11 +16,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -155,7 +151,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL MasslessRamboSamplingKernelDevice::MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] BufferMomenta& momenta, // output: momenta @@ -194,11 +189,9 @@ namespace mg5amcCpu throw std::runtime_error( sstr.str() ); } } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL __global__ void getMomentaInitialDevice( const fptype energy, fptype* momenta ) @@ -206,21 +199,17 @@ namespace mg5amcCpu constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; return getMomentaInitial( energy, momenta ); } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL void MasslessRamboSamplingKernelDevice::getMomentaInitial() { gpuLaunchKernel( getMomentaInitialDevice, m_gpublocks, m_gputhreads, m_energy, m_momenta.data() ); } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL __global__ void getMomentaFinalDevice( const fptype energy, const fptype* rndmom, @@ -230,17 +219,14 @@ namespace mg5amcCpu constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; return getMomentaFinal( energy, rndmom, momenta, wgts ); } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL void MasslessRamboSamplingKernelDevice::getMomentaFinal() { gpuLaunchKernel( getMomentaFinalDevice, m_gpublocks, m_gputhreads, m_energy, m_rndmom.data(), m_momenta.data(), m_weights.data() ); } -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h index 6244d55618..c7962754c0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h @@ -13,11 +13,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -132,7 +128,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating RAMBO phase space sampling on a GPU device class MasslessRamboSamplingKernelDevice final : public SamplingKernelBase, public NumberOfEvents { @@ -166,7 +161,6 @@ namespace mg5amcCpu // The number of threads in the GPU grid size_t m_gputhreads; }; -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h index 0fb880bb36..7fbde1edd0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h @@ -18,11 +18,7 @@ struct curandGenerator_st; struct rocrand_generator_base_type; typedef rocrand_generator_base_type hiprandGenerator_st; -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc index 68e93edb50..f0517bcb07 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc @@ -56,11 +56,7 @@ namespace { -#ifdef MGONGPUCPP_GPUIMPL using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif // Fixed physics inputs fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV @@ -203,7 +199,6 @@ namespace } } -#ifdef MGONGPUCPP_GPUIMPL __global__ void aosoa_to_umami_kernel( const fptype* aosoa, double* soa, @@ -213,7 +208,6 @@ namespace if( ievt >= nevt ) return; aosoa_to_umami_one( aosoa, soa, ievt, nevt ); } -#endif const char* backend_label() { @@ -284,29 +278,18 @@ namespace unsigned int nevt, mgOnGpu::TimerMap& timermap, double& wavetime, -#ifdef MGONGPUCPP_GPUIMPL const DeviceBufferBase& devUmamiMomenta, const DeviceBufferBase& devFlv, DeviceBufferBase& devUmamiMEs, std::vector& hstMEs -#else - const std::vector& umamiMomenta, - const std::vector& flvVec, - std::vector& umamiMEs -#endif ) { constexpr unsigned int UmamiInKeyNum = 2; timermap.start( "3a SigmaKin" ); UmamiInputKey in_keys[UmamiInKeyNum] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX }; UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; -#ifdef MGONGPUCPP_GPUIMPL const void* inputs[UmamiInKeyNum] = { devUmamiMomenta.data(), devFlv.data() }; void* outputs[1] = { devUmamiMEs.data() }; -#else - const void* inputs[UmamiInKeyNum] = { umamiMomenta.data(), flvVec.data() }; - void* outputs[1] = { umamiMEs.data() }; -#endif UmamiStatus st = umami_matrix_element( handle, nevt, nevt, 0, UmamiInKeyNum, in_keys, inputs, 1, out_keys, outputs ); wavetime += timermap.stop(); @@ -316,11 +299,9 @@ namespace return false; } -#ifdef MGONGPUCPP_GPUIMPL timermap.start( "3b CpDTHmes" ); gpuMemcpy( hstMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); wavetime += timermap.stop(); -#endif return true; } @@ -593,7 +574,6 @@ namespace mgOnGpu::TimerMap timermap; -#ifdef MGONGPUCPP_GPUIMPL timermap.start( "00 GpuInit" ); GpuRuntime gpuRuntime( false ); @@ -609,14 +589,6 @@ namespace std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); std::vector flvVec( nevt ); std::vector hstUmamiMEs( nevt ); -#else - HostBufferRndNumMomenta hstRndmom( nevt ); - HostBufferMomenta hstMomenta( nevt ); - HostBufferWeights hstWeights( nevt ); - std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - std::vector umamiMEs( nevt ); - std::vector flvVec( nevt ); -#endif UmamiHandle umami_handle = nullptr; if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) @@ -656,10 +628,8 @@ namespace SLHAReader slha( "../../Cards/param_card.dat", false ); const double alphaS = slha.get_block_entry( "sminputs", 3, 1.180000e-01 ); std::vector alphasVec( nevt, alphaS ); -#ifdef MGONGPUCPP_GPUIMPL DeviceBufferBase devAlphaS( nevt ); gpuMemcpy( devAlphaS.data(), alphasVec.data(), nevt * sizeof( double ), gpuMemcpyHostToDevice ); -#endif // Always massive RAMBO std::unique_ptr prsk( @@ -673,16 +643,11 @@ namespace for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) for( unsigned int ievt = 0; ievt < nevt; ++ievt ) umamiMomenta[(std::size_t)ip4 * CPPProcess::npar * nevt + (std::size_t)ipar * nevt + ievt] = point[ipar][ip4]; -#ifdef MGONGPUCPP_GPUIMPL gpuMemcpy( devUmamiMomenta.data(), umamiMomenta.data(), umamiMomenta.size() * sizeof( double ), gpuMemcpyHostToDevice ); // Host only implementation now (copy) copyDeviceFromHost( devMomenta, hstMomenta ); gpuLaunchKernel( aosoa_to_umami_kernel, kMatrixBlocks, kMatrixThreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); checkGpu( gpuPeekAtLastError() ); -#else - for( std::size_t ievt = 0; ievt < nevt; ++ievt ) - aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); -#endif if( verbose ) { @@ -710,19 +675,12 @@ namespace for( unsigned int iflav = 0; iflav < nFlavors; ++iflav ) { std::fill( flvVec.begin(), flvVec.end(), iflav ); -#ifdef MGONGPUCPP_GPUIMPL gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); -#endif timermap.start( "3a SigmaKin" ); UmamiInputKey in_keys[3] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S }; UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; -#ifdef MGONGPUCPP_GPUIMPL const void* inputs[3] = { devUmamiMomenta.data(), devFlv.data(), devAlphaS.data() }; void* outputs[1] = { devUmamiMEs.data() }; -#else - const void* inputs[3] = { umamiMomenta.data(), flvVec.data(), alphasVec.data() }; - void* outputs[1] = { umamiMEs.data() }; -#endif UmamiStatus st = umami_matrix_element( umami_handle, nevt, nevt, 0, 3, in_keys, inputs, 1, out_keys, outputs ); timermap.stop(); @@ -732,12 +690,8 @@ namespace umami_free( umami_handle ); return 3; } -#ifdef MGONGPUCPP_GPUIMPL gpuMemcpy( hstUmamiMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); const double* mes = hstUmamiMEs.data(); -#else - const double* mes = umamiMEs.data(); -#endif std::cout << " PDG"; for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) @@ -780,7 +734,6 @@ namespace mgOnGpu::TimerMap timermap; -#ifdef MGONGPUCPP_GPUIMPL timermap.start( "00 GpuInit" ); GpuRuntime gpuRuntime( false ); @@ -797,14 +750,6 @@ namespace std::vector hstUmamiMEs( nevt ); // perf-mode runs a single flavor, so the device-side flavor buffer is filled once. gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); -#else - HostBufferRndNumMomenta hstRndmom( nevt ); - HostBufferMomenta hstMomenta( nevt ); - HostBufferWeights hstWeights( nevt ); - std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - std::vector umamiMEs( nevt ); - std::vector flvVec( nevt, flavorID ); -#endif std::unique_ptr prnk( new CommonRandomNumberKernel( hstRndmom ) ); @@ -845,11 +790,7 @@ namespace } else { -#ifdef MGONGPUCPP_GPUIMPL prsk.reset( new MasslessRamboSamplingKernelDevice( kEnergy, devRndmom, devMomenta, devWeights, gpublocks, gputhreads ) ); -#else - prsk.reset( new MasslessRamboSamplingKernelHost( kEnergy, hstRndmom, hstMomenta, hstWeights, nevt ) ); -#endif } std::unique_ptr genrtimes( new double[niter] ); @@ -877,14 +818,12 @@ namespace timermap.start( "1b GenRnGen" ); prnk->generateRnarray(); genrtime += timermap.stop(); -#ifdef MGONGPUCPP_GPUIMPL if( ramboType == RAMBO_MASSLESS ) { timermap.start( "1c CpHTDrnd" ); copyDeviceFromHost( devRndmom, hstRndmom ); genrtime += timermap.stop(); } -#endif timermap.start( "2a RamboIni" ); prsk->getMomentaInitial(); @@ -892,7 +831,6 @@ namespace timermap.start( "2b RamboFin" ); prsk->getMomentaFinal(); rambtime += timermap.stop(); -#ifdef MGONGPUCPP_GPUIMPL // Massive host only (copy) if( ramboType != RAMBO_MASSLESS ) { @@ -900,7 +838,6 @@ namespace copyDeviceFromHost( devMomenta, hstMomenta ); rambtime += timermap.stop(); } -#endif } else { @@ -919,37 +856,25 @@ namespace MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar ) = (fptype)ev[ipar][ip4]; } rambtime += timermap.stop(); -#ifdef MGONGPUCPP_GPUIMPL timermap.start( "2c CpHTDmom" ); copyDeviceFromHost( devMomenta, hstMomenta ); rambtime += timermap.stop(); -#endif } timermap.start( "2d Aosoa2U " ); -#ifdef MGONGPUCPP_GPUIMPL gpuLaunchKernel( aosoa_to_umami_kernel, gpublocks, gputhreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); checkGpu( gpuPeekAtLastError() ); -#else - for( std::size_t ievt = 0; ievt < nevt; ++ievt ) - aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); -#endif rambtime += timermap.stop(); double wavetime = 0; if( !run_umami( umami_handle, nevt, timermap, wavetime, -#ifdef MGONGPUCPP_GPUIMPL devUmamiMomenta, devFlv, devUmamiMEs, hstUmamiMEs -#else - umamiMomenta, flvVec, umamiMEs -#endif ) ) { umami_free( umami_handle ); return 3; } -#ifdef MGONGPUCPP_GPUIMPL if( verbose ) { timermap.start( "3c CpDTHmom" ); @@ -957,9 +882,6 @@ namespace wavetime += timermap.stop(); } const double* mes = hstUmamiMEs.data(); -#else - const double* mes = umamiMEs.data(); -#endif timermap.start( "4@ UpdtStat" ); for( unsigned int ievt = 0; ievt < nreal; ++ievt ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc index 30c6799932..bb324fa3bd 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc @@ -10,11 +10,7 @@ #include "MemoryAccessMatrixElements.h" -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors @@ -23,7 +19,6 @@ namespace mg5amcCpu // *** COLOR MATRIX BELOW *** %(color_matrix_lines)s -#ifdef MGONGPUCPP_GPUIMPL // The normalized color matrix (divide each column by denom) template struct NormalizedColorMatrix @@ -39,11 +34,9 @@ namespace mg5amcCpu }; // The fptype2 version is the default used by kernels (supporting mixed floating point mode also in blas) static __device__ fptype2 s_pNormalizedColorMatrix2[ncolor * ncolor]; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL void createNormalizedColorMatrix() { static bool first = true; @@ -54,107 +47,12 @@ namespace mg5amcCpu gpuMemcpyToSymbol( s_pNormalizedColorMatrix2, normalizedColorMatrix2.value, ncolor * ncolor * sizeof( fptype2 ) ); } } -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL - void - color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity - const int ievt0 ) // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) - { - // Pre-compute a constexpr triangular color matrix properly normalized #475 - struct TriangularNormalizedColorMatrix - { - // See https://stackoverflow.com/a/34465458 - __host__ __device__ constexpr TriangularNormalizedColorMatrix() - : value() - { - for( int icol = 0; icol < ncolor; icol++ ) - { - // Diagonal terms - value[icol][icol] = colorMatrix[icol][icol] / colorDenom[icol]; - // Off-diagonal terms - for( int jcol = icol + 1; jcol < ncolor; jcol++ ) - value[icol][jcol] = 2 * colorMatrix[icol][jcol] / colorDenom[icol]; - } - } - fptype2 value[ncolor][ncolor]; - }; - static constexpr auto cf2 = TriangularNormalizedColorMatrix(); - // Use the property that M is a real matrix (see #475): - // we can rewrite the quadratic form (A-iB)(M)(A+iB) as AMA - iBMA + iBMA + BMB = AMA + BMB - // In addition, on C++ use the property that M is symmetric (see #475), - // and also use constexpr to compute "2*" and "/colorDenom[icol]" once and for all at compile time: - // we gain (not a factor 2...) in speed here as we only loop over the up diagonal part of the matrix. - // Strangely, CUDA is slower instead, so keep the old implementation for the moment. - fptype_sv deltaMEs = { 0 }; -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype_sv deltaMEs_next = { 0 }; - // Mixed mode: merge two neppV vectors into one neppV2 vector - fptype2_sv jampR_sv[ncolor]; - fptype2_sv jampI_sv[ncolor]; - for( int icol = 0; icol < ncolor; icol++ ) - { - jampR_sv[icol] = fpvmerge( cxreal( allJamp_sv[icol] ), cxreal( allJamp_sv[ncolor + icol] ) ); - jampI_sv[icol] = fpvmerge( cximag( allJamp_sv[icol] ), cximag( allJamp_sv[ncolor + icol] ) ); - } -#else - const cxtype_sv* jamp_sv = allJamp_sv; -#endif - // Loop over icol - for( int icol = 0; icol < ncolor; icol++ ) - { - // Diagonal terms -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype2_sv& jampRi_sv = jampR_sv[icol]; - fptype2_sv& jampIi_sv = jampI_sv[icol]; -#else - fptype2_sv jampRi_sv = (fptype2_sv)( cxreal( jamp_sv[icol] ) ); - fptype2_sv jampIi_sv = (fptype2_sv)( cximag( jamp_sv[icol] ) ); -#endif - fptype2_sv ztempR_sv = cf2.value[icol][icol] * jampRi_sv; - fptype2_sv ztempI_sv = cf2.value[icol][icol] * jampIi_sv; - // Loop over jcol - for( int jcol = icol + 1; jcol < ncolor; jcol++ ) - { - // Off-diagonal terms -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype2_sv& jampRj_sv = jampR_sv[jcol]; - fptype2_sv& jampIj_sv = jampI_sv[jcol]; -#else - fptype2_sv jampRj_sv = (fptype2_sv)( cxreal( jamp_sv[jcol] ) ); - fptype2_sv jampIj_sv = (fptype2_sv)( cximag( jamp_sv[jcol] ) ); -#endif - ztempR_sv += cf2.value[icol][jcol] * jampRj_sv; - ztempI_sv += cf2.value[icol][jcol] * jampIj_sv; - } - fptype2_sv deltaMEs2 = ( jampRi_sv * ztempR_sv + jampIi_sv * ztempI_sv ); // may underflow #831 -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - deltaMEs += fpvsplit0( deltaMEs2 ); - deltaMEs_next += fpvsplit1( deltaMEs2 ); -#else - deltaMEs += deltaMEs2; -#endif - } - // *** STORE THE RESULTS *** - using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events - fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); - // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) - fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); - MEs_sv += deltaMEs; // fix #435 -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype* MEs_next = E_ACCESS::ieventAccessRecord( allMEs, ievt0 + neppV ); - fptype_sv& MEs_sv_next = E_ACCESS::kernelAccess( MEs_next ); - MEs_sv_next += deltaMEs_next; -#endif - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL __global__ void color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity @@ -210,11 +108,9 @@ namespace mg5amcCpu // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) E_ACCESS::kernelAccess( allMEs ) += deltaMEs; // fix #435 } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL #ifndef MGONGPU_HAS_NO_BLAS #if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT __global__ void @@ -233,12 +129,10 @@ namespace mg5amcCpu allJamps[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt]; } #endif -#endif #endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL #ifndef MGONGPU_HAS_NO_BLAS #if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT __global__ void @@ -249,12 +143,10 @@ namespace mg5amcCpu allMEs[ievt] = allMEsFpt2[ievt]; } #endif -#endif #endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ #ifndef MGONGPU_HAS_NO_BLAS void color_sum_blas( fptype* ghelAllMEs, // output: allMEs super-buffer[nhel][nevt], add |M|^2 separately for each helicity @@ -371,11 +263,9 @@ namespace mg5amcCpu #endif } #endif /* clang-format on */ -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL void color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities @@ -427,7 +317,6 @@ namespace mg5amcCpu #endif } } -#endif //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h index 347184c4e1..07ef85ce10 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h @@ -14,15 +14,10 @@ #include "CPPProcess.h" #include "GpuAbstraction.h" -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL class DeviceAccessJamp { public: @@ -57,26 +52,16 @@ namespace mg5amcCpu buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); } }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL void createNormalizedColorMatrix(); -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL - void - color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity - const int ievt0 ); // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL void color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities @@ -87,17 +72,14 @@ namespace mg5amcCpu const int gpublocks, // input: cuda gpublocks const int gputhreads, // input: cuda gputhreads const bool processAllHelicities); // input: if true, use blockIdx.y to index helicities -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL __global__ void color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity const int nGoodHel, // input: number of good helicities const int nevtIfAllHelicities); // input: zero in single-helicity mode, number of events in multi-helicity mode -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h index 027f1aa44e..4a465e52b0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h @@ -30,11 +30,7 @@ namespace mgOnGpu // The number of channels in the channel2iconfig array below // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** constexpr unsigned int nchannels = %(nb_diag)i; -#ifdef MGONGPUCPP_GPUIMPL static_assert( nchannels <= mg5amcGpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 -#else - static_assert( nchannels <= mg5amcCpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 -#endif // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) @@ -45,13 +41,9 @@ namespace mgOnGpu }; // Host copy of the channel2iconfig array (this is needed in runTest #917) -#ifndef MGONGPUCPP_GPUIMPL - constexpr const int* hostChannel2iconfig = channel2iconfig; -#else constexpr int hostChannel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed %(channelc2iconfig_lines)s }; -#endif // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) constexpr unsigned int nconfigSDE = %(nb_channel)s; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h index 32c1bbb963..6864e7d7b0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h @@ -28,11 +28,7 @@ #endif // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc index 05c3b3e0c4..f7710943b4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc @@ -23,11 +23,7 @@ extern "C" * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. */ -#ifdef MGONGPUCPP_GPUIMPL using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif /** * The floating point precision used in Fortran arrays. @@ -47,9 +43,7 @@ extern "C" */ void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) { -#ifdef MGONGPUCPP_GPUIMPL GpuRuntime::setUp(); -#endif // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) // FIXME: disable OMP in Bridge when called from Fortran *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); @@ -66,9 +60,7 @@ extern "C" Bridge* pbridge = dynamic_cast*>( *ppbridge ); if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); delete pbridge; -#ifdef MGONGPUCPP_GPUIMPL GpuRuntime::tearDown(); -#endif } /** @@ -101,15 +93,9 @@ extern "C" Bridge* pbridge = dynamic_cast*>( *ppbridge ); //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); -#ifdef MGONGPUCPP_GPUIMPL // Use the device/GPU implementation in the CUDA library // (there is also a host implementation in this library) pbridge->gpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); -#else - // Use the host/CPU implementation in the C++ library - // (there is no device implementation in this library) - pbridge->cpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); -#endif } /** diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h index e676df0a33..5bd0b270d2 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h @@ -12,11 +12,7 @@ extern "C" { -#ifdef MGONGPUCPP_GPUIMPL using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif using FORTRANFPTYPE = double; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc index 8a4468fb3c..8745aa6cbf 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc @@ -14,11 +14,7 @@ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { template class Sampler final : public CppObjectInFortran @@ -41,15 +37,9 @@ namespace mg5amcCpu private: const int m_nevt; // The number of events in each iteration int m_iiter; // The iteration counter (for random number seeding) -#ifndef MGONGPUCPP_GPUIMPL - HostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers - HostBufferMomenta m_hstMomenta; // Memory buffers for momenta - HostBufferWeights m_hstWeights; // Memory buffers for sampling weights -#else PinnedHostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers PinnedHostBufferMomenta m_hstMomenta; // Memory buffers for momenta PinnedHostBufferWeights m_hstWeights; // Memory buffers for sampling weights -#endif std::unique_ptr m_prnk; // The appropriate RandomNumberKernel std::unique_ptr m_prsk; // The appropriate SamplingKernel // HARDCODED DEFAULTS @@ -106,11 +96,7 @@ namespace mg5amcCpu extern "C" { -#ifdef MGONGPUCPP_GPUIMPL using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif /** * The floating point precision used in Fortran arrays. diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h index bcd3990bed..0fc66e512e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h @@ -22,11 +22,7 @@ #include // Simplified rambo version for 2 to N (with N>=2) processes with massless particles -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { namespace massless_rambo { @@ -89,7 +85,6 @@ namespace massless_rambo { static bool first = true; if( first ) { -#ifdef MGONGPUCPP_GPUIMPL if constexpr( M_ACCESS::isOnDevice() ) // avoid { const int ievt0 = 0; @@ -98,7 +93,6 @@ namespace massless_rambo { printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); } else -#endif { printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); } @@ -172,20 +166,6 @@ namespace massless_rambo { wt = po2log; if( nparf != 2 ) wt = ( 2. * nparf - 4. ) * log( energy ) + z[nparf - 1]; -#ifndef MGONGPUCPP_GPUIMPL - // issue warnings if weight is too small or too large - static int iwarn[5] = { 0, 0, 0, 0, 0 }; - if( wt < -180. ) - { - if( iwarn[0] <= 5 ) std::cout << "Too small wt, risk for underflow: " << wt << std::endl; - iwarn[0] = iwarn[0] + 1; - } - if( wt > 174. ) - { - if( iwarn[1] <= 5 ) std::cout << "Too large wt, risk for overflow: " << wt << std::endl; - iwarn[1] = iwarn[1] + 1; - } -#endif // return for weighted massless momenta // nothing else to do in this event if all particles are massless (nm==0) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h index 09c76a6bd5..7c2512a7c6 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h @@ -152,11 +152,6 @@ #endif // SANITY CHECKS (C++ complex number implementation) -#ifndef MGONGPUCPP_GPUIMPL -#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX and defined MGONGPU_CPPCXTYPE_CXSMPL -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL for C++ -#endif -#endif // NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) namespace mgOnGpu @@ -192,9 +187,6 @@ namespace mgOnGpu // Alignment requirement for using reinterpret_cast with SIMD vectorized code // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) // Only needed for C++ code but can be enforced also in NVCC builds of C++ code using CUDA>=11.2 and C++17 (#318, #319, #333) -#ifndef MGONGPUCPP_GPUIMPL - constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) -#endif } @@ -208,41 +200,7 @@ using mgOnGpu::fptype2; #endif // C++ SIMD vectorization width (this will be used to set neppV) -#ifdef MGONGPUCPP_GPUIMPL // CUDA and HIP implementations have no SIMD -#undef MGONGPU_CPPSIMD -#elif defined __AVX512VL__ && defined MGONGPU_PVW512 // C++ "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) -#ifdef MGONGPU_FPTYPE_DOUBLE -#define MGONGPU_CPPSIMD 8 -#else -#define MGONGPU_CPPSIMD 16 -#endif -#elif defined __AVX512VL__ // C++ "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] -#ifdef MGONGPU_FPTYPE_DOUBLE -#define MGONGPU_CPPSIMD 4 -#else -#define MGONGPU_CPPSIMD 8 -#endif -#elif defined __AVX2__ // C++ "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] -#ifdef MGONGPU_FPTYPE_DOUBLE -#define MGONGPU_CPPSIMD 4 -#else -#define MGONGPU_CPPSIMD 8 -#endif -#elif defined __SSE4_2__ // C++ "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [Power9 default] -#ifdef MGONGPU_FPTYPE_DOUBLE -#define MGONGPU_CPPSIMD 2 -#else -#define MGONGPU_CPPSIMD 4 -#endif -#elif defined __ARM_NEON // C++ "sse4" ARM NEON (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [ARM default] -#ifdef MGONGPU_FPTYPE_DOUBLE -#define MGONGPU_CPPSIMD 2 -#else -#define MGONGPU_CPPSIMD 4 -#endif -#else // C++ "none" i.e. no SIMD #undef MGONGPU_CPPSIMD -#endif /* clang-format off */ // CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation [NB: CURRENTLY NO LONGER SUPPORTED!] @@ -260,11 +218,6 @@ using mgOnGpu::fptype2; //#endif /* clang-format on */ // Define empty CUDA/HIP declaration specifiers for C++ -#ifndef MGONGPUCPP_GPUIMPL -#define __global__ -#define __host__ -#define __device__ -#endif // For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two (https://stackoverflow.com/a/108360) inline constexpr bool diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h index d6e1d02e05..c185d2e14d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h @@ -163,11 +163,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { template inline __host__ std::ostream& @@ -297,11 +293,7 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { // --- Type definitions (complex type: cxtype) #ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) @@ -333,11 +325,7 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL @@ -699,11 +687,7 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h index a2489d05d5..34c6536323 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h @@ -13,15 +13,10 @@ #include // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // cuda namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { //========================================================================== -#ifdef MGONGPUCPP_GPUIMPL // cuda //------------------------------ // Floating point types - Cuda @@ -65,35 +60,9 @@ namespace mg5amcCpu #endif } -#endif // #ifdef MGONGPUCPP_GPUIMPL //========================================================================== -#ifndef MGONGPUCPP_GPUIMPL - - //------------------------------ - // Floating point types - C++ - //------------------------------ - - inline const fptype& - fpmax( const fptype& a, const fptype& b ) - { - return std::max( a, b ); - } - - inline const fptype& - fpmin( const fptype& a, const fptype& b ) - { - return std::min( a, b ); - } - - inline fptype - fpsqrt( const fptype& f ) - { - return std::sqrt( f ); - } - -#endif // #ifndef MGONGPUCPP_GPUIMPL //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h index 74d93f05a3..b2e3b61b8d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h @@ -33,113 +33,11 @@ #endif // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { -#ifdef MGONGPU_CPPSIMD - - const int neppV = MGONGPU_CPPSIMD; - - // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) - static_assert( mgOnGpu::cppAlign % ( neppV * sizeof( fptype ) ) == 0 ); - - // SANITY CHECK: check that neppV is a power of two - static_assert( ispoweroftwo( neppV ), "neppV is not a power of 2" ); - - // --- Type definition (using vector compiler extensions: need -march=...) - // For gcc: https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html - // For clang: https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors -#ifdef __clang__ - typedef fptype fptype_v __attribute__( ( ext_vector_type( neppV ) ) ); // RRRR -#else - typedef fptype fptype_v __attribute__( ( vector_size( neppV * sizeof(fptype) ), aligned( neppV * sizeof(fptype) ) ) ); // RRRR -#endif - - // Mixed fptypes #537: float for color algebra and double elsewhere -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - const int neppV2 = MGONGPU_CPPSIMD * 2; - static_assert( mgOnGpu::cppAlign % ( neppV2 * sizeof( fptype2 ) ) == 0 ); - static_assert( ispoweroftwo( neppV2 ), "neppV2 is not a power of 2" ); -#ifdef __clang__ - typedef fptype2 fptype2_v __attribute__( ( ext_vector_type( neppV2 ) ) ); // RRRRRRRR -#else - typedef fptype2 fptype2_v __attribute__( ( vector_size( neppV2 * sizeof( fptype2 ) ), aligned( neppV2 * sizeof( fptype2 ) ) ) ); // RRRRRRRR -#endif -#else - typedef fptype_v fptype2_v; -#endif - - // --- Type definition (using vector compiler extensions: need -march=...) - class cxtype_v // no need for "class alignas(2*sizeof(fptype_v)) cxtype_v" - { - public: - // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) - // See https://en.cppreference.com/w/c/language/array_initialization#Notes - cxtype_v() - : m_real{ 0 }, m_imag{ 0 } {} // RRRR=0000 IIII=0000 - cxtype_v( const cxtype_v& ) = default; - cxtype_v( cxtype_v&& ) = default; - cxtype_v( const fptype_v& r, const fptype_v& i ) - : m_real( r ), m_imag( i ) {} - cxtype_v( const fptype_v& r ) - : m_real( r ), m_imag{ 0 } {} // IIII=0000 - cxtype_v( const fptype& r ) - : m_real( fptype_v{} + r ), m_imag{ 0 } {} // IIII=0000 - cxtype_v& operator=( const cxtype_v& ) = default; - cxtype_v& operator=( cxtype_v&& ) = default; - cxtype_v& operator+=( const cxtype_v& c ) - { - m_real += c.real(); - m_imag += c.imag(); - return *this; - } - cxtype_v& operator-=( const cxtype_v& c ) - { - m_real -= c.real(); - m_imag -= c.imag(); - return *this; - } -#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK - // NB: THIS IS THE FUNDAMENTAL DIFFERENCE BETWEEN MGONGPU_HAS_CPPCXTYPEV_BRK DEFINED AND NOT DEFINED - // NB: the alternative "clang" implementation is simpler: it simply does not have any bracket operator[] - //cxtype_ref operator[]( size_t i ) const { return cxtype_ref( m_real[i], m_imag[i] ); } // gcc14.2 build fails #1004 - cxtype_ref operator[]( size_t i ) { return cxtype_ref( m_real[i], m_imag[i] ); } - cxtype operator[]( size_t i ) const { return cxtype( m_real[i], m_imag[i] ); } -#endif - const fptype_v& real() const - { - return m_real; - } - const fptype_v& imag() const { return m_imag; } - private: - fptype_v m_real, m_imag; // RRRRIIII - }; - - // --- Type definition (using vector compiler extensions: need -march=...) -#ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors - typedef unsigned int uint_v __attribute__( ( ext_vector_type( neppV ) ) ); -#if defined MGONGPU_FPTYPE_DOUBLE - typedef long int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb -#elif defined MGONGPU_FPTYPE_FLOAT - typedef int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb -#endif -#else // gcc - typedef unsigned int uint_v __attribute__( ( vector_size( neppV * sizeof( unsigned int ) ), aligned( neppV * sizeof( unsigned int ) ) ) ); -#if defined MGONGPU_FPTYPE_DOUBLE - typedef long int bool_v __attribute__( ( vector_size( neppV * sizeof( long int ) ), aligned( neppV * sizeof( long int ) ) ) ); // bbbb -#elif defined MGONGPU_FPTYPE_FLOAT - typedef int bool_v __attribute__( ( vector_size( neppV * sizeof( int ) ), aligned( neppV * sizeof( int ) ) ) ); // bbbb -#endif -#endif - -#else // i.e #ifndef MGONGPU_CPPSIMD (this includes #ifdef MGONGPUCPP_GPUIMPL) const int neppV = 1; -#endif // #ifdef MGONGPU_CPPSIMD } //-------------------------------------------------------------------------- @@ -157,685 +55,11 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { -#ifndef MGONGPUCPP_GPUIMPL - - // Printout to stream for user defined types - -#ifndef MGONGPU_CPPCXTYPE_CXSMPL // operator<< for cxsmpl has already been defined! - inline std::ostream& - operator<<( std::ostream& out, const cxtype& c ) - { - out << "[" << cxreal( c ) << "," << cximag( c ) << "]"; - //out << cxreal(c) << "+i" << cximag(c); - return out; - } -#endif - - /* -#ifdef MGONGPU_CPPSIMD - inline std::ostream& - operator<<( std::ostream& out, const bool_v& v ) - { - out << "{ " << v[0]; - for ( int i=1; i 0 ) outi = fpsqrt( (fptype)v[i] ); - out[i] = outi; - } - return out; - } - - inline fptype_v - fpsqrt( const fptype_v& v ) - { - // See https://stackoverflow.com/questions/18921049/gcc-vector-extensions-sqrt - fptype_v out = {}; // avoid warning 'out' may be used uninitialized: see #594 - for( int i = 0; i < neppV; i++ ) out[i] = fpsqrt( v[i] ); - return out; - } -#endif - - /* -#ifdef MGONGPU_CPPSIMD - inline fptype_v - fpvmake( const fptype v[neppV] ) - { - fptype_v out = {}; // see #594 - for ( int i=0; i -#ifdef MGONGPUCPP_GPUIMPL using namespace mg5amcGpu; -#else -using namespace mg5amcCpu; -#endif struct CUDA_CPU_TestBase : public TestDriverBase { @@ -78,147 +74,7 @@ struct CUDA_CPU_TestBase : public TestDriverBase } }; -#ifndef MGONGPUCPP_GPUIMPL -struct CPUTest : public CUDA_CPU_TestBase -{ - // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) - // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] - CPPProcess process; - HostBufferRndNumMomenta hstRndMom; - HostBufferChannelIds hstChannelIds; - HostBufferMomenta hstMomenta; - HostBufferGs hstGs; - HostBufferRndNumHelicity hstRndHel; - HostBufferRndNumColor hstRndCol; - HostBufferWeights hstWeights; - HostBufferMatrixElements hstMatrixElements; - HostBufferSelectedHelicity hstSelHel; - HostBufferSelectedColor hstSelCol; - HostBufferHelicityMask hstIsGoodHel; - std::unique_ptr pmek; - - // Create a process object - // Read param_card and set parameters - // ** WARNING EVIL EVIL ** - // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. - // Don't remove! - CPUTest( const std::string& refFileName ) - : CUDA_CPU_TestBase( refFileName ) - , process( /*verbose=*/false ) - , hstRndMom( nevt ) - , hstChannelIds( nevt ) - , hstMomenta( nevt ) - , hstGs( nevt ) - , hstRndHel( nevt ) - , hstRndCol( nevt ) - , hstWeights( nevt ) - , hstMatrixElements( nevt ) - , hstSelHel( nevt ) - , hstSelCol( nevt ) - , hstIsGoodHel( CPPProcess::ncomb ) - , pmek( new MatrixElementKernelHost( hstMomenta, hstGs, hstRndHel, hstRndCol, hstChannelIds, hstMatrixElements, hstSelHel, hstSelCol, nevt ) ) - { - // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? - process.initProc( "../../Cards/param_card.dat" ); - } - - virtual ~CPUTest() {} - - void prepareRandomNumbers( unsigned int iiter ) override - { - // Random numbers for momenta - CommonRandomNumberKernel rnk( hstRndMom ); - rnk.seedGenerator( 1337 + iiter ); - rnk.generateRnarray(); - // Random numbers for helicity and color selection (fix #931) - CommonRandomNumberKernel rnk2( hstRndHel ); - rnk2.seedGenerator( 1338 + iiter ); - rnk2.generateRnarray(); - CommonRandomNumberKernel rnk3( hstRndCol ); - rnk3.seedGenerator( 1339 + iiter ); - rnk3.generateRnarray(); - } - - void prepareMomenta( fptype energy ) override - { - RamboSamplingKernelHost rsk( energy, hstRndMom, hstMomenta, hstWeights, nevt ); - // --- 2a. Fill in momenta of initial state particles on the device - rsk.getMomentaInitial(); - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - rsk.getMomentaFinal(); - } - - void runSigmaKin( std::size_t iiter ) override - { - constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) - for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; - setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 - if( iiter == 0 ) pmek->computeGoodHelicities(); - pmek->computeMatrixElements( useChannelIds() ); - } - - fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override - { - assert( ipar < npar ); - assert( ip4 < np4 ); - return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); - } - - fptype getMatrixElement( std::size_t ievt ) const override - { - return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); - } - - int getChannelId( std::size_t ievt ) const override - { - return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); - } - - int getSelectedHelicity( std::size_t ievt ) const override - { - //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... - return hstSelHel.data()[ievt]; - } - - int getSelectedColor( std::size_t ievt ) const override - { - //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... - return hstSelCol.data()[ievt]; - } -}; - -// Old test with multi-channel disabled #466 -struct CPUTestNoMultiChannel : public CPUTest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return false; } - // Constructor - CPUTestNoMultiChannel( const std::string& refFileName ) - : CPUTest( refFileName ) {} // suffix .txt - - // Destructor - virtual ~CPUTestNoMultiChannel() {} -}; - -// New test with multi-channel enabled #896 -struct CPUTestMultiChannel : public CPUTest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return true; } - - // Constructor - CPUTestMultiChannel( const std::string& refFileName ) - : CPUTest( refFileName + "2" ) {} // suffix .txt2 - - // Destructor - virtual ~CPUTestMultiChannel() {} -}; -#endif - -#ifdef MGONGPUCPP_GPUIMPL struct CUDATest : public CUDA_CPU_TestBase { // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) @@ -390,24 +246,15 @@ struct CUDATestMultiChannel : public CUDATest // Destructor virtual ~CUDATestMultiChannel() {} }; -#endif /* clang-format off */ // AV July 2024 much simpler class structure without the presently-unnecessary googletest templates // This is meant as a workaround to prevent not-understood segfault #907 when adding a second test // Note: instantiate test2 first and test1 second to ensure that the channelid printout from the dtors comes from test1 first and test2 second -#ifdef MGONGPUCPP_GPUIMPL // CUDA test drivers CUDATestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); #define TESTID2( s ) s##_GPU_MULTICHANNEL CUDATestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); #define TESTID1( s ) s##_GPU_NOMULTICHANNEL -#else -// CPU test drivers -CPUTestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID2( s ) s##_CPU_MULTICHANNEL -CPUTestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID1( s ) s##_CPU_NOMULTICHANNEL -#endif // Madgraph tests MadgraphTest mgTest2( driver2 ); MadgraphTest mgTest1( driver1 ); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc index 74c4902dc5..622d26b010 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc @@ -23,26 +23,14 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL #define TESTID( s ) s##_GPU_MISC -#else -#define TESTID( s ) s##_CPU_MISC -#endif #define XTESTID( s ) TESTID( s ) // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { -#ifdef MGONGPU_CPPSIMD /* clang-format off */ -#define EXPECT_TRUE_sv( cond ) { bool_v mask( cond ); EXPECT_TRUE( maskand( mask ) ); } -#else #define EXPECT_TRUE_sv( cond ) { EXPECT_TRUE( cond ); } -#endif /* clang-format on */ inline const std::string boolTF( const bool& b ) @@ -50,26 +38,11 @@ namespace mg5amcCpu return ( b ? "T" : "F" ); } -#ifdef MGONGPU_CPPSIMD - inline const std::string - boolTF( const bool_v& v ) - { - std::stringstream out; - out << "{ " << ( v[0] ? "T" : "F" ); - for( int i = 1; i < neppV; i++ ) out << ", " << ( v[i] ? "T" : "F" ); - out << " }"; - return out.str(); - } -#endif } TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) { -#ifdef MGONGPUCPP_GPUIMPL using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif //-------------------------------------------------------------------------- @@ -92,34 +65,9 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) fptype_sv f{ 1 }; //std::cout << f << std::endl << boolTF( f == 1 ) << std::endl; //EXPECT_TRUE_sv( f == 1 ); // this fails for vectors! TFFF -#ifndef MGONGPU_CPPSIMD EXPECT_TRUE_sv( f == 1 ); // this succeds: T -#else - EXPECT_TRUE( ( f == 1 )[0] ); // this succeds: TFFF[0] - EXPECT_TRUE( ( f[0] == 1 ) ); - for( int i = 1; i < neppV; i++ ) - { - EXPECT_TRUE( !( ( f == 1 )[i] ) ); // this succeds: FTTT[i>=1] - EXPECT_TRUE( ( f[i] == 0 ) ); // equals 0, not 1 - } -#endif } -#ifdef MGONGPU_CPPSIMD - // Vector initialization for cxtype_sv - demonstrate fix for bug #339 - { - fptype_sv f1 = fptype_v{ 0 } + 1; - EXPECT_TRUE_sv( f1 == 1 ); - cxtype_v c12 = cxmake( f1, 2 ); - //std::cout << c12 << std::endl << boolTF( c12.real() == 1 ) << std::endl << boolTF( c12.imag() == 2 ) << std::endl; - EXPECT_TRUE_sv( c12.real() == 1 ); - EXPECT_TRUE_sv( c12.imag() == 2 ); - cxtype_v c21 = cxmake( 2, f1 ); - //std::cout << c21 << std::endl << boolTF( c21.real() == 2 ) << std::endl << boolTF( c21.imag() == 1 ) << std::endl; - EXPECT_TRUE_sv( c21.real() == 2 ); - EXPECT_TRUE_sv( c21.imag() == 1 ); - } -#endif // Vector initialization for cxtype_sv { @@ -256,48 +204,20 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) test_int_sv channelids1_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) fptype_sv absamp0_sv{}; // mimic cxabs2( amp_sv[0] ) fptype_sv absamp1_sv{}; // mimic cxabs2( amp_sv[0] ) -#ifdef MGONGPU_CPPSIMD - for( int i = 0; i < neppV; i++ ) - { - channelids0_sv[i] = i; // 0123 - channelids1_sv[i] = i; // 1234 - absamp0_sv[i] = 10. + i; // 10. 11. 12. 13. - absamp1_sv[i] = 11. + i; // 11. 12. 13. 14. - } -#else channelids0_sv = 0; channelids1_sv = 1; absamp0_sv = 10.; absamp1_sv = 11.; -#endif bool_sv mask0_sv = ( channelids0_sv % 2 == 0 ); // even channels 0123 -> TFTF (1010) bool_sv mask1_sv = ( channelids1_sv % 2 == 0 ); // even channels 1234 -> FTFT (0101) constexpr fptype_sv fpZERO_sv{}; // 0000 //fptype_sv numerators0_sv = mask0_sv * absamp0_sv; // invalid operands to binary * ('__vector(4) long int' and '__vector(4) double') fptype_sv numerators0_sv = fpternary( mask0_sv, absamp0_sv, fpZERO_sv ); // equivalent to "mask0_sv * absamp0_sv" fptype_sv numerators1_sv = fpternary( mask1_sv, absamp1_sv, fpZERO_sv ); // equivalent to "mask1_sv * absamp1_sv" -#ifdef MGONGPU_CPPSIMD - //std::cout << "numerators0_sv: " << numerators0_sv << std::endl; - //std::cout << "numerators1_sv: " << numerators1_sv << std::endl; - for( int i = 0; i < neppV; i++ ) - { - // Values of numerators0_sv: 10.*1 11.*0 12.*1 13.*0 - if( channelids0_sv[i] % 2 == 0 ) // even channels - EXPECT_TRUE( numerators0_sv[i] == ( 10. + i ) ); - else // odd channels - EXPECT_TRUE( numerators0_sv[i] == 0. ); - // Values of numerators1_sv: 11.*0 12.*1 13.*0 14.*1 - if( channelids1_sv[i] % 2 == 0 ) // even channels - EXPECT_TRUE( numerators1_sv[i] == ( 11. + i ) ); - else // odd channels - EXPECT_TRUE( numerators1_sv[i] == 0. ); - } -#else // Values of numerators0_sv: 10.*1 EXPECT_TRUE( numerators0_sv == 10. ); // Values of numerators1_sv: 11.*0 EXPECT_TRUE( numerators1_sv == 0. ); -#endif } //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc index 4fcf7d26b6..ccdcd8261a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc @@ -25,40 +25,24 @@ #include #include #include -#ifdef MGONGPUCPP_GPUIMPL #define TESTID( s ) s##_GPU_XXX -#else -#define TESTID( s ) s##_CPU_XXX -#endif #define XTESTID( s ) TESTID( s ) -#ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif { std::string fpeHandlerMessage = "unknown"; int fpeHandlerIevt = -1; inline void fpeHandlerTestxxx( int /*sig*/ ) { -#ifdef MGONGPUCPP_GPUIMPL std::cerr << "Floating Point Exception (GPU): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; -#else - std::cerr << "Floating Point Exception (CPU neppV=" << neppV << "): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; -#endif exit( 1 ); } } TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { -#ifdef MGONGPUCPP_GPUIMPL using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif #ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); #endif @@ -72,11 +56,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt %% neppM == 0 ); // nevt must be a multiple of neppM assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV // Fill in the input momenta -#ifdef MGONGPUCPP_GPUIMPL mg5amcGpu::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] -#else - mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] -#endif /* clang-format off */ // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! const fptype par0[np4 * nevt] = // AOS[nevt][np4] { @@ -167,19 +147,8 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) out << " // ---------" << std::endl; for( int iw6 = 0; iw6 < nw6; iw6++ ) { -#ifdef MGONGPU_CPPSIMD - const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration -#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK - out << std::setw( 26 ) << cxreal( wf[iw6][ieppV] ) << ", "; - out << std::setw( 22 ) << cximag( wf[iw6][ieppV] ); -#else - out << std::setw( 26 ) << wf[iw6].real()[ieppV] << ", "; - out << std::setw( 22 ) << wf[iw6].imag()[ieppV]; -#endif -#else out << std::setw( 26 ) << wf[iw6].real(); out << ", " << std::setw( 22 ) << wf[iw6].imag(); -#endif if( iw6 < nw6 - 1 ) out << ", "; else @@ -207,25 +176,10 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) const fptype expImag = expwf[iw6 * 2 + 1]; if( true ) { -#ifdef MGONGPU_CPPSIMD - const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration -#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK - EXPECT_NEAR( cxreal( wf[iw6][ieppV] ), expReal, std::abs( expReal * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt; - EXPECT_NEAR( cximag( wf[iw6][ieppV] ), expImag, std::abs( expImag * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt; -#else - EXPECT_NEAR( wf[iw6].real()[ieppV], expReal, std::abs( expReal * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt; - EXPECT_NEAR( wf[iw6].imag()[ieppV], expImag, std::abs( expImag * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt; -#endif -#else EXPECT_NEAR( cxreal( wf[iw6] ), expReal, std::abs( expReal * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt; EXPECT_NEAR( cximag( wf[iw6] ), expImag, std::abs( expImag * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt; -#endif } } } @@ -246,31 +200,12 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { if( true ) { -#ifdef MGONGPU_CPPSIMD - const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration -#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK - const fptype expReal = cxreal( expwf[iw6][ieppV] ); - const fptype expImag = cximag( expwf[iw6][ieppV] ); - EXPECT_NEAR( cxreal( wf[iw6][ieppV] ), expReal, std::abs( expReal * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; - EXPECT_NEAR( cximag( wf[iw6][ieppV] ), expImag, std::abs( expImag * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; -#else - const fptype expReal = expwf[iw6].real()[ieppV]; - const fptype expImag = expwf[iw6].imag()[ieppV]; - EXPECT_NEAR( wf[iw6].real()[ieppV], expReal, std::abs( expReal * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; - EXPECT_NEAR( wf[iw6].imag()[ieppV], expImag, std::abs( expImag * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; -#endif -#else const fptype expReal = cxreal( expwf[iw6] ); const fptype expImag = cximag( expwf[iw6] ); EXPECT_NEAR( cxreal( wf[iw6] ), expReal, std::abs( expReal * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; EXPECT_NEAR( cximag( wf[iw6] ), expImag, std::abs( expImag * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; -#endif } } } @@ -317,11 +252,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { for( int ievt = 0; ievt < nevt; ievt++ ) { -#ifdef MGONGPUCPP_GPUIMPL using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif if( debug ) { std::cout << std::endl; @@ -438,9 +369,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) void myexit() { -#ifdef MGONGPUCPP_GPUIMPL //checkGpu( gpuDeviceReset() ); // FIXME??? this still crashes! should systematically avoid CUDA calls in all destructors? -#endif } // Main function (see https://google.github.io/googletest/primer.html#writing-the-main-function) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc index d19c93bb9b..ee2f4f2afb 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc @@ -16,11 +16,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL using namespace mg5amcGpu; -#else -using namespace mg5amcCpu; -#endif namespace { @@ -30,9 +26,7 @@ namespace const fptype* couplings, const unsigned int* flavor_indices, fptype* matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL fptype* color_jamps, -#endif fptype* numerators, fptype* denominators, std::size_t count ) @@ -40,9 +34,7 @@ namespace bool is_good_hel[CPPProcess::ncomb]; sigmaKin_getGoodHel( momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, -#ifdef MGONGPUCPP_GPUIMPL color_jamps, -#endif is_good_hel, count ); sigmaKin_setGoodHel( is_good_hel ); @@ -54,26 +46,20 @@ namespace const fptype* couplings, const unsigned int* flavor_indices, fptype* matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL fptype* color_jamps, -#endif fptype* numerators, fptype* denominators, std::size_t count ) { // static local initialization is called exactly once in a thread-safe way static void* dummy = initialize_impl( momenta, couplings, flavor_indices, matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL color_jamps, -#endif numerators, denominators, count ); } -#ifdef MGONGPUCPP_GPUIMPL __device__ -#endif void transpose_momenta( const double* momenta_in, fptype* momenta_out, std::size_t i_event_in, std::size_t i_event_out, std::size_t stride ) { @@ -91,7 +77,6 @@ namespace } } -#ifdef MGONGPUCPP_GPUIMPL __global__ void copy_inputs( const double* momenta_in, @@ -154,7 +139,6 @@ namespace if( helicity_out ) helicity_out[i_event + offset] = helicity_index[i_event] - 1; } -#endif // MGONGPUCPP_GPUIMPL struct InterfaceInstance { @@ -174,14 +158,10 @@ extern "C" case UMAMI_META_DEVICE: { UmamiDevice& device = *static_cast( result ); -#ifdef MGONGPUCPP_GPUIMPL #ifdef __CUDACC__ device = UMAMI_DEVICE_CUDA; #elif defined( __HIPCC__ ) device = UMAMI_DEVICE_HIP; -#endif -#else - device = UMAMI_DEVICE_CPU; #endif break; } @@ -231,11 +211,7 @@ extern "C" UmamiStatus umami_supported_outputs( bool const** supported, int* count ) { // MATRIX_ELEMENT, DIAGRAM_AMP2, COLOR_INDEX, HELICITY_INDEX, DIAGRAM_INDEX, GPU_STREAM -#ifdef MGONGPUCPP_GPUIMPL static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true, true }; -#else - static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true }; -#endif *supported = data; *count = UMAMI_OUTPUT_KEY_COUNT; return UMAMI_SUCCESS; @@ -326,9 +302,7 @@ extern "C" } if( !momenta_in ) return UMAMI_ERROR_MISSING_INPUT; -#ifdef MGONGPUCPP_GPUIMPL gpuStream_t gpu_stream = nullptr; -#endif double* m2_out = nullptr; double* amp2_out = nullptr; int* diagram_out = nullptr; @@ -354,17 +328,14 @@ extern "C" case UMAMI_OUT_DIAGRAM_INDEX: diagram_out = static_cast( output ); break; -#ifdef MGONGPUCPP_GPUIMPL case UMAMI_OUT_GPU_STREAM: gpu_stream = static_cast( output ); break; -#endif default: return UMAMI_ERROR_UNSUPPORTED_OUTPUT; } } -#ifdef MGONGPUCPP_GPUIMPL std::size_t n_threads = 256; std::size_t n_blocks = ( count + n_threads - 1 ) / n_threads; std::size_t rounded_count = n_blocks * n_threads; @@ -479,182 +450,6 @@ extern "C" checkGpu( gpuPeekAtLastError() ); gpuFreeAsync( buffer, gpu_stream ); -#else // MGONGPUCPP_GPUIMPL - constexpr std::size_t vector_size = MemoryAccessMomentaBase::neppM; - // need to round to round to double page size for some reason - constexpr std::size_t page_size2 = 2 * vector_size; - std::vector permutation; - std::size_t rounded_count; - - constexpr std::size_t flavor_count = CPPProcess::nmaxflavor; - HostBufferBase flavor_indices( ((count + page_size2 - 1) / page_size2 + flavor_count) * page_size2 ); - bool sort_flavors = vector_size > 1 && flavor_count > 1 && flavor_indices_in; - if ( sort_flavors ) - { - permutation.resize(count); - std::size_t voffset = 0; - std::size_t vector_indices[flavor_count] = {}; - std::size_t vector_counts[flavor_count] = {}; - // determine permutation of inputs such that all entries in a SIMD vector - // have the same flavor index - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - unsigned int flav = flavor_indices_in[i_event + offset]; - auto& vcount = vector_counts[flav]; - auto& vindex = vector_indices[flav]; - if ( vcount == 0 ) - { - vindex = voffset * page_size2; - for ( std::size_t i = 0; i < page_size2; ++i) { - flavor_indices[voffset * page_size2 + i] = flav; - } - voffset += 1; - } - permutation[i_event] = vindex + vcount; - vcount = (vcount + 1) % page_size2; - } - rounded_count = voffset * page_size2; - } else { - rounded_count = ( count + page_size2 - 1 ) / page_size2 * page_size2; - } - - HostBufferBase momenta( rounded_count * CPPProcess::npar * 4 ); - HostBufferBase couplings( rounded_count * mg5amcCpu::Parameters_dependentCouplings::ndcoup * 2 ); - HostBufferBase g_s( rounded_count ); - HostBufferBase helicity_random( rounded_count ); - HostBufferBase color_random( rounded_count ); - HostBufferBase diagram_random( rounded_count ); - HostBufferBase matrix_elements( rounded_count ); - HostBufferBase diagram_index( rounded_count ); - HostBufferBase numerators( rounded_count * CPPProcess::ndiagrams ); - HostBufferBase denominators( rounded_count ); - HostBufferBase helicity_index( rounded_count ); - HostBufferBase color_index( rounded_count ); - if ( sort_flavors ) { - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - std::size_t i_sorted = permutation[i_event]; - transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_sorted, stride ); - helicity_random[i_sorted] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; - color_random[i_sorted] = random_color_in ? random_color_in[i_event + offset] : 0.5; - diagram_random[i_sorted] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; - g_s[i_sorted] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; - } - } else { - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_event, stride ); - helicity_random[i_event] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; - color_random[i_event] = random_color_in ? random_color_in[i_event + offset] : 0.5; - diagram_random[i_event] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; - g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; - flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; - } - for ( std::size_t i_event = count; i_event < rounded_count; ++i_event ) { - flavor_indices[i_event] = 0; - } - } - computeDependentCouplings( g_s.data(), couplings.data(), rounded_count ); - - InterfaceInstance* instance = static_cast( handle ); - if( !instance->initialized ) - { - initialize( - momenta.data(), - couplings.data(), - flavor_indices.data(), - matrix_elements.data(), - numerators.data(), - denominators.data(), - rounded_count ); - instance->initialized = true; - } - - sigmaKin( - momenta.data(), - couplings.data(), - flavor_indices.data(), - helicity_random.data(), - color_random.data(), - nullptr, - diagram_random.data(), - matrix_elements.data(), - helicity_index.data(), - color_index.data(), - numerators.data(), - denominators.data(), - diagram_index.data(), - false, - rounded_count ); - - if ( sort_flavors ) - { - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - std::size_t i_sorted = permutation[i_event]; - std::size_t page_size = MemoryAccessMomentaBase::neppM; - std::size_t i_page = i_sorted / page_size; - std::size_t i_vector = i_sorted % page_size; // vector lane - - double denominator = denominators[i_sorted]; - if( m2_out != nullptr ) - { - m2_out[i_event + offset] = matrix_elements[i_sorted]; - } - if( amp2_out != nullptr ) - { - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) - { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; - } - } - if( diagram_out != nullptr ) - { - diagram_out[i_event + offset] = diagram_index[i_sorted] - 1; - } - if( color_out != nullptr ) - { - color_out[i_event + offset] = color_index[i_sorted] - 1; - } - if( helicity_out != nullptr ) - { - helicity_out[i_event + offset] = helicity_index[i_sorted] - 1; - } - } - } else { - std::size_t page_size = MemoryAccessMomentaBase::neppM; - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - std::size_t i_page = i_event / page_size; - std::size_t i_vector = i_event % page_size; - - double denominator = denominators[i_event]; - if( m2_out != nullptr ) - { - m2_out[i_event + offset] = matrix_elements[i_event]; - } - if( amp2_out != nullptr ) - { - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) - { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; - } - } - if( diagram_out != nullptr ) - { - diagram_out[i_event + offset] = diagram_index[i_event] - 1; - } - if( color_out != nullptr ) - { - color_out[i_event + offset] = color_index[i_event] - 1; - } - if( helicity_out != nullptr ) - { - helicity_out[i_event + offset] = helicity_index[i_event] - 1; - } - } - } -#endif // MGONGPUCPP_GPUIMPL return UMAMI_SUCCESS; } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/Bridge.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/Bridge.h deleted file mode 100644 index 8417e3ad15..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/Bridge.h +++ /dev/null @@ -1,633 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Nov 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2021-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef BRIDGE_H -#define BRIDGE_H 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" // for CPPProcess -#include "CrossSectionKernels.h" // for flagAbnormalMEs -#include "MatrixElementKernels.h" // for MatrixElementKernelHost, MatrixElementKernelDevice -#include "MemoryAccessMomenta.h" // for MemoryAccessMomenta::neppM -#include "MemoryBuffers.h" // for HostBufferMomenta, DeviceBufferMomenta etc - -//#ifdef __HIPCC__ -//#include // see -//https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 #else #include -// // bypass this completely to ease portability on LUMI #803 #endif - -#include // bypass std::filesystem #803 - -#include -#include -#include -#include -#include -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - /** - * A base class for a class whose pointer is passed between Fortran and C++. - * This is not really necessary, but it allows minimal type checks on all such - * pointers. - */ - struct CppObjectInFortran - { - CppObjectInFortran() {} - virtual ~CppObjectInFortran() {} - }; - - //-------------------------------------------------------------------------- - /** - * A templated class for calling the CUDA/C++ matrix element calculations of the - * event generation workflow. The FORTRANFPTYPE template parameter indicates the - * precision of the Fortran momenta from MadEvent (float or double). The - * precision of the matrix element calculation is hardcoded in the fptype - * typedef in CUDA/C++. - * - * The Fortran momenta passed in are in the form of - * DOUBLE PRECISION P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) - * where the dimensions are , , - * . In memory, this is stored in a way that C reads as an array - * P_MULTI[nevtF][nparF][np4F]. The CUDA/C++ momenta are stored as an - * array[npagM][npar][np4][neppM] with nevt=npagM*neppM. The Bridge is - * configured to store nevt==nevtF events in CUDA/C++. It also checks that - * Fortran and C++ parameters match, nparF==npar and np4F==np4. - * - * The cpu/gpu sequences take FORTRANFPTYPE* (not fptype*) momenta/MEs. - * This allows mixing double in MadEvent Fortran with float in CUDA/C++ - * sigmaKin. In the fcheck_sa.f test, Fortran uses double while CUDA/C++ may use - * double or float. In the check_sa "--bridge" test, everything is implemented - * in fptype (double or float). - */ - template - class Bridge final : public CppObjectInFortran - { - public: - /** - * Constructor - * - * @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran array - * loops (VECSIZE_USED <= VECSIZE_MEMMAX) - * @param nparF (NEXTERNAL, nexternal.inc) number of external particles in - * Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F number of momenta components, usually 4, in Fortran arrays - * (KEPT FOR SANITY CHECKS ONLY) - */ - Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F ); - - /** - * Destructor - */ - virtual ~Bridge() {} - - // Delete copy/move constructors and assignment operators - Bridge( const Bridge& ) = delete; - Bridge( Bridge&& ) = delete; - Bridge& operator=( const Bridge& ) = delete; - Bridge& operator=( Bridge&& ) = delete; - -#ifdef MGONGPUCPP_GPUIMPL - /** - * Set the gpublocks and gputhreads for the gpusequence - throws if evnt != - * gpublocks*gputhreads (this is needed for BridgeKernel tests rather than for - * actual production use in Fortran) - * - * @param gpublocks number of gpublocks - * @param gputhreads number of gputhreads - */ - void set_gpugrid( const int gpublocks, const int gputhreads ); - - /** - * Sequence to be executed for the Cuda matrix element calculation - * - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant - * alphas) - * @param iflavorVec the index of the flavor combination - * @param rndhel the pointer to the input random numbers for helicity - * selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 - * to n - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void gpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); -#else - /** - * Sequence to be executed for the vectorized CPU matrix element calculation - * - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant - * alphas) - * @param iflavorVec the index of the flavor combination - * @param rndhel the pointer to the input random numbers for helicity - * selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 - * to n - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void cpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); -#endif - - // Return the number of good helicities (-1 initially when they have not yet - // been calculated) - int nGoodHel() const { return m_nGoodHel; } - - // Return the total number of helicities (expose cudacpp ncomb in the Bridge - // interface to Fortran) - constexpr int nTotHel() const { return CPPProcess::ncomb; } - - private: - unsigned int m_nevt; // number of events - int m_nGoodHel; // the number of good helicities (-1 initially when they have - // not yet been calculated) - -#ifdef MGONGPUCPP_GPUIMPL - int m_gputhreads; // number of gpu threads (default set from number of - // events, can be modified) - int m_gpublocks; // number of gpu blocks (default set from number of events, - // can be modified) - DeviceBuffer m_devMomentaF; - DeviceBufferMomenta m_devMomentaC; - DeviceBufferGs m_devGs; - DeviceBufferIflavorVec m_devIflavorVec; - DeviceBufferRndNumHelicity m_devRndHel; - DeviceBufferRndNumColor m_devRndCol; - DeviceBufferMatrixElements m_devMEs; - DeviceBufferSelectedHelicity m_devSelHel; - DeviceBufferSelectedColor m_devSelCol; - DeviceBufferChannelIds m_devChannelIds; - PinnedHostBufferIflavorVec m_hstIflavorVec; - PinnedHostBufferGs m_hstGs; - PinnedHostBufferRndNumHelicity m_hstRndHel; - PinnedHostBufferRndNumColor m_hstRndCol; - PinnedHostBufferMatrixElements m_hstMEs; - PinnedHostBufferSelectedHelicity m_hstSelHel; - PinnedHostBufferSelectedColor m_hstSelCol; - PinnedHostBufferChannelIds m_hstChannelIds; - std::unique_ptr m_pmek; - // static constexpr int s_gputhreadsmin = 16; // minimum number of gpu threads - // (TEST VALUE FOR MADEVENT) - static constexpr int s_gputhreadsmin = - 32; // minimum number of gpu threads (DEFAULT) -#else - HostBufferMomenta m_hstMomentaC; - HostBufferGs m_hstGs; - HostBufferIflavorVec m_hstIflavorVec; - HostBufferRndNumHelicity m_hstRndHel; - HostBufferRndNumColor m_hstRndCol; - HostBufferMatrixElements m_hstMEs; - HostBufferSelectedHelicity m_hstSelHel; - HostBufferSelectedColor m_hstSelCol; - HostBufferChannelIds m_hstChannelIds; - std::unique_ptr m_pmek; -#endif - }; - - //-------------------------------------------------------------------------- - // - // Forward declare transposition methods - // - -#ifdef MGONGPUCPP_GPUIMPL - - template - __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); - -#endif // MGONGPUCPP_GPUIMPL - - template - void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); - - template - void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ); - - //-------------------------------------------------------------------------- - // - // Implementations of member functions of class Bridge - // - - template - Bridge::Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F) - : m_nevt( nevtF ), m_nGoodHel( -1 ) -#ifdef MGONGPUCPP_GPUIMPL - , m_gputhreads( 256 ) // default number of gpu threads - , m_gpublocks( m_nevt / m_gputhreads ) // this ensures m_nevt <= m_gpublocks*m_gputhreads - , m_devMomentaF( m_nevt ) - , m_devMomentaC( m_nevt ) - , m_devIflavorVec( m_nevt ) - , m_devGs( m_nevt ) - , m_devRndHel( m_nevt ) - , m_devRndCol( m_nevt ) - , m_devMEs( m_nevt ) - , m_devSelHel( m_nevt ) - , m_devSelCol( m_nevt ) - , m_devChannelIds( m_nevt ) -#else - , m_hstMomentaC( m_nevt ) -#endif - , m_hstGs( m_nevt ) - , m_hstIflavorVec( m_nevt ) - , m_hstRndHel( m_nevt ) - , m_hstRndCol( m_nevt ) - , m_hstMEs( m_nevt ) - , m_hstSelHel( m_nevt ) - , m_hstSelCol( m_nevt ) - , m_hstChannelIds( m_nevt ) - , m_pmek( nullptr ) - { - if( nparF != CPPProcess::npar ) - throw std::runtime_error( "Bridge constructor: npar mismatch" ); - if( np4F != CPPProcess::np4 ) - throw std::runtime_error( "Bridge constructor: np4 mismatch" ); -#ifdef MGONGPUCPP_GPUIMPL - if( ( m_nevt < s_gputhreadsmin ) || ( m_nevt % s_gputhreadsmin != 0 ) ) - throw std::runtime_error( - "Bridge constructor: nevt should be a multiple of " + - std::to_string( s_gputhreadsmin ) ); - while( m_nevt != m_gpublocks * m_gputhreads ) - { - m_gputhreads /= 2; - if( m_gputhreads < s_gputhreadsmin ) - throw std::logic_error( - "Bridge constructor: FIXME! cannot choose gputhreads" ); // this - // should - // never - // happen! - m_gpublocks = m_nevt / m_gputhreads; - } -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Instantiate device Bridge (nevt=" << m_nevt - << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads - << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" - << std::endl; -#endif - m_pmek.reset( new MatrixElementKernelDevice( - m_devMomentaC, m_devGs, m_devIflavorVec, m_devRndHel, m_devRndCol, m_devChannelIds, m_devMEs, m_devSelHel, m_devSelCol, m_gpublocks, m_gputhreads) ); -#else -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Instantiate host Bridge (nevt=" << m_nevt << ")" - << std::endl; -#endif - m_pmek.reset( new MatrixElementKernelHost( - m_hstMomentaC, m_hstGs, m_hstIflavorVec, m_hstRndHel, m_hstRndCol, m_hstChannelIds, m_hstMEs, m_hstSelHel, m_hstSelCol, m_nevt ) ); -#endif // MGONGPUCPP_GPUIMPL - // Create a process object, read param card and set parameters - // FIXME: the process instance can happily go out of scope because it is only - // needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? what if fbridgecreate - // is called from several Fortran threads? - CPPProcess process( /*verbose=*/false ); - std::string paramCard = - "../Cards/param_card.dat"; // ZW: change default param_card.dat location - // to one dir down - /* -#ifdef __HIPCC__ - if( !std::experimental::filesystem::exists( paramCard ) ) paramCard = "../" + -paramCard; #else if( !std::filesystem::exists( paramCard ) ) paramCard = "../" + -paramCard; #endif - */ - // struct stat dummybuffer; // bypass std::filesystem #803 - // if( !( stat( paramCard.c_str(), &dummyBuffer ) == 0 ) ) paramCard = "../" + - // paramCard; // - auto fileExists = []( std::string& fileName ) - { - struct stat buffer; - return stat( fileName.c_str(), &buffer ) == 0; - }; - size_t paramCardCheck = 2; // ZW: check for paramCard up to 2 directories up - for( size_t k = 0; k < paramCardCheck; ++k ) - { - if( fileExists( paramCard ) ) break; // bypass std::filesystem #803 - paramCard = "../" + paramCard; - } - process.initProc( paramCard ); - } - -#ifdef MGONGPUCPP_GPUIMPL - template - void Bridge::set_gpugrid( const int gpublocks, - const int gputhreads ) - { - if( m_nevt != gpublocks * gputhreads ) - throw std::runtime_error( - "Bridge: gpublocks*gputhreads must equal m_nevt in set_gpugrid" ); - m_gpublocks = gpublocks; - m_gputhreads = gputhreads; -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Set grid in Bridge (nevt=" << m_nevt - << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads - << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" - << std::endl; -#endif - m_pmek->setGrid( m_gpublocks, m_gputhreads ); - } -#endif - -#ifdef MGONGPUCPP_GPUIMPL - template - void Bridge::gpu_sequence( const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool goodHelOnly ) - { - constexpr int neppM = MemoryAccessMomenta::neppM; - if constexpr( neppM == 1 && std::is_same_v ) - { - gpuMemcpy( m_devMomentaC.data(), momenta, m_devMomentaC.bytes(), gpuMemcpyHostToDevice ); - } - else - { - gpuMemcpy( m_devMomentaF.data(), momenta, m_devMomentaF.bytes(), gpuMemcpyHostToDevice ); - const int thrPerEvt = - CPPProcess::npar * - CPPProcess::np4; // AV: transpose alg does 1 element per thread (NOT 1 - // event per thread) - // const int thrPerEvt = 1; // AV: try new alg with 1 event per thread... - // this seems slower - gpuLaunchKernel( dev_transposeMomentaF2C, m_gpublocks * thrPerEvt, m_gputhreads, m_devMomentaF.data(), m_devMomentaC.data(), m_nevt ); - } - if constexpr( std::is_same_v ) - { - memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); - } - else - { - std::copy( gs, gs + m_nevt, m_hstGs.data() ); - std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); - std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); - } - const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); - if( useChannelIds ) - memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); - // else ... // no need to initialize m_hstChannel: it is allocated with - // gpuMallocHost and NOT initialized in PinnedHostBufferBase, but it is NOT - // used later on - // initialise iflavorVec - memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); - copyDeviceFromHost( m_devGs, m_hstGs ); - copyDeviceFromHost( m_devRndHel, m_hstRndHel ); - copyDeviceFromHost( m_devRndCol, m_hstRndCol ); - if( useChannelIds ) copyDeviceFromHost( m_devChannelIds, m_hstChannelIds ); - copyDeviceFromHost( m_devIflavorVec, m_hstIflavorVec ); - if( m_nGoodHel < 0 ) - { - m_nGoodHel = m_pmek->computeGoodHelicities(); - if( m_nGoodHel < 0 ) - throw std::runtime_error( - "Bridge gpu_sequence: computeGoodHelicities returned nGoodHel<0" ); - } - if( goodHelOnly ) return; - m_pmek->computeMatrixElements( useChannelIds ); - copyHostFromDevice( m_hstMEs, m_devMEs ); -#ifdef MGONGPUCPP_VERBOSE - flagAbnormalMEs( m_hstMEs.data(), m_nevt ); -#endif - copyHostFromDevice( m_hstSelHel, m_devSelHel ); - copyHostFromDevice( m_hstSelCol, m_devSelCol ); - if constexpr( std::is_same_v ) - { - memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); - memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); - memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); - } - else - { - std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); - std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); - std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); - } - } -#endif - -#ifndef MGONGPUCPP_GPUIMPL - template - void Bridge::cpu_sequence( const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool goodHelOnly ) - { - hst_transposeMomentaF2C( momenta, m_hstMomentaC.data(), m_nevt ); - if constexpr( std::is_same_v ) - { - memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); - } - else - { - std::copy( gs, gs + m_nevt, m_hstGs.data() ); - std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); - std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); - } - const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); - if( useChannelIds ) - memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); - // else ... // no need to initialize m_hstChannel: it is allocated and default - // initialized in HostBufferBase (and it is not used later on anyway) - // initialise iflavorVec - memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); - if( m_nGoodHel < 0 ) - { - m_nGoodHel = m_pmek->computeGoodHelicities(); - if( m_nGoodHel < 0 ) - throw std::runtime_error( - "Bridge cpu_sequence: computeGoodHelicities returned nGoodHel<0" ); - } - if( goodHelOnly ) return; - m_pmek->computeMatrixElements( useChannelIds ); -#ifdef MGONGPUCPP_VERBOSE - flagAbnormalMEs( m_hstMEs.data(), m_nevt ); -#endif - if constexpr( std::is_same_v ) - { - memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); - memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); - memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); - } - else - { - std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); - std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); - std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); - } - } -#endif - - //-------------------------------------------------------------------------- - // - // Implementations of transposition methods - // - FORTRAN arrays: P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) ==> - // p_multi[nevtF][nparF][np4F] in C++ (AOS) - // - C++ array: momenta[npagM][npar][np4][neppM] with nevt=npagM*neppM (AOSOA) - // - -#ifdef MGONGPUCPP_GPUIMPL - template - __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool oldImplementation = true; // default: use old implementation - if constexpr( oldImplementation ) - { - // SR initial implementation - constexpr int part = CPPProcess::npar; - constexpr int mome = CPPProcess::np4; - constexpr int strd = MemoryAccessMomenta::neppM; - int pos = blockDim.x * blockIdx.x + threadIdx.x; - int arrlen = nevt * part * mome; - if( pos < arrlen ) - { - int page_i = pos / ( strd * mome * part ); - int rest_1 = pos % ( strd * mome * part ); - int part_i = rest_1 / ( strd * mome ); - int rest_2 = rest_1 % ( strd * mome ); - int mome_i = rest_2 / strd; - int strd_i = rest_2 % strd; - int inpos = ( page_i * strd + strd_i ) // event number - * ( part * mome ) // event size (pos of event) - + part_i * mome // particle inside event - + mome_i; // momentum inside particle - out[pos] = in[inpos]; // F2C (Fortran to C) - } - } - else - { - // AV attempt another implementation with 1 event per thread: this seems - // slower... F-style: AOS[nevtF][nparF][np4F] C-style: - // AOSOA[npagM][npar][np4][neppM] with nevt=npagM*neppM - constexpr int npar = CPPProcess::npar; - constexpr int np4 = CPPProcess::np4; - constexpr int neppM = MemoryAccessMomenta::neppM; - assert( nevt % neppM == - 0 ); // number of events is not a multiple of neppM??? - int ievt = blockDim.x * blockIdx.x + threadIdx.x; - int ipagM = ievt / neppM; - int ieppM = ievt % neppM; - for( int ip4 = 0; ip4 < np4; ip4++ ) - for( int ipar = 0; ipar < npar; ipar++ ) - { - int cpos = ipagM * npar * np4 * neppM + ipar * np4 * neppM + - ip4 * neppM + ieppM; - int fpos = ievt * npar * np4 + ipar * np4 + ip4; - out[cpos] = in[fpos]; // F2C (Fortran to C) - } - } - } -#endif - - template - void hst_transposeMomenta( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool oldImplementation = false; // default: use new implementation - if constexpr( oldImplementation ) - { - // SR initial implementation - constexpr unsigned int part = CPPProcess::npar; - constexpr unsigned int mome = CPPProcess::np4; - constexpr unsigned int strd = MemoryAccessMomenta::neppM; - unsigned int arrlen = nevt * part * mome; - for( unsigned int pos = 0; pos < arrlen; ++pos ) - { - unsigned int page_i = pos / ( strd * mome * part ); - unsigned int rest_1 = pos % ( strd * mome * part ); - unsigned int part_i = rest_1 / ( strd * mome ); - unsigned int rest_2 = rest_1 % ( strd * mome ); - unsigned int mome_i = rest_2 / strd; - unsigned int strd_i = rest_2 % strd; - unsigned int inpos = ( page_i * strd + strd_i ) // event number - * ( part * mome ) // event size (pos of event) - + part_i * mome // particle inside event - + mome_i; // momentum inside particle - if constexpr( F2C ) // needs c++17 and cuda >=11.2 (#333) - out[pos] = in[inpos]; // F2C (Fortran to C) - else - out[inpos] = in[pos]; // C2F (C to Fortran) - } - } - else - { - // AV attempt another implementation: this is slightly faster (better c++ - // pipelining?) [NB! this is not a transposition, it is an AOS to AOSOA - // conversion: if neppM=1, a memcpy is enough] F-style: - // AOS[nevtF][nparF][np4F] C-style: AOSOA[npagM][npar][np4][neppM] with - // nevt=npagM*neppM - constexpr unsigned int npar = CPPProcess::npar; - constexpr unsigned int np4 = CPPProcess::np4; - constexpr unsigned int neppM = MemoryAccessMomenta::neppM; - if constexpr( neppM == 1 && std::is_same_v ) - { - memcpy( out, in, nevt * npar * np4 * sizeof( Tin ) ); - } - else - { - const unsigned int npagM = nevt / neppM; - assert( nevt % neppM == - 0 ); // number of events is not a multiple of neppM??? - for( unsigned int ipagM = 0; ipagM < npagM; ipagM++ ) - for( unsigned int ip4 = 0; ip4 < np4; ip4++ ) - for( unsigned int ipar = 0; ipar < npar; ipar++ ) - for( unsigned int ieppM = 0; ieppM < neppM; ieppM++ ) - { - unsigned int ievt = ipagM * neppM + ieppM; - unsigned int cpos = ipagM * npar * np4 * neppM + - ipar * np4 * neppM + ip4 * neppM + ieppM; - unsigned int fpos = ievt * npar * np4 + ipar * np4 + ip4; - if constexpr( F2C ) - out[cpos] = in[fpos]; // F2C (Fortran to C) - else - out[fpos] = in[cpos]; // C2F (C to Fortran) - } - } - } - } - - template - void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool F2C = true; - hst_transposeMomenta( in, out, nevt ); - } - - template - void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool F2C = false; - hst_transposeMomenta( in, out, nevt ); - } - - //-------------------------------------------------------------------------- -} // namespace mg5amcGpu -#endif // BRIDGE_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.cc deleted file mode 100644 index 65c97616d8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.cc +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "BridgeKernels.h" - -#include "GpuAbstraction.h" -#include "MemoryAccessMomenta.h" - -#include - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - - //-------------------------------------------------------------------------- - - BridgeKernelBase::BridgeKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( nevt ) - , m_bridge( nevt, npar, np4 ) - { - if( m_momenta.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: momenta must be a host array" ); - if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: matrixElements must be a host array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with matrixElements" ); - } - - //-------------------------------------------------------------------------- -} - -//============================================================================ - -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu -{ - - //-------------------------------------------------------------------------- - - BridgeKernelHost::BridgeKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: Gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt) - : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, nevt) - , m_fortranMomenta( nevt ) - { - } - - //-------------------------------------------------------------------------- - - void BridgeKernelHost::transposeInputMomentaC2F() - { - hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); - } - - //-------------------------------------------------------------------------- - - int BridgeKernelHost::computeGoodHelicities() - { - constexpr bool goodHelOnly = true; - constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering - m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - return m_bridge.nGoodHel(); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelHost::computeMatrixElements( const bool useChannelIds ) - { - constexpr bool goodHelOnly = false; - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - //-------------------------------------------------------------------------- - - BridgeKernelDevice::BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: Gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads) - : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, gpublocks * gputhreads) - , m_fortranMomenta( nevt() ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if( m_gpublocks == 0 ) throw std::runtime_error( "BridgeKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "BridgeKernelDevice: gputhreads must be > 0" ); - m_bridge.set_gpugrid( gpublocks, gputhreads ); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelDevice::transposeInputMomentaC2F() - { - hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); - } - - //-------------------------------------------------------------------------- - - int BridgeKernelDevice::computeGoodHelicities() - { - constexpr bool goodHelOnly = true; - constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering - m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - return m_bridge.nGoodHel(); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelDevice::computeMatrixElements( const bool useChannelIds ) - { - constexpr bool goodHelOnly = false; - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.h deleted file mode 100644 index 2d873486a8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/BridgeKernels.h +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally originally by: A. Valassi (Jan 2022). -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef BRIDGEKERNELS_H -#define BRIDGEKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "Bridge.h" -#include "MatrixElementKernels.h" -#include "MemoryBuffers.h" - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - // A Bridge wrapper base class encapsulating matrix element calculations on a CPU host - class BridgeKernelBase : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - BridgeKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~BridgeKernelBase() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - virtual void transposeInputMomentaC2F() = 0; - - protected: - - // The wrapped bridge - Bridge m_bridge; - }; - - //-------------------------------------------------------------------------- - -#ifndef MGONGPUCPP_GPUIMPL - // A Bridge wrapper class encapsulating matrix element calculations on a CPU host - class BridgeKernelHost final : public BridgeKernelBase - { - public: - - // Constructor from existing input and output buffers - BridgeKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~BridgeKernelHost() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - void transposeInputMomentaC2F() override final; - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // The buffer for the input momenta, transposed to Fortran array indexing - HostBufferMomenta m_fortranMomenta; - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A Bridge wrapper class encapsulating matrix element calculations on a GPU device - class BridgeKernelDevice : public BridgeKernelBase - { - public: - - // Constructor from existing input and output buffers - BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads); - - // Destructor - virtual ~BridgeKernelDevice() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - void transposeInputMomentaC2F() override final; - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The buffer for the input momenta, transposed to Fortran array indexing - PinnedHostBufferMomenta m_fortranMomenta; - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; -#endif - - //-------------------------------------------------------------------------- -} -#endif // BRIDGEKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc index 16671b5686..14c1ceeaf5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc @@ -11,11 +11,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc index 7a3e0e0519..861bca8940 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc @@ -79,11 +79,7 @@ debug_me_is_abnormal( const fptype& me, size_t ievtALL ) //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -187,52 +183,5 @@ namespace mg5amcCpu //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - /* - //-------------------------------------------------------------------------- - - CrossSectionKernelDevice::CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats, // output: event statistics - const size_t gpublocks, - const size_t gputhreads ) - : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) - , NumberOfEvents( gpublocks*gputhreads ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if ( ! m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: samplingWeights must be a device array" ); - if ( ! m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: matrixElements must be a device array" ); - if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0" ); - if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0" ); - if ( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with samplingWeights" ); - if ( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with matrixElements" ); - } - - //-------------------------------------------------------------------------- - - void CrossSectionKernelDevice::setGrid( const size_t gpublocks, const size_t gputhreads ) - { - if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0 in setGrid" ); - if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0 in setGrid" ); - if ( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch in setGrid" ); - } - - //-------------------------------------------------------------------------- - - void CrossSectionKernelDevice::updateEventStatistics( const bool debug ) - { - // Increment the iterations counter - m_iter++; - } - - //-------------------------------------------------------------------------- - */ - -} -#endif //============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h index d1e29b7999..9fb3c3fce2 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h @@ -14,11 +14,7 @@ //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc index 0c8e6e6445..24ee78772b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc @@ -24,11 +24,7 @@ inline void assertCurand( curandStatus_t code, const char *file, int line, bool } #endif /* clang-format on */ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_CURAND @@ -38,12 +34,7 @@ namespace mg5amcCpu { if( m_isOnDevice ) { -#ifdef MGONGPUCPP_GPUIMPL - if( !m_rnarray.isOnDevice() ) - throw std::runtime_error( "CurandRandomNumberKernel on device with a host random number array" ); -#else throw std::runtime_error( "CurandRandomNumberKernel does not support CurandDevice on CPU host" ); -#endif } else { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h index 4a1fd8464d..831d477bb6 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h @@ -17,11 +17,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h index 2c8ae1a804..7366509821 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h @@ -17,85 +17,12 @@ //-------------------------------------------------------------------------- // See https://stackoverflow.com/a/14038590 -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ -#define checkGpu( code ) { assertGpu( code, __FILE__, __LINE__ ); } -inline void assertGpu( gpuError_t code, const char* file, int line, bool abort = true ) -{ - if( code != gpuSuccess ) - { - printf( "ERROR! assertGpu: '%s' (%d) in %s:%d\n", gpuGetErrorString( code ), code, file, line ); - if( abort ) assert( code == gpuSuccess ); - } -} -#endif /* clang-format on */ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ -#ifndef MGONGPU_HAS_NO_BLAS -#define checkGpuBlas( code ){ assertGpuBlas( code, __FILE__, __LINE__ ); } -inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, bool abort = true ) -{ - if ( code != GPUBLAS_STATUS_SUCCESS ) - { - printf( "ERROR! assertGpuBlas: '%d' in %s:%d\n", code, file, line ); - if( abort ) assert( code == GPUBLAS_STATUS_SUCCESS ); - } -} -#endif -#endif /* clang-format on */ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - // Instantiate a GpuRuntime at the beginnining of the application's main to - // invoke gpuSetDevice(0) in the constructor and book a gpuDeviceReset() call in the destructor - // *** FIXME! This will all need to be designed differently when going to multi-GPU nodes! *** - struct GpuRuntime final - { - GpuRuntime( const bool debug = true ) - : m_debug( debug ) { setUp( m_debug ); } - ~GpuRuntime() { tearDown( m_debug ); } - GpuRuntime( const GpuRuntime& ) = delete; - GpuRuntime( GpuRuntime&& ) = delete; - GpuRuntime& operator=( const GpuRuntime& ) = delete; - GpuRuntime& operator=( GpuRuntime&& ) = delete; - bool m_debug; - - // Set up CUDA application - // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** - // Calling cudaSetDevice on startup is useful to properly book-keep the time spent in CUDA initialization - static void setUp( const bool debug = false ) // ZW: changed debug default to false - { - // ** NB: it is useful to call cudaSetDevice, or cudaFree, to properly book-keep the time spent in CUDA initialization - // ** NB: otherwise, the first CUDA operation (eg a cudaMemcpyToSymbol in CPPProcess ctor) appears to take much longer! - /* - // [We initially added cudaFree(0) to "ease profile analysis" only because it shows up as a big recognizable block!] - // No explicit initialization is needed: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#initialization - // It is not clear what cudaFree(0) does at all: https://stackoverflow.com/questions/69967813/ - if ( debug ) std::cout << "__CudaRuntime: calling cudaFree(0)" << std::endl; - checkCuda( cudaFree( 0 ) ); // SLOW! - */ - // Replace cudaFree(0) by cudaSetDevice(0), even if it is not really needed either - // (but see https://developer.nvidia.com/blog/cuda-pro-tip-always-set-current-device-avoid-multithreading-bugs) - if( debug ) std::cout << "__GpuRuntime: calling GpuSetDevice(0)" << std::endl; - checkGpu( gpuSetDevice( 0 ) ); // SLOW! - } - - // Tear down CUDA application (call cudaDeviceReset) - // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** - // Calling cudaDeviceReset on shutdown is only needed for checking memory leaks in cuda-memcheck - // See https://docs.nvidia.com/cuda/cuda-memcheck/index.html#leak-checking - static void tearDown( const bool debug = false ) // ZW: changed debug default to false - { - if( debug ) std::cout << "__GpuRuntime: calling GpuDeviceReset()" << std::endl; - checkGpu( gpuDeviceReset() ); - } - }; -} -#endif //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc index b806da2e09..2fd64a2dce 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc @@ -28,11 +28,7 @@ inline void assertHiprand( hiprandStatus_t code, const char *file, int line, boo } #endif /* clang-format on */ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_HIPRAND @@ -42,12 +38,7 @@ namespace mg5amcCpu { if( m_isOnDevice ) { -#ifdef MGONGPUCPP_GPUIMPL - if( !m_rnarray.isOnDevice() ) - throw std::runtime_error( "HiprandRandomNumberKernel on device with a host random number array" ); -#else throw std::runtime_error( "HiprandRandomNumberKernel does not support HiprandDevice on CPU host" ); -#endif } else { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h index 12b7d45adf..641917d8ff 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h @@ -27,11 +27,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -using mg5amcGpu::CPPProcess; -#else using mg5amcCpu::CPPProcess; -#endif namespace { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc index 872e4795e3..fb19139f2d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc @@ -17,11 +17,7 @@ //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -146,7 +142,6 @@ namespace mg5amcCpu //============================================================================ -#ifndef MGONGPUCPP_GPUIMPL namespace mg5amcCpu { @@ -282,228 +277,8 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- } -#endif //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - //-------------------------------------------------------------------------- - - MatrixElementKernelDevice::MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( gpublocks * gputhreads ) - , m_couplings( this->nevt() ) - , m_pHelMEs() - , m_pHelJamps() - , m_pHelNumerators() - , m_pHelDenominators() - , m_colJamp2s( CPPProcess::ncolor * this->nevt() ) -#ifdef MGONGPU_CHANNELID_DEBUG - , m_hstChannelIds( this->nevt() ) -#endif -#ifndef MGONGPU_HAS_NO_BLAS - , m_blasColorSum( false ) - , m_blasTf32Tensor( false ) - , m_pHelBlasTmp() - , m_blasHandle() -#endif - , m_helStreams() - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - //std::cout << "DEBUG: MatrixElementKernelDevice::ctor " << this << std::endl; - if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: momenta must be a device array" ); - if( !m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: matrixElements must be a device array" ); - if( !m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: channelIds must be a device array" ); // FIXME?! - if( !m_iflavorVec.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: iflavorVec must be a device array" ); - if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with matrixElements" ); - if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with channelIds" ); - if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with iflavorVec" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( m_gputhreads % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MatrixElementKernelHost: gputhreads should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Create the "one-helicity" jamp buffer that will be used for helicity filtering - m_pHelJamps.reset( new DeviceBufferSimple( CPPProcess::ncolor * mgOnGpu::nx2 * this->nevt() ) ); - // Create the "one-helicity" numerator and denominator buffers that will be used for helicity filtering - m_pHelNumerators.reset( new DeviceBufferSimple( this->nevt() * CPPProcess::ndiagrams ) ); - m_pHelDenominators.reset( new DeviceBufferSimple( this->nevt() ) ); - // Decide at runtime whether to use BLAS for color sums - // Decide at runtime whether TF32TENSOR math should be used in cuBLAS - static bool first = true; - if( first ) - { - first = false; - // Analyse environment variable CUDACPP_RUNTIME_BLASCOLORSUM - const char* blasEnv = getenv( "CUDACPP_RUNTIME_BLASCOLORSUM" ); - if( blasEnv && std::string( blasEnv ) != "" ) - { -#ifndef MGONGPU_HAS_NO_BLAS - m_blasColorSum = true; // fixme? eventually set default=true and decode "Y" and "N" choices? - std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty: enable BLAS" << std::endl; -#else - throw std::runtime_error( "Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty, but BLAS was disabled at build time" ); -#endif - } - else - { -#ifndef MGONGPU_HAS_NO_BLAS - std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is empty or not set: disable BLAS" << std::endl; -#else - std::cout << "INFO: BLAS was disabled at build time" << std::endl; -#endif - } -#ifndef MGONGPU_HAS_NO_BLAS -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) - // Analyse environment variable CUDACPP_RUNTIME_CUBLASTF32TENSOR - const char* blasEnv2 = getenv( "CUDACPP_RUNTIME_CUBLASTF32TENSOR" ); - if( blasEnv2 && std::string( blasEnv2 ) != "" ) - { - if( m_blasColorSum ) - { -#ifdef MGONGPU_FPTYPE2_FLOAT - m_blasTf32Tensor = true; - std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty: enable CUBLAS_TF32_TENSOR_OP_MATH" << std::endl; -#else - std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but color sums use FP64" << std::endl; -#endif - } - else - std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but BLAS was disabled at runtime" << std::endl; - } -#ifdef MGONGPU_FPTYPE2_FLOAT - else - { - if( m_blasColorSum ) - std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is empty or not set: keep cuBLAS math defaults" << std::endl; - } -#endif -#endif -#endif - } - } - - //-------------------------------------------------------------------------- - - MatrixElementKernelDevice::~MatrixElementKernelDevice() - { - //std::cout << "DEBUG: MatrixElementKernelDevice::dtor " << this << std::endl; -#ifndef MGONGPU_HAS_NO_BLAS - if( m_blasHandle ) gpuBlasDestroy( m_blasHandle ); -#endif - for( int ihel = 0; ihel < CPPProcess::ncomb; ihel++ ) - { - if( m_helStreams[ihel] ) gpuStreamDestroy( m_helStreams[ihel] ); // do not destroy if nullptr - } - } - - //-------------------------------------------------------------------------- - - // FIXME! The relevance of this function should be reassessed (#543 and #902) - void MatrixElementKernelDevice::setGrid( const int /*gpublocks*/, const int /*gputhreads*/ ) - { - if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0 in setGrid" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0 in setGrid" ); - if( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch in setGrid" ); - } - - //-------------------------------------------------------------------------- - - int MatrixElementKernelDevice::computeGoodHelicities() - { - PinnedHostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); - // ... 0d1. Compute good helicity mask (a host variable) on the device - gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); - const int nevt = m_gpublocks * m_gputhreads; - sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_pHelJamps->data(), m_pHelNumerators->data(), m_pHelDenominators->data(), hstIsGoodHel.data(), nevt ); - // ... 0d3. Set good helicity list in host static memory - int nGoodHel = sigmaKin_setGoodHel( hstIsGoodHel.data() ); - assert( nGoodHel > 0 ); // SANITY CHECK: there should be at least one good helicity - // Create one GPU stream for each good helicity - for( int ighel = 0; ighel < nGoodHel; ighel++ ) - gpuStreamCreate( &m_helStreams[ighel] ); -#ifndef MGONGPU_HAS_NO_BLAS - // Create one cuBLAS/hipBLAS handle for each good helicity (attached to the default stream) - if( m_blasColorSum ) - { - checkGpuBlas( gpuBlasCreate( &m_blasHandle ) ); -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) - if( m_blasTf32Tensor ) - checkGpuBlas( cublasSetMathMode( m_blasHandle, CUBLAS_TF32_TENSOR_OP_MATH ) ); // enable TF32 tensor cores -#endif - } -#endif - // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) - m_pHelMEs.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); - // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) - // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) - m_pHelJamps.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); - // ... Create the "many-helicity" super-buffers of nGoodHel numerator and denominator buffers (dynamically allocated) - // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) - m_pHelNumerators.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ndiagrams * nevt ) ); - m_pHelDenominators.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); -#ifndef MGONGPU_HAS_NO_BLAS - // Create the "many-helicity" super-buffers of real/imag ncolor*nevt temporary buffers for cuBLAS/hipBLAS intermediate results in color_sum_blas -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Mixed precision mode: need two fptype2[ncolor*2*nevt] buffers and one fptype2[nevt] buffer per good helicity - if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ( 2 * CPPProcess::ncolor * mgOnGpu::nx2 + 1 ) * nevt ) ); -#else - // Standard single/double precision mode: need one fptype2[ncolor*2*nevt] buffer per good helicity - if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); -#endif -#endif - // Return the number of good helicities - return nGoodHel; - } - - //-------------------------------------------------------------------------- - - void MatrixElementKernelDevice::computeMatrixElements( const bool useChannelIds ) - { - gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); -#ifndef MGONGPU_HAS_NO_BLAS - fptype2* ghelAllBlasTmp = ( m_blasColorSum ? m_pHelBlasTmp->data() : nullptr ); - gpuBlasHandle_t* pBlasHandle = ( m_blasColorSum ? &m_blasHandle : nullptr ); -#else - fptype2* ghelAllBlasTmp = nullptr; - gpuBlasHandle_t* pBlasHandle = nullptr; -#endif - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_colJamp2s.data(), m_pHelNumerators->data(), m_pHelDenominators->data(), nullptr, true, m_pHelMEs->data(), m_pHelJamps->data(), ghelAllBlasTmp, pBlasHandle, m_helStreams, false, m_gpublocks, m_gputhreads ); -#ifdef MGONGPU_CHANNELID_DEBUG - //std::cout << "DEBUG: MatrixElementKernelDevice::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; - copyHostFromDevice( m_hstChannelIds, m_channelIds ); // FIXME?! - const unsigned int* pHstChannelIds = ( useChannelIds ? m_hstChannelIds.data() : nullptr ); - MatrixElementKernelBase::updateNevtProcessedByChannel( pHstChannelIds, nevt() ); -#endif - checkGpu( gpuPeekAtLastError() ); // is this needed? - checkGpu( gpuDeviceSynchronize() ); // probably not needed? but it avoids errors in sigmaKin above from appearing later on in random places... - } - - //-------------------------------------------------------------------------- - -} -#endif //============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h index 01041e43b7..9cfb1c6247 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h @@ -16,11 +16,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -109,7 +105,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating matrix element calculations on a CPU host class MatrixElementKernelHost final : public MatrixElementKernelBase, public NumberOfEvents { @@ -156,94 +151,9 @@ namespace mg5amcCpu // The buffer for the event-by-event denominators of multichannel factors HostBufferDenominators m_denominators; }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating matrix element calculations on a GPU device - class MatrixElementKernelDevice : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~MatrixElementKernelDevice(); - - // Reset gpublocks and gputhreads - void setGrid( const int gpublocks, const int gputhreads ); - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The buffer for the event-by-event couplings that depends on alphas QCD - DeviceBufferCouplings m_couplings; - - // The super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelMEs; - - // The super-buffer of nGoodHel jamp buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelJamps; - - // The super-buffer of nGoodHel numerator buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelNumerators; - - // The super-buffer of nGoodHel denominator buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelDenominators; - - // The super-buffer of ncolor jamp2 buffers - DeviceBufferSimple m_colJamp2s; - -#ifdef MGONGPU_CHANNELID_DEBUG - // The **host** buffer for the channelId array - // FIXME? MEKD should accept a host buffer as an argument instead of a device buffer, so that a second copy can be avoided? - PinnedHostBufferChannelIds m_hstChannelIds; -#endif - -#ifndef MGONGPU_HAS_NO_BLAS - // Decide at runtime whether to use BLAS for color sums - bool m_blasColorSum; - - // Decide at runtime whether TF32TENSOR math should be used in cuBLAS - bool m_blasTf32Tensor; - - // The super-buffer of nGoodHel cuBLAS/hipBLAS temporary buffers - std::unique_ptr m_pHelBlasTmp; - - // The cuBLAS/hipBLAS handle (a single one for all good helicities) - gpuBlasHandle_t m_blasHandle; -#endif - - // The array of GPU streams (one for each good helicity) - gpuStream_t m_helStreams[CPPProcess::ncomb]; // reserve ncomb streams (but only nGoodHel <= ncomb will be used) - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h index f41e40bf82..9cd0e794c0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h @@ -16,11 +16,7 @@ #define MGONGPU_TRIVIAL_AMPLITUDES 1 // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h index 331e7f12c3..afdca9d788 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h @@ -14,11 +14,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -103,14 +99,10 @@ namespace mg5amcCpu kernelAccessConst( const unsigned int* buffer ) { const unsigned int& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) static_assert( mg5amcCpu::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } }; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h index fd3fb80c65..faabc3204e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h @@ -16,11 +16,7 @@ #include "MemoryBuffers.h" // for HostBufferCouplings::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -189,9 +185,6 @@ namespace mg5amcCpu const int ix2 ) { fptype& out = kernelAccessIx2_s( buffer, ix2 ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS @@ -199,7 +192,6 @@ namespace mg5amcCpu static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h index 757de7b6f2..fd1e1abadf 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h @@ -15,11 +15,7 @@ //#include "MemoryAccessHelpers.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h index 205952e514..01611fe92d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h @@ -10,11 +10,7 @@ #include "MemoryAccessGs.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h index bfcb3e3222..484a48116a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h @@ -14,11 +14,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -119,14 +115,10 @@ namespace mg5amcCpu kernelAccess( fptype* buffer ) { fptype& out = kernelAccess_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) @@ -148,14 +140,10 @@ namespace mg5amcCpu kernelAccessConst( const fptype* buffer ) { const fptype& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } }; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h index 8b5bd27f4d..525196d770 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h @@ -106,13 +106,7 @@ class KernelAccessHelper : public MemoryAccessHelper } else { -#ifdef MGONGPUCPP_GPUIMPL - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid - //printf( "kernelAccessRecord: ievt=%d threadId=%d\n", ievt, threadIdx.x ); - return T::ieventAccessRecord( buffer, ievt ); // NB fptype and fptype_sv coincide for CUDA -#else throw std::runtime_error( "kernelAccessRecord on device is only implemented in CUDA" ); -#endif } } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h index ffc74b0f0c..fcd1d2d724 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h @@ -13,11 +13,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -102,14 +98,10 @@ namespace mg5amcCpu kernelAccessConst( const unsigned int* buffer ) { const unsigned int& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) static_assert( mg5amcCpu::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } }; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h index 4236e20602..8f18847d56 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h @@ -14,11 +14,7 @@ #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -119,14 +115,10 @@ namespace mg5amcCpu kernelAccess( fptype* buffer ) { fptype& out = kernelAccess_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) static_assert( mg5amcCpu::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h index 3ef4d76fbd..92811a2ddf 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h @@ -14,11 +14,7 @@ #include "MemoryAccessVectors.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- @@ -31,32 +27,17 @@ namespace mg5amcCpu // Number of Events Per Page in the momenta AOSOA memory buffer layout // (these are all best kept as a compile-time constants: see issue #23) -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - // ----------------------------------------------------------------------------------------------- - // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline - // --- This is relevant to ensure coalesced access to momenta in global memory - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------------- - //static constexpr int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) - static constexpr int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) - //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 1.03E9 instead of 1.11E9 in eemumu) -#else // ----------------------------------------------------------------------------------------------- // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register // --- This is relevant to ensure faster access to momenta from C++ memory cache lines // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded // --- In practice, neppR, neppM and neppV could now (in principle) all be different // ----------------------------------------------------------------------------------------------- -#ifdef MGONGPU_CPPSIMD static constexpr int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance //static constexpr int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) //static constexpr int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 4.66E6 instead of 5.09E9 in eemumu) //static constexpr int neppM = MGONGPU_CPPSIMD*2; // FOR TESTS -#else - static constexpr int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) -#endif -#endif /* clang-format on */ // SANITY CHECK: check that neppM is a power of two static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); @@ -208,9 +189,6 @@ namespace mg5amcCpu const int ipar ) { const fptype& out = kernelAccessIp4IparConst_s( buffer, ip4, ipar ); -#ifndef MGONGPU_CPPSIMD - return out; -#else constexpr int neppM = MemoryAccessMomentaBase::neppM; constexpr bool useContiguousEventsIfPossible = true; // DEFAULT //constexpr bool useContiguousEventsIfPossible = false; // FOR PERFORMANCE TESTS (treat as arbitrary array even if it is an AOSOA) @@ -252,7 +230,6 @@ namespace mg5amcCpu { return MemoryAccessMomenta::ieventAccessIp4IparConst( buffer, ievt0 + ieppV, ip4, ipar ); }; return mg5amcCpu::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) } -#endif } // Is this a HostAccess or DeviceAccess class? diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h index c8d0343b7e..42356061e7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h @@ -10,11 +10,7 @@ #include "MemoryAccessGs.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h index 57f1d02081..ebdffe9c84 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h @@ -12,11 +12,7 @@ #include "CPPProcess.h" #include "MemoryAccessHelpers.h" -#ifdef MGONGPUCPP_GPUIMPL -using mg5amcGpu::CPPProcess; -#else using mg5amcCpu::CPPProcess; -#endif //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h index 8563d8f2f7..628ae29cc4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h @@ -11,11 +11,9 @@ #include "mgOnGpuVectors.h" -#ifndef MGONGPUCPP_GPUIMPL namespace mg5amcCpu // this is only needed for CPU SIMD vectorization { -#ifdef MGONGPU_CPPSIMD //-------------------------------------------------------------------------- // Cast one non-const fptype_v reference (one vector of neppV fptype values) from one non-const fptype reference (#435), @@ -46,43 +44,8 @@ namespace mg5amcCpu // this is only needed for CPU SIMD vectorization // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", but that the arrays are not aligned inline fptype_v fptypevFromUnalignedArray( const fptype& ref ) { -#if MGONGPU_CPPSIMD == 2 return fptype_v{ *( &ref ), // explicit initialization of all array elements (2) *( &ref + 1 ) }; -#elif MGONGPU_CPPSIMD == 4 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (4) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ) }; -#elif MGONGPU_CPPSIMD == 8 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (8) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ), - *( &ref + 4 ), - *( &ref + 5 ), - *( &ref + 6 ), - *( &ref + 7 ) }; -#elif MGONGPU_CPPSIMD == 16 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (16) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ), - *( &ref + 4 ), - *( &ref + 5 ), - *( &ref + 6 ), - *( &ref + 7 ), - *( &ref + 8 ), - *( &ref + 9 ), - *( &ref + 10 ), - *( &ref + 11 ), - *( &ref + 12 ), - *( &ref + 13 ), - *( &ref + 14 ), - *( &ref + 15 ) }; -#else -#error Internal error! Unknown MGONGPU_CPPSIMD value -#endif } // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, @@ -90,49 +53,12 @@ namespace mg5amcCpu // this is only needed for CPU SIMD vectorization template inline fptype_v fptypevFromArbitraryArray( Functor decoderIeppv ) { -#if MGONGPU_CPPSIMD == 2 return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (2) decoderIeppv( 1 ) }; -#elif MGONGPU_CPPSIMD == 4 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (4) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ) }; -#elif MGONGPU_CPPSIMD == 8 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (8) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ), - decoderIeppv( 4 ), - decoderIeppv( 5 ), - decoderIeppv( 6 ), - decoderIeppv( 7 ) }; -#elif MGONGPU_CPPSIMD == 16 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (16) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ), - decoderIeppv( 4 ), - decoderIeppv( 5 ), - decoderIeppv( 6 ), - decoderIeppv( 7 ), - decoderIeppv( 8 ), - decoderIeppv( 9 ), - decoderIeppv( 10 ), - decoderIeppv( 11 ), - decoderIeppv( 12 ), - decoderIeppv( 13 ), - decoderIeppv( 14 ), - decoderIeppv( 15 ) }; -#else -#error Internal error! Unknown MGONGPU_CPPSIMD value -#endif } //-------------------------------------------------------------------------- -#endif } // end namespace -#endif #endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h index 14a37c7fea..672a66bc09 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h @@ -16,11 +16,7 @@ #define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h index c56527f581..0a82fe49d1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h @@ -12,11 +12,7 @@ #include "MemoryAccessHelpers.h" // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //---------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h index d259aa7456..c229fc0a56 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h @@ -18,11 +18,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -90,7 +86,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL constexpr bool HostBufferALIGNED = false; // ismisaligned=false constexpr bool HostBufferMISALIGNED = true; // ismisaligned=true @@ -118,51 +113,15 @@ namespace mg5amcCpu public: static constexpr size_t cppAlign = mgOnGpu::cppAlign; }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA pinned host buffer - template - class PinnedHostBufferBase : public BufferBase - { - public: - PinnedHostBufferBase( const size_t size ) - : BufferBase( size, false ) - { - gpuMallocHost( &( this->m_data ), this->bytes() ); - } - virtual ~PinnedHostBufferBase() - { - gpuFreeHost( this->m_data ); - } - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA device buffer - template - class DeviceBufferBase : public BufferBase - { - public: - DeviceBufferBase( const size_t size ) - : BufferBase( size, true ) - { - gpuMalloc( &( this->m_data ), this->bytes() ); - } - virtual ~DeviceBufferBase() - { - gpuFree( this->m_data ); - } - }; -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for a given number of events template class HostBuffer : public HostBufferBase, virtual private NumberOfEvents @@ -180,53 +139,15 @@ namespace mg5amcCpu } virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA pinned host buffer for a given number of events - template - class PinnedHostBuffer : public PinnedHostBufferBase, virtual private NumberOfEvents - { - public: - PinnedHostBuffer( const size_t nevt ) - : NumberOfEvents( nevt ) - , PinnedHostBufferBase( sizePerEvent * nevt ) {} - virtual ~PinnedHostBuffer() {} - virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA device buffer for a given number of events - template - class DeviceBuffer : public DeviceBufferBase, virtual protected NumberOfEvents - { - public: - DeviceBuffer( const size_t nevt ) - : NumberOfEvents( nevt ) - , DeviceBufferBase( sizePerEvent * nevt ) - { - //std::cout << "DeviceBuffer::ctor " << this << " " << nevt << std::endl; - } - virtual ~DeviceBuffer() - { - //std::cout << "DeviceBuffer::dtor " << this << std::endl; - } - virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a simple CUDA device buffer managed on an ad-hoc basis - typedef DeviceBuffer DeviceBufferSimple; - typedef DeviceBuffer DeviceBufferSimple2; -#endif //-------------------------------------------------------------------------- @@ -236,15 +157,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for momenta random numbers constexpr size_t sizePerEventRndNumMomenta = MemoryBuffers::np4 * MemoryBuffers::nparf; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for momenta random numbers typedef HostBuffer HostBufferRndNumMomenta; -#else - // A class encapsulating a CUDA pinned host buffer for momenta random numbers - typedef PinnedHostBuffer PinnedHostBufferRndNumMomenta; - // A class encapsulating a CUDA device buffer for momenta random numbers - typedef DeviceBuffer DeviceBufferRndNumMomenta; -#endif //-------------------------------------------------------------------------- @@ -280,15 +194,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for Gs constexpr size_t sizePerEventGs = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for gs typedef HostBuffer HostBufferGs; -#else - // A class encapsulating a CUDA pinned host buffer for gs - typedef PinnedHostBuffer PinnedHostBufferGs; - // A class encapsulating a CUDA device buffer for gs - typedef DeviceBuffer DeviceBufferGs; -#endif //-------------------------------------------------------------------------- @@ -299,15 +206,8 @@ namespace mg5amcCpu // (should be equal to the number of diagrams in the process) constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for numerators typedef HostBuffer HostBufferNumerators; -#else - // A class encapsulating a CUDA pinned host buffer for numerators - typedef PinnedHostBuffer PinnedHostBufferNumerators; - // A class encapsulating a CUDA device buffer for numerators - typedef DeviceBuffer DeviceBufferNumerators; -#endif //-------------------------------------------------------------------------- @@ -317,15 +217,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for denominators constexpr size_t sizePerEventDenominators = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for denominators typedef HostBuffer HostBufferDenominators; -#else - // A class encapsulating a CUDA pinned host buffer for denominators - typedef PinnedHostBuffer PinnedHostBufferDenominators; - // A class encapsulating a CUDA device buffer for denominators - typedef DeviceBuffer DeviceBufferDenominators; -#endif //-------------------------------------------------------------------------- @@ -335,15 +228,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for random numbers constexpr size_t sizePerEventCouplings = MemoryBuffers::ndcoup * MemoryBuffers::nx2; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for couplings typedef HostBuffer HostBufferCouplings; -#else - // A class encapsulating a CUDA pinned host buffer for couplings - typedef PinnedHostBuffer PinnedHostBufferCouplings; - // A class encapsulating a CUDA device buffer for couplings - typedef DeviceBuffer DeviceBufferCouplings; -#endif //-------------------------------------------------------------------------- @@ -353,16 +239,9 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for momenta constexpr size_t sizePerEventMomenta = MemoryBuffers::np4 * MemoryBuffers::npar; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for momenta typedef HostBuffer HostBufferMomenta; //typedef HostBuffer HostBufferMomenta; // TEST MISALIGNMENT! -#else - // A class encapsulating a CUDA pinned host buffer for momenta - typedef PinnedHostBuffer PinnedHostBufferMomenta; - // A class encapsulating a CUDA device buffer for momenta - typedef DeviceBuffer DeviceBufferMomenta; -#endif //-------------------------------------------------------------------------- @@ -372,15 +251,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for sampling weights constexpr size_t sizePerEventWeights = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for sampling weights typedef HostBuffer HostBufferWeights; -#else - // A class encapsulating a CUDA pinned host buffer for sampling weights - typedef PinnedHostBuffer PinnedHostBufferWeights; - // A class encapsulating a CUDA device buffer for sampling weights - typedef DeviceBuffer DeviceBufferWeights; -#endif //-------------------------------------------------------------------------- @@ -390,30 +262,16 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for matrix elements constexpr size_t sizePerEventMatrixElements = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for matrix elements typedef HostBuffer HostBufferMatrixElements; -#else - // A class encapsulating a CUDA pinned host buffer for matrix elements - typedef PinnedHostBuffer PinnedHostBufferMatrixElements; - // A class encapsulating a CUDA device buffer for matrix elements - typedef DeviceBuffer DeviceBufferMatrixElements; -#endif //-------------------------------------------------------------------------- // A base class encapsulating a memory buffer for the helicity mask typedef BufferBase BufferHelicityMask; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for the helicity mask typedef HostBufferBase HostBufferHelicityMask; -#else - // A class encapsulating a CUDA pinned host buffer for the helicity mask - typedef PinnedHostBufferBase PinnedHostBufferHelicityMask; - // A class encapsulating a CUDA device buffer for the helicity mask - typedef DeviceBufferBase DeviceBufferHelicityMask; -#endif //-------------------------------------------------------------------------- @@ -423,15 +281,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for wavefunctions constexpr size_t sizePerEventWavefunctions = MemoryBuffers::nw6 * MemoryBuffers::nx2; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for wavefunctions typedef HostBuffer HostBufferWavefunctions; -#else - // A class encapsulating a CUDA pinned host buffer for wavefunctions - typedef PinnedHostBuffer PinnedHostBufferWavefunctions; - // A class encapsulating a CUDA device buffer for wavefunctions - typedef DeviceBuffer DeviceBufferWavefunctions; -#endif //-------------------------------------------------------------------------- @@ -441,15 +292,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for helicity random numbers constexpr size_t sizePerEventRndNumHelicity = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for helicity random numbers typedef HostBuffer HostBufferRndNumHelicity; -#else - // A class encapsulating a CUDA pinned host buffer for helicity random numbers - typedef PinnedHostBuffer PinnedHostBufferRndNumHelicity; - // A class encapsulating a CUDA device buffer for helicity random numbers - typedef DeviceBuffer DeviceBufferRndNumHelicity; -#endif //-------------------------------------------------------------------------- @@ -459,15 +303,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for color random numbers constexpr size_t sizePerEventRndNumColor = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for color random numbers typedef HostBuffer HostBufferRndNumColor; -#else - // A class encapsulating a CUDA pinned host buffer for color random numbers - typedef PinnedHostBuffer PinnedHostBufferRndNumColor; - // A class encapsulating a CUDA device buffer for color random numbers - typedef DeviceBuffer DeviceBufferRndNumColor; -#endif //-------------------------------------------------------------------------- @@ -477,15 +314,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for channel ids constexpr size_t sizePerEventChannelId = 1; -#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) // A class encapsulating a C++ host buffer for channel ids typedef HostBuffer HostBufferChannelIds; -#else - // A class encapsulating a CUDA pinned host buffer for channel ids - typedef PinnedHostBuffer PinnedHostBufferChannelIds; - // A class encapsulating a CUDA device buffer for channel ids - typedef DeviceBuffer DeviceBufferChannelIds; -#endif //-------------------------------------------------------------------------- @@ -495,15 +325,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for channel ids constexpr size_t sizePerEventIflavorVec = 1; -#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) // A class encapsulating a C++ host buffer for channel ids typedef HostBuffer HostBufferIflavorVec; -#else - // A class encapsulating a CUDA pinned host buffer for channel ids - typedef PinnedHostBuffer PinnedHostBufferIflavorVec; - // A class encapsulating a CUDA device buffer for channel ids - typedef DeviceBuffer DeviceBufferIflavorVec; -#endif //-------------------------------------------------------------------------- @@ -513,15 +336,8 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for helicity selection constexpr size_t sizePerEventSelectedHelicity = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for helicity selection typedef HostBuffer HostBufferSelectedHelicity; -#else - // A class encapsulating a CUDA pinned host buffer for helicity selection - typedef PinnedHostBuffer PinnedHostBufferSelectedHelicity; - // A class encapsulating a CUDA device buffer for helicity selection - typedef DeviceBuffer DeviceBufferSelectedHelicity; -#endif //-------------------------------------------------------------------------- @@ -531,71 +347,17 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for color selection constexpr size_t sizePerEventSelectedColor = 1; -#ifndef MGONGPUCPP_GPUIMPL // A class encapsulating a C++ host buffer for color selection typedef HostBuffer HostBufferSelectedColor; -#else - // A class encapsulating a CUDA pinned host buffer for color selection - typedef PinnedHostBuffer PinnedHostBufferSelectedColor; - // A class encapsulating a CUDA device buffer for color selection - typedef DeviceBuffer DeviceBufferSelectedColor; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // The size (number of elements) per event in a memory buffer for jamps - constexpr size_t sizePerEventJamps = MemoryBuffers::ncolor * MemoryBuffers::nx2; - - // A class encapsulating a CUDA device buffer for color selection - typedef DeviceBuffer DeviceBufferJamps; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - template - void copyDeviceFromHost( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy - { - if( dst.size() != src.size() ) - { - std::ostringstream sstr; - sstr << "Size (#elements) mismatch in copyDeviceFromHost: dst=" << dst.size() << ", src=" << src.size(); - throw std::runtime_error( sstr.str() ); - } - if( dst.bytes() != src.bytes() ) - { - std::ostringstream sstr; - sstr << "Size (#bytes) mismatch in copyDeviceFromHost: dst=" << dst.bytes() << ", src=" << src.bytes(); - throw std::runtime_error( sstr.str() ); - } - // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array - gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyHostToDevice ); - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - template - void copyHostFromDevice( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy - { - if( dst.size() != src.size() ) - { - std::ostringstream sstr; - sstr << "Size (#elements) mismatch in copyHostFromDevice: dst=" << dst.size() << ", src=" << src.size(); - throw std::runtime_error( sstr.str() ); - } - if( dst.bytes() != src.bytes() ) - { - std::ostringstream sstr; - sstr << "Size (#bytes) mismatch in copyHostFromDevice: dst=" << dst.bytes() << ", src=" << src.bytes(); - throw std::runtime_error( sstr.str() ); - } - // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array - gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyDeviceToHost ); - } -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc index d40b1b4174..e1e17d6247 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc @@ -16,11 +16,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -155,92 +151,18 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - MasslessRamboSamplingKernelDevice::MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t gpublocks, - const size_t gputhreads ) - : SamplingKernelBase( energy, rndmom, momenta, weights ) - , NumberOfEvents( gpublocks * gputhreads ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if( !m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: rndmom must be a device array" ); - if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: momenta must be a device array" ); - if( !m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: weights must be a device array" ); - if( m_gpublocks == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gputhreads must be > 0" ); - if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with rndmom" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with momenta" ); - if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with weights" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( m_gputhreads % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelHost: gputhreads should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Sanity checks for memory access (random number buffer) - constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout - static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); - if( m_gputhreads % neppR != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelDevice: gputhreads should be a multiple of neppR=" << neppR; - throw std::runtime_error( sstr.str() ); - } - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - getMomentaInitialDevice( const fptype energy, - fptype* momenta ) - { - constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; - return getMomentaInitial( energy, momenta ); - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void - MasslessRamboSamplingKernelDevice::getMomentaInitial() - { - gpuLaunchKernel( getMomentaInitialDevice, m_gpublocks, m_gputhreads, m_energy, m_momenta.data() ); - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - getMomentaFinalDevice( const fptype energy, - const fptype* rndmom, - fptype* momenta, - fptype* wgts ) - { - constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; - return getMomentaFinal( energy, rndmom, momenta, wgts ); - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void - MasslessRamboSamplingKernelDevice::getMomentaFinal() - { - gpuLaunchKernel( getMomentaFinalDevice, m_gpublocks, m_gputhreads, m_energy, m_rndmom.data(), m_momenta.data(), m_weights.data() ); - } -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h index 6244d55618..e0aa2aea7f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h @@ -13,11 +13,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- @@ -132,41 +128,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating RAMBO phase space sampling on a GPU device - class MasslessRamboSamplingKernelDevice final : public SamplingKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~MasslessRamboSamplingKernelDevice() {} - - // Get momenta of initial state particles - void getMomentaInitial() override final; - - // Get momenta of final state particles and weights - void getMomentaFinal() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h index 0fb880bb36..480824a7e5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h @@ -18,11 +18,7 @@ struct curandGenerator_st; struct rocrand_generator_base_type; typedef rocrand_generator_base_type hiprandGenerator_st; -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc index 68e93edb50..ebce23a97b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc @@ -56,11 +56,7 @@ namespace { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif // Fixed physics inputs fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV @@ -203,17 +199,6 @@ namespace } } -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - aosoa_to_umami_kernel( const fptype* aosoa, - double* soa, - std::size_t nevt ) - { - std::size_t ievt = blockDim.x * blockIdx.x + threadIdx.x; - if( ievt >= nevt ) return; - aosoa_to_umami_one( aosoa, soa, ievt, nevt ); - } -#endif const char* backend_label() { @@ -284,29 +269,17 @@ namespace unsigned int nevt, mgOnGpu::TimerMap& timermap, double& wavetime, -#ifdef MGONGPUCPP_GPUIMPL - const DeviceBufferBase& devUmamiMomenta, - const DeviceBufferBase& devFlv, - DeviceBufferBase& devUmamiMEs, - std::vector& hstMEs -#else const std::vector& umamiMomenta, const std::vector& flvVec, std::vector& umamiMEs -#endif ) { constexpr unsigned int UmamiInKeyNum = 2; timermap.start( "3a SigmaKin" ); UmamiInputKey in_keys[UmamiInKeyNum] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX }; UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; -#ifdef MGONGPUCPP_GPUIMPL - const void* inputs[UmamiInKeyNum] = { devUmamiMomenta.data(), devFlv.data() }; - void* outputs[1] = { devUmamiMEs.data() }; -#else const void* inputs[UmamiInKeyNum] = { umamiMomenta.data(), flvVec.data() }; void* outputs[1] = { umamiMEs.data() }; -#endif UmamiStatus st = umami_matrix_element( handle, nevt, nevt, 0, UmamiInKeyNum, in_keys, inputs, 1, out_keys, outputs ); wavetime += timermap.stop(); @@ -316,11 +289,6 @@ namespace return false; } -#ifdef MGONGPUCPP_GPUIMPL - timermap.start( "3b CpDTHmes" ); - gpuMemcpy( hstMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); - wavetime += timermap.stop(); -#endif return true; } @@ -593,30 +561,12 @@ namespace mgOnGpu::TimerMap timermap; -#ifdef MGONGPUCPP_GPUIMPL - timermap.start( "00 GpuInit" ); - GpuRuntime gpuRuntime( false ); - - PinnedHostBufferRndNumMomenta hstRndmom( nevt ); - PinnedHostBufferMomenta hstMomenta( nevt ); - PinnedHostBufferWeights hstWeights( nevt ); - DeviceBufferRndNumMomenta devRndmom( nevt ); - DeviceBufferMomenta devMomenta( nevt ); - DeviceBufferWeights devWeights( nevt ); - DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - DeviceBufferBase devUmamiMEs( nevt ); - DeviceBufferBase devFlv( nevt ); - std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - std::vector flvVec( nevt ); - std::vector hstUmamiMEs( nevt ); -#else HostBufferRndNumMomenta hstRndmom( nevt ); HostBufferMomenta hstMomenta( nevt ); HostBufferWeights hstWeights( nevt ); std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); std::vector umamiMEs( nevt ); std::vector flvVec( nevt ); -#endif UmamiHandle umami_handle = nullptr; if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) @@ -656,10 +606,6 @@ namespace SLHAReader slha( "../../Cards/param_card.dat", false ); const double alphaS = slha.get_block_entry( "sminputs", 3, 1.180000e-01 ); std::vector alphasVec( nevt, alphaS ); -#ifdef MGONGPUCPP_GPUIMPL - DeviceBufferBase devAlphaS( nevt ); - gpuMemcpy( devAlphaS.data(), alphasVec.data(), nevt * sizeof( double ), gpuMemcpyHostToDevice ); -#endif // Always massive RAMBO std::unique_ptr prsk( @@ -673,16 +619,8 @@ namespace for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) for( unsigned int ievt = 0; ievt < nevt; ++ievt ) umamiMomenta[(std::size_t)ip4 * CPPProcess::npar * nevt + (std::size_t)ipar * nevt + ievt] = point[ipar][ip4]; -#ifdef MGONGPUCPP_GPUIMPL - gpuMemcpy( devUmamiMomenta.data(), umamiMomenta.data(), umamiMomenta.size() * sizeof( double ), gpuMemcpyHostToDevice ); - // Host only implementation now (copy) - copyDeviceFromHost( devMomenta, hstMomenta ); - gpuLaunchKernel( aosoa_to_umami_kernel, kMatrixBlocks, kMatrixThreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); - checkGpu( gpuPeekAtLastError() ); -#else for( std::size_t ievt = 0; ievt < nevt; ++ievt ) aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); -#endif if( verbose ) { @@ -710,19 +648,11 @@ namespace for( unsigned int iflav = 0; iflav < nFlavors; ++iflav ) { std::fill( flvVec.begin(), flvVec.end(), iflav ); -#ifdef MGONGPUCPP_GPUIMPL - gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); -#endif timermap.start( "3a SigmaKin" ); UmamiInputKey in_keys[3] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S }; UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; -#ifdef MGONGPUCPP_GPUIMPL - const void* inputs[3] = { devUmamiMomenta.data(), devFlv.data(), devAlphaS.data() }; - void* outputs[1] = { devUmamiMEs.data() }; -#else const void* inputs[3] = { umamiMomenta.data(), flvVec.data(), alphasVec.data() }; void* outputs[1] = { umamiMEs.data() }; -#endif UmamiStatus st = umami_matrix_element( umami_handle, nevt, nevt, 0, 3, in_keys, inputs, 1, out_keys, outputs ); timermap.stop(); @@ -732,12 +662,7 @@ namespace umami_free( umami_handle ); return 3; } -#ifdef MGONGPUCPP_GPUIMPL - gpuMemcpy( hstUmamiMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); - const double* mes = hstUmamiMEs.data(); -#else const double* mes = umamiMEs.data(); -#endif std::cout << " PDG"; for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) @@ -780,31 +705,12 @@ namespace mgOnGpu::TimerMap timermap; -#ifdef MGONGPUCPP_GPUIMPL - timermap.start( "00 GpuInit" ); - GpuRuntime gpuRuntime( false ); - - PinnedHostBufferRndNumMomenta hstRndmom( nevt ); - PinnedHostBufferMomenta hstMomenta( nevt ); - PinnedHostBufferWeights hstWeights( nevt ); - DeviceBufferRndNumMomenta devRndmom( nevt ); - DeviceBufferMomenta devMomenta( nevt ); - DeviceBufferWeights devWeights( nevt ); - DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - DeviceBufferBase devUmamiMEs( nevt ); - DeviceBufferBase devFlv( nevt ); - std::vector flvVec( nevt, flavorID ); - std::vector hstUmamiMEs( nevt ); - // perf-mode runs a single flavor, so the device-side flavor buffer is filled once. - gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); -#else HostBufferRndNumMomenta hstRndmom( nevt ); HostBufferMomenta hstMomenta( nevt ); HostBufferWeights hstWeights( nevt ); std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); std::vector umamiMEs( nevt ); std::vector flvVec( nevt, flavorID ); -#endif std::unique_ptr prnk( new CommonRandomNumberKernel( hstRndmom ) ); @@ -845,11 +751,7 @@ namespace } else { -#ifdef MGONGPUCPP_GPUIMPL - prsk.reset( new MasslessRamboSamplingKernelDevice( kEnergy, devRndmom, devMomenta, devWeights, gpublocks, gputhreads ) ); -#else prsk.reset( new MasslessRamboSamplingKernelHost( kEnergy, hstRndmom, hstMomenta, hstWeights, nevt ) ); -#endif } std::unique_ptr genrtimes( new double[niter] ); @@ -877,14 +779,6 @@ namespace timermap.start( "1b GenRnGen" ); prnk->generateRnarray(); genrtime += timermap.stop(); -#ifdef MGONGPUCPP_GPUIMPL - if( ramboType == RAMBO_MASSLESS ) - { - timermap.start( "1c CpHTDrnd" ); - copyDeviceFromHost( devRndmom, hstRndmom ); - genrtime += timermap.stop(); - } -#endif timermap.start( "2a RamboIni" ); prsk->getMomentaInitial(); @@ -892,15 +786,6 @@ namespace timermap.start( "2b RamboFin" ); prsk->getMomentaFinal(); rambtime += timermap.stop(); -#ifdef MGONGPUCPP_GPUIMPL - // Massive host only (copy) - if( ramboType != RAMBO_MASSLESS ) - { - timermap.start( "2c CpHTDmom" ); - copyDeviceFromHost( devMomenta, hstMomenta ); - rambtime += timermap.stop(); - } -#endif } else { @@ -919,47 +804,23 @@ namespace MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar ) = (fptype)ev[ipar][ip4]; } rambtime += timermap.stop(); -#ifdef MGONGPUCPP_GPUIMPL - timermap.start( "2c CpHTDmom" ); - copyDeviceFromHost( devMomenta, hstMomenta ); - rambtime += timermap.stop(); -#endif } timermap.start( "2d Aosoa2U " ); -#ifdef MGONGPUCPP_GPUIMPL - gpuLaunchKernel( aosoa_to_umami_kernel, gpublocks, gputhreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); - checkGpu( gpuPeekAtLastError() ); -#else for( std::size_t ievt = 0; ievt < nevt; ++ievt ) aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); -#endif rambtime += timermap.stop(); double wavetime = 0; if( !run_umami( umami_handle, nevt, timermap, wavetime, -#ifdef MGONGPUCPP_GPUIMPL - devUmamiMomenta, devFlv, devUmamiMEs, hstUmamiMEs -#else umamiMomenta, flvVec, umamiMEs -#endif ) ) { umami_free( umami_handle ); return 3; } -#ifdef MGONGPUCPP_GPUIMPL - if( verbose ) - { - timermap.start( "3c CpDTHmom" ); - copyHostFromDevice( hstMomenta, devMomenta ); - wavetime += timermap.stop(); - } - const double* mes = hstUmamiMEs.data(); -#else const double* mes = umamiMEs.data(); -#endif timermap.start( "4@ UpdtStat" ); for( unsigned int ievt = 0; ievt < nreal; ++ievt ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc index 30c6799932..2cbe2e38cc 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc @@ -10,11 +10,7 @@ #include "MemoryAccessMatrixElements.h" -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors @@ -23,42 +19,12 @@ namespace mg5amcCpu // *** COLOR MATRIX BELOW *** %(color_matrix_lines)s -#ifdef MGONGPUCPP_GPUIMPL - // The normalized color matrix (divide each column by denom) - template - struct NormalizedColorMatrix - { - constexpr __host__ __device__ NormalizedColorMatrix() - : value() - { - for( int icol = 0; icol < ncolor; icol++ ) - for( int jcol = 0; jcol < ncolor; jcol++ ) - value[icol * ncolor + jcol] = colorMatrix[icol][jcol] / colorDenom[icol]; - } - T value[ncolor * ncolor]; - }; - // The fptype2 version is the default used by kernels (supporting mixed floating point mode also in blas) - static __device__ fptype2 s_pNormalizedColorMatrix2[ncolor * ncolor]; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void createNormalizedColorMatrix() - { - static bool first = true; - if( first ) - { - first = false; - constexpr NormalizedColorMatrix normalizedColorMatrix2; - gpuMemcpyToSymbol( s_pNormalizedColorMatrix2, normalizedColorMatrix2.value, ncolor * ncolor * sizeof( fptype2 ) ); - } - } -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL void color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity @@ -150,284 +116,21 @@ namespace mg5amcCpu MEs_sv_next += deltaMEs_next; #endif } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity - const int nGoodHel, // input: number of good helicities - const int nevtIfAllHelicities ) // input: zero in single-helicity mode, number of events in multi-helicity mode - { - if (nevtIfAllHelicities) { - int ighel = blockIdx.y; - allMEs = allMEs + ighel * nevtIfAllHelicities; // MEs for one specific helicity ighel - allJamps = allJamps + ighel * nevtIfAllHelicities; // Jamps for one specific helicity ighel - } - using J_ACCESS = DeviceAccessJamp; - fptype jampR[ncolor]; - fptype jampI[ncolor]; - for( int icol = 0; icol < ncolor; icol++ ) - { - constexpr int ihel0 = 0; // the input buffer allJamps already points to a specific helicity - cxtype jamp = J_ACCESS::kernelAccessIcolIhelNhelConst( allJamps, icol, ihel0, nGoodHel ); - jampR[icol] = jamp.real(); - jampI[icol] = jamp.imag(); - } - // Loop over icol - fptype deltaMEs = { 0 }; - for( int icol = 0; icol < ncolor; icol++ ) - { - fptype2 ztempR = { 0 }; - fptype2 ztempI = { 0 }; - fptype2 jampRi = jampR[icol]; - fptype2 jampIi = jampI[icol]; - // OLD IMPLEMENTATION (ihel3: symmetric square matrix) - Loop over all jcol - //for( int jcol = 0; jcol < ncolor; jcol++ ) - //{ - // fptype2 jampRj = jampR[jcol]; - // fptype2 jampIj = jampI[jcol]; - // ztempR += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix - // ztempI += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix - //} - // NEW IMPLEMENTATION #475 (ihel3p1: triangular lower diagonal matrix) - Loop over jcol < icol - ztempR += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampRi; // use fptype2 version of color matrix - ztempI += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampIi; // use fptype2 version of color matrix - for( int jcol = 0; jcol < icol; jcol++ ) - { - fptype2 jampRj = jampR[jcol]; - fptype2 jampIj = jampI[jcol]; - ztempR += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix - ztempI += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix - } - deltaMEs += ztempR * jampRi; - deltaMEs += ztempI * jampIi; - } - // *** STORE THE RESULTS *** - using E_ACCESS = DeviceAccessMatrixElements; // non-trivial access: buffer includes all events - // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) - E_ACCESS::kernelAccess( allMEs ) += deltaMEs; // fix #435 - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -#ifndef MGONGPU_HAS_NO_BLAS -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - __global__ void - convertD2F_Jamps( fptype2* allJampsFpt2, // output: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel - const fptype* allJamps, // input: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel - const int nhel ) // input: number of good helicities nGoodHel - { - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - constexpr int ihel = 0; // the input buffer allJamps already points to a specific helicity - // NB! From a functional point of view, any striding will be ok here as long as ncolor*2*nevt elements are all correctly copied! - // NB! Just in case this may be better for performance reasons, however, the same striding as in compute_jamps and cuBLAS is used here - for( int ix2 = 0; ix2 < mgOnGpu::nx2; ix2++ ) - for( int icol = 0; icol < ncolor; icol++ ) - allJampsFpt2[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] = - allJamps[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt]; - } -#endif -#endif -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -#ifndef MGONGPU_HAS_NO_BLAS -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - __global__ void - convertF2D_MEs( fptype* allMEs, // output: allMEs[nevt] for one specific helicity - const fptype2* allMEsFpt2 ) // input: allMEs[nevt] for one specific helicity - { - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - allMEs[ievt] = allMEsFpt2[ievt]; - } -#endif -#endif -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ -#ifndef MGONGPU_HAS_NO_BLAS - void - color_sum_blas( fptype* ghelAllMEs, // output: allMEs super-buffer[nhel][nevt], add |M|^2 separately for each helicity - const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nhel][nevt] for nhel good helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nhel good helicities - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) -#else - gpuStream_t* /*ghelStreams*/, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) -#endif - const int nhel, // input: number of good helicities (nhel == nGoodHel) - const int gpublocks, // input: cuda gpublocks - const int gputhreads ) // input: cuda gputhreads - { - const int nevt = gpublocks * gputhreads; - - // Get the address associated with the normalized color matrix in device memory - static fptype2* devNormColMat = nullptr; - if( !devNormColMat ) gpuGetSymbolAddress( (void**)&devNormColMat, s_pNormalizedColorMatrix2 ); - -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Mixed precision mode: need two fptype2[2*ncolor*nhel*nevt] buffers and one fptype2[nhel*nevt] buffers for the nhel helicities - fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of first fptype2[ncolor*2*nhel*nevt] buffer - fptype2* ghelAllJampsFpt2 = ghelAllBlasTmp + ncolor * mgOnGpu::nx2 * nhel * nevt; // start of second fptype2[ncolor*2*nhel*nevt] buffer - fptype2* ghelAllMEsFpt2 = ghelAllBlasTmp + 2 * ncolor * mgOnGpu::nx2 * nhel * nevt; // start of fptype2[nhel*nevt] buffer - // Convert jamps from double to float - for( int ighel = 0; ighel < nhel; ighel++ ) - { - const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // jamps for a single helicity ihel - fptype2* hAllJampsFpt2 = ghelAllJampsFpt2 + ighel * nevt; // jamps for a single helicity ihel - gpuLaunchKernelStream( convertD2F_Jamps, gpublocks, gputhreads, ghelStreams[ighel], hAllJampsFpt2, hAllJamps, nhel ); - } - // Real and imaginary components - const fptype2* ghelAllJampsReal = ghelAllJampsFpt2; - const fptype2* ghelAllJampsImag = ghelAllJampsFpt2 + ncolor * nhel * nevt; -#else - // Standard single or double precision mode: need one fptype2[ncolor*2*nhel*nevt] buffer - static_assert( std::is_same::value ); - fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of fptype2[ncolor*2*nhel*nevt] buffer - fptype2* ghelAllMEsFpt2 = ghelAllMEs; - // Real and imaginary components - const fptype2* ghelAllJampsReal = ghelAllJamps; // this is not a cast (the two types are identical) - const fptype2* ghelAllJampsImag = ghelAllJamps + ncolor * nhel * nevt; // this is not a cast (the two types are identical) -#endif - // Real and imaginary components - fptype2* ghelAllZtempReal = ghelAllZtempBoth; - fptype2* ghelAllZtempImag = ghelAllZtempBoth + ncolor * nhel * nevt; - - // Note: striding for cuBLAS from DeviceAccessJamp: - // - ghelAllJamps(icol,ihel,ievt).real is ghelAllJamps[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] - // - ghelAllJamps(icol,ihel,ievt).imag is ghelAllJamps[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] - - // Step 1: Compute Ztemp[ncolor][nhel*nevt] = ColorMatrix[ncolor][ncolor] * JampsVector[ncolor][nhel*nevt] for both real and imag - // In this case alpha=1 and beta=0: the operation is Ztemp = alpha * ColorMatrix * JampsVector + beta * Ztemp - fptype2 alpha1 = 1; - fptype2 beta1 = 0; - const int ncolorM = ncolor; - const int nevtN = nhel*nevt; - const int ncolorK = ncolor; - checkGpuBlas( gpuBlasTgemm( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose ColMat - GPUBLAS_OP_T, // transpose JampsV (new1) - ncolorM, nevtN, ncolorK, - &alpha1, - devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK - ghelAllJampsReal, nevtN, // JampsV is nevtN x ncolorK - &beta1, - ghelAllZtempReal, ncolorM ) ); // Ztemp is ncolorM x nevtN - checkGpuBlas( gpuBlasTgemm( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose ColMat - GPUBLAS_OP_T, // transpose JampsV (new1) - ncolorM, nevtN, ncolorK, - &alpha1, - devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK - ghelAllJampsImag, nevtN, // JampsV is nevtN x ncolorK (new1) - &beta1, - ghelAllZtempImag, ncolorM ) ); // Ztemp is ncolorM x nevtN - - // Step 2: For each ievt, compute the dot product of JampsVector[ncolor][ievt] dot tmp[ncolor][ievt] - // In this case alpha=1 and beta=1: the operation is ME = alpha * ( Tmp dot JampsVector ) + beta * ME - // Use cublasSgemmStridedBatched to perform these batched dot products in one call - fptype2 alpha2 = 1; - fptype2 beta2 = 1; - checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose JampsV (new1) - GPUBLAS_OP_N, // do not transpose Tmp - 1, 1, ncolor, // result is 1x1 (dot product) - &alpha2, - ghelAllJampsReal, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column - ghelAllZtempReal, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column - &beta2, - ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) - nevtN ) ); // there are nevtN (nhel*nevt) "batches" - checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose JampsV (new1) - GPUBLAS_OP_N, // do not transpose Tmp - 1, 1, ncolor, // result is 1x1 (dot product) - &alpha2, - ghelAllJampsImag, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column (new1) - ghelAllZtempImag, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column - &beta2, - ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) - nevtN ) ); // there are nevt (nhel*nevt) "batches" - -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Convert MEs from float to double - for( int ighel = 0; ighel < nhel; ighel++ ) - { - fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for a single helicity ihel - fptype2* hAllMEsFpt2 = ghelAllMEsFpt2 + ighel * nevt; // MEs for a single helicity ihel - gpuLaunchKernelStream( convertF2D_MEs, gpublocks, gputhreads, ghelStreams[ighel], hAllMEs, hAllMEsFpt2 ); - } -#endif - } -#endif /* clang-format on */ -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void - color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) - const int nGoodHel, // input: number of good helicities - const int gpublocks, // input: cuda gpublocks - const int gputhreads, // input: cuda gputhreads - const bool processAllHelicities ) // input: if true, use blockIdx.y to index helicities - { - const int nevt = gpublocks * gputhreads; - // CASE 1: KERNEL - if( !pBlasHandle ) - { - assert( ghelAllBlasTmp == nullptr ); // sanity check for HASBLAS=hasNoBlas or CUDACPP_RUNTIME_BLASCOLORSUM not set - if (processAllHelicities) { - gpuLaunchKernel2D( color_sum_kernel, gpublocks, nGoodHel, gputhreads, ghelStreams[0], ghelAllMEs, ghelAllJamps, nGoodHel, nevt ); - } else { - // Loop over helicities - for( int ighel = 0; ighel < nGoodHel; ighel++ ) - { - fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for one specific helicity ighel - const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // Jamps for one specific helicity ighel - gpuStream_t hStream = ghelStreams[ighel]; - gpuLaunchKernelStream( color_sum_kernel, gpublocks, gputhreads, hStream, hAllMEs, hAllJamps, nGoodHel, 0 ); - } - } - } - // CASE 2: BLAS - else - { -#ifdef MGONGPU_HAS_NO_BLAS - assert( false ); // sanity check: no path to this statement for HASBLAS=hasNoBlas -#else - if (processAllHelicities) { - assert( false ); // BLAS in async mode not supported for now - } else { - checkGpu( gpuDeviceSynchronize() ); // do not start the BLAS color sum for all helicities until the loop over helicities has completed - // Reset the tmp buffer -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( 2 * ncolor * mgOnGpu::nx2 + 1 ) * sizeof( fptype2 ) ); -#else - gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( ncolor * mgOnGpu::nx2 ) * sizeof( fptype2 ) ); -#endif - // Delegate the color sum to BLAS for - color_sum_blas( ghelAllMEs, ghelAllJamps, ghelAllBlasTmp, pBlasHandle, ghelStreams, nGoodHel, gpublocks, gputhreads ); - } -#endif - } - } -#endif //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h index 347184c4e1..2e043fa128 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h @@ -14,90 +14,26 @@ #include "CPPProcess.h" #include "GpuAbstraction.h" -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - class DeviceAccessJamp - { - public: - static __device__ inline cxtype_ref - kernelAccessIcolIhelNhel( fptype* buffer, const int icol, const int ihel, const int nhel ) - { - const int ncolor = CPPProcess::ncolor; // the number of leading colors - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) - //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" - // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) - // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS - //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" - // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) - return cxtype_ref( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], - buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); - } - static __device__ inline const cxtype - kernelAccessIcolIhelNhelConst( const fptype* buffer, const int icol, const int ihel, const int nhel ) - { - const int ncolor = CPPProcess::ncolor; // the number of leading colors - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) - //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" - // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) - // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS - //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" - // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) - return cxtype( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], - buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); - } - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void createNormalizedColorMatrix(); -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL void color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity const int ievt0 ); // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - void - color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) - const int nGoodHel, // input: number of good helicities - const int gpublocks, // input: cuda gpublocks - const int gputhreads, // input: cuda gputhreads - const bool processAllHelicities); // input: if true, use blockIdx.y to index helicities -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity - const int nGoodHel, // input: number of good helicities - const int nevtIfAllHelicities); // input: zero in single-helicity mode, number of events in multi-helicity mode -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h index 027f1aa44e..8ff67e63ff 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h @@ -30,11 +30,7 @@ namespace mgOnGpu // The number of channels in the channel2iconfig array below // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** constexpr unsigned int nchannels = %(nb_diag)i; -#ifdef MGONGPUCPP_GPUIMPL - static_assert( nchannels <= mg5amcGpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 -#else static_assert( nchannels <= mg5amcCpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 -#endif // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) @@ -45,13 +41,7 @@ namespace mgOnGpu }; // Host copy of the channel2iconfig array (this is needed in runTest #917) -#ifndef MGONGPUCPP_GPUIMPL constexpr const int* hostChannel2iconfig = channel2iconfig; -#else - constexpr int hostChannel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed -%(channelc2iconfig_lines)s - }; -#endif // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) constexpr unsigned int nconfigSDE = %(nb_channel)s; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h index 32c1bbb963..356f034c92 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h @@ -28,11 +28,7 @@ #endif // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc index 05c3b3e0c4..d0b13c937f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc @@ -23,11 +23,7 @@ extern "C" * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. */ -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif /** * The floating point precision used in Fortran arrays. @@ -47,9 +43,6 @@ extern "C" */ void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) { -#ifdef MGONGPUCPP_GPUIMPL - GpuRuntime::setUp(); -#endif // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) // FIXME: disable OMP in Bridge when called from Fortran *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); @@ -66,9 +59,6 @@ extern "C" Bridge* pbridge = dynamic_cast*>( *ppbridge ); if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); delete pbridge; -#ifdef MGONGPUCPP_GPUIMPL - GpuRuntime::tearDown(); -#endif } /** @@ -101,15 +91,9 @@ extern "C" Bridge* pbridge = dynamic_cast*>( *ppbridge ); //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); -#ifdef MGONGPUCPP_GPUIMPL - // Use the device/GPU implementation in the CUDA library - // (there is also a host implementation in this library) - pbridge->gpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); -#else // Use the host/CPU implementation in the C++ library // (there is no device implementation in this library) pbridge->cpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); -#endif } /** diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h index e676df0a33..4815fdb027 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h @@ -12,11 +12,7 @@ extern "C" { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif using FORTRANFPTYPE = double; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc index 8a4468fb3c..4048cce912 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc @@ -14,11 +14,7 @@ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { template class Sampler final : public CppObjectInFortran @@ -41,15 +37,9 @@ namespace mg5amcCpu private: const int m_nevt; // The number of events in each iteration int m_iiter; // The iteration counter (for random number seeding) -#ifndef MGONGPUCPP_GPUIMPL HostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers HostBufferMomenta m_hstMomenta; // Memory buffers for momenta HostBufferWeights m_hstWeights; // Memory buffers for sampling weights -#else - PinnedHostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers - PinnedHostBufferMomenta m_hstMomenta; // Memory buffers for momenta - PinnedHostBufferWeights m_hstWeights; // Memory buffers for sampling weights -#endif std::unique_ptr m_prnk; // The appropriate RandomNumberKernel std::unique_ptr m_prsk; // The appropriate SamplingKernel // HARDCODED DEFAULTS @@ -106,11 +96,7 @@ namespace mg5amcCpu extern "C" { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif /** * The floating point precision used in Fortran arrays. diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h index bcd3990bed..2b7aaafc19 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h @@ -22,11 +22,7 @@ #include // Simplified rambo version for 2 to N (with N>=2) processes with massless particles -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { namespace massless_rambo { @@ -89,16 +85,6 @@ namespace massless_rambo { static bool first = true; if( first ) { -#ifdef MGONGPUCPP_GPUIMPL - if constexpr( M_ACCESS::isOnDevice() ) // avoid - { - const int ievt0 = 0; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid - if( ievt == ievt0 ) - printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); - } - else -#endif { printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); } @@ -172,7 +158,6 @@ namespace massless_rambo { wt = po2log; if( nparf != 2 ) wt = ( 2. * nparf - 4. ) * log( energy ) + z[nparf - 1]; -#ifndef MGONGPUCPP_GPUIMPL // issue warnings if weight is too small or too large static int iwarn[5] = { 0, 0, 0, 0, 0 }; if( wt < -180. ) @@ -185,7 +170,6 @@ namespace massless_rambo { if( iwarn[1] <= 5 ) std::cout << "Too large wt, risk for overflow: " << wt << std::endl; iwarn[1] = iwarn[1] + 1; } -#endif // return for weighted massless momenta // nothing else to do in this event if all particles are massless (nm==0) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h index 09c76a6bd5..53b41f3b80 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h @@ -152,11 +152,9 @@ #endif // SANITY CHECKS (C++ complex number implementation) -#ifndef MGONGPUCPP_GPUIMPL #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX and defined MGONGPU_CPPCXTYPE_CXSMPL #error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL for C++ #endif -#endif // NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) namespace mgOnGpu @@ -192,9 +190,7 @@ namespace mgOnGpu // Alignment requirement for using reinterpret_cast with SIMD vectorized code // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) // Only needed for C++ code but can be enforced also in NVCC builds of C++ code using CUDA>=11.2 and C++17 (#318, #319, #333) -#ifndef MGONGPUCPP_GPUIMPL constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) -#endif } @@ -208,9 +204,7 @@ using mgOnGpu::fptype2; #endif // C++ SIMD vectorization width (this will be used to set neppV) -#ifdef MGONGPUCPP_GPUIMPL // CUDA and HIP implementations have no SIMD -#undef MGONGPU_CPPSIMD -#elif defined __AVX512VL__ && defined MGONGPU_PVW512 // C++ "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#if defined __AVX512VL__ && defined MGONGPU_PVW512 // C++ "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 8 #else @@ -260,11 +254,9 @@ using mgOnGpu::fptype2; //#endif /* clang-format on */ // Define empty CUDA/HIP declaration specifiers for C++ -#ifndef MGONGPUCPP_GPUIMPL #define __global__ #define __host__ #define __device__ -#endif // For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two (https://stackoverflow.com/a/108360) inline constexpr bool diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h index d6e1d02e05..900a496b7d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h @@ -163,11 +163,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { template inline __host__ std::ostream& @@ -297,11 +293,7 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { // --- Type definitions (complex type: cxtype) #ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) @@ -333,11 +325,7 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL @@ -699,11 +687,7 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h index a2489d05d5..fa465e7658 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h @@ -13,63 +13,13 @@ #include // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // cuda -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { //========================================================================== -#ifdef MGONGPUCPP_GPUIMPL // cuda - - //------------------------------ - // Floating point types - Cuda - //------------------------------ - - /* - inline __host__ __device__ fptype - fpmax( const fptype& a, const fptype& b ) - { - return max( a, b ); - } - - inline __host__ __device__ fptype - fpmin( const fptype& a, const fptype& b ) - { - return min( a, b ); - } - */ - - inline __host__ __device__ const fptype& - fpmax( const fptype& a, const fptype& b ) - { - return ( ( b < a ) ? a : b ); - } - - inline __host__ __device__ const fptype& - fpmin( const fptype& a, const fptype& b ) - { - return ( ( a < b ) ? a : b ); - } - - inline __host__ __device__ fptype - fpsqrt( const fptype& f ) - { -#if defined MGONGPU_FPTYPE_FLOAT - // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__SINGLE.html - return sqrtf( f ); -#else - // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__DOUBLE.html - return sqrt( f ); -#endif - } - -#endif // #ifdef MGONGPUCPP_GPUIMPL //========================================================================== -#ifndef MGONGPUCPP_GPUIMPL //------------------------------ // Floating point types - C++ @@ -93,7 +43,6 @@ namespace mg5amcCpu return std::sqrt( f ); } -#endif // #ifndef MGONGPUCPP_GPUIMPL //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h index 74d93f05a3..ce5a07fb19 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h @@ -33,13 +33,8 @@ #endif // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { -#ifdef MGONGPU_CPPSIMD const int neppV = MGONGPU_CPPSIMD; @@ -135,11 +130,6 @@ namespace mg5amcCpu #endif #endif -#else // i.e #ifndef MGONGPU_CPPSIMD (this includes #ifdef MGONGPUCPP_GPUIMPL) - - const int neppV = 1; - -#endif // #ifdef MGONGPU_CPPSIMD } //-------------------------------------------------------------------------- @@ -157,13 +147,8 @@ namespace mg5amcCpu //========================================================================== // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { -#ifndef MGONGPUCPP_GPUIMPL // Printout to stream for user defined types @@ -190,7 +175,6 @@ namespace mg5amcCpu #endif */ -#ifdef MGONGPU_CPPSIMD inline std::ostream& operator<<( std::ostream& out, const fptype_v& v ) { @@ -199,7 +183,6 @@ namespace mg5amcCpu out << " }"; return out; } -#endif #if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT inline std::ostream& @@ -212,7 +195,6 @@ namespace mg5amcCpu } #endif -#ifdef MGONGPU_CPPSIMD inline std::ostream& operator<<( std::ostream& out, const cxtype_v& v ) { @@ -226,9 +208,7 @@ namespace mg5amcCpu out << " }"; return out; } -#endif -#ifdef MGONGPU_CPPSIMD inline std::ostream& operator<<( std::ostream& out, const uint_v& v ) { @@ -237,7 +217,6 @@ namespace mg5amcCpu out << " }"; return out; } -#endif //-------------------------------------------------------------------------- @@ -261,7 +240,6 @@ namespace mg5amcCpu // Functions and operators for fptype_v -#ifdef MGONGPU_CPPSIMD inline fptype_v fpsqrt( const volatile fptype_v& v ) // volatile fixes #736 { @@ -284,7 +262,6 @@ namespace mg5amcCpu for( int i = 0; i < neppV; i++ ) out[i] = fpsqrt( v[i] ); return out; } -#endif /* #ifdef MGONGPU_CPPSIMD @@ -302,7 +279,6 @@ namespace mg5amcCpu // Functions and operators for cxtype_v -#ifdef MGONGPU_CPPSIMD /* inline cxtype_v @@ -507,13 +483,11 @@ namespace mg5amcCpu return cxtype_v( a.real() / b, a.imag() / b ); } -#endif // #ifdef MGONGPU_CPPSIMD //-------------------------------------------------------------------------- // Functions and operators for bool_v (ternary and masks) -#ifdef MGONGPU_CPPSIMD inline fptype_v fpternary( const bool_v& mask, const fptype_v& a, const fptype_v& b ) @@ -642,41 +616,11 @@ namespace mg5amcCpu return out; } -#else // i.e. #ifndef MGONGPU_CPPSIMD - - inline fptype - fpternary( const bool& mask, const fptype& a, const fptype& b ) - { - return ( mask ? a : b ); - } - - inline cxtype - cxternary( const bool& mask, const cxtype& a, const cxtype& b ) - { - return ( mask ? a : b ); - } - - /* - inline bool - maskor( const bool& mask ) - { - return mask; - } - */ - - inline bool - maskand( const bool& mask ) - { - return mask; - } - -#endif // #ifdef MGONGPU_CPPSIMD //-------------------------------------------------------------------------- // Functions and operators for fptype_v (min/max) -#ifdef MGONGPU_CPPSIMD inline fptype_v fpmax( const fptype_v& a, const fptype_v& b ) @@ -743,7 +687,6 @@ namespace mg5amcCpu fptype_v *m_preal, *m_pimag; // RRRRIIII }; -#endif // #ifdef MGONGPU_CPPSIMD //-------------------------------------------------------------------------- @@ -770,16 +713,8 @@ namespace mg5amcCpu } return out; */ -#if MGONGPU_CPPSIMD == 2 fptype2_v out = { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v2[0], (fptype2)v2[1] }; -#elif MGONGPU_CPPSIMD == 4 - fptype2_v out = - { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v1[2], (fptype2)v1[3], (fptype2)v2[0], (fptype2)v2[1], (fptype2)v2[2], (fptype2)v2[3] }; -#elif MGONGPU_CPPSIMD == 8 - fptype2_v out = - { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v1[2], (fptype2)v1[3], (fptype2)v1[4], (fptype2)v1[5], (fptype2)v1[6], (fptype2)v1[7], (fptype2)v2[0], (fptype2)v2[1], (fptype2)v2[2], (fptype2)v2[3], (fptype2)v2[4], (fptype2)v2[5], (fptype2)v2[6], (fptype2)v2[7] }; -#endif return out; } @@ -793,16 +728,8 @@ namespace mg5amcCpu out[ieppV] = v[ieppV]; } */ -#if MGONGPU_CPPSIMD == 2 fptype_v out = { (fptype)v[0], (fptype)v[1] }; -#elif MGONGPU_CPPSIMD == 4 - fptype_v out = - { (fptype)v[0], (fptype)v[1], (fptype)v[2], (fptype)v[3] }; -#elif MGONGPU_CPPSIMD == 8 - fptype_v out = - { (fptype)v[0], (fptype)v[1], (fptype)v[2], (fptype)v[3], (fptype)v[4], (fptype)v[5], (fptype)v[6], (fptype)v[7] }; -#endif return out; } @@ -816,105 +743,29 @@ namespace mg5amcCpu out[ieppV] = v[ieppV+neppV]; } */ -#if MGONGPU_CPPSIMD == 2 fptype_v out = { (fptype)v[2], (fptype)v[3] }; -#elif MGONGPU_CPPSIMD == 4 - fptype_v out = - { (fptype)v[4], (fptype)v[5], (fptype)v[6], (fptype)v[7] }; -#elif MGONGPU_CPPSIMD == 8 - fptype_v out = - { (fptype)v[8], (fptype)v[9], (fptype)v[10], (fptype)v[11], (fptype)v[12], (fptype)v[13], (fptype)v[14], (fptype)v[15] }; -#endif return out; } #endif // #if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT -#endif // #ifndef MGONGPUCPP_GPUIMPL //========================================================================== -#ifdef MGONGPUCPP_GPUIMPL - - //------------------------------ - // Vector types - CUDA - //------------------------------ - - // Printout to std::cout for user defined types - inline __host__ __device__ void - print( const fptype& f ) - { - printf( "%f\n", f ); - } - inline __host__ __device__ void - print( const cxtype& c ) - { - printf( "[%f, %f]\n", cxreal( c ), cximag( c ) ); - } - - /* - inline __host__ __device__ const cxtype& - cxvmake( const cxtype& c ) - { - return c; - } - */ - - inline __host__ __device__ fptype - fpternary( const bool& mask, const fptype& a, const fptype& b ) - { - return ( mask ? a : b ); - } - - inline __host__ __device__ cxtype - cxternary( const bool& mask, const cxtype& a, const cxtype& b ) - { - return ( mask ? a : b ); - } - - inline __host__ __device__ bool - maskand( const bool& mask ) - { - return mask; - } - -#endif // #ifdef MGONGPUCPP_GPUIMPL //========================================================================== // Scalar-or-vector types: scalar in CUDA, vector or scalar in C++ -#ifdef MGONGPUCPP_GPUIMPL - typedef bool bool_sv; - typedef fptype fptype_sv; - typedef fptype2 fptype2_sv; - typedef unsigned int uint_sv; - typedef cxtype cxtype_sv; - typedef cxtype_ref cxtype_sv_ref; -#elif defined MGONGPU_CPPSIMD typedef bool_v bool_sv; typedef fptype_v fptype_sv; typedef fptype2_v fptype2_sv; typedef uint_v uint_sv; typedef cxtype_v cxtype_sv; typedef cxtype_v_ref cxtype_sv_ref; -#else - typedef bool bool_sv; - typedef fptype fptype_sv; - typedef fptype2 fptype2_sv; - typedef unsigned int uint_sv; - typedef cxtype cxtype_sv; - typedef cxtype_ref cxtype_sv_ref; -#endif // Scalar-or-vector zeros: scalar in CUDA, vector or scalar in C++ -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - inline __host__ __device__ cxtype cxzero_sv(){ return cxtype( 0, 0 ); } -#elif defined MGONGPU_CPPSIMD inline cxtype_v cxzero_sv() { return cxtype_v(); } // RRRR=0000 IIII=0000 -#else - inline cxtype cxzero_sv() { return cxtype( 0, 0 ); } -#endif /* clang-format on */ //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/runTest.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/runTest.cc index 369d7197cc..7bd143de4a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/runTest.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/runTest.cc @@ -23,11 +23,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif struct CUDA_CPU_TestBase : public TestDriverBase { @@ -78,7 +74,6 @@ struct CUDA_CPU_TestBase : public TestDriverBase } }; -#ifndef MGONGPUCPP_GPUIMPL struct CPUTest : public CUDA_CPU_TestBase { // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) @@ -216,198 +211,16 @@ struct CPUTestMultiChannel : public CPUTest // Destructor virtual ~CPUTestMultiChannel() {} }; -#endif - -#ifdef MGONGPUCPP_GPUIMPL -struct CUDATest : public CUDA_CPU_TestBase -{ - // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) - // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] - CPPProcess process; - PinnedHostBufferRndNumMomenta hstRndMom; - PinnedHostBufferMomenta hstMomenta; - PinnedHostBufferGs hstGs; - PinnedHostBufferRndNumHelicity hstRndHel; - PinnedHostBufferRndNumColor hstRndCol; - PinnedHostBufferWeights hstWeights; - PinnedHostBufferChannelIds hstChannelIds; - PinnedHostBufferMatrixElements hstMatrixElements; - PinnedHostBufferSelectedHelicity hstSelHel; - PinnedHostBufferSelectedColor hstSelCol; - PinnedHostBufferHelicityMask hstIsGoodHel; - DeviceBufferRndNumMomenta devRndMom; - DeviceBufferChannelIds devChannelIds; - DeviceBufferMomenta devMomenta; - DeviceBufferGs devGs; - DeviceBufferRndNumHelicity devRndHel; - DeviceBufferRndNumColor devRndCol; - DeviceBufferWeights devWeights; - DeviceBufferMatrixElements devMatrixElements; - DeviceBufferSelectedHelicity devSelHel; - DeviceBufferSelectedColor devSelCol; - DeviceBufferHelicityMask devIsGoodHel; - std::unique_ptr pmek; - - // Create a process object - // Read param_card and set parameters - // ** WARNING EVIL EVIL ** - // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. - // Don't remove! - CUDATest( const std::string& refFileName ) - : CUDA_CPU_TestBase( refFileName ) - , process( /*verbose=*/false ) - , hstRndMom( nevt ) - , hstChannelIds( nevt ) - , hstMomenta( nevt ) - , hstGs( nevt ) - , hstRndHel( nevt ) - , hstRndCol( nevt ) - , hstWeights( nevt ) - , hstMatrixElements( nevt ) - , hstSelHel( nevt ) - , hstSelCol( nevt ) - , hstIsGoodHel( CPPProcess::ncomb ) - , devRndMom( nevt ) - , devChannelIds( nevt ) - , devMomenta( nevt ) - , devGs( nevt ) - , devRndHel( nevt ) - , devRndCol( nevt ) - , devWeights( nevt ) - , devMatrixElements( nevt ) - , devSelHel( nevt ) - , devSelCol( nevt ) - , devIsGoodHel( CPPProcess::ncomb ) - , pmek( new MatrixElementKernelDevice( devMomenta, devGs, devRndHel, devRndCol, devChannelIds, devMatrixElements, devSelHel, devSelCol, gpublocks, gputhreads ) ) - { - // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? - process.initProc( "../../Cards/param_card.dat" ); - } - - virtual ~CUDATest() {} - - void prepareRandomNumbers( unsigned int iiter ) override - { - // Random numbers for momenta - CommonRandomNumberKernel rnk( hstRndMom ); - rnk.seedGenerator( 1337 + iiter ); - rnk.generateRnarray(); - copyDeviceFromHost( devRndMom, hstRndMom ); - // Random numbers for helicity and color selection (fix #931) - CommonRandomNumberKernel rnk2( hstRndHel ); - rnk2.seedGenerator( 1338 + iiter ); - rnk2.generateRnarray(); - copyDeviceFromHost( devRndHel, hstRndHel ); - CommonRandomNumberKernel rnk3( hstRndCol ); - rnk3.seedGenerator( 1339 + iiter ); - rnk3.generateRnarray(); - copyDeviceFromHost( devRndCol, hstRndCol ); - } - - void prepareMomenta( fptype energy ) override - { - RamboSamplingKernelDevice rsk( energy, devRndMom, devMomenta, devWeights, gpublocks, gputhreads ); - // --- 2a. Fill in momenta of initial state particles on the device - rsk.getMomentaInitial(); - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - rsk.getMomentaFinal(); - // --- 2c. CopyDToH Weights - copyHostFromDevice( hstWeights, devWeights ); - // --- 2d. CopyDToH Momenta - copyHostFromDevice( hstMomenta, devMomenta ); - } - - void runSigmaKin( std::size_t iiter ) override - { - constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) - for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; - copyDeviceFromHost( devGs, hstGs ); // BUG FIX #566 - setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 - copyDeviceFromHost( devChannelIds, hstChannelIds ); - if( iiter == 0 ) pmek->computeGoodHelicities(); - pmek->computeMatrixElements( useChannelIds() ); - copyHostFromDevice( hstMatrixElements, devMatrixElements ); - copyHostFromDevice( hstSelHel, devSelHel ); - copyHostFromDevice( hstSelCol, devSelCol ); - } - - fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override - { - assert( ipar < npar ); - assert( ip4 < np4 ); - return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); - } - - fptype getMatrixElement( std::size_t ievt ) const override - { - return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); - } - - int getChannelId( std::size_t ievt ) const override - { - return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); - } - - int getSelectedHelicity( std::size_t ievt ) const override - { - //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... - return hstSelHel.data()[ievt]; - } - - int getSelectedColor( std::size_t ievt ) const override - { - //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... - return hstSelCol.data()[ievt]; - } -}; -// Old test with multi-channel disabled #466 -struct CUDATestNoMultiChannel : public CUDATest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return false; } - - // Constructor - CUDATestNoMultiChannel( const std::string& refFileName ) - : CUDATest( refFileName ) {} // suffix .txt - - // Destructor - virtual ~CUDATestNoMultiChannel() {} -}; - -// New test with multi-channel enabled #896 -struct CUDATestMultiChannel : public CUDATest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return true; } - - // Constructor - CUDATestMultiChannel( const std::string& refFileName ) - : CUDATest( refFileName + "2" ) {} // suffix .txt2 - - // Destructor - virtual ~CUDATestMultiChannel() {} -}; -#endif /* clang-format off */ // AV July 2024 much simpler class structure without the presently-unnecessary googletest templates // This is meant as a workaround to prevent not-understood segfault #907 when adding a second test // Note: instantiate test2 first and test1 second to ensure that the channelid printout from the dtors comes from test1 first and test2 second -#ifdef MGONGPUCPP_GPUIMPL -// CUDA test drivers -CUDATestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID2( s ) s##_GPU_MULTICHANNEL -CUDATestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID1( s ) s##_GPU_NOMULTICHANNEL -#else // CPU test drivers CPUTestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); #define TESTID2( s ) s##_CPU_MULTICHANNEL CPUTestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); #define TESTID1( s ) s##_CPU_NOMULTICHANNEL -#endif // Madgraph tests MadgraphTest mgTest2( driver2 ); MadgraphTest mgTest1( driver1 ); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc index 74c4902dc5..ce876db2cf 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc @@ -23,26 +23,14 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -#define TESTID( s ) s##_GPU_MISC -#else #define TESTID( s ) s##_CPU_MISC -#endif #define XTESTID( s ) TESTID( s ) // NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { -#ifdef MGONGPU_CPPSIMD /* clang-format off */ #define EXPECT_TRUE_sv( cond ) { bool_v mask( cond ); EXPECT_TRUE( maskand( mask ) ); } -#else -#define EXPECT_TRUE_sv( cond ) { EXPECT_TRUE( cond ); } -#endif /* clang-format on */ inline const std::string boolTF( const bool& b ) @@ -50,7 +38,6 @@ namespace mg5amcCpu return ( b ? "T" : "F" ); } -#ifdef MGONGPU_CPPSIMD inline const std::string boolTF( const bool_v& v ) { @@ -60,16 +47,11 @@ namespace mg5amcCpu out << " }"; return out.str(); } -#endif } TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif //-------------------------------------------------------------------------- @@ -92,9 +74,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) fptype_sv f{ 1 }; //std::cout << f << std::endl << boolTF( f == 1 ) << std::endl; //EXPECT_TRUE_sv( f == 1 ); // this fails for vectors! TFFF -#ifndef MGONGPU_CPPSIMD - EXPECT_TRUE_sv( f == 1 ); // this succeds: T -#else EXPECT_TRUE( ( f == 1 )[0] ); // this succeds: TFFF[0] EXPECT_TRUE( ( f[0] == 1 ) ); for( int i = 1; i < neppV; i++ ) @@ -102,10 +81,8 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) EXPECT_TRUE( !( ( f == 1 )[i] ) ); // this succeds: FTTT[i>=1] EXPECT_TRUE( ( f[i] == 0 ) ); // equals 0, not 1 } -#endif } -#ifdef MGONGPU_CPPSIMD // Vector initialization for cxtype_sv - demonstrate fix for bug #339 { fptype_sv f1 = fptype_v{ 0 } + 1; @@ -119,7 +96,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) EXPECT_TRUE_sv( c21.real() == 2 ); EXPECT_TRUE_sv( c21.imag() == 1 ); } -#endif // Vector initialization for cxtype_sv { @@ -256,7 +232,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) test_int_sv channelids1_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) fptype_sv absamp0_sv{}; // mimic cxabs2( amp_sv[0] ) fptype_sv absamp1_sv{}; // mimic cxabs2( amp_sv[0] ) -#ifdef MGONGPU_CPPSIMD for( int i = 0; i < neppV; i++ ) { channelids0_sv[i] = i; // 0123 @@ -264,19 +239,12 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) absamp0_sv[i] = 10. + i; // 10. 11. 12. 13. absamp1_sv[i] = 11. + i; // 11. 12. 13. 14. } -#else - channelids0_sv = 0; - channelids1_sv = 1; - absamp0_sv = 10.; - absamp1_sv = 11.; -#endif bool_sv mask0_sv = ( channelids0_sv % 2 == 0 ); // even channels 0123 -> TFTF (1010) bool_sv mask1_sv = ( channelids1_sv % 2 == 0 ); // even channels 1234 -> FTFT (0101) constexpr fptype_sv fpZERO_sv{}; // 0000 //fptype_sv numerators0_sv = mask0_sv * absamp0_sv; // invalid operands to binary * ('__vector(4) long int' and '__vector(4) double') fptype_sv numerators0_sv = fpternary( mask0_sv, absamp0_sv, fpZERO_sv ); // equivalent to "mask0_sv * absamp0_sv" fptype_sv numerators1_sv = fpternary( mask1_sv, absamp1_sv, fpZERO_sv ); // equivalent to "mask1_sv * absamp1_sv" -#ifdef MGONGPU_CPPSIMD //std::cout << "numerators0_sv: " << numerators0_sv << std::endl; //std::cout << "numerators1_sv: " << numerators1_sv << std::endl; for( int i = 0; i < neppV; i++ ) @@ -292,12 +260,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) else // odd channels EXPECT_TRUE( numerators1_sv[i] == 0. ); } -#else - // Values of numerators0_sv: 10.*1 - EXPECT_TRUE( numerators0_sv == 10. ); - // Values of numerators1_sv: 11.*0 - EXPECT_TRUE( numerators1_sv == 0. ); -#endif } //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc index 4fcf7d26b6..3ddffd4559 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc @@ -25,40 +25,24 @@ #include #include #include -#ifdef MGONGPUCPP_GPUIMPL -#define TESTID( s ) s##_GPU_XXX -#else #define TESTID( s ) s##_CPU_XXX -#endif #define XTESTID( s ) TESTID( s ) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else namespace mg5amcCpu -#endif { std::string fpeHandlerMessage = "unknown"; int fpeHandlerIevt = -1; inline void fpeHandlerTestxxx( int /*sig*/ ) { -#ifdef MGONGPUCPP_GPUIMPL - std::cerr << "Floating Point Exception (GPU): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; -#else std::cerr << "Floating Point Exception (CPU neppV=" << neppV << "): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; -#endif exit( 1 ); } } TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif #ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); #endif @@ -72,11 +56,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt %% neppM == 0 ); // nevt must be a multiple of neppM assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV // Fill in the input momenta -#ifdef MGONGPUCPP_GPUIMPL - mg5amcGpu::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] -#else mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] -#endif /* clang-format off */ // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! const fptype par0[np4 * nevt] = // AOS[nevt][np4] { @@ -167,7 +147,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) out << " // ---------" << std::endl; for( int iw6 = 0; iw6 < nw6; iw6++ ) { -#ifdef MGONGPU_CPPSIMD const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration #ifdef MGONGPU_HAS_CPPCXTYPEV_BRK out << std::setw( 26 ) << cxreal( wf[iw6][ieppV] ) << ", "; @@ -175,10 +154,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) #else out << std::setw( 26 ) << wf[iw6].real()[ieppV] << ", "; out << std::setw( 22 ) << wf[iw6].imag()[ieppV]; -#endif -#else - out << std::setw( 26 ) << wf[iw6].real(); - out << ", " << std::setw( 22 ) << wf[iw6].imag(); #endif if( iw6 < nw6 - 1 ) out << ", "; @@ -207,7 +182,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) const fptype expImag = expwf[iw6 * 2 + 1]; if( true ) { -#ifdef MGONGPU_CPPSIMD const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration #ifdef MGONGPU_HAS_CPPCXTYPEV_BRK EXPECT_NEAR( cxreal( wf[iw6][ieppV] ), expReal, std::abs( expReal * toleranceXXXs ) ) @@ -219,12 +193,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) << " itest=" << itest << ": " << xxx << "#" << ievt; EXPECT_NEAR( wf[iw6].imag()[ieppV], expImag, std::abs( expImag * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt; -#endif -#else - EXPECT_NEAR( cxreal( wf[iw6] ), expReal, std::abs( expReal * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt; - EXPECT_NEAR( cximag( wf[iw6] ), expImag, std::abs( expImag * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt; #endif } } @@ -246,7 +214,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { if( true ) { -#ifdef MGONGPU_CPPSIMD const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration #ifdef MGONGPU_HAS_CPPCXTYPEV_BRK const fptype expReal = cxreal( expwf[iw6][ieppV] ); @@ -262,14 +229,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; EXPECT_NEAR( wf[iw6].imag()[ieppV], expImag, std::abs( expImag * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; -#endif -#else - const fptype expReal = cxreal( expwf[iw6] ); - const fptype expImag = cximag( expwf[iw6] ); - EXPECT_NEAR( cxreal( wf[iw6] ), expReal, std::abs( expReal * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; - EXPECT_NEAR( cximag( wf[iw6] ), expImag, std::abs( expImag * toleranceXXXs ) ) - << " itest=" << itest << ": " << xxx << "#" << ievt << " against " << xxxFull; #endif } } @@ -317,11 +276,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { for( int ievt = 0; ievt < nevt; ievt++ ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif if( debug ) { std::cout << std::endl; @@ -438,9 +393,6 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) void myexit() { -#ifdef MGONGPUCPP_GPUIMPL - //checkGpu( gpuDeviceReset() ); // FIXME??? this still crashes! should systematically avoid CUDA calls in all destructors? -#endif } // Main function (see https://google.github.io/googletest/primer.html#writing-the-main-function) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc index d19c93bb9b..0cfb07ed26 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc @@ -16,11 +16,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else using namespace mg5amcCpu; -#endif namespace { @@ -30,9 +26,6 @@ namespace const fptype* couplings, const unsigned int* flavor_indices, fptype* matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL - fptype* color_jamps, -#endif fptype* numerators, fptype* denominators, std::size_t count ) @@ -40,9 +33,6 @@ namespace bool is_good_hel[CPPProcess::ncomb]; sigmaKin_getGoodHel( momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, -#ifdef MGONGPUCPP_GPUIMPL - color_jamps, -#endif is_good_hel, count ); sigmaKin_setGoodHel( is_good_hel ); @@ -54,26 +44,17 @@ namespace const fptype* couplings, const unsigned int* flavor_indices, fptype* matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL - fptype* color_jamps, -#endif fptype* numerators, fptype* denominators, std::size_t count ) { // static local initialization is called exactly once in a thread-safe way static void* dummy = initialize_impl( momenta, couplings, flavor_indices, matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL - color_jamps, -#endif numerators, denominators, count ); } -#ifdef MGONGPUCPP_GPUIMPL - __device__ -#endif void transpose_momenta( const double* momenta_in, fptype* momenta_out, std::size_t i_event_in, std::size_t i_event_out, std::size_t stride ) { @@ -91,70 +72,6 @@ namespace } } -#ifdef MGONGPUCPP_GPUIMPL - - __global__ void copy_inputs( - const double* momenta_in, - const double* helicity_random_in, - const double* color_random_in, - const double* diagram_random_in, - const double* alpha_s_in, - const unsigned int* flavor_indices_in, - fptype* momenta, - fptype* helicity_random, - fptype* color_random, - fptype* diagram_random, - fptype* g_s, - unsigned int* flavor_indices, - std::size_t count, - std::size_t stride, - std::size_t offset ) - { - std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; - if( i_event >= count ) return; - - transpose_momenta( &momenta_in[offset], momenta, i_event, i_event, stride ); - diagram_random[i_event] = diagram_random_in ? diagram_random_in[i_event + offset] : 0.5; - helicity_random[i_event] = helicity_random_in ? helicity_random_in[i_event + offset] : 0.5; - color_random[i_event] = color_random_in ? color_random_in[i_event + offset] : 0.5; - g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; - flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; - } - - __global__ void copy_outputs( - fptype* denominators, - fptype* numerators, - fptype* matrix_elements, - unsigned int* diagram_index, - int* color_index, - int* helicity_index, - double* m2_out, - double* amp2_out, - int* diagram_out, - int* color_out, - int* helicity_out, - std::size_t count, - std::size_t stride, - std::size_t offset ) - { - std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; - if( i_event >= count ) return; - - if( m2_out ) m2_out[i_event + offset] = matrix_elements[i_event]; - if( amp2_out ) - { - double denominator = denominators[i_event]; - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) - { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_event * CPPProcess::ndiagrams + i_diag] / denominator; - } - } - if( diagram_out ) diagram_out[i_event + offset] = diagram_index[i_event] - 1; - if( color_out ) color_out[i_event + offset] = color_index[i_event] - 1; - if( helicity_out ) helicity_out[i_event + offset] = helicity_index[i_event] - 1; - } - -#endif // MGONGPUCPP_GPUIMPL struct InterfaceInstance { @@ -174,15 +91,7 @@ extern "C" case UMAMI_META_DEVICE: { UmamiDevice& device = *static_cast( result ); -#ifdef MGONGPUCPP_GPUIMPL -#ifdef __CUDACC__ - device = UMAMI_DEVICE_CUDA; -#elif defined( __HIPCC__ ) - device = UMAMI_DEVICE_HIP; -#endif -#else device = UMAMI_DEVICE_CPU; -#endif break; } case UMAMI_META_PARTICLE_COUNT: @@ -231,11 +140,7 @@ extern "C" UmamiStatus umami_supported_outputs( bool const** supported, int* count ) { // MATRIX_ELEMENT, DIAGRAM_AMP2, COLOR_INDEX, HELICITY_INDEX, DIAGRAM_INDEX, GPU_STREAM -#ifdef MGONGPUCPP_GPUIMPL - static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true, true }; -#else static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true }; -#endif *supported = data; *count = UMAMI_OUTPUT_KEY_COUNT; return UMAMI_SUCCESS; @@ -326,9 +231,6 @@ extern "C" } if( !momenta_in ) return UMAMI_ERROR_MISSING_INPUT; -#ifdef MGONGPUCPP_GPUIMPL - gpuStream_t gpu_stream = nullptr; -#endif double* m2_out = nullptr; double* amp2_out = nullptr; int* diagram_out = nullptr; @@ -354,132 +256,11 @@ extern "C" case UMAMI_OUT_DIAGRAM_INDEX: diagram_out = static_cast( output ); break; -#ifdef MGONGPUCPP_GPUIMPL - case UMAMI_OUT_GPU_STREAM: - gpu_stream = static_cast( output ); - break; -#endif default: return UMAMI_ERROR_UNSUPPORTED_OUTPUT; } } -#ifdef MGONGPUCPP_GPUIMPL - std::size_t n_threads = 256; - std::size_t n_blocks = ( count + n_threads - 1 ) / n_threads; - std::size_t rounded_count = n_blocks * n_threads; - - fptype *momenta, *couplings, *g_s, *helicity_random, *color_random, *diagram_random, *color_jamps; - fptype *matrix_elements, *numerators, *denominators, *ghel_matrix_elements, *ghel_jamps; - int *helicity_index, *color_index; - unsigned int *flavor_indices, *diagram_index; - - std::size_t n_coup = mg5amcGpu::Parameters_dependentCouplings::ndcoup; - std::array, 16> ptrs_and_sizes = {{ - {reinterpret_cast(&momenta), rounded_count * CPPProcess::npar * 4 * sizeof( fptype )}, - {reinterpret_cast(&couplings), rounded_count * n_coup * 2 * sizeof( fptype )}, - {reinterpret_cast(&g_s), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&flavor_indices), rounded_count * sizeof( unsigned int )}, - {reinterpret_cast(&helicity_random), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&color_random), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&diagram_random), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&matrix_elements), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&diagram_index), rounded_count * sizeof( unsigned int )}, - {reinterpret_cast(&color_jamps), rounded_count * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, - {reinterpret_cast(&numerators), rounded_count * CPPProcess::ndiagrams * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&denominators), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&helicity_index), rounded_count * sizeof( int )}, - {reinterpret_cast(&color_index), rounded_count * sizeof( int )}, - {reinterpret_cast(&ghel_matrix_elements), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&ghel_jamps), rounded_count * CPPProcess::ncomb * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, - }}; - std::size_t total_size = 0; - constexpr std::size_t MAX_SIZE = std::max(sizeof(fptype), sizeof(int)); - for (auto [ptr, size] : ptrs_and_sizes) { - std::size_t aligned_size = (size + MAX_SIZE - 1) / MAX_SIZE * MAX_SIZE; - total_size += aligned_size; - } - uint8_t* buffer; - // we can consider caching this between matrix element calls - gpuMallocAsync( &buffer, total_size, gpu_stream ); - std::size_t buf_offset = 0; - for (auto [ptr, size] : ptrs_and_sizes) { - std::size_t aligned_size = (size + 7) / 8 * 8; - *ptr = buffer + buf_offset; - buf_offset += aligned_size; - } - - copy_inputs<<>>( - momenta_in, - random_helicity_in, - random_color_in, - random_diagram_in, - alpha_s_in, - flavor_indices_in, - momenta, - helicity_random, - color_random, - diagram_random, - g_s, - flavor_indices, - count, - stride, - offset ); - computeDependentCouplings<<>>( g_s, couplings ); - checkGpu( gpuPeekAtLastError() ); - - InterfaceInstance* instance = static_cast( handle ); - if( !instance->initialized ) - { - initialize( - momenta, couplings, flavor_indices, matrix_elements, color_jamps, numerators, denominators, rounded_count ); - instance->initialized = true; - } - - sigmaKin( - momenta, - couplings, - flavor_indices, - helicity_random, - color_random, - nullptr, - diagram_random, - matrix_elements, - helicity_index, - color_index, - color_jamps, - numerators, - denominators, - diagram_index, - false, - ghel_matrix_elements, - ghel_jamps, - nullptr, - nullptr, - &gpu_stream, - true, - n_blocks, - n_threads ); - - copy_outputs<<>>( - denominators, - numerators, - matrix_elements, - diagram_index, - color_index, - helicity_index, - m2_out, - amp2_out, - diagram_out, - color_out, - helicity_out, - count, - stride, - offset ); - checkGpu( gpuPeekAtLastError() ); - - gpuFreeAsync( buffer, gpu_stream ); -#else // MGONGPUCPP_GPUIMPL constexpr std::size_t vector_size = MemoryAccessMomentaBase::neppM; // need to round to round to double page size for some reason constexpr std::size_t page_size2 = 2 * vector_size; @@ -654,7 +435,6 @@ extern "C" } } } -#endif // MGONGPUCPP_GPUIMPL return UMAMI_SUCCESS; } From 929654b7aeadf17260b59448bf34716268337e78 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 14:18:48 +0200 Subject: [PATCH 03/25] Put the process specific constants in the files ProcessData.h ColorMatrixData.h process specic data --- .../madmatrix/ColorMatrixData.h | 23 ++++++++++++ .../madmatrix/backend/cpu/EventStatistics.h | 4 +- .../backend/cpu/MatrixElementKernels.cc | 17 +++++---- .../backend/cpu/MatrixElementKernels.h | 1 - .../backend/cpu/MemoryAccessMomenta.h | 6 +-- .../backend/cpu/MemoryAccessRandomNumbers.h | 8 ++-- .../madmatrix/backend/cpu/MemoryBuffers.h | 12 +++--- .../madmatrix/backend/cpu/color_sum.cc | 11 +----- .../madmatrix/backend/cpu/color_sum.h | 2 +- .../madmatrix/backend/cpu/coloramps.h | 4 +- .../madmatrix/backend/cpu/massless_rambo.h | 10 ++--- .../madmatrix/backend/cpu/umami.cc | 35 +++++++++--------- .../madmatrix/backend/gpu/EventStatistics.h | 4 +- .../backend/gpu/MatrixElementKernels.cc | 31 ++++++++-------- .../backend/gpu/MatrixElementKernels.h | 4 +- .../backend/gpu/MemoryAccessMomenta.h | 6 +-- .../backend/gpu/MemoryAccessRandomNumbers.h | 8 ++-- .../madmatrix/backend/gpu/MemoryBuffers.h | 12 +++--- .../madmatrix/backend/gpu/color_sum.cc | 6 +-- .../madmatrix/backend/gpu/color_sum.h | 6 +-- .../madmatrix/backend/gpu/coloramps.h | 4 +- .../madmatrix/backend/gpu/massless_rambo.h | 10 ++--- .../madmatrix/backend/gpu/umami.cc | 37 ++++++++++--------- .../madmatrix/backend/simd/EventStatistics.h | 4 +- .../backend/simd/MatrixElementKernels.cc | 17 +++++---- .../backend/simd/MatrixElementKernels.h | 1 - .../backend/simd/MemoryAccessMomenta.h | 6 +-- .../backend/simd/MemoryAccessRandomNumbers.h | 8 ++-- .../madmatrix/backend/simd/MemoryBuffers.h | 12 +++--- .../madmatrix/backend/simd/color_sum.cc | 11 +----- .../madmatrix/backend/simd/color_sum.h | 2 +- .../madmatrix/backend/simd/coloramps.h | 4 +- .../madmatrix/backend/simd/massless_rambo.h | 10 ++--- .../madmatrix/backend/simd/umami.cc | 35 +++++++++--------- .../madmatrix/process_class.inc | 28 +++++--------- .../template_files/madmatrix/process_h.inc | 1 + madmatrix/model_handling.py | 33 +++++++++++++++-- 37 files changed, 228 insertions(+), 205 deletions(-) create mode 100644 madgraph/iolibs/template_files/madmatrix/ColorMatrixData.h diff --git a/madgraph/iolibs/template_files/madmatrix/ColorMatrixData.h b/madgraph/iolibs/template_files/madmatrix/ColorMatrixData.h new file mode 100644 index 0000000000..7a8f7e7ca6 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/ColorMatrixData.h @@ -0,0 +1,23 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Process-specific color matrix, generated once per subprocess. Kept as a +// header (not a .cc) so the backend-owned color_sum_cpu/color_sum_gpu +// (backend/{cpu,simd,gpu}/color_sum.cc) can #include it and still constexpr- +// evaluate the normalized color matrix at compile time. + +#ifndef COLORMATRIXDATA_H +#define COLORMATRIXDATA_H 1 + +#include "mgOnGpuConfig.h" +#include "ProcessData.h" + +namespace ColorMatrixData +{ + constexpr int ncolor = ProcessData::ncolor; + +%(color_matrix_lines)s +} + +#endif // COLORMATRIXDATA_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h index 831d477bb6..1be807a9bd 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" // for npar (meGeVexponent) +#include "ProcessData.h" // for npar (meGeVexponent) #include #include @@ -138,7 +138,7 @@ namespace mg5amcCpu void printout( std::ostream& out ) const { const EventStatistics& s = *this; - constexpr int meGeVexponent = -( 2 * CPPProcess::npar - 8 ); + constexpr int meGeVexponent = -( 2 * ProcessData::npar - 8 ); out << s.tag << "NumMatrixElems(notAbnormal) = " << s.nevtOK() << std::endl << std::scientific // fixed format: affects all floats (default precision: 6) << s.tag << "MeanMatrixElemValue = ( " << s.meanME() diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc index fb19139f2d..a2de4d93ed 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc @@ -6,7 +6,8 @@ #include "MatrixElementKernels.h" -#include "CPPProcess.h" +#include "ProcessData.h" +#include "CPPProcess.h" // TODO(backend_separation): drop once sigmaKin/getGoodHel/computeDependentCouplings move to backend/ #include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" @@ -46,7 +47,7 @@ namespace mg5amcCpu { //std::cout << "DEBUG: MatrixElementKernelBase ctor " << this << std::endl; #ifdef MGONGPU_CHANNELID_DEBUG - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) m_nevtProcessedByChannel[channelId] = 0; #endif } @@ -76,7 +77,7 @@ namespace mg5amcCpu { const size_t channelId = pHstChannelIds[ievt]; // Fortran indexing //assert( channelId > 0 ); - //assert( channelId < CPPProcess::ndiagrams ); + //assert( channelId < ProcessData::ndiagrams ); m_nevtProcessedByChannel[channelId]++; } } @@ -94,11 +95,11 @@ namespace mg5amcCpu void MatrixElementKernelBase::dumpNevtProcessedByChannel() { size_t nevtProcessed = 0; - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) nevtProcessed += m_nevtProcessedByChannel[channelId]; std::ostringstream sstr; sstr << " {"; - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) { if( m_nevtProcessedByChannel[channelId] > 0 ) { @@ -113,7 +114,7 @@ namespace mg5amcCpu sstr << " }"; std::cout << "DEBUG: MEK " << this; if( m_tag != "" ) std::cout << " " << m_tag; - std::cout << " processed " << nevtProcessed << " events across " << CPPProcess::ndiagrams << " channels" << sstr.str() << std::endl; + std::cout << " processed " << nevtProcessed << " events across " << ProcessData::ndiagrams << " channels" << sstr.str() << std::endl; } #endif @@ -160,7 +161,7 @@ namespace mg5amcCpu : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) , NumberOfEvents( nevt ) , m_couplings( nevt ) - , m_numerators( nevt * CPPProcess::ndiagrams ) + , m_numerators( nevt * ProcessData::ndiagrams ) , m_denominators( nevt ) { //std::cout << "DEBUG: MatrixElementKernelHost::ctor " << this << std::endl; @@ -198,7 +199,7 @@ namespace mg5amcCpu int MatrixElementKernelHost::computeGoodHelicities() { - HostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); + HostBufferHelicityMask hstIsGoodHel( ProcessData::ncomb ); // ... 0d1. Compute good helicity mask on the host computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_numerators.data(), m_denominators.data(), hstIsGoodHel.data(), nevt() ); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h index 9cfb1c6247..58cd323e39 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h @@ -9,7 +9,6 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" #include "GpuAbstraction.h" #include "MemoryBuffers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h index a9e428999c..b7c175fe51 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "MemoryAccessHelpers.h" #include "MemoryAccessVectors.h" @@ -45,10 +45,10 @@ namespace mg5amcCpu friend class KernelAccessHelper; // The number of components of a 4-momentum - static constexpr int np4 = CPPProcess::np4; + static constexpr int np4 = ProcessData::np4; // The number of particles in this physics process - static constexpr int npar = CPPProcess::npar; + static constexpr int npar = ProcessData::npar; //-------------------------------------------------------------------------- // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h index ebdffe9c84..a20726428b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h @@ -9,11 +9,9 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "MemoryAccessHelpers.h" -using mg5amcCpu::CPPProcess; - //---------------------------------------------------------------------------- // A class describing the internal layout of memory buffers for random numbers @@ -36,10 +34,10 @@ class MemoryAccessRandomNumbersBase //_AOSOAv1 friend class KernelAccessHelper; // The number of components of a 4-momentum - static constexpr int np4 = CPPProcess::np4; + static constexpr int np4 = ProcessData::np4; // The number of final state particles in this physics process - static constexpr int nparf = CPPProcess::nparf; + static constexpr int nparf = ProcessData::nparf; //-------------------------------------------------------------------------- // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h index c229fc0a56..0dd7244541 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h @@ -11,7 +11,7 @@ #include "mgOnGpuCxtypes.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "GpuRuntime.h" #include "Parameters.h" #include "processConfig.h" @@ -25,14 +25,14 @@ namespace mg5amcCpu namespace MemoryBuffers { // Process-independent compile-time constants - static constexpr size_t np4 = CPPProcess::np4; - static constexpr size_t nw6 = CPPProcess::nw6; + static constexpr size_t np4 = ProcessData::np4; + static constexpr size_t nw6 = ProcessData::nw6; static constexpr size_t nx2 = mgOnGpu::nx2; // Process-dependent compile-time constants - static constexpr size_t nparf = CPPProcess::nparf; - static constexpr size_t npar = CPPProcess::npar; + static constexpr size_t nparf = ProcessData::nparf; + static constexpr size_t npar = ProcessData::npar; static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; - static constexpr size_t ncolor = CPPProcess::ncolor; + static constexpr size_t ncolor = ProcessData::ncolor; } //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc index 2cbe2e38cc..d9d8b15aa4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc @@ -8,19 +8,12 @@ #include "mgOnGpuConfig.h" +#include "ColorMatrixData.h" // P1-generated: colorMatrix/colorDenom #include "MemoryAccessMatrixElements.h" namespace mg5amcCpu { - constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors - - //-------------------------------------------------------------------------- - - // *** COLOR MATRIX BELOW *** -%(color_matrix_lines)s - - - //-------------------------------------------------------------------------- + using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h index 2e043fa128..c88f717621 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h @@ -11,7 +11,7 @@ #include "mgOnGpuVectors.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "GpuAbstraction.h" namespace mg5amcCpu diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h index 8ff67e63ff..01ca72ac96 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h @@ -7,7 +7,7 @@ #ifndef COLORAMPS_H #define COLORAMPS_H 1 -#include "CPPProcess.h" +#include "ProcessData.h" namespace mgOnGpu @@ -30,7 +30,7 @@ namespace mgOnGpu // The number of channels in the channel2iconfig array below // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** constexpr unsigned int nchannels = %(nb_diag)i; - static_assert( nchannels <= mg5amcCpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 + static_assert( nchannels <= ProcessData::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h index 2b7aaafc19..d466affa16 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h @@ -15,7 +15,7 @@ #include "mgOnGpuFptypes.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include #include @@ -26,10 +26,10 @@ namespace mg5amcCpu { namespace massless_rambo { - constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npari = CPPProcess::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- - constexpr int nparf = CPPProcess::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- - constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- + constexpr int np4 = ProcessData::np4; // dimensions of 4-momenta (E,px,py,pz) + constexpr int npari = ProcessData::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- + constexpr int nparf = ProcessData::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- + constexpr int npar = ProcessData::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc index 0cfb07ed26..95aa04f718 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc @@ -6,7 +6,8 @@ #include "umami.h" -#include "CPPProcess.h" +#include "ProcessData.h" +#include "CPPProcess.h" // needed to construct/initProc the process object (umami_initialize) #include "GpuRuntime.h" #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" @@ -30,7 +31,7 @@ namespace fptype* denominators, std::size_t count ) { - bool is_good_hel[CPPProcess::ncomb]; + bool is_good_hel[ProcessData::ncomb]; sigmaKin_getGoodHel( momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, is_good_hel, @@ -62,12 +63,12 @@ namespace std::size_t i_page = i_event_out / page_size; std::size_t i_vector = i_event_out % page_size; - for( std::size_t i_part = 0; i_part < CPPProcess::npar; ++i_part ) + for( std::size_t i_part = 0; i_part < ProcessData::npar; ++i_part ) { for( std::size_t i_mom = 0; i_mom < 4; ++i_mom ) { - momenta_out[i_page * CPPProcess::npar * 4 * page_size + - i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( CPPProcess::npar * i_mom + i_part ) + i_event_in]; + momenta_out[i_page * ProcessData::npar * 4 * page_size + + i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( ProcessData::npar * i_mom + i_part ) + i_event_in]; } } } @@ -95,21 +96,21 @@ extern "C" break; } case UMAMI_META_PARTICLE_COUNT: - *static_cast( result ) = CPPProcess::npar; + *static_cast( result ) = ProcessData::npar; break; case UMAMI_META_DIAGRAM_COUNT: - *static_cast( result ) = CPPProcess::ndiagrams; + *static_cast( result ) = ProcessData::ndiagrams; break; case UMAMI_META_HELICITY_COUNT: - *static_cast( result ) = CPPProcess::ncomb; + *static_cast( result ) = ProcessData::ncomb; break; case UMAMI_META_COLOR_COUNT: return UMAMI_ERROR_UNSUPPORTED_META; case UMAMI_META_MASSES: { - if( g_externalMasses.size() != (size_t)CPPProcess::npar ) return UMAMI_ERROR_UNINITIALIZED_META; + if( g_externalMasses.size() != (size_t)ProcessData::npar ) return UMAMI_ERROR_UNINITIALIZED_META; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + for( int ipar = 0; ipar < ProcessData::npar; ++ipar ) static_cast( result )[ipar] = g_externalMasses[ipar]; break; } @@ -267,7 +268,7 @@ extern "C" std::vector permutation; std::size_t rounded_count; - constexpr std::size_t flavor_count = CPPProcess::nmaxflavor; + constexpr std::size_t flavor_count = ProcessData::nmaxflavor; HostBufferBase flavor_indices( ((count + page_size2 - 1) / page_size2 + flavor_count) * page_size2 ); bool sort_flavors = vector_size > 1 && flavor_count > 1 && flavor_indices_in; if ( sort_flavors ) @@ -299,7 +300,7 @@ extern "C" rounded_count = ( count + page_size2 - 1 ) / page_size2 * page_size2; } - HostBufferBase momenta( rounded_count * CPPProcess::npar * 4 ); + HostBufferBase momenta( rounded_count * ProcessData::npar * 4 ); HostBufferBase couplings( rounded_count * mg5amcCpu::Parameters_dependentCouplings::ndcoup * 2 ); HostBufferBase g_s( rounded_count ); HostBufferBase helicity_random( rounded_count ); @@ -307,7 +308,7 @@ extern "C" HostBufferBase diagram_random( rounded_count ); HostBufferBase matrix_elements( rounded_count ); HostBufferBase diagram_index( rounded_count ); - HostBufferBase numerators( rounded_count * CPPProcess::ndiagrams ); + HostBufferBase numerators( rounded_count * ProcessData::ndiagrams ); HostBufferBase denominators( rounded_count ); HostBufferBase helicity_index( rounded_count ); HostBufferBase color_index( rounded_count ); @@ -384,9 +385,9 @@ extern "C" } if( amp2_out != nullptr ) { - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + for( std::size_t i_diag = 0; i_diag < ProcessData::ndiagrams; ++i_diag ) { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * ProcessData::ndiagrams + i_diag * page_size + i_vector] / denominator; } } if( diagram_out != nullptr ) @@ -416,9 +417,9 @@ extern "C" } if( amp2_out != nullptr ) { - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + for( std::size_t i_diag = 0; i_diag < ProcessData::ndiagrams; ++i_diag ) { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * ProcessData::ndiagrams + i_diag * page_size + i_vector] / denominator; } } if( diagram_out != nullptr ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h index 9e334c5c47..c8a25e2369 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" // for npar (meGeVexponent) +#include "ProcessData.h" // for npar (meGeVexponent) #include #include @@ -138,7 +138,7 @@ namespace mg5amcGpu void printout( std::ostream& out ) const { const EventStatistics& s = *this; - constexpr int meGeVexponent = -( 2 * CPPProcess::npar - 8 ); + constexpr int meGeVexponent = -( 2 * ProcessData::npar - 8 ); out << s.tag << "NumMatrixElems(notAbnormal) = " << s.nevtOK() << std::endl << std::scientific // fixed format: affects all floats (default precision: 6) << s.tag << "MeanMatrixElemValue = ( " << s.meanME() diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc index e6aaa2f763..83214ab225 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc @@ -6,7 +6,8 @@ #include "MatrixElementKernels.h" -#include "CPPProcess.h" +#include "ProcessData.h" +#include "CPPProcess.h" // TODO(backend_separation): drop once sigmaKin/getGoodHel/computeDependentCouplings move to backend/ #include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" @@ -46,7 +47,7 @@ namespace mg5amcGpu { //std::cout << "DEBUG: MatrixElementKernelBase ctor " << this << std::endl; #ifdef MGONGPU_CHANNELID_DEBUG - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) m_nevtProcessedByChannel[channelId] = 0; #endif } @@ -76,7 +77,7 @@ namespace mg5amcGpu { const size_t channelId = pHstChannelIds[ievt]; // Fortran indexing //assert( channelId > 0 ); - //assert( channelId < CPPProcess::ndiagrams ); + //assert( channelId < ProcessData::ndiagrams ); m_nevtProcessedByChannel[channelId]++; } } @@ -94,11 +95,11 @@ namespace mg5amcGpu void MatrixElementKernelBase::dumpNevtProcessedByChannel() { size_t nevtProcessed = 0; - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) nevtProcessed += m_nevtProcessedByChannel[channelId]; std::ostringstream sstr; sstr << " {"; - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) { if( m_nevtProcessedByChannel[channelId] > 0 ) { @@ -113,7 +114,7 @@ namespace mg5amcGpu sstr << " }"; std::cout << "DEBUG: MEK " << this; if( m_tag != "" ) std::cout << " " << m_tag; - std::cout << " processed " << nevtProcessed << " events across " << CPPProcess::ndiagrams << " channels" << sstr.str() << std::endl; + std::cout << " processed " << nevtProcessed << " events across " << ProcessData::ndiagrams << " channels" << sstr.str() << std::endl; } #endif @@ -168,7 +169,7 @@ namespace mg5amcGpu , m_pHelJamps() , m_pHelNumerators() , m_pHelDenominators() - , m_colJamp2s( CPPProcess::ncolor * this->nevt() ) + , m_colJamp2s( ProcessData::ncolor * this->nevt() ) #ifdef MGONGPU_CHANNELID_DEBUG , m_hstChannelIds( this->nevt() ) #endif @@ -203,9 +204,9 @@ namespace mg5amcGpu throw std::runtime_error( sstr.str() ); } // Create the "one-helicity" jamp buffer that will be used for helicity filtering - m_pHelJamps.reset( new DeviceBufferSimple( CPPProcess::ncolor * mgOnGpu::nx2 * this->nevt() ) ); + m_pHelJamps.reset( new DeviceBufferSimple( ProcessData::ncolor * mgOnGpu::nx2 * this->nevt() ) ); // Create the "one-helicity" numerator and denominator buffers that will be used for helicity filtering - m_pHelNumerators.reset( new DeviceBufferSimple( this->nevt() * CPPProcess::ndiagrams ) ); + m_pHelNumerators.reset( new DeviceBufferSimple( this->nevt() * ProcessData::ndiagrams ) ); m_pHelDenominators.reset( new DeviceBufferSimple( this->nevt() ) ); // Decide at runtime whether to use BLAS for color sums // Decide at runtime whether TF32TENSOR math should be used in cuBLAS @@ -270,7 +271,7 @@ namespace mg5amcGpu #ifndef MGONGPU_HAS_NO_BLAS if( m_blasHandle ) gpuBlasDestroy( m_blasHandle ); #endif - for( int ihel = 0; ihel < CPPProcess::ncomb; ihel++ ) + for( int ihel = 0; ihel < ProcessData::ncomb; ihel++ ) { if( m_helStreams[ihel] ) gpuStreamDestroy( m_helStreams[ihel] ); // do not destroy if nullptr } @@ -290,7 +291,7 @@ namespace mg5amcGpu int MatrixElementKernelDevice::computeGoodHelicities() { - PinnedHostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); + PinnedHostBufferHelicityMask hstIsGoodHel( ProcessData::ncomb ); // ... 0d1. Compute good helicity mask (a host variable) on the device gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); const int nevt = m_gpublocks * m_gputhreads; @@ -316,19 +317,19 @@ namespace mg5amcGpu m_pHelMEs.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) - m_pHelJamps.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); + m_pHelJamps.reset( new DeviceBufferSimple( nGoodHel * ProcessData::ncolor * mgOnGpu::nx2 * nevt ) ); // ... Create the "many-helicity" super-buffers of nGoodHel numerator and denominator buffers (dynamically allocated) // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) - m_pHelNumerators.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ndiagrams * nevt ) ); + m_pHelNumerators.reset( new DeviceBufferSimple( nGoodHel * ProcessData::ndiagrams * nevt ) ); m_pHelDenominators.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); #ifndef MGONGPU_HAS_NO_BLAS // Create the "many-helicity" super-buffers of real/imag ncolor*nevt temporary buffers for cuBLAS/hipBLAS intermediate results in color_sum_blas #if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT // Mixed precision mode: need two fptype2[ncolor*2*nevt] buffers and one fptype2[nevt] buffer per good helicity - if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ( 2 * CPPProcess::ncolor * mgOnGpu::nx2 + 1 ) * nevt ) ); + if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ( 2 * ProcessData::ncolor * mgOnGpu::nx2 + 1 ) * nevt ) ); #else // Standard single/double precision mode: need one fptype2[ncolor*2*nevt] buffer per good helicity - if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); + if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ProcessData::ncolor * mgOnGpu::nx2 * nevt ) ); #endif #endif // Return the number of good helicities diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h index 5e30723a3f..3be385f40e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "GpuAbstraction.h" #include "MemoryBuffers.h" @@ -182,7 +182,7 @@ namespace mg5amcGpu #endif // The array of GPU streams (one for each good helicity) - gpuStream_t m_helStreams[CPPProcess::ncomb]; // reserve ncomb streams (but only nGoodHel <= ncomb will be used) + gpuStream_t m_helStreams[ProcessData::ncomb]; // reserve ncomb streams (but only nGoodHel <= ncomb will be used) // The number of blocks in the GPU grid size_t m_gpublocks; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h index 6301587740..0f76adb03c 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "MemoryAccessHelpers.h" #include "MemoryAccessVectors.h" @@ -46,10 +46,10 @@ namespace mg5amcGpu friend class KernelAccessHelper; // The number of components of a 4-momentum - static constexpr int np4 = CPPProcess::np4; + static constexpr int np4 = ProcessData::np4; // The number of particles in this physics process - static constexpr int npar = CPPProcess::npar; + static constexpr int npar = ProcessData::npar; //-------------------------------------------------------------------------- // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h index d553211648..a20726428b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h @@ -9,11 +9,9 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "MemoryAccessHelpers.h" -using mg5amcGpu::CPPProcess; - //---------------------------------------------------------------------------- // A class describing the internal layout of memory buffers for random numbers @@ -36,10 +34,10 @@ class MemoryAccessRandomNumbersBase //_AOSOAv1 friend class KernelAccessHelper; // The number of components of a 4-momentum - static constexpr int np4 = CPPProcess::np4; + static constexpr int np4 = ProcessData::np4; // The number of final state particles in this physics process - static constexpr int nparf = CPPProcess::nparf; + static constexpr int nparf = ProcessData::nparf; //-------------------------------------------------------------------------- // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h index d27c3ebbbc..cfff3d5d60 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h @@ -11,7 +11,7 @@ #include "mgOnGpuCxtypes.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "GpuRuntime.h" #include "Parameters.h" #include "processConfig.h" @@ -25,14 +25,14 @@ namespace mg5amcGpu namespace MemoryBuffers { // Process-independent compile-time constants - static constexpr size_t np4 = CPPProcess::np4; - static constexpr size_t nw6 = CPPProcess::nw6; + static constexpr size_t np4 = ProcessData::np4; + static constexpr size_t nw6 = ProcessData::nw6; static constexpr size_t nx2 = mgOnGpu::nx2; // Process-dependent compile-time constants - static constexpr size_t nparf = CPPProcess::nparf; - static constexpr size_t npar = CPPProcess::npar; + static constexpr size_t nparf = ProcessData::nparf; + static constexpr size_t npar = ProcessData::npar; static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; - static constexpr size_t ncolor = CPPProcess::ncolor; + static constexpr size_t ncolor = ProcessData::ncolor; } //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc index bb324fa3bd..7d8dd2c31a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc @@ -8,17 +8,15 @@ #include "mgOnGpuConfig.h" +#include "ColorMatrixData.h" // P1-generated: colorMatrix/colorDenom #include "MemoryAccessMatrixElements.h" namespace mg5amcGpu { - constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors + using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor //-------------------------------------------------------------------------- - // *** COLOR MATRIX BELOW *** -%(color_matrix_lines)s - // The normalized color matrix (divide each column by denom) template struct NormalizedColorMatrix diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h index 07ef85ce10..8d9fea5577 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h @@ -11,7 +11,7 @@ #include "mgOnGpuVectors.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "GpuAbstraction.h" namespace mg5amcGpu @@ -24,7 +24,7 @@ namespace mg5amcGpu static __device__ inline cxtype_ref kernelAccessIcolIhelNhel( fptype* buffer, const int icol, const int ihel, const int nhel ) { - const int ncolor = CPPProcess::ncolor; // the number of leading colors + const int ncolor = ProcessData::ncolor; // the number of leading colors const int nevt = gridDim.x * blockDim.x; const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) @@ -39,7 +39,7 @@ namespace mg5amcGpu static __device__ inline const cxtype kernelAccessIcolIhelNhelConst( const fptype* buffer, const int icol, const int ihel, const int nhel ) { - const int ncolor = CPPProcess::ncolor; // the number of leading colors + const int ncolor = ProcessData::ncolor; // the number of leading colors const int nevt = gridDim.x * blockDim.x; const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h index 4a465e52b0..72dc0d4b8a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h @@ -7,7 +7,7 @@ #ifndef COLORAMPS_H #define COLORAMPS_H 1 -#include "CPPProcess.h" +#include "ProcessData.h" namespace mgOnGpu @@ -30,7 +30,7 @@ namespace mgOnGpu // The number of channels in the channel2iconfig array below // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** constexpr unsigned int nchannels = %(nb_diag)i; - static_assert( nchannels <= mg5amcGpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 + static_assert( nchannels <= ProcessData::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h index 0fc66e512e..6a1180f7be 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h @@ -15,7 +15,7 @@ #include "mgOnGpuFptypes.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include #include @@ -26,10 +26,10 @@ namespace mg5amcGpu { namespace massless_rambo { - constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npari = CPPProcess::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- - constexpr int nparf = CPPProcess::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- - constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- + constexpr int np4 = ProcessData::np4; // dimensions of 4-momenta (E,px,py,pz) + constexpr int npari = ProcessData::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- + constexpr int nparf = ProcessData::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- + constexpr int npar = ProcessData::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc index ee2f4f2afb..329ce57593 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc @@ -6,7 +6,8 @@ #include "umami.h" -#include "CPPProcess.h" +#include "ProcessData.h" +#include "CPPProcess.h" // needed to construct/initProc the process object (umami_initialize) #include "GpuRuntime.h" #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" @@ -31,7 +32,7 @@ namespace fptype* denominators, std::size_t count ) { - bool is_good_hel[CPPProcess::ncomb]; + bool is_good_hel[ProcessData::ncomb]; sigmaKin_getGoodHel( momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, color_jamps, @@ -67,12 +68,12 @@ namespace std::size_t i_page = i_event_out / page_size; std::size_t i_vector = i_event_out % page_size; - for( std::size_t i_part = 0; i_part < CPPProcess::npar; ++i_part ) + for( std::size_t i_part = 0; i_part < ProcessData::npar; ++i_part ) { for( std::size_t i_mom = 0; i_mom < 4; ++i_mom ) { - momenta_out[i_page * CPPProcess::npar * 4 * page_size + - i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( CPPProcess::npar * i_mom + i_part ) + i_event_in]; + momenta_out[i_page * ProcessData::npar * 4 * page_size + + i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( ProcessData::npar * i_mom + i_part ) + i_event_in]; } } } @@ -129,9 +130,9 @@ namespace if( amp2_out ) { double denominator = denominators[i_event]; - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + for( std::size_t i_diag = 0; i_diag < ProcessData::ndiagrams; ++i_diag ) { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_event * CPPProcess::ndiagrams + i_diag] / denominator; + amp2_out[stride * i_diag + i_event + offset] = numerators[i_event * ProcessData::ndiagrams + i_diag] / denominator; } } if( diagram_out ) diagram_out[i_event + offset] = diagram_index[i_event] - 1; @@ -166,21 +167,21 @@ extern "C" break; } case UMAMI_META_PARTICLE_COUNT: - *static_cast( result ) = CPPProcess::npar; + *static_cast( result ) = ProcessData::npar; break; case UMAMI_META_DIAGRAM_COUNT: - *static_cast( result ) = CPPProcess::ndiagrams; + *static_cast( result ) = ProcessData::ndiagrams; break; case UMAMI_META_HELICITY_COUNT: - *static_cast( result ) = CPPProcess::ncomb; + *static_cast( result ) = ProcessData::ncomb; break; case UMAMI_META_COLOR_COUNT: return UMAMI_ERROR_UNSUPPORTED_META; case UMAMI_META_MASSES: { - if( g_externalMasses.size() != (size_t)CPPProcess::npar ) return UMAMI_ERROR_UNINITIALIZED_META; + if( g_externalMasses.size() != (size_t)ProcessData::npar ) return UMAMI_ERROR_UNINITIALIZED_META; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + for( int ipar = 0; ipar < ProcessData::npar; ++ipar ) static_cast( result )[ipar] = g_externalMasses[ipar]; break; } @@ -347,7 +348,7 @@ extern "C" std::size_t n_coup = mg5amcGpu::Parameters_dependentCouplings::ndcoup; std::array, 16> ptrs_and_sizes = {{ - {reinterpret_cast(&momenta), rounded_count * CPPProcess::npar * 4 * sizeof( fptype )}, + {reinterpret_cast(&momenta), rounded_count * ProcessData::npar * 4 * sizeof( fptype )}, {reinterpret_cast(&couplings), rounded_count * n_coup * 2 * sizeof( fptype )}, {reinterpret_cast(&g_s), rounded_count * sizeof( fptype )}, {reinterpret_cast(&flavor_indices), rounded_count * sizeof( unsigned int )}, @@ -356,13 +357,13 @@ extern "C" {reinterpret_cast(&diagram_random), rounded_count * sizeof( fptype )}, {reinterpret_cast(&matrix_elements), rounded_count * sizeof( fptype )}, {reinterpret_cast(&diagram_index), rounded_count * sizeof( unsigned int )}, - {reinterpret_cast(&color_jamps), rounded_count * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, - {reinterpret_cast(&numerators), rounded_count * CPPProcess::ndiagrams * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&denominators), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&color_jamps), rounded_count * ProcessData::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, + {reinterpret_cast(&numerators), rounded_count * ProcessData::ndiagrams * ProcessData::ncomb * sizeof( fptype )}, + {reinterpret_cast(&denominators), rounded_count * ProcessData::ncomb * sizeof( fptype )}, {reinterpret_cast(&helicity_index), rounded_count * sizeof( int )}, {reinterpret_cast(&color_index), rounded_count * sizeof( int )}, - {reinterpret_cast(&ghel_matrix_elements), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&ghel_jamps), rounded_count * CPPProcess::ncomb * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, + {reinterpret_cast(&ghel_matrix_elements), rounded_count * ProcessData::ncomb * sizeof( fptype )}, + {reinterpret_cast(&ghel_jamps), rounded_count * ProcessData::ncomb * ProcessData::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, }}; std::size_t total_size = 0; constexpr std::size_t MAX_SIZE = std::max(sizeof(fptype), sizeof(int)); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h index 831d477bb6..1be807a9bd 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" // for npar (meGeVexponent) +#include "ProcessData.h" // for npar (meGeVexponent) #include #include @@ -138,7 +138,7 @@ namespace mg5amcCpu void printout( std::ostream& out ) const { const EventStatistics& s = *this; - constexpr int meGeVexponent = -( 2 * CPPProcess::npar - 8 ); + constexpr int meGeVexponent = -( 2 * ProcessData::npar - 8 ); out << s.tag << "NumMatrixElems(notAbnormal) = " << s.nevtOK() << std::endl << std::scientific // fixed format: affects all floats (default precision: 6) << s.tag << "MeanMatrixElemValue = ( " << s.meanME() diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc index fb19139f2d..a2de4d93ed 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc @@ -6,7 +6,8 @@ #include "MatrixElementKernels.h" -#include "CPPProcess.h" +#include "ProcessData.h" +#include "CPPProcess.h" // TODO(backend_separation): drop once sigmaKin/getGoodHel/computeDependentCouplings move to backend/ #include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" @@ -46,7 +47,7 @@ namespace mg5amcCpu { //std::cout << "DEBUG: MatrixElementKernelBase ctor " << this << std::endl; #ifdef MGONGPU_CHANNELID_DEBUG - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) m_nevtProcessedByChannel[channelId] = 0; #endif } @@ -76,7 +77,7 @@ namespace mg5amcCpu { const size_t channelId = pHstChannelIds[ievt]; // Fortran indexing //assert( channelId > 0 ); - //assert( channelId < CPPProcess::ndiagrams ); + //assert( channelId < ProcessData::ndiagrams ); m_nevtProcessedByChannel[channelId]++; } } @@ -94,11 +95,11 @@ namespace mg5amcCpu void MatrixElementKernelBase::dumpNevtProcessedByChannel() { size_t nevtProcessed = 0; - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) nevtProcessed += m_nevtProcessedByChannel[channelId]; std::ostringstream sstr; sstr << " {"; - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) { if( m_nevtProcessedByChannel[channelId] > 0 ) { @@ -113,7 +114,7 @@ namespace mg5amcCpu sstr << " }"; std::cout << "DEBUG: MEK " << this; if( m_tag != "" ) std::cout << " " << m_tag; - std::cout << " processed " << nevtProcessed << " events across " << CPPProcess::ndiagrams << " channels" << sstr.str() << std::endl; + std::cout << " processed " << nevtProcessed << " events across " << ProcessData::ndiagrams << " channels" << sstr.str() << std::endl; } #endif @@ -160,7 +161,7 @@ namespace mg5amcCpu : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) , NumberOfEvents( nevt ) , m_couplings( nevt ) - , m_numerators( nevt * CPPProcess::ndiagrams ) + , m_numerators( nevt * ProcessData::ndiagrams ) , m_denominators( nevt ) { //std::cout << "DEBUG: MatrixElementKernelHost::ctor " << this << std::endl; @@ -198,7 +199,7 @@ namespace mg5amcCpu int MatrixElementKernelHost::computeGoodHelicities() { - HostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); + HostBufferHelicityMask hstIsGoodHel( ProcessData::ncomb ); // ... 0d1. Compute good helicity mask on the host computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_numerators.data(), m_denominators.data(), hstIsGoodHel.data(), nevt() ); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h index 9cfb1c6247..58cd323e39 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h @@ -9,7 +9,6 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" #include "GpuAbstraction.h" #include "MemoryBuffers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h index 92811a2ddf..b591a33624 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "MemoryAccessHelpers.h" #include "MemoryAccessVectors.h" @@ -49,10 +49,10 @@ namespace mg5amcCpu friend class KernelAccessHelper; // The number of components of a 4-momentum - static constexpr int np4 = CPPProcess::np4; + static constexpr int np4 = ProcessData::np4; // The number of particles in this physics process - static constexpr int npar = CPPProcess::npar; + static constexpr int npar = ProcessData::npar; //-------------------------------------------------------------------------- // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h index ebdffe9c84..a20726428b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h @@ -9,11 +9,9 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "MemoryAccessHelpers.h" -using mg5amcCpu::CPPProcess; - //---------------------------------------------------------------------------- // A class describing the internal layout of memory buffers for random numbers @@ -36,10 +34,10 @@ class MemoryAccessRandomNumbersBase //_AOSOAv1 friend class KernelAccessHelper; // The number of components of a 4-momentum - static constexpr int np4 = CPPProcess::np4; + static constexpr int np4 = ProcessData::np4; // The number of final state particles in this physics process - static constexpr int nparf = CPPProcess::nparf; + static constexpr int nparf = ProcessData::nparf; //-------------------------------------------------------------------------- // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h index c229fc0a56..0dd7244541 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h @@ -11,7 +11,7 @@ #include "mgOnGpuCxtypes.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "GpuRuntime.h" #include "Parameters.h" #include "processConfig.h" @@ -25,14 +25,14 @@ namespace mg5amcCpu namespace MemoryBuffers { // Process-independent compile-time constants - static constexpr size_t np4 = CPPProcess::np4; - static constexpr size_t nw6 = CPPProcess::nw6; + static constexpr size_t np4 = ProcessData::np4; + static constexpr size_t nw6 = ProcessData::nw6; static constexpr size_t nx2 = mgOnGpu::nx2; // Process-dependent compile-time constants - static constexpr size_t nparf = CPPProcess::nparf; - static constexpr size_t npar = CPPProcess::npar; + static constexpr size_t nparf = ProcessData::nparf; + static constexpr size_t npar = ProcessData::npar; static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; - static constexpr size_t ncolor = CPPProcess::ncolor; + static constexpr size_t ncolor = ProcessData::ncolor; } //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc index 2cbe2e38cc..d9d8b15aa4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc @@ -8,19 +8,12 @@ #include "mgOnGpuConfig.h" +#include "ColorMatrixData.h" // P1-generated: colorMatrix/colorDenom #include "MemoryAccessMatrixElements.h" namespace mg5amcCpu { - constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors - - //-------------------------------------------------------------------------- - - // *** COLOR MATRIX BELOW *** -%(color_matrix_lines)s - - - //-------------------------------------------------------------------------- + using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h index 2e043fa128..c88f717621 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h @@ -11,7 +11,7 @@ #include "mgOnGpuVectors.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "GpuAbstraction.h" namespace mg5amcCpu diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h index 8ff67e63ff..01ca72ac96 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h @@ -7,7 +7,7 @@ #ifndef COLORAMPS_H #define COLORAMPS_H 1 -#include "CPPProcess.h" +#include "ProcessData.h" namespace mgOnGpu @@ -30,7 +30,7 @@ namespace mgOnGpu // The number of channels in the channel2iconfig array below // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** constexpr unsigned int nchannels = %(nb_diag)i; - static_assert( nchannels <= mg5amcCpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 + static_assert( nchannels <= ProcessData::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h index 2b7aaafc19..d466affa16 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h @@ -15,7 +15,7 @@ #include "mgOnGpuFptypes.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include #include @@ -26,10 +26,10 @@ namespace mg5amcCpu { namespace massless_rambo { - constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npari = CPPProcess::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- - constexpr int nparf = CPPProcess::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- - constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- + constexpr int np4 = ProcessData::np4; // dimensions of 4-momenta (E,px,py,pz) + constexpr int npari = ProcessData::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- + constexpr int nparf = ProcessData::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- + constexpr int npar = ProcessData::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc index 0cfb07ed26..95aa04f718 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc @@ -6,7 +6,8 @@ #include "umami.h" -#include "CPPProcess.h" +#include "ProcessData.h" +#include "CPPProcess.h" // needed to construct/initProc the process object (umami_initialize) #include "GpuRuntime.h" #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" @@ -30,7 +31,7 @@ namespace fptype* denominators, std::size_t count ) { - bool is_good_hel[CPPProcess::ncomb]; + bool is_good_hel[ProcessData::ncomb]; sigmaKin_getGoodHel( momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, is_good_hel, @@ -62,12 +63,12 @@ namespace std::size_t i_page = i_event_out / page_size; std::size_t i_vector = i_event_out % page_size; - for( std::size_t i_part = 0; i_part < CPPProcess::npar; ++i_part ) + for( std::size_t i_part = 0; i_part < ProcessData::npar; ++i_part ) { for( std::size_t i_mom = 0; i_mom < 4; ++i_mom ) { - momenta_out[i_page * CPPProcess::npar * 4 * page_size + - i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( CPPProcess::npar * i_mom + i_part ) + i_event_in]; + momenta_out[i_page * ProcessData::npar * 4 * page_size + + i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( ProcessData::npar * i_mom + i_part ) + i_event_in]; } } } @@ -95,21 +96,21 @@ extern "C" break; } case UMAMI_META_PARTICLE_COUNT: - *static_cast( result ) = CPPProcess::npar; + *static_cast( result ) = ProcessData::npar; break; case UMAMI_META_DIAGRAM_COUNT: - *static_cast( result ) = CPPProcess::ndiagrams; + *static_cast( result ) = ProcessData::ndiagrams; break; case UMAMI_META_HELICITY_COUNT: - *static_cast( result ) = CPPProcess::ncomb; + *static_cast( result ) = ProcessData::ncomb; break; case UMAMI_META_COLOR_COUNT: return UMAMI_ERROR_UNSUPPORTED_META; case UMAMI_META_MASSES: { - if( g_externalMasses.size() != (size_t)CPPProcess::npar ) return UMAMI_ERROR_UNINITIALIZED_META; + if( g_externalMasses.size() != (size_t)ProcessData::npar ) return UMAMI_ERROR_UNINITIALIZED_META; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + for( int ipar = 0; ipar < ProcessData::npar; ++ipar ) static_cast( result )[ipar] = g_externalMasses[ipar]; break; } @@ -267,7 +268,7 @@ extern "C" std::vector permutation; std::size_t rounded_count; - constexpr std::size_t flavor_count = CPPProcess::nmaxflavor; + constexpr std::size_t flavor_count = ProcessData::nmaxflavor; HostBufferBase flavor_indices( ((count + page_size2 - 1) / page_size2 + flavor_count) * page_size2 ); bool sort_flavors = vector_size > 1 && flavor_count > 1 && flavor_indices_in; if ( sort_flavors ) @@ -299,7 +300,7 @@ extern "C" rounded_count = ( count + page_size2 - 1 ) / page_size2 * page_size2; } - HostBufferBase momenta( rounded_count * CPPProcess::npar * 4 ); + HostBufferBase momenta( rounded_count * ProcessData::npar * 4 ); HostBufferBase couplings( rounded_count * mg5amcCpu::Parameters_dependentCouplings::ndcoup * 2 ); HostBufferBase g_s( rounded_count ); HostBufferBase helicity_random( rounded_count ); @@ -307,7 +308,7 @@ extern "C" HostBufferBase diagram_random( rounded_count ); HostBufferBase matrix_elements( rounded_count ); HostBufferBase diagram_index( rounded_count ); - HostBufferBase numerators( rounded_count * CPPProcess::ndiagrams ); + HostBufferBase numerators( rounded_count * ProcessData::ndiagrams ); HostBufferBase denominators( rounded_count ); HostBufferBase helicity_index( rounded_count ); HostBufferBase color_index( rounded_count ); @@ -384,9 +385,9 @@ extern "C" } if( amp2_out != nullptr ) { - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + for( std::size_t i_diag = 0; i_diag < ProcessData::ndiagrams; ++i_diag ) { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * ProcessData::ndiagrams + i_diag * page_size + i_vector] / denominator; } } if( diagram_out != nullptr ) @@ -416,9 +417,9 @@ extern "C" } if( amp2_out != nullptr ) { - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + for( std::size_t i_diag = 0; i_diag < ProcessData::ndiagrams; ++i_diag ) { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * ProcessData::ndiagrams + i_diag * page_size + i_vector] / denominator; } } if( diagram_out != nullptr ) diff --git a/madgraph/iolibs/template_files/madmatrix/process_class.inc b/madgraph/iolibs/template_files/madmatrix/process_class.inc index 59a6d07333..44a853d4bb 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_class.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_class.inc @@ -45,24 +45,16 @@ public: - // Process-independent compile-time constants - static constexpr int np4 = 4; // dimensions of 4-momenta (E,px,py,pz) - static constexpr int nw6 = %(nwave)d; // dimensions of each wavefunction (HELAS KEK 91-11): e.g. 6 for e+ e- -> mu+ mu- (fermions and vectors) - - // Process-dependent compile-time constants - static constexpr int npari = %(nincoming)d; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- - static constexpr int nparf = %(noutcoming)d; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- - static constexpr int npar = npari + nparf; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - static constexpr int ncomb = %(nbhel)d; // #helicity combinations: e.g. 16 for e+ e- -> mu+ mu- (2**4 = fermion spin up/down ** npar) - static constexpr int ndiagrams = %(ndiagrams)d; // #Feynman diagrams: e.g. 3 for e+ e- -> mu+ mu- - static constexpr int ncolor = %(ncolor)s; // the number of leading colors: e.g. 1 for e+ e- -> mu+ mu- - static constexpr int nmaxflavor = %(nmaxflavor)d; // the maximum number of flavor combinations - - // Hardcoded parameters for this process (constant class variables) - // [NB: this class assumes nprocesses==1 i.e. a single DSIG1 and no DSIG2 in Fortran (#272 and #343)] - // [NB: these parameters (e.g. nwf) are P1-specific, i.e. they are different for different P1 subdirectories (#644)] - // [NB: I am currently unable to get the right value of nwf in CPPProcess.h - will hardcode it in CPPProcess.cc instead (#644)] - //static const int nwf = ??; // #wavefunctions = #external (npar) + #internal: e.g. 5 for e+ e- -> mu+ mu- (1 internal is gamma or Z) + // Compile-time constants: single source of truth is ProcessData.h + static constexpr int np4 = ProcessData::np4; + static constexpr int nw6 = ProcessData::nw6; + static constexpr int npari = ProcessData::npari; + static constexpr int nparf = ProcessData::nparf; + static constexpr int npar = ProcessData::npar; + static constexpr int ncomb = ProcessData::ncomb; + static constexpr int ndiagrams = ProcessData::ndiagrams; + static constexpr int ncolor = ProcessData::ncolor; + static constexpr int nmaxflavor = ProcessData::nmaxflavor; private: /* clang-format on */ diff --git a/madgraph/iolibs/template_files/madmatrix/process_h.inc b/madgraph/iolibs/template_files/madmatrix/process_h.inc index fb715a8778..61b47c9848 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_h.inc @@ -20,6 +20,7 @@ #include "GpuAbstraction.h" #include "Parameters.h" +#include "ProcessData.h" #include diff --git a/madmatrix/model_handling.py b/madmatrix/model_handling.py index b4d1bac787..d7edb941db 100644 --- a/madmatrix/model_handling.py +++ b/madmatrix/model_handling.py @@ -1952,10 +1952,35 @@ def generate_process_files(self): self.edit_memorybuffers() # AV new file (NB this is generic in Subprocesses and then linked in Sigma-specific) self.edit_memoryaccesscouplings() # AV new file (NB this is generic in Subprocesses and then linked in Sigma-specific) super().generate_process_files() + # needs to be after get_matrix_element_calls to have nwf ready + self.edit_processdata() # NB: symlink of cudacpp.mk to makefile is overwritten by madevent makefile if this exists (#480) # NB: this relies on the assumption that cudacpp code is generated before madevent code files.ln(pjoin(self.path, "..", "makefile"), self.path, "makefile") + # seperate process constants to one truth file + def edit_processdata(self): + """Generate ProcessData.h""" + template = open(pjoin(self.template_path, 'madmatrix', 'ProcessData.h'), 'r').read() + me = self.matrix_elements[0] + replace_dict = {} + nexternal, nincoming = me.get_nexternal_ninitial() + replace_dict['nincoming'] = nincoming + replace_dict['noutcoming'] = nexternal - nincoming + replace_dict['nbhel'] = me.get_helicity_combinations() + replace_dict['ndiagrams'] = len(me.get('diagrams')) + replace_dict['nmaxflavor'] = len(me.get_external_flavors_with_iden()) + replace_dict['nwave'] = 4 + (1 if fd_gauge else 0) + replace_dict['ncolor'] = len(me.get_color_amplitudes()) + replace_dict['nwf'] = me.get_number_of_wavefunctions() + replace_dict['nproc'] = sum(2 if m.get('has_mirror_process') else 1 for m in self.matrix_elements) + replace_dict['proc_id'] = self.proc_id if self.proc_id > 0 else 1 + den_factors = [str(m.get_denominator_factor()) for m in self.matrix_elements] + replace_dict['den_factors'] = ",".join(den_factors) + ff = open(pjoin(self.path, 'ProcessData.h'), 'w') + ff.write(template % replace_dict) + ff.close() + # AV - replace the export_cpp.OneProcessExporterCPP method (add debug printouts and multichannel handling #473) def edit_mgonGPU(self): """Generate mgOnGpuConfig.h""" @@ -1984,15 +2009,15 @@ def edit_processidfile(self): ff.write(template % replace_dict) ff.close() - # AV - new method + # generate process specific color matrix data - algo is backend owned def edit_colorsum(self): - """Generate color_sum.cc""" + """Generate ColorMatrixData.h""" ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_colorsum') - template = open(pjoin(self.template_path,'madmatrix','color_sum.cc'),'r').read() + template = open(pjoin(self.template_path,'madmatrix','ColorMatrixData.h'),'r').read() replace_dict = {} # Extract color matrix again (this was also in get_matrix_single_process called within get_all_sigmaKin_lines) replace_dict['color_matrix_lines'] = self.get_color_matrix_lines(self.matrix_elements[0]) - ff = open(pjoin(self.path, 'color_sum.cc'),'w') + ff = open(pjoin(self.path, 'ColorMatrixData.h'),'w') ff.write(template % replace_dict) ff.close() From 8d8c4af347c2574d25e33790d89d81f5612fa5b3 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 17:52:11 +0200 Subject: [PATCH 04/25] Backend algos from CPPProcess to /backend Relocate sigmaKin, calculate_jamps, computeDependentCouplings, getGoodHel/ setGoodHel, and color_sum_cpu/color_sum_gpu out of the per-process CPPProcess.cc into backend-owned SigmaKin.{h,cc} and color_sum.cc under backend/{cpu,simd,gpu}/. The storage they operate on (cHel/cFlavors/cIPD/ cIPC/cIPF_*/bsmIndepParam/good-helicity cache) moves with them as generic, backend-owned declarations sized via nIPD/nIPC/ncomb/etc. - constexpr constants pulled from each process's own ProcessData.h/ProcessTables.h at compile time, the same pattern already used by color_sum.cc (ncolor from ColorMatrixData.h). CPPProcess's constructor/initProc populates this storage through new setters (setHelicitiesAndFlavors, setIndependentParams, etc.) instead of writing it directly. + more resolution of #ifdefs paths to anhilatethem --- .../backend/cpu/MatrixElementKernels.cc | 2 +- .../backend/cpu/MemoryAccessCouplings.h | 10 +- .../madmatrix/backend/cpu/SigmaKin.cc | 517 ++++++++++++ .../madmatrix/backend/cpu/SigmaKin.h | 64 ++ .../madmatrix/backend/cpu/color_sum.cc | 47 -- .../madmatrix/backend/cpu/mgOnGpuConfig.h | 4 +- .../madmatrix/backend/cpu/testxxx.cc | 4 +- .../madmatrix/backend/cpu/umami.cc | 1 + .../backend/gpu/MatrixElementKernels.cc | 2 +- .../backend/gpu/MemoryAccessCouplings.h | 10 +- .../madmatrix/backend/gpu/SigmaKin.cc | 569 +++++++++++++ .../madmatrix/backend/gpu/SigmaKin.h | 72 ++ .../madmatrix/backend/gpu/mgOnGpuConfig.h | 4 +- .../madmatrix/backend/gpu/testxxx.cc | 4 +- .../madmatrix/backend/gpu/umami.cc | 1 + .../backend/simd/MatrixElementKernels.cc | 2 +- .../backend/simd/MemoryAccessCouplings.h | 14 +- .../madmatrix/backend/simd/SigmaKin.cc | 600 ++++++++++++++ .../madmatrix/backend/simd/SigmaKin.h | 64 ++ .../madmatrix/backend/simd/color_sum.cc | 25 +- .../madmatrix/backend/simd/mgOnGpuConfig.h | 4 +- .../madmatrix/backend/simd/testxxx.cc | 10 +- .../madmatrix/backend/simd/umami.cc | 1 + .../process_function_definitions.inc | 764 +----------------- .../template_files/madmatrix/process_h.inc | 92 +-- madmatrix/model_handling.py | 427 ++++------ 26 files changed, 2090 insertions(+), 1224 deletions(-) create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc create mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc index a2de4d93ed..4d8d01c60e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc @@ -7,7 +7,7 @@ #include "MatrixElementKernels.h" #include "ProcessData.h" -#include "CPPProcess.h" // TODO(backend_separation): drop once sigmaKin/getGoodHel/computeDependentCouplings move to backend/ +#include "SigmaKin.h" #include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h index 844ee5a5ed..97435bd038 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h @@ -88,7 +88,7 @@ namespace mg5amcCpu const int ievt ) { const int ipagC = ievt / neppC; // #event "C-page" - const int ieppC = ievt %% neppC; // #event in the current event C-page + const int ieppC = ievt % neppC; // #event in the current event C-page constexpr int idcoup = 0; constexpr int ix2 = 0; return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] @@ -211,9 +211,9 @@ namespace mg5amcCpu // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast #endif } @@ -227,7 +227,7 @@ namespace mg5amcCpu /* fptype_sv& real = kernelAccessIx2( buffer, 0 ); fptype_sv& imag = kernelAccessIx2( buffer, 1 ); - printf( "C_ACCESS::kernelAccess: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + printf( "C_ACCESS::kernelAccess: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); return cxtype_sv_ref( real, imag ); */ return cxtype_sv_ref( kernelAccessIx2( buffer, 0 ), @@ -242,7 +242,7 @@ namespace mg5amcCpu /* const fptype_sv& real = kernelAccessIx2Const( buffer, 0 ); const fptype_sv& imag = kernelAccessIx2Const( buffer, 1 ); - printf( "C_ACCESS::kernelAccessConst: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + printf( "C_ACCESS::kernelAccessConst: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); return cxtype_sv( real, imag ); */ return cxtype_sv( kernelAccessIx2Const( buffer, 0 ), diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc new file mode 100644 index 0000000000..9536d8abb6 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc @@ -0,0 +1,517 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin, calculate_jamps, good-helicity filtering. +// The diagram/vertex-call sequence (helas_calls) is process-specific and +// lives in the P1-generated EvaluateDiagrams.inc, #include'd below. + +#include "SigmaKin.h" + +#include "CPPProcess.h" // ProcessData.h, Parameters.h, HelAmps_.h transitively +#include "ProcessTables.h" + +#include "GpuRuntime.h" +#include "MemoryAccessAmplitudes.h" +#include "MemoryAccessChannelIds.h" +#include "MemoryAccessCouplings.h" +#include "MemoryAccessCouplingsFixed.h" +#include "MemoryAccessDenominators.h" +#include "MemoryAccessGs.h" +#include "MemoryAccessIflavorVec.h" +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessNumerators.h" +#include "MemoryAccessWavefunctions.h" +#include "color_sum.h" +#include "coloramps.h" +#include "processConfig.h" + +namespace mg5amcCpu +{ + using namespace ProcessData; + using namespace ProcessTables; + using Parameters_dependentCouplings::ndcoup; // #couplings that vary event by event (depend on running alphas QCD) + using Parameters_independentCouplings::nicoup; // #couplings that are fixed for all events (do not depend on running alphas QCD) + + // The number of SIMD vectors of events processed by calculate_jamps + constexpr int nParity = 1; + + // Helicity/flavor tables and SM parameter/coupling storage, populated once + // by CPPProcess's constructor/initProc via the setters below. + static short cHel[ncomb][npar]; + static short cFlavors[nmaxflavor][npar]; + static int cNGoodHel; + static int cGoodHel[ncomb]; + static fptype cIPD[nIPD > 0 ? nIPD : 1]; + static fptype cIPC[nIPC > 0 ? nIPC * 2 : 1]; + static int cIPF_partner1[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + static int cIPF_partner2[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + static fptype cIPF_value[ProcessTables::nMF * nIPF * 2 > 0 ? ProcessTables::nMF * nIPF * 2 : 1]; + static double bsmIndepParam[Parameters::nBsmIndepParam > 0 ? Parameters::nBsmIndepParam : 1]; + + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ) + { + memcpy( cHel, tHel, ncomb * npar * sizeof( short ) ); + memcpy( cFlavors, tFlavors, nmaxflavor * npar * sizeof( short ) ); + } + + void setIndependentParams( const fptype* tIPD ) + { + if( nIPD > 0 ) memcpy( cIPD, tIPD, nIPD * sizeof( fptype ) ); + } + + void setIndependentCouplings( const cxtype* tIPC ) + { + if( nIPC > 0 ) memcpy( cIPC, tIPC, nIPC * sizeof( cxtype ) ); + } + + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ) + { + if( nIPF == 0 ) return; + memcpy( cIPF_partner1, tIPF_partner1, nMF * nIPF * sizeof( int ) ); + memcpy( cIPF_partner2, tIPF_partner2, nMF * nIPF * sizeof( int ) ); + memcpy( cIPF_value, tIPF_value, nMF * nIPF * sizeof( cxtype ) ); + } + + void setBsmIndepParam( const double* values, int n ) + { + if( n > 0 ) memcpy( bsmIndepParam, values, n * sizeof( double ) ); + } + + //-------------------------------------------------------------------------- + + __device__ INLINE unsigned int + getChannelId( const unsigned int* allChannelIds, const int ievt00, bool sanityCheckMixedPrecision = true ) + { + unsigned int channelId = 0; // disable multichannel single-diagram enhancement unless allChannelIds != nullptr + using CID_ACCESS = HostAccessChannelIds; // non-trivial access: buffer includes all events + // SCALAR channelId for the whole SIMD neppV2 event page (C++), i.e. one or two neppV event page(s) + // The cudacpp implementation ASSUMES (and checks! #898) that all channelIds are the same in a neppV2 SIMD event page + // **NB! in "mixed" precision, using SIMD, calculate_wavefunctions computes MEs for TWO neppV pages with a single channelId! #924 + if( allChannelIds != nullptr ) + { + // First - and/or only - neppV page of channels (iParity=0 => ievt0 = ievt00 + 0 * neppV) + const unsigned int* channelIds = CID_ACCESS::ieventAccessRecordConst( allChannelIds, ievt00 ); // fix bug #899/#911 + uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) + // NB: channelIds_sv is a scalar in no-SIMD C++ + channelId = channelIds_sv; + assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) + (void)sanityCheckMixedPrecision; // no second SIMD page to cross-check in no-SIMD C++ + } + return channelId; + } + + //-------------------------------------------------------------------------- + + __global__ void + computeDependentCouplings( const fptype* allgs, fptype* allcouplings, const int nevt ) + { + using G_ACCESS = HostAccessGs; + using C_ACCESS = HostAccessCouplings; + for( int ipagV = 0; ipagV < nevt / neppV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + const fptype* gs = MemoryAccessGs::ieventAccessRecordConst( allgs, ievt0 ); + fptype* couplings = MemoryAccessCouplings::ieventAccessRecord( allcouplings, ievt0 ); + G2COUP( gs, couplings, bsmIndepParam ); + } + } + + //-------------------------------------------------------------------------- + + // Evaluate QCD partial amplitudes jamps for this given helicity from Feynman diagrams. + // This function processes a single event "page" or SIMD vector (or for two in "mixed" + // precision mode, nParity=2). Accepts a SCALAR channelId because it is GUARANTEED that + // all events in a SIMD vector have the same channelId #898. + void + calculate_jamps( int ihel, + const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + cxtype_sv* allJamp_sv, + bool storeChannelWeights, + fptype* allNumerators, + fptype* allDenominators, + fptype_sv* jamp2_sv, + const int ievt00 ) + { + using M_ACCESS = HostAccessMomenta; + using W_ACCESS = HostAccessWavefunctions; + using A_ACCESS = HostAccessAmplitudes; + using CD_ACCESS = HostAccessCouplings; + using CI_ACCESS = HostAccessCouplingsFixed; + using F_ACCESS = HostAccessIflavorVec; + using NUM_ACCESS = HostAccessNumerators; + using DEN_ACCESS = HostAccessDenominators; + mgDebug( 0, __FUNCTION__ ); + + fptype_sv pvec_sv[nwf][np4]; + cxtype_sv w_sv[nwf][nw6]; // particle wavefunctions within Feynman diagrams + cxtype_sv amp_sv[1]; // invariant amplitude for one given Feynman diagram + ALOHAOBJ aloha_obj[nwf]; + for( int iwf = 0; iwf < nwf; iwf++ ) aloha_obj[iwf] = ALOHAOBJ{ pvec_sv[iwf], w_sv[iwf] }; + fptype* amp_fp = reinterpret_cast( amp_sv ); + + // jamp: sum (for one event or event page) of the invariant amplitudes for + // all Feynman diagrams in a given color combination + cxtype_sv jamp_sv[ncolor] = {}; // all zeros + + for( int iParity = 0; iParity < nParity; ++iParity ) + { + const int ievt0 = ievt00 + iParity * neppV; +#include "EvaluateDiagrams.inc" + + // *** COLOR CHOICE BELOW *** + // Store the leading color flows for choice of color + if( jamp2_sv ) // disable color choice if nullptr + { + for( int icol = 0; icol < ncolor; icol++ ) + jamp2_sv[ncolor * iParity + icol] += cxabs2( jamp_sv[icol] ); // may underflow #831 + } + + // *** PREPARE OUTPUT JAMPS *** + // In C++, copy the local jamp to the output array passed as function argument + for( int icol = 0; icol < ncolor; icol++ ) + allJamp_sv[iParity * ncolor + icol] = jamp_sv[icol]; + } + // END LOOP ON IPARITY + + mgDebug( 1, __FUNCTION__ ); + return; + } + + //-------------------------------------------------------------------------- + + void + sigmaKin_getGoodHel( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + fptype* allMEs, + fptype* allNumerators, + fptype* allDenominators, + bool* isGoodHel, + const int nevt ) + { + // Allocate arrays at build time to contain at least 16 events (or at least neppV events if neppV>16, e.g. in future VPUs) + constexpr int maxtry0 = std::max( 16, neppV ); // 16, but at least neppV (otherwise the npagV loop does not even start) + assert( nevt >= neppV ); + const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt 1 ) + { + for( int i = broken_sym_component_starts[icomp] - 1; i < broken_sym_component_ends[icomp]; i++ ) + { + if( pid_work[i] == 0 ) + continue; + int n_tot = 1; + for( int j = i + 1; j < broken_sym_component_ends[icomp]; j++ ) + { + if( pid_work[i] != pid_work[j] ) + continue; + bool same_block = ( broken_sym_block_lengths[i] == broken_sym_block_lengths[j] ); + for( int k = 0; same_block && k < broken_sym_block_lengths[i]; k++ ) + { + if( cFlavors[iflavor][broken_sym_block_starts[i] - 1 + k] != cFlavors[iflavor][broken_sym_block_starts[j] - 1 + k] ) + same_block = false; + } + if( same_block ) + { + pid_work[j] = 0; + n_tot = n_tot + 1; + old_factor = old_factor / n_tot; + } + } + } + } + total_factor = total_factor * old_factor; + } + return total_factor; + } + + //-------------------------------------------------------------------------- + // Evaluate |M|^2, part independent of incoming flavour + + void + sigmaKin( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + const fptype* allrndhel, + const fptype* allrndcol, + const unsigned int* allChannelIds, + const fptype* allrnddiagram, + fptype* allMEs, + int* allselhel, + int* allselcol, + fptype* allNumerators, + fptype* allDenominators, + unsigned int* allDiagramIdsOut, + bool mulChannelWeight, + const int nevt ) + { + mgDebugInitialise(); + + // SANITY CHECKS for cudacpp code generation (see issues #272 and #343 and PRs #619, #626, #360, #396 and #754) + { + // nprocesses == 2 may happen for "mirror processes" such as P0_uux_ttx within pp_tt012j (see PR #754) + static_assert( nproc == 1 || nproc == 2, "Assume nprocesses == 1 or 2" ); + static_assert( proc_id == 1, "Assume process_id == 1" ); + } + + using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events + using NUM_ACCESS = HostAccessNumerators; // non-trivial access: buffer includes all events + using DEN_ACCESS = HostAccessDenominators; // non-trivial access: buffer includes all events + + // Reset the "matrix elements" - running sums of |M|^2 over helicities for the given event + const int npagV = nevt / neppV; + for( int ipagV = 0; ipagV < npagV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv = fptype_sv{ 0 }; + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + for( int i = 0; i < processConfig::ndiagrams; ++i ) + { + numerators_sv[i] = fptype_sv{ 0 }; + } + denominators_sv = fptype_sv{ 0 }; + } + + // HELICITY LOOP: CALCULATE WAVEFUNCTIONS (using precomputed good helicities) + const int npagV2 = npagV; // loop on one SIMD page (neppV events) at a time +#ifdef _OPENMP + // OMP multithreading #575 (NB: tested only with gcc11 so far) +#define _OMPLIST0 allcouplings, allMEs, allmomenta, allrndcol, allrndhel, allselcol, allselhel, cGoodHel, cNGoodHel, npagV2 +#define _OMPLIST1 , allDenominators, allNumerators, allChannelIds, mgOnGpu::icolamp, mgOnGpu::channel2iconfig +#pragma omp parallel for default( none ) shared( _OMPLIST0 _OMPLIST1 ) +#undef _OMPLIST0 +#undef _OMPLIST1 +#endif // _OPENMP + for( int ipagV2 = 0; ipagV2 < npagV2; ++ipagV2 ) + { + const int ievt00 = ipagV2 * neppV; // loop on one SIMD page (neppV events) at a time + // Running sum of partial amplitudes squared for event by event color selection (#402) + fptype_sv jamp2_sv[nParity * ncolor] = {}; + fptype_sv MEs_ighel[ncomb] = {}; // sum of MEs for all good helicities up to ighel (for the first - and/or only - neppV page) + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + const int ihel = cGoodHel[ighel]; + cxtype_sv jamp_sv[nParity * ncolor] = {}; // fixed nasty bug (omitting 'nParity' caused memory corruptions after calling calculate_jamps) + bool storeChannelWeights = allChannelIds != nullptr || allrnddiagram != nullptr; + calculate_jamps( ihel, allmomenta, allcouplings, iflavorVec, jamp_sv, storeChannelWeights, allNumerators, allDenominators, jamp2_sv, ievt00 ); + color_sum_cpu( allMEs, jamp_sv, ievt00 ); + MEs_ighel[ighel] = E_ACCESS::kernelAccess( E_ACCESS::ieventAccessRecord( allMEs, ievt00 ) ); + } + // Event-by-event random choice of helicity #403 + for( int ieppV = 0; ieppV < neppV; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + const bool okhel = allrndhel[ievt] < ( MEs_ighel[ighel] / MEs_ighel[cNGoodHel - 1] ); + if( okhel ) + { + const int ihelF = cGoodHel[ighel] + 1; // NB Fortran [1,ncomb], cudacpp [0,ncomb-1] + allselhel[ievt] = ihelF; + break; + } + } + } + const int vecsize = neppV; + unsigned int channelIdVec[vecsize]; + if( allChannelIds != nullptr ) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + channelIdVec[ieppV] = allChannelIds[ievt]; + } + } + + // Event-by-event random choice of channel + if( allrnddiagram != nullptr ) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + fptype numerator_sum = 0., normalization = 0.; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + normalization += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + ichan * neppV + ieppV % neppV]; + } + channelIdVec[ieppV] = mgOnGpu::nchannels; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + numerator_sum += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + ichan * neppV + ieppV % neppV]; + if( allrnddiagram[ievt] < numerator_sum / normalization ) + { + channelIdVec[ieppV] = ichan + 1; + break; + } + } + allDiagramIdsOut[ievt] = channelIdVec[ieppV]; + } + } + + // Event-by-event random choice of color #402 + if( allChannelIds != nullptr || allrnddiagram != nullptr ) // no event-by-event choice of color if channelId == 0 (fix FPE #783) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + unsigned int channelId = channelIdVec[ieppV]; + if( channelId > mgOnGpu::nchannels ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which is greater than nchannels=%d\n", channelId, mgOnGpu::nchannels ); + assert( channelId <= mgOnGpu::nchannels ); // SANITY CHECK #919 #910 + } + const int iconfig = mgOnGpu::channel2iconfig[channelId - 1]; // map N_diagrams to N_config <= N_diagrams configs (fix LHE color mismatch #856: see also #826, #852, #853) + if( iconfig <= 0 ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which has no associated SDE iconfig\n", channelId ); + assert( iconfig > 0 ); // SANITY CHECK #917 + } + else if( iconfig > (int)mgOnGpu::nconfigSDE ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d (invalid SDE iconfig=%d\n > nconfig=%d)", channelId, iconfig, mgOnGpu::nconfigSDE ); + assert( iconfig <= (int)mgOnGpu::nconfigSDE ); // SANITY CHECK #917 + } + fptype targetamp[ncolor] = { 0 }; + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( icolC == 0 ) + targetamp[icolC] = 0; + else + targetamp[icolC] = targetamp[icolC - 1]; + if( mgOnGpu::icolamp[iconfig - 1][icolC] ) targetamp[icolC] += + jamp2_sv[icolC + ncolor * ( ieppV / neppV )]; + } + const int ievt = ievt00 + ieppV; + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( allrndcol[ievt] < ( targetamp[icolC] / targetamp[ncolor - 1] ) ) + { + allselcol[ievt] = icolC + 1; // NB Fortran [1,ncolor], cudacpp [0,ncolor-1] + break; + } + } + } + } + else + { + for( int ieppV = 0; ieppV < neppV; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + allselcol[ievt] = 0; // no color selected in Fortran range [1,ncolor] if channelId == 0 (see #931) + } + } + } + // *** END OF PART 1b - C++ (loop on event pages) + + // PART 2 - FINALISATION (after calculate_jamps) + // Get the final |M|^2 as an average over helicities/colors of the running sum of |M|^2 over helicities for the given event + for( int ipagV = 0; ipagV < npagV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv = MEs_sv * broken_symmetry_factor( iflavorVec[ievt0] ) / helcolDenominators[0]; + if( mulChannelWeight && allChannelIds != nullptr ) // fix segfault #892 (not 'channelIds[0] != 0') + { + const unsigned int channelId = getChannelId( allChannelIds, ievt0, false ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + MEs_sv *= numerators_sv[channelId - 1] / denominators_sv; + } + } + mgDebugFinalise(); + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h new file mode 100644 index 0000000000..725ff931d2 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h @@ -0,0 +1,64 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin and everything it calls (calculate_jamps, +// good-helicity filtering, color/channel selection). Declared here so +// CPPProcess.cc's constructor/initProc (P1-generated) can call the setters +// that populate this file's storage, and so umami.cc/MatrixElementKernels.cc +// can call sigmaKin/computeDependentCouplings. + +#ifndef SIGMAKIN_H +#define SIGMAKIN_H 1 + +#include "mgOnGpuConfig.h" +#include "mgOnGpuCxtypes.h" // for cxtype + +namespace mg5amcCpu +{ + __global__ void + computeDependentCouplings( const fptype* allgs, + fptype* allcouplings, + const int nevt ); + + void + sigmaKin_getGoodHel( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + fptype* allMEs, + fptype* allNumerators, + fptype* allDenominators, + bool* isGoodHel, + const int nevt ); + + int + sigmaKin_setGoodHel( const bool* isGoodHel ); + + void + sigmaKin( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + const fptype* allrndhel, + const fptype* allrndcol, + const unsigned int* allChannelIds, + const fptype* allrnddiagram, + fptype* allMEs, + int* allselhel, + int* allselcol, + fptype* allNumerators, + fptype* allDenominators, + unsigned int* allDiagramIdsOut, + bool mulChannelWeight, + const int nevt ); + + // Setters: called once by CPPProcess (P1-generated) to populate this file's + // otherwise-internal storage, since it can no longer be written directly + // from a different translation unit. + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ); + void setIndependentParams( const fptype* tIPD ); + void setIndependentCouplings( const cxtype* tIPC ); + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ); + void setBsmIndepParam( const double* values, int n ); +} + +#endif // SIGMAKIN_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc index d9d8b15aa4..94bd2733f7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc @@ -49,53 +49,26 @@ namespace mg5amcCpu // we gain (not a factor 2...) in speed here as we only loop over the up diagonal part of the matrix. // Strangely, CUDA is slower instead, so keep the old implementation for the moment. fptype_sv deltaMEs = { 0 }; -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype_sv deltaMEs_next = { 0 }; - // Mixed mode: merge two neppV vectors into one neppV2 vector - fptype2_sv jampR_sv[ncolor]; - fptype2_sv jampI_sv[ncolor]; - for( int icol = 0; icol < ncolor; icol++ ) - { - jampR_sv[icol] = fpvmerge( cxreal( allJamp_sv[icol] ), cxreal( allJamp_sv[ncolor + icol] ) ); - jampI_sv[icol] = fpvmerge( cximag( allJamp_sv[icol] ), cximag( allJamp_sv[ncolor + icol] ) ); - } -#else const cxtype_sv* jamp_sv = allJamp_sv; -#endif // Loop over icol for( int icol = 0; icol < ncolor; icol++ ) { // Diagonal terms -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype2_sv& jampRi_sv = jampR_sv[icol]; - fptype2_sv& jampIi_sv = jampI_sv[icol]; -#else fptype2_sv jampRi_sv = (fptype2_sv)( cxreal( jamp_sv[icol] ) ); fptype2_sv jampIi_sv = (fptype2_sv)( cximag( jamp_sv[icol] ) ); -#endif fptype2_sv ztempR_sv = cf2.value[icol][icol] * jampRi_sv; fptype2_sv ztempI_sv = cf2.value[icol][icol] * jampIi_sv; // Loop over jcol for( int jcol = icol + 1; jcol < ncolor; jcol++ ) { // Off-diagonal terms -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype2_sv& jampRj_sv = jampR_sv[jcol]; - fptype2_sv& jampIj_sv = jampI_sv[jcol]; -#else fptype2_sv jampRj_sv = (fptype2_sv)( cxreal( jamp_sv[jcol] ) ); fptype2_sv jampIj_sv = (fptype2_sv)( cximag( jamp_sv[jcol] ) ); -#endif ztempR_sv += cf2.value[icol][jcol] * jampRj_sv; ztempI_sv += cf2.value[icol][jcol] * jampIj_sv; } fptype2_sv deltaMEs2 = ( jampRi_sv * ztempR_sv + jampIi_sv * ztempI_sv ); // may underflow #831 -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - deltaMEs += fpvsplit0( deltaMEs2 ); - deltaMEs_next += fpvsplit1( deltaMEs2 ); -#else deltaMEs += deltaMEs2; -#endif } // *** STORE THE RESULTS *** using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events @@ -103,28 +76,8 @@ namespace mg5amcCpu // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); MEs_sv += deltaMEs; // fix #435 -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype* MEs_next = E_ACCESS::ieventAccessRecord( allMEs, ievt0 + neppV ); - fptype_sv& MEs_sv_next = E_ACCESS::kernelAccess( MEs_next ); - MEs_sv_next += deltaMEs_next; -#endif } //-------------------------------------------------------------------------- - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - } // end namespace diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h index 53b41f3b80..dd13522513 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h @@ -79,7 +79,7 @@ ////#define MGONGPU_INLINE_HELAMPS 1 // Choose whether to hardcode the cIPD physics parameters rather than reading them from user cards -// This optimization can gain 20%% in CUDA in eemumu (issue #39) +// This optimization can gain 20% in CUDA in eemumu (issue #39) // By default, do not hardcode, but allow this macro to be set from outside with e.g. -DMGONGPU_HARDCODE_PARAM // ** NB: The option to use hardcoded cIPD physics parameters is supported again even now when alphas is running (#373) // ** NB: Note however that it now only refers to cIPD parameters (cIPC parameters are always accessed through global memory) @@ -245,7 +245,7 @@ using mgOnGpu::fptype2; //#define mgDebugDeclare() __shared__ float mgDebugCounter[mgOnGpu::ntpbMAX]; //#define mgDebugInitialise() { mgDebugCounter[threadIdx.x] = 0; } //#define mgDebug( code, text ) { mgDebugCounter[threadIdx.x] += 1; } -//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%%f\n", mgDebugCounter[threadIdx.x] ); } +//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%f\n", mgDebugCounter[threadIdx.x] ); } //#else #define mgDebugDeclare() /*noop*/ #define mgDebugInitialise() /*noop*/ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc index 46786a40aa..7fe572468a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc @@ -53,8 +53,8 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout constexpr int np4 = CPPProcess::np4; const int nevt = 32; // 12 independent tests plus 20 duplicates (need a multiple of 16 for floats '512z') - assert( nevt %% neppM == 0 ); // nevt must be a multiple of neppM - assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV + assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM + assert( nevt % neppV == 0 ); // nevt must be a multiple of neppV // Fill in the input momenta mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc index 95aa04f718..7fcdb3d91f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc @@ -8,6 +8,7 @@ #include "ProcessData.h" #include "CPPProcess.h" // needed to construct/initProc the process object (umami_initialize) +#include "SigmaKin.h" // sigmaKin_getGoodHel/setGoodHel #include "GpuRuntime.h" #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc index 83214ab225..7c91a9929e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc @@ -7,7 +7,7 @@ #include "MatrixElementKernels.h" #include "ProcessData.h" -#include "CPPProcess.h" // TODO(backend_separation): drop once sigmaKin/getGoodHel/computeDependentCouplings move to backend/ +#include "SigmaKin.h" #include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h index a5e1fcc262..8b191722a1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h @@ -88,7 +88,7 @@ namespace mg5amcGpu const int ievt ) { const int ipagC = ievt / neppC; // #event "C-page" - const int ieppC = ievt %% neppC; // #event in the current event C-page + const int ieppC = ievt % neppC; // #event in the current event C-page constexpr int idcoup = 0; constexpr int ix2 = 0; return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] @@ -211,9 +211,9 @@ namespace mg5amcGpu // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast #endif } @@ -227,7 +227,7 @@ namespace mg5amcGpu /* fptype_sv& real = kernelAccessIx2( buffer, 0 ); fptype_sv& imag = kernelAccessIx2( buffer, 1 ); - printf( "C_ACCESS::kernelAccess: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + printf( "C_ACCESS::kernelAccess: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); return cxtype_sv_ref( real, imag ); */ return cxtype_sv_ref( kernelAccessIx2( buffer, 0 ), @@ -242,7 +242,7 @@ namespace mg5amcGpu /* const fptype_sv& real = kernelAccessIx2Const( buffer, 0 ); const fptype_sv& imag = kernelAccessIx2Const( buffer, 1 ); - printf( "C_ACCESS::kernelAccessConst: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + printf( "C_ACCESS::kernelAccessConst: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); return cxtype_sv( real, imag ); */ return cxtype_sv( kernelAccessIx2Const( buffer, 0 ), diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc new file mode 100644 index 0000000000..605ff0b89a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc @@ -0,0 +1,569 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin, calculate_jamps, good-helicity filtering. +// The diagram/vertex-call sequence (helas_calls) is process-specific and +// lives in the P1-generated EvaluateDiagrams.inc, #include'd below. + +#include "SigmaKin.h" + +#include "CPPProcess.h" // ProcessData.h, Parameters.h, HelAmps_.h transitively +#include "ProcessTables.h" + +#include "GpuRuntime.h" +#include "MemoryAccessAmplitudes.h" +#include "MemoryAccessChannelIds.h" +#include "MemoryAccessCouplings.h" +#include "MemoryAccessCouplingsFixed.h" +#include "MemoryAccessDenominators.h" +#include "MemoryAccessGs.h" +#include "MemoryAccessIflavorVec.h" +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessNumerators.h" +#include "MemoryAccessWavefunctions.h" +#include "color_sum.h" +#include "coloramps.h" +#include "processConfig.h" + +namespace mg5amcGpu +{ + using namespace ProcessData; + using namespace ProcessTables; + using Parameters_dependentCouplings::ndcoup; // #couplings that vary event by event (depend on running alphas QCD) + using Parameters_independentCouplings::nicoup; // #couplings that are fixed for all events (do not depend on running alphas QCD) + constexpr int nParity = 1; // CUDA/HIP process one event per thread, no host-side SIMD paging + + // Helicity/flavor tables and SM parameter/coupling storage, populated once + // by CPPProcess's constructor/initProc via the setters below. + __device__ __constant__ short cHel[ncomb][npar]; + __device__ __constant__ int dcNGoodHel; + __device__ __constant__ int dcGoodHel[ncomb]; + __device__ __constant__ short cFlavors[nmaxflavor][npar]; + static int cNGoodHel; + static int cGoodHel[ncomb]; + __device__ __constant__ fptype cIPD[nIPD > 0 ? nIPD : 1]; + __device__ __constant__ fptype cIPC[nIPC > 0 ? nIPC * 2 : 1]; + __device__ __constant__ int cIPF_partner1[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + __device__ __constant__ int cIPF_partner2[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + __device__ __constant__ fptype cIPF_value[ProcessTables::nMF * nIPF * 2 > 0 ? ProcessTables::nMF * nIPF * 2 : 1]; + __device__ __constant__ double bsmIndepParam[Parameters::nBsmIndepParam > 0 ? Parameters::nBsmIndepParam : 1]; + + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ) + { + gpuMemcpyToSymbol( cHel, tHel, ncomb * npar * sizeof( short ) ); + gpuMemcpyToSymbol( cFlavors, tFlavors, nmaxflavor * npar * sizeof( short ) ); + } + + void setIndependentParams( const fptype* tIPD ) + { + if( nIPD > 0 ) gpuMemcpyToSymbol( cIPD, tIPD, nIPD * sizeof( fptype ) ); + } + + void setIndependentCouplings( const cxtype* tIPC ) + { + if( nIPC > 0 ) gpuMemcpyToSymbol( cIPC, tIPC, nIPC * sizeof( cxtype ) ); + } + + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ) + { + if( nIPF == 0 ) return; + gpuMemcpyToSymbol( cIPF_partner1, tIPF_partner1, nMF * nIPF * sizeof( int ) ); + gpuMemcpyToSymbol( cIPF_partner2, tIPF_partner2, nMF * nIPF * sizeof( int ) ); + gpuMemcpyToSymbol( cIPF_value, tIPF_value, nMF * nIPF * sizeof( cxtype ) ); + } + + void setBsmIndepParam( const double* values, int n ) + { + if( n > 0 ) gpuMemcpyToSymbol( bsmIndepParam, values, n * sizeof( double ) ); + } + + //-------------------------------------------------------------------------- + + class DeviceAccessJamp2 + { + public: + static __device__ inline fptype& + kernelAccessIcol( fptype* buffer, const int icol ) + { + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + return buffer[icol * nevt + ievt]; + } + static __device__ inline const fptype& + kernelAccessIcolConst( const fptype* buffer, const int icol ) + { + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + return buffer[icol * nevt + ievt]; + } + }; + + //-------------------------------------------------------------------------- + + __device__ INLINE unsigned int + gpu_channelId( const unsigned int* allChannelIds ) + { + unsigned int channelId = 0; // disable multichannel single-diagram enhancement unless allChannelIds != nullptr + using CID_ACCESS = DeviceAccessChannelIds; // non-trivial access: buffer includes all events + // SCALAR channelId for the current event (CUDA) + if( allChannelIds != nullptr ) + { + const unsigned int* channelIds = allChannelIds; // fix #899 (distinguish channelIds and allChannelIds) + const uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) + // NB: channelIds_sv is a scalar in CUDA + channelId = channelIds_sv; + assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) + } + return channelId; + } + + //-------------------------------------------------------------------------- + + __global__ void /* clang-format off */ + computeDependentCouplings( const fptype* allgs, // input: Gs[nevt] + fptype* allcouplings // output: couplings[nevt*ndcoup*2] + ) /* clang-format on */ + { + using G_ACCESS = DeviceAccessGs; + using C_ACCESS = DeviceAccessCouplings; + G2COUP( allgs, allcouplings, bsmIndepParam ); + } + + //-------------------------------------------------------------------------- + + // Evaluate QCD partial amplitudes jamps for this given helicity from Feynman diagrams. + // This function processes a single event (one CUDA thread). + __global__ void /* clang-format off */ + calculate_jamps( int ihel, + const fptype* allmomenta, // input: momenta[nevt*npar*4] + const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] + const unsigned int* iflavorVec, // input: indices of the flavor combinations + fptype* allJamps, // output: jamp[2*ncolor*nevt] buffer for one helicity _within a super-buffer for dcNGoodHel helicities_ + bool storeChannelWeights, + fptype* allNumerators, // input/output: multichannel numerators[nevt], add helicity ihel + fptype* allDenominators, // input/output: multichannel denominators[nevt], add helicity ihel + fptype* colAllJamp2s, // output: allJamp2s[ncolor][nevt] super-buffer, sum over col/hel (nullptr to disable) + const int nevt, // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + const bool processAllHelicities // input: if true, use blockIdx.y to index helicities + ) /* clang-format on */ + { + using M_ACCESS = DeviceAccessMomenta; // non-trivial access: buffer includes all events + using W_ACCESS = DeviceAccessWavefunctions; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event + using A_ACCESS = DeviceAccessAmplitudes; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event + using CD_ACCESS = DeviceAccessCouplings; // non-trivial access (dependent couplings): buffer includes all events + using CI_ACCESS = DeviceAccessCouplingsFixed; // TRIVIAL access (independent couplings): buffer for one event + using F_ACCESS = DeviceAccessIflavorVec; // non-trivial access: buffer includes all events + using NUM_ACCESS = DeviceAccessNumerators; // non-trivial access: buffer includes all events + using DEN_ACCESS = DeviceAccessDenominators; // non-trivial access: buffer includes all events + mgDebug( 0, __FUNCTION__ ); + if( processAllHelicities ) + { + int ighel = blockIdx.y; + ihel = dcGoodHel[ighel]; + allJamps = allJamps + ighel * nevt; + allNumerators = allNumerators + ighel * nevt * processConfig::ndiagrams; + allDenominators = allDenominators + ighel * nevt; + } + + fptype_sv pvec_sv[nwf][np4]; + cxtype_sv w_sv[nwf][nw6]; // particle wavefunctions within Feynman diagrams + cxtype_sv amp_sv[1]; // invariant amplitude for one given Feynman diagram + ALOHAOBJ aloha_obj[nwf]; + for( int iwf = 0; iwf < nwf; iwf++ ) aloha_obj[iwf] = ALOHAOBJ{ pvec_sv[iwf], w_sv[iwf] }; + fptype* amp_fp = reinterpret_cast( amp_sv ); + + // jamp: sum (for one event) of the invariant amplitudes for all Feynman diagrams in a given color combination + cxtype_sv jamp_sv[ncolor] = {}; // all zeros + + for( int iParity = 0; iParity < nParity; ++iParity ) + { +#include "EvaluateDiagrams.inc" + + // *** COLOR CHOICE BELOW *** + // Store the leading color flows for choice of color + assert( iParity == 0 ); // sanity check for J2_ACCESS + using J2_ACCESS = DeviceAccessJamp2; + if( colAllJamp2s ) // disable color choice if nullptr + { + for( int icol = 0; icol < ncolor; icol++ ) + // NB: atomicAdd is needed after moving to cuda streams with one helicity per stream! + atomicAdd( &J2_ACCESS::kernelAccessIcol( colAllJamp2s, icol ), cxabs2( jamp_sv[icol] ) ); + } + + // *** PREPARE OUTPUT JAMPS *** + // In CUDA, copy the local jamp to the output global-memory jamp + constexpr int ihel0 = 0; // the allJamps buffer already points to a specific helicity _within a super-buffer for dcNGoodHel helicities_ + using J_ACCESS = DeviceAccessJamp; + for( int icol = 0; icol < ncolor; icol++ ) + J_ACCESS::kernelAccessIcolIhelNhel( allJamps, icol, ihel0, dcNGoodHel ) = jamp_sv[icol]; + } + // END LOOP ON IPARITY + + mgDebug( 1, __FUNCTION__ ); + return; + } + + //-------------------------------------------------------------------------- + + void /* clang-format off */ + sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta[nevt*npar*4] + const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] + const unsigned int* iflavorVec, // input: index of the flavor combination + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + fptype* allNumerators, // output: multichannel numerators[nevt], running_sum_over_helicities + fptype* allDenominators, // output: multichannel denominators[nevt], running_sum_over_helicities + fptype_sv* allJamps, // tmp: jamp[ncolor*2*nevt] _for one helicity_ (reused in the getGoodHel helicity loop) + bool* isGoodHel, // output: isGoodHel[ncomb] - host array + const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + { /* clang-format on */ + const int maxtry0 = 16; + fptype hstMEs[maxtry0]; + const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt 1 ) + { + for( int i = broken_sym_component_starts[icomp] - 1; i < broken_sym_component_ends[icomp]; i++ ) + { + if( pid_work[i] == 0 ) + continue; + int n_tot = 1; + for( int j = i + 1; j < broken_sym_component_ends[icomp]; j++ ) + { + if( pid_work[i] != pid_work[j] ) + continue; + bool same_block = ( broken_sym_block_lengths[i] == broken_sym_block_lengths[j] ); + for( int k = 0; same_block && k < broken_sym_block_lengths[i]; k++ ) + { + if( cFlavors[iflavor][broken_sym_block_starts[i] - 1 + k] != cFlavors[iflavor][broken_sym_block_starts[j] - 1 + k] ) + same_block = false; + } + if( same_block ) + { + pid_work[j] = 0; + n_tot = n_tot + 1; + old_factor = old_factor / n_tot; + } + } + } + } + total_factor = total_factor * old_factor; + } + return total_factor; + } + + //-------------------------------------------------------------------------- + + __global__ void + normalise_output( fptype* allMEs, // output: allMEs[nevt], |M|^2 running_sum_over_helicities + const unsigned int* iflavorVec, + fptype* ghelAllNumerators, // input/tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + fptype* ghelAllDenominators, // input/tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable SDE enhancement (fix #899/#911) + bool storeChannelWeights, // if true, compute final multichannel weights + bool mulChannelWeight, // if true, multiply matrix element by channel weight + const fptype globaldenom) + { + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) + allMEs[ievt] = allMEs[ievt] * broken_symmetry_factor(iflavorVec[ievt]) / globaldenom; + const int nevt = gridDim.x * blockDim.x; + if( storeChannelWeights ) // fix segfault #892 (not 'channelIds[0] != 0') + { + fptype* totAllNumerators = ghelAllNumerators; // reuse "helicity #0" buffer to compute the total over all helicities + fptype* totAllDenominators = ghelAllDenominators; // reuse "helicity #0" buffer to compute the total over all helicities + for( int ighel = 1; ighel < dcNGoodHel; ighel++ ) // NB: the loop starts at ighel=1 + { + fptype* hAllDenominators = ghelAllDenominators + ighel * nevt; + totAllDenominators[ievt] += hAllDenominators[ievt]; + fptype* hAllNumerators = ghelAllNumerators + ( ievt + ighel * nevt ) * processConfig::ndiagrams; + fptype* firstNumerator = ghelAllNumerators + ievt * processConfig::ndiagrams; + for( int idiag = 0; idiag < processConfig::ndiagrams; ++idiag ) + { + firstNumerator[idiag] += hAllNumerators[idiag]; + } + } + if( mulChannelWeight ) + { + unsigned int channelId = allChannelIds[ievt]; + allMEs[ievt] *= totAllNumerators[channelId - 1 + ievt * processConfig::ndiagrams] / totAllDenominators[ievt]; + } + } + return; + } + + //-------------------------------------------------------------------------- + + __global__ void + add_and_select_hel( int* allselhel, // output: helicity selection[nevt] + const fptype* allrndhel, // input: random numbers[nevt] for helicity selection + fptype* ghelAllMEs, // input/tmp: allMEs for nGoodHel <= ncomb individual/runningsum helicities (index is ighel) + fptype* allMEs, // output: allMEs[nevt], final sum over helicities + const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + { + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) + // Compute the sum of MEs over all good helicities (defer this after the helicity loop to avoid breaking streams parallelism) + for( int ighel = 0; ighel < dcNGoodHel; ighel++ ) + { + allMEs[ievt] += ghelAllMEs[ighel * nevt + ievt]; + ghelAllMEs[ighel * nevt + ievt] = allMEs[ievt]; // reuse the buffer to store the running sum for helicity selection + } + // Event-by-event random choice of helicity #403 + for( int ighel = 0; ighel < dcNGoodHel; ighel++ ) + { + if( allrndhel[ievt] < ( ghelAllMEs[ighel * nevt + ievt] / allMEs[ievt] ) ) + { + const int ihelF = dcGoodHel[ighel] + 1; // NB Fortran [1,ncomb], cudacpp [0,ncomb-1] + allselhel[ievt] = ihelF; + break; + } + } + return; + } + + //-------------------------------------------------------------------------- + + __global__ void + select_col_and_diag( int* allselcol, // output: color selection[nevt] + unsigned int* allDiagramIdsOut, // output: sampled diagram ids + const fptype* allrndcol, // input: random numbers[nevt] for color selection + const fptype* allrnddiagram, // input: random numbers[nevt] for diagram selection + const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable SDE enhancement (fix #899/#911) + const fptype_sv* allJamp2s, // input: jamp2[ncolor][nevt] for color choice (nullptr if disabled) + const fptype* allNumerators, // input: all numerators + const fptype* allDenominators, // input: all denominators + const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + { + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) + // SCALAR channelId for the current event (CUDA) + unsigned int channelId = gpu_channelId( allChannelIds ); + // Event-by-event random choice of channel + if( allrnddiagram != nullptr ) + { + fptype numerator_sum = 0., normalization = 0.; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + normalization += allNumerators[ievt * processConfig::ndiagrams + ichan]; + } + channelId = mgOnGpu::nchannels; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + numerator_sum += allNumerators[ievt * processConfig::ndiagrams + ichan]; + if( allrnddiagram[ievt] < numerator_sum / normalization ) + { + channelId = ichan + 1; + break; + } + } + allDiagramIdsOut[ievt] = channelId; + } + + if( channelId != 0 ) // no event-by-event choice of color if channelId == 0 (fix FPE #783) + { + if( channelId > mgOnGpu::nchannels ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which is greater than nchannels=%d\n", channelId, mgOnGpu::nchannels ); + assert( channelId <= mgOnGpu::nchannels ); // SANITY CHECK #919 #910 + } + // Determine the jamp2 for this event + fptype_sv jamp2_sv[ncolor] = { 0 }; + assert( allJamp2s != nullptr ); // sanity check + using J2_ACCESS = DeviceAccessJamp2; + for( int icolC = 0; icolC < ncolor; icolC++ ) + jamp2_sv[icolC] = J2_ACCESS::kernelAccessIcolConst( allJamp2s, icolC ); + const int iconfig = mgOnGpu::channel2iconfig[channelId - 1]; // map N_diagrams to N_config <= N_diagrams configs (fix LHE color mismatch #856: see also #826, #852, #853) + if( iconfig <= 0 ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which has no associated SDE iconfig\n", channelId ); + assert( iconfig > 0 ); // SANITY CHECK #917 + } + else if( iconfig > (int)mgOnGpu::nconfigSDE ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d (invalid SDE iconfig=%d\n > nconfig=%d)", channelId, iconfig, mgOnGpu::nconfigSDE ); + assert( iconfig <= (int)mgOnGpu::nconfigSDE ); // SANITY CHECK #917 + } + fptype targetamp[ncolor] = { 0 }; + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( icolC == 0 ) + targetamp[icolC] = 0; + else + targetamp[icolC] = targetamp[icolC - 1]; + if( mgOnGpu::icolamp[iconfig - 1][icolC] ) targetamp[icolC] += jamp2_sv[icolC]; + } + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( allrndcol[ievt] < ( targetamp[icolC] / targetamp[ncolor - 1] ) ) + { + allselcol[ievt] = icolC + 1; // NB Fortran [1,ncolor], cudacpp [0,ncolor-1] + break; + } + } + } + else + { + allselcol[ievt] = 0; // no color selected in Fortran range [1,ncolor] if channelId == 0 (see #931) + } + return; + } + + //-------------------------------------------------------------------------- + // Evaluate |M|^2, part independent of incoming flavour + + void /* clang-format off */ + sigmaKin( const fptype* allmomenta, // input: momenta[nevt*npar*4] + const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] + const unsigned int* iflavorVec, // input: indices of the flavor combinations + const fptype* allrndhel, // input: random numbers[nevt] for helicity selection + const fptype* allrndcol, // input: random numbers[nevt] for color selection + const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable single-diagram enhancement (fix #899/#911) + const fptype* allrnddiagram, // input: random numbers[nevt] for channel sampling + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + int* allselhel, // output: helicity selection[nevt] + int* allselcol, // output: helicity selection[nevt] + fptype* colAllJamp2s, // tmp: allJamp2s super-buffer for ncolor individual colors, running sum over colors and helicities + fptype* ghelAllNumerators, // tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + fptype* ghelAllDenominators, // tmp: allDenominators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + unsigned int* allDiagramIdsOut, // output: multichannel channelIds[nevt] (1 to #diagrams) + bool mulChannelWeight, // if true, multiply channel weight to ME output + fptype* ghelAllMEs, // tmp: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + fptype* ghelAllJamps, // tmp: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) + const bool async, // input: if true, run everything asynchronously in first stream in ghelStreams + const int gpublocks, // input: cuda gpublocks + const int gputhreads // input: cuda gputhreads + ) /* clang-format on */ + { + mgDebugInitialise(); + + // SANITY CHECKS for cudacpp code generation (see issues #272 and #343 and PRs #619, #626, #360, #396 and #754) + { + // nprocesses == 2 may happen for "mirror processes" such as P0_uux_ttx within pp_tt012j (see PR #754) + static_assert( nproc == 1 || nproc == 2, "Assume nprocesses == 1 or 2" ); + static_assert( proc_id == 1, "Assume process_id == 1" ); + } + + const int nevt = gpublocks * gputhreads; + gpuMemset( allMEs, 0, nevt * sizeof( fptype ) ); + gpuMemset( ghelAllJamps, 0, cNGoodHel * ncolor * mgOnGpu::nx2 * nevt * sizeof( fptype ) ); + gpuMemset( colAllJamp2s, 0, ncolor * nevt * sizeof( fptype ) ); + gpuMemset( ghelAllNumerators, 0, cNGoodHel * processConfig::ndiagrams * nevt * sizeof( fptype ) ); + gpuMemset( ghelAllDenominators, 0, cNGoodHel * nevt * sizeof( fptype ) ); + gpuMemset( ghelAllMEs, 0, cNGoodHel * nevt * sizeof( fptype ) ); + + // *** HELICITY LOOP: CALCULATE WAVEFUNCTIONS (one event per GPU thread) *** + // Use CUDA/HIP streams to process different helicities in parallel (one good helicity per stream) + // (1) First, within each helicity stream, compute the QCD partial amplitudes jamp's for each helicity + // In multichannel mode, also compute the running sums over helicities of numerators, denominators and squared jamp2s + bool storeChannelWeights = allChannelIds != nullptr || allrnddiagram != nullptr; + if( async ) + { + gpuLaunchKernel2D( calculate_jamps, gpublocks, cNGoodHel, gputhreads, ghelStreams[0], 0, allmomenta, allcouplings, iflavorVec, ghelAllJamps, storeChannelWeights, ghelAllNumerators, ghelAllDenominators, colAllJamp2s, nevt, true ); + color_sum_gpu( ghelAllMEs, ghelAllJamps, ghelAllBlasTmp, pBlasHandle, ghelStreams, cNGoodHel, gpublocks, gputhreads, true ); + } + else + { + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + const int ihel = cGoodHel[ighel]; + fptype* hAllJamps = ghelAllJamps + ighel * nevt; // HACK: bypass DeviceAccessJamp (consistent with layout defined there) + fptype* hAllNumerators = ghelAllNumerators + ighel * nevt * processConfig::ndiagrams; + fptype* hAllDenominators = ghelAllDenominators + ighel * nevt; + gpuLaunchKernelStream( calculate_jamps, gpublocks, gputhreads, ghelStreams[ighel], ihel, allmomenta, allcouplings, iflavorVec, hAllJamps, storeChannelWeights, hAllNumerators, hAllDenominators, colAllJamp2s, nevt, false ); + } + // (2) Then compute the ME for that helicity from the color sum of QCD partial amplitudes jamps + color_sum_gpu( ghelAllMEs, ghelAllJamps, ghelAllBlasTmp, pBlasHandle, ghelStreams, cNGoodHel, gpublocks, gputhreads, false ); + checkGpu( gpuDeviceSynchronize() ); // do not start helicity/color selection until the loop over helicities has completed + // (3) Wait for all helicity streams to complete, then finally compute the ME sum over all helicities and choose one helicity and one color + } + // Event-by-event random choice of helicity #403 and ME sum over helicities (defer this after the helicity loop to avoid breaking streams parallelism) + gpuLaunchKernel( add_and_select_hel, gpublocks, gputhreads, allselhel, allrndhel, ghelAllMEs, allMEs, gpublocks * gputhreads ); + + gpuLaunchKernel( normalise_output, gpublocks, gputhreads, allMEs, iflavorVec, ghelAllNumerators, ghelAllDenominators, allChannelIds, storeChannelWeights, mulChannelWeight, helcolDenominators[0] ); + + // Event-by-event random choice of color and diagram #402 + gpuLaunchKernel( select_col_and_diag, gpublocks, gputhreads, allselcol, allDiagramIdsOut, allrndcol, allrnddiagram, allChannelIds, colAllJamp2s, ghelAllNumerators, ghelAllDenominators, gpublocks * gputhreads ); + + mgDebugFinalise(); + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h new file mode 100644 index 0000000000..6ba1ec9097 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h @@ -0,0 +1,72 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin and everything it calls (calculate_jamps, +// good-helicity filtering, color/channel selection). Declared here so +// CPPProcess.cc's constructor/initProc (P1-generated) can call the setters +// that populate this file's storage, and so umami.cc/MatrixElementKernels.cc +// can call sigmaKin/computeDependentCouplings. + +#ifndef SIGMAKIN_H +#define SIGMAKIN_H 1 + +#include "mgOnGpuConfig.h" +#include "mgOnGpuCxtypes.h" // for cxtype + +namespace mg5amcGpu +{ + __global__ void + computeDependentCouplings( const fptype* allgs, + fptype* allcouplings ); + + void + sigmaKin_getGoodHel( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + fptype* allMEs, + fptype* allNumerators, + fptype* allDenominators, + fptype_sv* allJamps, + bool* isGoodHel, + const int nevt ); + + int + sigmaKin_setGoodHel( const bool* isGoodHel ); + + void + sigmaKin( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + const fptype* allrndhel, + const fptype* allrndcol, + const unsigned int* allChannelIds, + const fptype* allrnddiagram, + fptype* allMEs, + int* allselhel, + int* allselcol, + fptype* colAllJamp2s, + fptype* ghelAllNumerators, + fptype* ghelAllDenominators, + unsigned int* allDiagramIdsOut, + bool mulChannelWeight, + fptype* ghelAllMEs, + fptype* ghelAllJamps, + fptype2* ghelAllBlasTmp, + gpuBlasHandle_t* pBlasHandle, + gpuStream_t* ghelStreams, + const bool async, + const int gpublocks, + const int gputhreads ); + + // Setters: called once by CPPProcess (P1-generated) to populate this file's + // otherwise-internal storage, since it can no longer be written directly + // from a different translation unit. + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ); + void setIndependentParams( const fptype* tIPD ); + void setIndependentCouplings( const cxtype* tIPC ); + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ); + void setBsmIndepParam( const double* values, int n ); +} + +#endif // SIGMAKIN_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h index 7c2512a7c6..bc3607d686 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h @@ -79,7 +79,7 @@ ////#define MGONGPU_INLINE_HELAMPS 1 // Choose whether to hardcode the cIPD physics parameters rather than reading them from user cards -// This optimization can gain 20%% in CUDA in eemumu (issue #39) +// This optimization can gain 20% in CUDA in eemumu (issue #39) // By default, do not hardcode, but allow this macro to be set from outside with e.g. -DMGONGPU_HARDCODE_PARAM // ** NB: The option to use hardcoded cIPD physics parameters is supported again even now when alphas is running (#373) // ** NB: Note however that it now only refers to cIPD parameters (cIPC parameters are always accessed through global memory) @@ -209,7 +209,7 @@ using mgOnGpu::fptype2; //#define mgDebugDeclare() __shared__ float mgDebugCounter[mgOnGpu::ntpbMAX]; //#define mgDebugInitialise() { mgDebugCounter[threadIdx.x] = 0; } //#define mgDebug( code, text ) { mgDebugCounter[threadIdx.x] += 1; } -//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%%f\n", mgDebugCounter[threadIdx.x] ); } +//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%f\n", mgDebugCounter[threadIdx.x] ); } //#else #define mgDebugDeclare() /*noop*/ #define mgDebugInitialise() /*noop*/ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc index ccdcd8261a..fc2c912b5d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc @@ -53,8 +53,8 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout constexpr int np4 = CPPProcess::np4; const int nevt = 32; // 12 independent tests plus 20 duplicates (need a multiple of 16 for floats '512z') - assert( nevt %% neppM == 0 ); // nevt must be a multiple of neppM - assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV + assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM + assert( nevt % neppV == 0 ); // nevt must be a multiple of neppV // Fill in the input momenta mg5amcGpu::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc index 329ce57593..ed8c9fd301 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc @@ -8,6 +8,7 @@ #include "ProcessData.h" #include "CPPProcess.h" // needed to construct/initProc the process object (umami_initialize) +#include "SigmaKin.h" // sigmaKin_getGoodHel/setGoodHel #include "GpuRuntime.h" #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc index a2de4d93ed..4d8d01c60e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc @@ -7,7 +7,7 @@ #include "MatrixElementKernels.h" #include "ProcessData.h" -#include "CPPProcess.h" // TODO(backend_separation): drop once sigmaKin/getGoodHel/computeDependentCouplings move to backend/ +#include "SigmaKin.h" #include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h index faabc3204e..00de06527a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h @@ -88,7 +88,7 @@ namespace mg5amcCpu const int ievt ) { const int ipagC = ievt / neppC; // #event "C-page" - const int ieppC = ievt %% neppC; // #event in the current event C-page + const int ieppC = ievt % neppC; // #event in the current event C-page constexpr int idcoup = 0; constexpr int ix2 = 0; return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] @@ -188,9 +188,9 @@ namespace mg5amcCpu // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } @@ -217,9 +217,9 @@ namespace mg5amcCpu // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast #endif } @@ -233,7 +233,7 @@ namespace mg5amcCpu /* fptype_sv& real = kernelAccessIx2( buffer, 0 ); fptype_sv& imag = kernelAccessIx2( buffer, 1 ); - printf( "C_ACCESS::kernelAccess: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + printf( "C_ACCESS::kernelAccess: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); return cxtype_sv_ref( real, imag ); */ return cxtype_sv_ref( kernelAccessIx2( buffer, 0 ), @@ -248,7 +248,7 @@ namespace mg5amcCpu /* const fptype_sv& real = kernelAccessIx2Const( buffer, 0 ); const fptype_sv& imag = kernelAccessIx2Const( buffer, 1 ); - printf( "C_ACCESS::kernelAccessConst: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + printf( "C_ACCESS::kernelAccessConst: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); return cxtype_sv( real, imag ); */ return cxtype_sv( kernelAccessIx2Const( buffer, 0 ), diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc new file mode 100644 index 0000000000..2d04ded182 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc @@ -0,0 +1,600 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin, calculate_jamps, good-helicity filtering. +// The diagram/vertex-call sequence (helas_calls) is process-specific and +// lives in the P1-generated EvaluateDiagrams.inc, #include'd below. + +#include "SigmaKin.h" + +#include "CPPProcess.h" // ProcessData.h, Parameters.h, HelAmps_.h transitively +#include "ProcessTables.h" + +#include "GpuRuntime.h" +#include "MemoryAccessAmplitudes.h" +#include "MemoryAccessChannelIds.h" +#include "MemoryAccessCouplings.h" +#include "MemoryAccessCouplingsFixed.h" +#include "MemoryAccessDenominators.h" +#include "MemoryAccessGs.h" +#include "MemoryAccessIflavorVec.h" +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessNumerators.h" +#include "MemoryAccessWavefunctions.h" +#include "color_sum.h" +#include "coloramps.h" +#include "processConfig.h" + +namespace mg5amcCpu +{ + using namespace ProcessData; + using namespace ProcessTables; + using Parameters_dependentCouplings::ndcoup; // #couplings that vary event by event (depend on running alphas QCD) + using Parameters_independentCouplings::nicoup; // #couplings that are fixed for all events (do not depend on running alphas QCD) + + // The number of SIMD vectors of events processed by calculate_jamps +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + constexpr int nParity = 2; +#else + constexpr int nParity = 1; +#endif + + // Helicity/flavor tables and SM parameter/coupling storage, populated once + // by CPPProcess's constructor/initProc via the setters below. + static short cHel[ncomb][npar]; + static short cFlavors[nmaxflavor][npar]; + static int cNGoodHel; + static int cGoodHel[ncomb]; + static fptype cIPD[nIPD > 0 ? nIPD : 1]; + static fptype cIPC[nIPC > 0 ? nIPC * 2 : 1]; + static int cIPF_partner1[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + static int cIPF_partner2[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + static fptype cIPF_value[ProcessTables::nMF * nIPF * 2 > 0 ? ProcessTables::nMF * nIPF * 2 : 1]; + static double bsmIndepParam[Parameters::nBsmIndepParam > 0 ? Parameters::nBsmIndepParam : 1]; + + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ) + { + memcpy( cHel, tHel, ncomb * npar * sizeof( short ) ); + memcpy( cFlavors, tFlavors, nmaxflavor * npar * sizeof( short ) ); + } + + void setIndependentParams( const fptype* tIPD ) + { + if( nIPD > 0 ) memcpy( cIPD, tIPD, nIPD * sizeof( fptype ) ); + } + + void setIndependentCouplings( const cxtype* tIPC ) + { + if( nIPC > 0 ) memcpy( cIPC, tIPC, nIPC * sizeof( cxtype ) ); + } + + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ) + { + if( nIPF == 0 ) return; + memcpy( cIPF_partner1, tIPF_partner1, nMF * nIPF * sizeof( int ) ); + memcpy( cIPF_partner2, tIPF_partner2, nMF * nIPF * sizeof( int ) ); + memcpy( cIPF_value, tIPF_value, nMF * nIPF * sizeof( cxtype ) ); + } + + void setBsmIndepParam( const double* values, int n ) + { + if( n > 0 ) memcpy( bsmIndepParam, values, n * sizeof( double ) ); + } + + //-------------------------------------------------------------------------- + + __device__ INLINE unsigned int + getChannelId( const unsigned int* allChannelIds, const int ievt00, bool sanityCheckMixedPrecision = true ) + { + unsigned int channelId = 0; // disable multichannel single-diagram enhancement unless allChannelIds != nullptr + using CID_ACCESS = HostAccessChannelIds; // non-trivial access: buffer includes all events + // SCALAR channelId for the whole SIMD neppV2 event page (C++), i.e. one or two neppV event page(s) + // The cudacpp implementation ASSUMES (and checks! #898) that all channelIds are the same in a neppV2 SIMD event page + // **NB! in "mixed" precision, using SIMD, calculate_wavefunctions computes MEs for TWO neppV pages with a single channelId! #924 + if( allChannelIds != nullptr ) + { + // First - and/or only - neppV page of channels (iParity=0 => ievt0 = ievt00 + 0 * neppV) + const unsigned int* channelIds = CID_ACCESS::ieventAccessRecordConst( allChannelIds, ievt00 ); // fix bug #899/#911 + uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) + // NB: channelIds_sv is a vector in SIMD C++ + channelId = channelIds_sv[0]; // element[0] + for( int i = 1; i < neppV; ++i ) // elements[1...neppV-1] + { + assert( channelId == channelIds_sv[i] ); // SANITY CHECK #898: check that all events in a SIMD vector have the same channelId + } + assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) + if( sanityCheckMixedPrecision ) + { +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Second neppV page of channels (iParity=1 => ievt0 = ievt00 + 1 * neppV) + const unsigned int* channelIds2 = CID_ACCESS::ieventAccessRecordConst( allChannelIds, ievt00 + neppV ); // fix bug #899/#911 + uint_v channelIds2_v = CID_ACCESS::kernelAccessConst( channelIds2 ); // fix #895 (compute this only once for all diagrams) + // **NB! in "mixed" precision, using SIMD, calculate_wavefunctions computes MEs for TWO neppV pages with a single channelId! #924 + for( int i = 0; i < neppV; ++i ) + { + assert( channelId == channelIds2_v[i] ); // SANITY CHECKS #898 #924: all events in the 2nd SIMD vector have the same channelId as that of the 1st SIMD vector + } +#endif + } + } + return channelId; + } + + //-------------------------------------------------------------------------- + + __global__ void + computeDependentCouplings( const fptype* allgs, fptype* allcouplings, const int nevt ) + { + using G_ACCESS = HostAccessGs; + using C_ACCESS = HostAccessCouplings; + for( int ipagV = 0; ipagV < nevt / neppV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + const fptype* gs = MemoryAccessGs::ieventAccessRecordConst( allgs, ievt0 ); + fptype* couplings = MemoryAccessCouplings::ieventAccessRecord( allcouplings, ievt0 ); + G2COUP( gs, couplings, bsmIndepParam ); + } + } + + //-------------------------------------------------------------------------- + + // Evaluate QCD partial amplitudes jamps for this given helicity from Feynman diagrams. + // This function processes a single event "page" or SIMD vector (or for two in "mixed" + // precision mode, nParity=2). Accepts a SCALAR channelId because it is GUARANTEED that + // all events in a SIMD vector have the same channelId #898. + void + calculate_jamps( int ihel, + const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + cxtype_sv* allJamp_sv, + bool storeChannelWeights, + fptype* allNumerators, + fptype* allDenominators, + fptype_sv* jamp2_sv, + const int ievt00 ) + { + using M_ACCESS = HostAccessMomenta; + using W_ACCESS = HostAccessWavefunctions; + using A_ACCESS = HostAccessAmplitudes; + using CD_ACCESS = HostAccessCouplings; + using CI_ACCESS = HostAccessCouplingsFixed; + using F_ACCESS = HostAccessIflavorVec; + using NUM_ACCESS = HostAccessNumerators; + using DEN_ACCESS = HostAccessDenominators; + mgDebug( 0, __FUNCTION__ ); + + fptype_sv pvec_sv[nwf][np4]; + cxtype_sv w_sv[nwf][nw6]; // particle wavefunctions within Feynman diagrams + cxtype_sv amp_sv[1]; // invariant amplitude for one given Feynman diagram + ALOHAOBJ aloha_obj[nwf]; + for( int iwf = 0; iwf < nwf; iwf++ ) aloha_obj[iwf] = ALOHAOBJ{ pvec_sv[iwf], w_sv[iwf] }; + fptype* amp_fp = reinterpret_cast( amp_sv ); + + // jamp: sum (for one event or event page) of the invariant amplitudes for + // all Feynman diagrams in a given color combination + cxtype_sv jamp_sv[ncolor] = {}; // all zeros + + for( int iParity = 0; iParity < nParity; ++iParity ) + { + const int ievt0 = ievt00 + iParity * neppV; +#include "EvaluateDiagrams.inc" + + // *** COLOR CHOICE BELOW *** + // Store the leading color flows for choice of color + if( jamp2_sv ) // disable color choice if nullptr + { + for( int icol = 0; icol < ncolor; icol++ ) + jamp2_sv[ncolor * iParity + icol] += cxabs2( jamp_sv[icol] ); // may underflow #831 + } + + // *** PREPARE OUTPUT JAMPS *** + // In C++, copy the local jamp to the output array passed as function argument + for( int icol = 0; icol < ncolor; icol++ ) + allJamp_sv[iParity * ncolor + icol] = jamp_sv[icol]; + } + // END LOOP ON IPARITY + + mgDebug( 1, __FUNCTION__ ); + return; + } + + //-------------------------------------------------------------------------- + + void + sigmaKin_getGoodHel( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + fptype* allMEs, + fptype* allNumerators, + fptype* allDenominators, + bool* isGoodHel, + const int nevt ) + { + // Allocate arrays at build time to contain at least 16 events (or at least neppV events if neppV>16, e.g. in future VPUs) + constexpr int maxtry0 = std::max( 16, neppV ); // 16, but at least neppV (otherwise the npagV loop does not even start) + assert( nevt >= neppV ); + const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt 1 ) + { + for( int i = broken_sym_component_starts[icomp] - 1; i < broken_sym_component_ends[icomp]; i++ ) + { + if( pid_work[i] == 0 ) + continue; + int n_tot = 1; + for( int j = i + 1; j < broken_sym_component_ends[icomp]; j++ ) + { + if( pid_work[i] != pid_work[j] ) + continue; + bool same_block = ( broken_sym_block_lengths[i] == broken_sym_block_lengths[j] ); + for( int k = 0; same_block && k < broken_sym_block_lengths[i]; k++ ) + { + if( cFlavors[iflavor][broken_sym_block_starts[i] - 1 + k] != cFlavors[iflavor][broken_sym_block_starts[j] - 1 + k] ) + same_block = false; + } + if( same_block ) + { + pid_work[j] = 0; + n_tot = n_tot + 1; + old_factor = old_factor / n_tot; + } + } + } + } + total_factor = total_factor * old_factor; + } + return total_factor; + } + + //-------------------------------------------------------------------------- + // Evaluate |M|^2, part independent of incoming flavour + + void + sigmaKin( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + const fptype* allrndhel, + const fptype* allrndcol, + const unsigned int* allChannelIds, + const fptype* allrnddiagram, + fptype* allMEs, + int* allselhel, + int* allselcol, + fptype* allNumerators, + fptype* allDenominators, + unsigned int* allDiagramIdsOut, + bool mulChannelWeight, + const int nevt ) + { + mgDebugInitialise(); + + // SANITY CHECKS for cudacpp code generation (see issues #272 and #343 and PRs #619, #626, #360, #396 and #754) + { + // nprocesses == 2 may happen for "mirror processes" such as P0_uux_ttx within pp_tt012j (see PR #754) + static_assert( nproc == 1 || nproc == 2, "Assume nprocesses == 1 or 2" ); + static_assert( proc_id == 1, "Assume process_id == 1" ); + } + + using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events + using NUM_ACCESS = HostAccessNumerators; // non-trivial access: buffer includes all events + using DEN_ACCESS = HostAccessDenominators; // non-trivial access: buffer includes all events + + // Reset the "matrix elements" - running sums of |M|^2 over helicities for the given event + const int npagV = nevt / neppV; + for( int ipagV = 0; ipagV < npagV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv = fptype_sv{ 0 }; + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + for( int i = 0; i < processConfig::ndiagrams; ++i ) + { + numerators_sv[i] = fptype_sv{ 0 }; + } + denominators_sv = fptype_sv{ 0 }; + } + + // HELICITY LOOP: CALCULATE WAVEFUNCTIONS (using precomputed good helicities) +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Mixed fptypes #537: float for color algebra and double elsewhere + // Delay color algebra and ME updates (only on even pages) + assert( npagV % 2 == 0 ); // SANITY CHECK for mixed fptypes: two neppV-pages are merged to one 2*neppV-page + const int npagV2 = npagV / 2; // loop on two SIMD pages (neppV events) at a time +#else + const int npagV2 = npagV; // loop on one SIMD page (neppV events) at a time +#endif +#ifdef _OPENMP + // OMP multithreading #575 (NB: tested only with gcc11 so far) +#define _OMPLIST0 allcouplings, allMEs, allmomenta, allrndcol, allrndhel, allselcol, allselhel, cGoodHel, cNGoodHel, npagV2 +#define _OMPLIST1 , allDenominators, allNumerators, allChannelIds, mgOnGpu::icolamp, mgOnGpu::channel2iconfig +#pragma omp parallel for default( none ) shared( _OMPLIST0 _OMPLIST1 ) +#undef _OMPLIST0 +#undef _OMPLIST1 +#endif // _OPENMP + for( int ipagV2 = 0; ipagV2 < npagV2; ++ipagV2 ) + { +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int ievt00 = ipagV2 * neppV * 2; // loop on two SIMD pages (neppV events) at a time +#else + const int ievt00 = ipagV2 * neppV; // loop on one SIMD page (neppV events) at a time +#endif + // Running sum of partial amplitudes squared for event by event color selection (#402) + fptype_sv jamp2_sv[nParity * ncolor] = {}; + fptype_sv MEs_ighel[ncomb] = {}; // sum of MEs for all good helicities up to ighel (for the first - and/or only - neppV page) +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype_sv MEs_ighel2[ncomb] = {}; // sum of MEs for all good helicities up to ighel (for the second neppV page) +#endif + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + const int ihel = cGoodHel[ighel]; + cxtype_sv jamp_sv[nParity * ncolor] = {}; // fixed nasty bug (omitting 'nParity' caused memory corruptions after calling calculate_jamps) + bool storeChannelWeights = allChannelIds != nullptr || allrnddiagram != nullptr; + calculate_jamps( ihel, allmomenta, allcouplings, iflavorVec, jamp_sv, storeChannelWeights, allNumerators, allDenominators, jamp2_sv, ievt00 ); + color_sum_cpu( allMEs, jamp_sv, ievt00 ); + MEs_ighel[ighel] = E_ACCESS::kernelAccess( E_ACCESS::ieventAccessRecord( allMEs, ievt00 ) ); +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + MEs_ighel2[ighel] = E_ACCESS::kernelAccess( E_ACCESS::ieventAccessRecord( allMEs, ievt00 + neppV ) ); +#endif + } + // Event-by-event random choice of helicity #403 + for( int ieppV = 0; ieppV < neppV; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + const bool okhel = allrndhel[ievt] < ( MEs_ighel[ighel][ieppV] / MEs_ighel[cNGoodHel - 1][ieppV] ); + if( okhel ) + { + const int ihelF = cGoodHel[ighel] + 1; // NB Fortran [1,ncomb], cudacpp [0,ncomb-1] + allselhel[ievt] = ihelF; + break; + } + } +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int ievt2 = ievt00 + ieppV + neppV; + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + if( allrndhel[ievt2] < ( MEs_ighel2[ighel][ieppV] / MEs_ighel2[cNGoodHel - 1][ieppV] ) ) + { + const int ihelF = cGoodHel[ighel] + 1; // NB Fortran [1,ncomb], cudacpp [0,ncomb-1] + allselhel[ievt2] = ihelF; + break; + } + } +#endif + } +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int vecsize = 2 * neppV; +#else + const int vecsize = neppV; +#endif + unsigned int channelIdVec[vecsize]; + if( allChannelIds != nullptr ) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + channelIdVec[ieppV] = allChannelIds[ievt]; + } + } + + // Event-by-event random choice of channel + if( allrnddiagram != nullptr ) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + fptype numerator_sum = 0., normalization = 0.; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + normalization += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + ichan * neppV + ieppV % neppV]; + } + channelIdVec[ieppV] = mgOnGpu::nchannels; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + numerator_sum += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + ichan * neppV + ieppV % neppV]; + if( allrnddiagram[ievt] < numerator_sum / normalization ) + { + channelIdVec[ieppV] = ichan + 1; + break; + } + } + allDiagramIdsOut[ievt] = channelIdVec[ieppV]; + } + } + + // Event-by-event random choice of color #402 + if( allChannelIds != nullptr || allrnddiagram != nullptr ) // no event-by-event choice of color if channelId == 0 (fix FPE #783) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + unsigned int channelId = channelIdVec[ieppV]; + if( channelId > mgOnGpu::nchannels ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which is greater than nchannels=%d\n", channelId, mgOnGpu::nchannels ); + assert( channelId <= mgOnGpu::nchannels ); // SANITY CHECK #919 #910 + } + const int iconfig = mgOnGpu::channel2iconfig[channelId - 1]; // map N_diagrams to N_config <= N_diagrams configs (fix LHE color mismatch #856: see also #826, #852, #853) + if( iconfig <= 0 ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which has no associated SDE iconfig\n", channelId ); + assert( iconfig > 0 ); // SANITY CHECK #917 + } + else if( iconfig > (int)mgOnGpu::nconfigSDE ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d (invalid SDE iconfig=%d\n > nconfig=%d)", channelId, iconfig, mgOnGpu::nconfigSDE ); + assert( iconfig <= (int)mgOnGpu::nconfigSDE ); // SANITY CHECK #917 + } + fptype targetamp[ncolor] = { 0 }; + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( icolC == 0 ) + targetamp[icolC] = 0; + else + targetamp[icolC] = targetamp[icolC - 1]; + if( mgOnGpu::icolamp[iconfig - 1][icolC] ) targetamp[icolC] += + jamp2_sv[icolC + ncolor * ( ieppV / neppV )][ieppV % neppV]; + } + const int ievt = ievt00 + ieppV; + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( allrndcol[ievt] < ( targetamp[icolC] / targetamp[ncolor - 1] ) ) + { + allselcol[ievt] = icolC + 1; // NB Fortran [1,ncolor], cudacpp [0,ncolor-1] + break; + } + } + } + } + else + { + for( int ieppV = 0; ieppV < neppV; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + allselcol[ievt] = 0; // no color selected in Fortran range [1,ncolor] if channelId == 0 (see #931) +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int ievt2 = ievt00 + ieppV + neppV; + allselcol[ievt2] = 0; // no color selected in Fortran range [1,ncolor] if channelId == 0 (see #931) +#endif + } + } + } + // *** END OF PART 1b - C++ (loop on event pages) + + // PART 2 - FINALISATION (after calculate_jamps) + // Get the final |M|^2 as an average over helicities/colors of the running sum of |M|^2 over helicities for the given event + for( int ipagV = 0; ipagV < npagV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv = MEs_sv * broken_symmetry_factor( iflavorVec[ievt0] ) / helcolDenominators[0]; + if( mulChannelWeight && allChannelIds != nullptr ) // fix segfault #892 (not 'channelIds[0] != 0') + { + const unsigned int channelId = getChannelId( allChannelIds, ievt0, false ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + MEs_sv *= numerators_sv[channelId - 1] / denominators_sv; + } + } + mgDebugFinalise(); + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h new file mode 100644 index 0000000000..725ff931d2 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h @@ -0,0 +1,64 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin and everything it calls (calculate_jamps, +// good-helicity filtering, color/channel selection). Declared here so +// CPPProcess.cc's constructor/initProc (P1-generated) can call the setters +// that populate this file's storage, and so umami.cc/MatrixElementKernels.cc +// can call sigmaKin/computeDependentCouplings. + +#ifndef SIGMAKIN_H +#define SIGMAKIN_H 1 + +#include "mgOnGpuConfig.h" +#include "mgOnGpuCxtypes.h" // for cxtype + +namespace mg5amcCpu +{ + __global__ void + computeDependentCouplings( const fptype* allgs, + fptype* allcouplings, + const int nevt ); + + void + sigmaKin_getGoodHel( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + fptype* allMEs, + fptype* allNumerators, + fptype* allDenominators, + bool* isGoodHel, + const int nevt ); + + int + sigmaKin_setGoodHel( const bool* isGoodHel ); + + void + sigmaKin( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + const fptype* allrndhel, + const fptype* allrndcol, + const unsigned int* allChannelIds, + const fptype* allrnddiagram, + fptype* allMEs, + int* allselhel, + int* allselcol, + fptype* allNumerators, + fptype* allDenominators, + unsigned int* allDiagramIdsOut, + bool mulChannelWeight, + const int nevt ); + + // Setters: called once by CPPProcess (P1-generated) to populate this file's + // otherwise-internal storage, since it can no longer be written directly + // from a different translation unit. + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ); + void setIndependentParams( const fptype* tIPD ); + void setIndependentCouplings( const cxtype* tIPC ); + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ); + void setBsmIndepParam( const double* values, int n ); +} + +#endif // SIGMAKIN_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc index d9d8b15aa4..8ec6f8045a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc @@ -49,7 +49,7 @@ namespace mg5amcCpu // we gain (not a factor 2...) in speed here as we only loop over the up diagonal part of the matrix. // Strangely, CUDA is slower instead, so keep the old implementation for the moment. fptype_sv deltaMEs = { 0 }; -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT fptype_sv deltaMEs_next = { 0 }; // Mixed mode: merge two neppV vectors into one neppV2 vector fptype2_sv jampR_sv[ncolor]; @@ -66,7 +66,7 @@ namespace mg5amcCpu for( int icol = 0; icol < ncolor; icol++ ) { // Diagonal terms -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT fptype2_sv& jampRi_sv = jampR_sv[icol]; fptype2_sv& jampIi_sv = jampI_sv[icol]; #else @@ -79,7 +79,7 @@ namespace mg5amcCpu for( int jcol = icol + 1; jcol < ncolor; jcol++ ) { // Off-diagonal terms -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT fptype2_sv& jampRj_sv = jampR_sv[jcol]; fptype2_sv& jampIj_sv = jampI_sv[jcol]; #else @@ -90,7 +90,7 @@ namespace mg5amcCpu ztempI_sv += cf2.value[icol][jcol] * jampIj_sv; } fptype2_sv deltaMEs2 = ( jampRi_sv * ztempR_sv + jampIi_sv * ztempI_sv ); // may underflow #831 -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT deltaMEs += fpvsplit0( deltaMEs2 ); deltaMEs_next += fpvsplit1( deltaMEs2 ); #else @@ -103,7 +103,7 @@ namespace mg5amcCpu // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); MEs_sv += deltaMEs; // fix #435 -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT fptype* MEs_next = E_ACCESS::ieventAccessRecord( allMEs, ievt0 + neppV ); fptype_sv& MEs_sv_next = E_ACCESS::kernelAccess( MEs_next ); MEs_sv_next += deltaMEs_next; @@ -112,19 +112,4 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - } // end namespace diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h index 53b41f3b80..dd13522513 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h @@ -79,7 +79,7 @@ ////#define MGONGPU_INLINE_HELAMPS 1 // Choose whether to hardcode the cIPD physics parameters rather than reading them from user cards -// This optimization can gain 20%% in CUDA in eemumu (issue #39) +// This optimization can gain 20% in CUDA in eemumu (issue #39) // By default, do not hardcode, but allow this macro to be set from outside with e.g. -DMGONGPU_HARDCODE_PARAM // ** NB: The option to use hardcoded cIPD physics parameters is supported again even now when alphas is running (#373) // ** NB: Note however that it now only refers to cIPD parameters (cIPC parameters are always accessed through global memory) @@ -245,7 +245,7 @@ using mgOnGpu::fptype2; //#define mgDebugDeclare() __shared__ float mgDebugCounter[mgOnGpu::ntpbMAX]; //#define mgDebugInitialise() { mgDebugCounter[threadIdx.x] = 0; } //#define mgDebug( code, text ) { mgDebugCounter[threadIdx.x] += 1; } -//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%%f\n", mgDebugCounter[threadIdx.x] ); } +//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%f\n", mgDebugCounter[threadIdx.x] ); } //#else #define mgDebugDeclare() /*noop*/ #define mgDebugInitialise() /*noop*/ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc index 3ddffd4559..9d5963fb58 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc @@ -53,8 +53,8 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout constexpr int np4 = CPPProcess::np4; const int nevt = 32; // 12 independent tests plus 20 duplicates (need a multiple of 16 for floats '512z') - assert( nevt %% neppM == 0 ); // nevt must be a multiple of neppM - assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV + assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM + assert( nevt % neppV == 0 ); // nevt must be a multiple of neppV // Fill in the input momenta mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! @@ -147,7 +147,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) out << " // ---------" << std::endl; for( int iw6 = 0; iw6 < nw6; iw6++ ) { - const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration + const int ieppV = ievt % neppV; // #event in the current event vector in this iteration #ifdef MGONGPU_HAS_CPPCXTYPEV_BRK out << std::setw( 26 ) << cxreal( wf[iw6][ieppV] ) << ", "; out << std::setw( 22 ) << cximag( wf[iw6][ieppV] ); @@ -182,7 +182,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) const fptype expImag = expwf[iw6 * 2 + 1]; if( true ) { - const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration + const int ieppV = ievt % neppV; // #event in the current event vector in this iteration #ifdef MGONGPU_HAS_CPPCXTYPEV_BRK EXPECT_NEAR( cxreal( wf[iw6][ieppV] ), expReal, std::abs( expReal * toleranceXXXs ) ) << " itest=" << itest << ": " << xxx << "#" << ievt; @@ -214,7 +214,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { if( true ) { - const int ieppV = ievt %% neppV; // #event in the current event vector in this iteration + const int ieppV = ievt % neppV; // #event in the current event vector in this iteration #ifdef MGONGPU_HAS_CPPCXTYPEV_BRK const fptype expReal = cxreal( expwf[iw6][ieppV] ); const fptype expImag = cximag( expwf[iw6][ieppV] ); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc index 95aa04f718..7fcdb3d91f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc @@ -8,6 +8,7 @@ #include "ProcessData.h" #include "CPPProcess.h" // needed to construct/initProc the process object (umami_initialize) +#include "SigmaKin.h" // sigmaKin_getGoodHel/setGoodHel #include "GpuRuntime.h" #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc index 082c373aa2..09acde6a72 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc @@ -10,6 +10,14 @@ //========================================================================== // Class member functions for calculating the matrix elements for %(process_lines)s +// +// sigmaKin and everything it calls (calculate_jamps, good-helicity +// filtering, color/channel selection) are backend-owned, see +// backend/{cpu,simd,gpu}/SigmaKin.cc, which #includes the process-specific +// EvaluateDiagrams.inc generated alongside this file. This file only +// contains what's genuinely process-specific: the class constructor, +// physics-parameter/coupling setup (initProc), and the flavor accessor. +//========================================================================== #ifdef MGONGPUCPP_GPUIMPL namespace mg5amcGpu @@ -17,165 +25,7 @@ namespace mg5amcGpu namespace mg5amcCpu #endif { - __device__ INLINE unsigned int getChannelId( const unsigned int* allChannelIds -#ifndef MGONGPUCPP_GPUIMPL - , - const int ievt00, - bool sanityCheckMixedPrecision = true -#endif - ) - { - unsigned int channelId = 0; // disable multichannel single-diagram enhancement unless allChannelIds != nullptr -#ifdef MGONGPUCPP_GPUIMPL - using CID_ACCESS = DeviceAccessChannelIds; // non-trivial access: buffer includes all events - // SCALAR channelId for the current event (CUDA) or for the whole SIMD event page (C++) - // The cudacpp implementation ASSUMES (and checks! #898) that all channelIds are the same in a SIMD event page - if( allChannelIds != nullptr ) - { - const unsigned int* channelIds = allChannelIds; // fix #899 (distinguish channelIds and allChannelIds) - const uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) - // NB: channelIds_sv is a scalar in CUDA - channelId = channelIds_sv; - assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) - } -#else // Cuda or C++ - using CID_ACCESS = HostAccessChannelIds; // non-trivial access: buffer includes all events - // SCALAR channelId for the whole SIMD neppV2 event page (C++), i.e. one or two neppV event page(s) - // The cudacpp implementation ASSUMES (and checks! #898) that all channelIds are the same in a neppV2 SIMD event page - // **NB! in "mixed" precision, using SIMD, calculate_wavefunctions computes MEs for TWO neppV pages with a single channelId! #924 - if( allChannelIds != nullptr ) - { - // First - and/or only - neppV page of channels (iParity=0 => ievt0 = ievt00 + 0 * neppV) - const unsigned int* channelIds = CID_ACCESS::ieventAccessRecordConst( allChannelIds, ievt00 ); // fix bug #899/#911 - uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) -#ifndef MGONGPU_CPPSIMD - // NB: channelIds_sv is a scalar in no-SIMD C++ - channelId = channelIds_sv; -#else - // NB: channelIds_sv is a vector in SIMD C++ - channelId = channelIds_sv[0]; // element[0] - for( int i = 1; i < neppV; ++i ) // elements[1...neppV-1] - { - assert( channelId == channelIds_sv[i] ); // SANITY CHECK #898: check that all events in a SIMD vector have the same channelId - } -#endif - assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) - if( sanityCheckMixedPrecision ) - { -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Second neppV page of channels (iParity=1 => ievt0 = ievt00 + 1 * neppV) - const unsigned int* channelIds2 = CID_ACCESS::ieventAccessRecordConst( allChannelIds, ievt00 + neppV ); // fix bug #899/#911 - uint_v channelIds2_v = CID_ACCESS::kernelAccessConst( channelIds2 ); // fix #895 (compute this only once for all diagrams) - // **NB! in "mixed" precision, using SIMD, calculate_wavefunctions computes MEs for TWO neppV pages with a single channelId! #924 - for( int i = 0; i < neppV; ++i ) - { - assert( channelId == channelIds2_v[i] ); // SANITY CHECKS #898 #924: all events in the 2nd SIMD vector have the same channelId as that of the 1st SIMD vector - } -#endif - } - } -#endif // MGONGPUCPP_GPUIMPL - return channelId; - } - - constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E, px, py, pz) - constexpr int nw6 = CPPProcess::nw6; // dimensions of each wavefunction (HELAS KEK 91-11): e.g. 6 for e+ e- -> mu+ mu- (fermions and vectors) - constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - constexpr int ncomb = CPPProcess::ncomb; // #helicity combinations: e.g. 16 for e+ e- -> mu+ mu- (2**4 = fermion spin up/down ** npar) - constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors - constexpr int nmaxflavor = CPPProcess::nmaxflavor; // the maximum number of flavor combinations - - // [NB: I am currently unable to get the right value of nwf in CPPProcess.h - will hardcode it in CPPProcess.cc instead (#644)] - //using CPPProcess::nwf; // #wavefunctions = #external (npar) + #internal: e.g. 5 for e+ e- -> mu+ mu- (1 internal is gamma or Z) - - using Parameters_dependentCouplings::ndcoup; // #couplings that vary event by event (depend on running alphas QCD) - using Parameters_independentCouplings::nicoup; // #couplings that are fixed for all events (do not depend on running alphas QCD) - - // The number of SIMD vectors of events processed by calculate_jamps -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - constexpr int nParity = 2; -#else - constexpr int nParity = 1; -#endif - - // Physics parameters (masses, coupling, etc...) - // For CUDA performance, hardcoded constexpr's would be better: fewer registers and a tiny throughput increase - // However, physics parameters are user-defined through card files: use CUDA constant memory instead (issue #39) - // [NB if hardcoded parameters are used, it's better to define them here to avoid silent shadowing (issue #263)] - constexpr int nIPD = %(nipd)i; // SM independent parameters used in this CPPProcess.cc (FIXME? rename as sm_IndepParam?) - // Note: in the Python code generator, nIPD == nparam, while nIPC <= nicoup, because (see #823) - // nIPC may vary from one P*/CPPProcess.cc to another, while nicoup is defined in src/Param.h and is common to all P* - constexpr int nIPC = %(nipc)i; // SM independent couplings used in this CPPProcess.cc (FIXME? rename as sm_IndepCoupl?) - // nIPF are the number of SM independent flavor couplings, of type FLV_COUPLING - constexpr int nMF = FLV_COUPLING::max_flavor; - constexpr int nIPF = %(nipf)i; - // nDPF are the number of dependent (event-by-event, running-alphas) flavor couplings - constexpr int nDPF = %(ndpf)i; - static_assert( nIPC <= nicoup ); - static_assert( nIPD >= 0 ); // Hack to avoid build warnings when nIPD==0 is unused - static_assert( nIPC >= 0 ); // Hack to avoid build warnings when nIPC==0 is unused - static_assert( nMF >= 0 ); // Hack to avoid build warnings when nMF ==0 is unused - static_assert( nIPF >= 0 ); // Hack to avoid build warnings when nIPF==0 is unused - static_assert( nDPF >= 0 ); // Hack to avoid build warnings when nDPF==0 is unused -#ifdef MGONGPU_HARDCODE_PARAM - %(cipdhrdcod)s - %(cipchrdcod)s - %(cipfhrdcod)s -#else -#ifdef MGONGPUCPP_GPUIMPL - %(cipddevice)s - %(cipcdevice)s - %(cipfdevice)s -#else - %(cipdstatic)s - %(cipcstatic)s - %(cipfstatic)s -#endif -#endif - - // Dependent (event-by-event, running-alphas) flavor couplings: partner indices and - // the per-flavor idcoup are pure compile-time constants (the complex values are - // gathered per event page in calculate_jamps), so they are emitted the same way in - // all build modes (no CUDA constant memory copy needed). - %(cdpfdecl)s - - // AV Jan 2024 (PR #625): this ugly #define was the only way I found to avoid creating arrays[nBsm] in CPPProcess.cc if nBsm is 0 - // The problem is that nBsm is determined when generating Parameters.h, which happens after CPPProcess.cc has already been generated - // For simplicity, keep this code hardcoded also for SM processes (a nullptr is needed as in the case nBsm == 0) -#ifdef MGONGPUCPP_NBSMINDEPPARAM_GT_0 -#ifdef MGONGPU_HARDCODE_PARAM - __device__ const double* bsmIndepParam = Parameters::mdl_bsmIndepParam; -#else -#ifdef MGONGPUCPP_GPUIMPL - __device__ __constant__ double bsmIndepParam[Parameters::nBsmIndepParam]; -#else - static double bsmIndepParam[Parameters::nBsmIndepParam]; -#endif -#endif -#else -#ifdef MGONGPU_HARDCODE_PARAM - __device__ const double* bsmIndepParam = nullptr; -#else -#ifdef MGONGPUCPP_GPUIMPL - __device__ __constant__ double* bsmIndepParam = nullptr; -#else - static double* bsmIndepParam = nullptr; -#endif -#endif -#endif - - // Helicity combinations (and filtering of "good" helicity combinations) -#ifdef MGONGPUCPP_GPUIMPL - __device__ __constant__ short cHel[ncomb][npar]; - __device__ __constant__ int dcNGoodHel; - __device__ __constant__ int dcGoodHel[ncomb]; - __device__ __constant__ short cFlavors[nmaxflavor][npar]; -#else - static short cHel[ncomb][npar]; - static short cFlavors[nmaxflavor][npar]; -#endif - static int cNGoodHel; - static int cGoodHel[ncomb]; + using namespace ProcessData; // Host-side flavor table: single source of truth for PDG ids (used by both the // constructor copy into cFlavors and the public CPPProcess::flavorPDG accessor). @@ -183,52 +33,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - class DeviceAccessJamp2 - { - public: - static __device__ inline fptype& - kernelAccessIcol( fptype* buffer, const int icol ) - { - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - return buffer[icol * nevt + ievt]; - } - static __device__ inline const fptype& - kernelAccessIcolConst( const fptype* buffer, const int icol ) - { - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - return buffer[icol * nevt + ievt]; - } - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - __device__ INLINE unsigned int - gpu_channelId( const unsigned int* allChannelIds ) - { - unsigned int channelId = 0; // disable multichannel single-diagram enhancement unless allChannelIds != nullptr - using CID_ACCESS = DeviceAccessChannelIds; // non-trivial access: buffer includes all events - // SCALAR channelId for the current event (CUDA) - if( allChannelIds != nullptr ) - { - const unsigned int* channelIds = allChannelIds; // fix #899 (distinguish channelIds and allChannelIds) - const uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) - // NB: channelIds_sv is a scalar in CUDA - channelId = channelIds_sv; - assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) - } - return channelId; - } -#endif - - //-------------------------------------------------------------------------- -%(all_sigmaKin)s - //-------------------------------------------------------------------------- - CPPProcess::CPPProcess( bool verbose, bool debug ) : m_verbose( verbose ) @@ -241,13 +45,7 @@ namespace mg5amcCpu // Helicities for the process [NB do keep 'static' for this constexpr array, see issue #283] // *** NB There is no automatic check yet that these are in the same order as Fortran! #569 *** %(all_helicities)s -#ifdef MGONGPUCPP_GPUIMPL - gpuMemcpyToSymbol( cHel, tHel, ncomb * npar * sizeof( short ) ); - gpuMemcpyToSymbol( cFlavors, tFlavors, nmaxflavor * npar * sizeof( short ) ); -#else - memcpy( cHel, tHel, ncomb * npar * sizeof( short ) ); - memcpy( cFlavors, tFlavors, nmaxflavor * npar * sizeof( short ) ); -#endif + setHelicitiesAndFlavors( &tHel[0][0], &tFlavors[0][0] ); // Enable SIGFPE traps for Floating Point Exceptions #ifdef MGONGPUCPP_DEBUG @@ -294,29 +92,12 @@ namespace mg5amcCpu // Create the normalized color matrix in device memory createNormalizedColorMatrix(); #endif - // Read physics parameters like masses and couplings from user configuration files (static: initialize once) - // Then copy them to CUDA constant memory (issue #39) or its C++ emulation in file-scope static memory + // Read physics parameters like masses and couplings from user configuration files, + // then hand them to the backend-owned storage via the setters (see SigmaKin.cc). %(cipdassign)s %(cipcassign)s %(cipfassign)s -#ifdef MGONGPUCPP_GPUIMPL - %(cipd2tipdSym)s - %(cipc2tipcSym)s - %(cipf2tipfSym)s -#ifdef MGONGPUCPP_NBSMINDEPPARAM_GT_0 - if( Parameters::nBsmIndepParam > 0 ) - gpuMemcpyToSymbol( bsmIndepParam, m_pars->mdl_bsmIndepParam, Parameters::nBsmIndepParam * sizeof( double ) ); -#endif -#else - %(cipd2tipd)s - %(cipc2tipc)s - %(cipf2tipf)s -#ifdef MGONGPUCPP_NBSMINDEPPARAM_GT_0 - if( Parameters::nBsmIndepParam > 0 ) - memcpy( bsmIndepParam, m_pars->mdl_bsmIndepParam, Parameters::nBsmIndepParam * sizeof( double ) ); -#endif -#endif%(cipddump)s%(cipcdump)s%(cipfdump)s - //for ( int i=0; imdl_bsmIndepParam[i] = " << m_pars->mdl_bsmIndepParam[i] << std::endl; + %(bsmassign)s } #else // Initialize process (with hardcoded parameters) @@ -336,6 +117,10 @@ namespace mg5amcCpu // Create the normalized color matrix in device memory createNormalizedColorMatrix(); #endif + %(cipdhrdassign)s + %(cipchrdassign)s + %(cipfhrdassign)s + %(bsmhrdassign)s } #endif @@ -388,25 +173,6 @@ namespace mg5amcCpu out << "Apple clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; #else out << "clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; - /* - // === AV 26-Jan-2024 DISABLE THIS CODE (START) - // === AV 26-Jan-2024 First, it is totally wrong to assume that the CXX environment variable is used in the build! - // === AV 26-Jan-2024 Second and worse, here we need build time values, while CXX in this code is evaluated at runtime! - // GCC toolchain version inside CLANG - std::string tchainout; - std::string tchaincmd = "readelf -p .comment $(${CXX} -print-libgcc-file-name) |& grep 'GCC: (GNU)' | grep -v Warning | sort -u | awk '{print $5}'"; - std::unique_ptr tchainpipe( popen( tchaincmd.c_str(), "r" ), pclose ); - if( !tchainpipe ) throw std::runtime_error( "`readelf ...` failed?" ); - std::array tchainbuf; - while( fgets( tchainbuf.data(), tchainbuf.size(), tchainpipe.get() ) != nullptr ) tchainout += tchainbuf.data(); - tchainout.pop_back(); // remove trailing newline -#if defined __NVCC__ or defined __INTEL_LLVM_COMPILER - out << ", gcc " << tchainout; -#else - out << " (gcc " << tchainout << ")"; -#endif - // === AV 26-Jan-2024 DISABLE THIS CODE (END) - */ #endif #else out << "clang UNKNOWKN"; @@ -426,500 +192,4 @@ namespace mg5amcCpu } //-------------------------------------------------------------------------- - - __global__ void /* clang-format off */ - computeDependentCouplings( const fptype* allgs, // input: Gs[nevt] - fptype* allcouplings // output: couplings[nevt*ndcoup*2] -#ifndef MGONGPUCPP_GPUIMPL - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#endif - ) /* clang-format on */ - { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; - using G_ACCESS = DeviceAccessGs; - using C_ACCESS = DeviceAccessCouplings; - G2COUP( allgs, allcouplings, bsmIndepParam ); -#else - using namespace mg5amcCpu; - using G_ACCESS = HostAccessGs; - using C_ACCESS = HostAccessCouplings; - for( int ipagV = 0; ipagV < nevt / neppV; ++ipagV ) - { - const int ievt0 = ipagV * neppV; - const fptype* gs = MemoryAccessGs::ieventAccessRecordConst( allgs, ievt0 ); - fptype* couplings = MemoryAccessCouplings::ieventAccessRecord( allcouplings, ievt0 ); - G2COUP( gs, couplings, bsmIndepParam ); - } -#endif - } - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - void /* clang-format off */ - sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: indices of the flavor combinations - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - fptype* allNumerators, // output: multichannel numerators[nevt], running_sum_over_helicities - fptype* allDenominators, // output: multichannel denominators[nevt], running_sum_over_helicities - fptype_sv* allJamps, // tmp: jamp[ncolor*2*nevt] _for one helicity_ (reused in the getGoodHel helicity loop) - bool* isGoodHel, // output: isGoodHel[ncomb] - host array - const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) - { /* clang-format on */ - const int maxtry0 = 16; - fptype hstMEs[maxtry0]; - const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt16, e.g. in future VPUs) - constexpr int maxtry0 = std::max( 16, neppV ); // 16, but at least neppV (otherwise the npagV loop does not even start) - // Loop over only nevt events if nevt is < 16 (note that nevt is always >= neppV) - assert( nevt >= neppV ); - const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt 1) - { - for (int i = comp_beg[icomp] - 1; i < comp_end[icomp]; i++) - { - if (pid_work[i] == 0) - continue; - int n_tot = 1; - for (int j = i + 1; j < comp_end[icomp]; j++) - { - if (pid_work[i] != pid_work[j]) - continue; - bool same_block = (block_len[i] == block_len[j]); - for (int k = 0; same_block && k < block_len[i]; k++) - { - if (cFlavors[iflavor][block_start[i] - 1 + k] != cFlavors[iflavor][block_start[j] - 1 + k]) - same_block = false; - } - if (same_block) - { - pid_work[j] = 0; - n_tot = n_tot + 1; - old_factor = old_factor / n_tot; - } - } - } - } - total_factor = total_factor * old_factor; - } - return total_factor; - } - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - __global__ void - normalise_output( fptype* allMEs, // output: allMEs[nevt], |M|^2 running_sum_over_helicities - const unsigned int* iflavorVec, - fptype* ghelAllNumerators, // input/tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - fptype* ghelAllDenominators, // input/tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable SDE enhancement (fix #899/#911) - bool storeChannelWeights, // if true, compute final multichannel weights - bool mulChannelWeight, // if true, multiply matrix element by channel weight - const fptype globaldenom) /* clang-format on */ - { - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) - allMEs[ievt] = allMEs[ievt] * broken_symmetry_factor(iflavorVec[ievt]) / globaldenom; - const int nevt = gridDim.x * blockDim.x; - if( storeChannelWeights ) // fix segfault #892 (not 'channelIds[0] != 0') - { - fptype* totAllNumerators = ghelAllNumerators; // reuse "helicity #0" buffer to compute the total over all helicities - fptype* totAllDenominators = ghelAllDenominators; // reuse "helicity #0" buffer to compute the total over all helicities - for( int ighel = 1; ighel < dcNGoodHel; ighel++ ) // NB: the loop starts at ighel=1 - { - fptype* hAllDenominators = ghelAllDenominators + ighel * nevt; - totAllDenominators[ievt] += hAllDenominators[ievt]; - fptype* hAllNumerators = ghelAllNumerators + ( ievt + ighel * nevt ) * processConfig::ndiagrams; - fptype* firstNumerator = ghelAllNumerators + ievt * processConfig::ndiagrams; - for( int idiag = 0; idiag < processConfig::ndiagrams; ++idiag ) - { - firstNumerator[idiag] += hAllNumerators[idiag]; - } - } - if( mulChannelWeight ) - { - unsigned int channelId = allChannelIds[ievt]; - allMEs[ievt] *= totAllNumerators[channelId - 1 + ievt * processConfig::ndiagrams] / totAllDenominators[ievt]; - } - } - return; - } -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - add_and_select_hel( int* allselhel, // output: helicity selection[nevt] - const fptype* allrndhel, // input: random numbers[nevt] for helicity selection - fptype* ghelAllMEs, // input/tmp: allMEs for nGoodHel <= ncomb individual/runningsum helicities (index is ighel) - fptype* allMEs, // output: allMEs[nevt], final sum over helicities - const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) - { - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) - // Compute the sum of MEs over all good helicities (defer this after the helicity loop to avoid breaking streams parall> - for( int ighel = 0; ighel < dcNGoodHel; ighel++ ) - { - allMEs[ievt] += ghelAllMEs[ighel * nevt + ievt]; - ghelAllMEs[ighel * nevt + ievt] = allMEs[ievt]; // reuse the buffer to store the running sum for helicity selection - } - // Event-by-event random choice of helicity #403 - //printf( "select_hel: ievt=%%4d rndhel=%%f\n", ievt, allrndhel[ievt] ); - for( int ighel = 0; ighel < dcNGoodHel; ighel++ ) - { - if( allrndhel[ievt] < ( ghelAllMEs[ighel * nevt + ievt] / allMEs[ievt] ) ) - { - const int ihelF = dcGoodHel[ighel] + 1; // NB Fortran [1,ncomb], cudacpp [0,ncomb-1] - allselhel[ievt] = ihelF; - //printf( "select_hel: ievt=%%4d ihel=%%4d\n", ievt, ihelF ); - break; - } - } - return; - } -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - select_col_and_diag( int* allselcol, // output: color selection[nevt] - unsigned int* allDiagramIdsOut, // output: sampled diagram ids - const fptype* allrndcol, // input: random numbers[nevt] for color selection - const fptype* allrnddiagram, // input: random numbers[nevt] for diagram selection - const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable SDE enhancement (fix #899/#911) - const fptype_sv* allJamp2s, // input: jamp2[ncolor][nevt] for color choice (nullptr if disabled) - const fptype* allNumerators, // input: all numerators - const fptype* allDenominators, // input: all denominators - const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) - { - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) - // SCALAR channelId for the current event (CUDA) - unsigned int channelId = gpu_channelId( allChannelIds ); - // Event-by-event random choice of color #402 - - // Event-by-event random choice of channel - if( allrnddiagram != nullptr ) - { - fptype numerator_sum = 0., normalization = 0.; - for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) - { - if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - normalization += allNumerators[ievt * processConfig::ndiagrams + ichan]; - } - channelId = mgOnGpu::nchannels; - for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) - { - if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - numerator_sum += allNumerators[ievt * processConfig::ndiagrams + ichan]; - if( allrnddiagram[ievt] < numerator_sum / normalization ) - { - channelId = ichan + 1; - break; - } - } - allDiagramIdsOut[ievt] = channelId; - } - - if( channelId != 0 ) // no event-by-event choice of color if channelId == 0 (fix FPE #783) - { - if( channelId > mgOnGpu::nchannels ) - { - printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%%d which is greater than nchannels=%%d\n", channelId, mgOnGpu::nchannels ); - assert( channelId <= mgOnGpu::nchannels ); // SANITY CHECK #919 #910 - } - // Determine the jamp2 for this event (TEMPORARY? could do this with a dedicated memory accessor instead...) - fptype_sv jamp2_sv[ncolor] = { 0 }; - assert( allJamp2s != nullptr ); // sanity check - using J2_ACCESS = DeviceAccessJamp2; - for( int icolC = 0; icolC < ncolor; icolC++ ) - jamp2_sv[icolC] = J2_ACCESS::kernelAccessIcolConst( allJamp2s, icolC ); - // NB (see #877): in the array channel2iconfig, the input index uses C indexing (channelId -1), the output index uses F indexing (iconfig) - // NB (see #917): mgOnGpu::channel2iconfig returns an int (which may be -1), not an unsigned int! - const int iconfig = mgOnGpu::channel2iconfig[channelId - 1]; // map N_diagrams to N_config <= N_diagrams configs (fix LHE color mismatch #856: see also #826, #852, #853) - if( iconfig <= 0 ) - { - printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%%d which has no associated SDE iconfig\n", channelId ); - assert( iconfig > 0 ); // SANITY CHECK #917 - } - else if( iconfig > (int)mgOnGpu::nconfigSDE ) - { - printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%%d (invalid SDE iconfig=%%d\n > nconfig=%%d)", channelId, iconfig, mgOnGpu::nconfigSDE ); - assert( iconfig <= (int)mgOnGpu::nconfigSDE ); // SANITY CHECK #917 - } - fptype targetamp[ncolor] = { 0 }; - // NB (see #877): explicitly use 'icolC' rather than 'icol' to indicate that icolC uses C indexing in [0, N_colors-1] - for( int icolC = 0; icolC < ncolor; icolC++ ) - { - if( icolC == 0 ) - targetamp[icolC] = 0; - else - targetamp[icolC] = targetamp[icolC - 1]; - // NB (see #877): in the array icolamp, the input index uses C indexing (iconfig -1) - if( mgOnGpu::icolamp[iconfig - 1][icolC] ) targetamp[icolC] += jamp2_sv[icolC]; - } - //printf( "sigmaKin: ievt=%%4d rndcol=%%f\n", ievt, allrndcol[ievt] ); - for( int icolC = 0; icolC < ncolor; icolC++ ) - { - if( allrndcol[ievt] < ( targetamp[icolC] / targetamp[ncolor - 1] ) ) - { - allselcol[ievt] = icolC + 1; // NB Fortran [1,ncolor], cudacpp [0,ncolor-1] - //printf( "sigmaKin: ievt=%%d icol=%%d\n", ievt, icolC+1 ); - break; - } - } - } - else - { - allselcol[ievt] = 0; // no color selected in Fortran range [1,ncolor] if channelId == 0 (see #931) - } - return; - } -#endif - - //-------------------------------------------------------------------------- - // Evaluate |M|^2, part independent of incoming flavour - - void /* clang-format off */ - sigmaKin( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: indices of the flavor combinations - const fptype* allrndhel, // input: random numbers[nevt] for helicity selection - const fptype* allrndcol, // input: random numbers[nevt] for color selection - const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable single-diagram enhancement (fix #899/#911) - const fptype* allrnddiagram, // input: random numbers[nevt] for diagram sampling - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - int* allselhel, // output: helicity selection[nevt] -#ifdef MGONGPUCPP_GPUIMPL - int* allselcol, // output: helicity selection[nevt] - fptype* colAllJamp2s, // tmp: allJamp2s super-buffer for ncolor individual colors, running sum over colors and helicities - fptype* ghelAllNumerators, // tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - fptype* ghelAllDenominators, // tmp: allDenominators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - unsigned int* allDiagramIdsOut, // output: multichannel channelIds[nevt] (1 to #diagrams) - bool mulChannelWeight, // if true, multiply channel weight to ME output - fptype* ghelAllMEs, // tmp: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - fptype* ghelAllJamps, // tmp: jamp[2*ncolor*nGoodHel*nevt] super-buffer for nGoodHel <= ncomb individual helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) - const bool async, - const int gpublocks, // input: cuda gpublocks - const int gputhreads // input: cuda gputhreads -#else - int* allselcol, // output: helicity selection[nevt] - fptype* allNumerators, // output: multichannel numerators[nevt], running_sum_over_helicities - fptype* allDenominators, // output: multichannel denominators[nevt], running_sum_over_helicities - unsigned int* allDiagramIdsOut, // output: multichannel channelIds[nevt] (1 to #diagrams) - bool mulChannelWeight, // if true, multiply channel weight to ME output - const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#endif - ) /* clang-format on */ - { - mgDebugInitialise(); - - // SANITY CHECKS for cudacpp code generation (see issues #272 and #343 and PRs #619, #626, #360, #396 and #754) - // These variable are not used anywhere else in the code and their scope is limited to this sanity check - { - // nprocesses == 2 may happen for "mirror processes" such as P0_uux_ttx within pp_tt012j (see PR #754) - constexpr int nprocesses = %(nproc)i; - static_assert( nprocesses == 1 || nprocesses == 2, "Assume nprocesses == 1 or 2" ); - constexpr int process_id = %(proc_id)i; // code generation source: %(proc_id_source)s - static_assert( process_id == 1, "Assume process_id == 1" ); - } - - // Denominators: spins, colors and identical particles - constexpr int helcolDenominators[1] = { %(den_factors)s }; // assume nprocesses == 1 (#272 and #343) - -#ifndef MGONGPUCPP_GPUIMPL - //assert( (size_t)(allmomenta) %% mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment [COMMENT OUT TO TEST MISALIGNED ACCESS] - //assert( (size_t)(allMEs) %% mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment [COMMENT OUT TO TEST MISALIGNED ACCESS] - using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events - using NUM_ACCESS = HostAccessNumerators; // non-trivial access: buffer includes all events - using DEN_ACCESS = HostAccessDenominators; // non-trivial access: buffer includes all events -#endif - - // Start sigmaKin_lines -%(sigmaKin_lines)s } // end namespace diff --git a/madgraph/iolibs/template_files/madmatrix/process_h.inc b/madgraph/iolibs/template_files/madmatrix/process_h.inc index 61b47c9848..1396a8fa61 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_h.inc @@ -21,6 +21,8 @@ #include "GpuAbstraction.h" #include "Parameters.h" #include "ProcessData.h" +#include "SigmaKin.h" // for the setters CPPProcess's constructor/initProc call +%(helamps_h)s #include @@ -34,94 +36,8 @@ namespace mg5amcCpu { %(process_class_definitions)s //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - computeDependentCouplings( const fptype* allgs, // input: Gs[nevt] - fptype* allcouplings ); // output: couplings[nevt*ndcoup*2] -#else - __global__ void - computeDependentCouplings( const fptype* allgs, // input: Gs[nevt] - fptype* allcouplings, // output: couplings[nevt*ndcoup*2] - const int nevt ); // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - void - sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: index of the flavor combination - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - fptype* allNumerators, // output: multichannel numerators[nevt], running_sum_over_helicities - fptype* allDenominators, // output: multichannel denominators[nevt], running_sum_over_helicities - fptype_sv* allJamps, // output: jamp[ncolor*2*nevt] - bool* isGoodHel, // output: isGoodHel[ncomb] - device array (GPU device implementation) - const int nevt ); // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#else - void - sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: index of the flavor combination - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - fptype* allNumerators, // output: multichannel numerators[nevt], running_sum_over_helicities - fptype* allDenominators, // output: multichannel denominators[nevt], running_sum_over_helicities - bool* isGoodHel, // output: isGoodHel[ncomb] - host array (C++ implementation) - const int nevt ); // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#endif /* clang-format on */ - - //-------------------------------------------------------------------------- - - int // output: nGoodHel (the number of good helicity combinations out of ncomb) - sigmaKin_setGoodHel( const bool* isGoodHel ); // input: isGoodHel[ncomb] - host array - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - void - sigmaKin( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: index of the flavor combination - const fptype* allrndhel, // input: random numbers[nevt] for helicity selection - const fptype* allrndcol, // input: random numbers[nevt] for color selection - const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable single-diagram enhancement (fix #899/#911) - const fptype* allrnddiagram, // input: random numbers[nevt] for channel sampling - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - int* allselhel, // output: helicity selection[nevt] - int* allselcol, // output: helicity selection[nevt] - fptype* colAllJamp2s, // tmp: allJamp2s super-buffer for ncolor individual colors, running sum over colors and helicities - fptype* ghelAllNumerators, // tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - fptype* ghelAllDenominators, // tmp: allDenominators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - unsigned int* allDiagramIdsOut, // output: multichannel channelIds[nevt] (1 to #diagrams) - bool mulChannelWeight, // if true, multiply channel weight to ME output - fptype* ghelAllMEs, // tmp: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - fptype* ghelAllJamps, // tmp: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) - const bool async, // input: if true, run everything asynchronously in first stream in ghelStreams - const int gpublocks, // input: cuda gpublocks - const int gputhreads ); // input: cuda gputhreads -#else - void - sigmaKin( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: index of the flavor combination - const fptype* allrndhel, // input: random numbers[nevt] for helicity selection - const fptype* allrndcol, // input: random numbers[nevt] for color selection - const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable single-diagram enhancement (fix #899) - const fptype* allrnddiagram, // input: random numbers[nevt] for channel sampling - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - int* allselhel, // output: helicity selection[nevt] - int* allselcol, // output: helicity selection[nevt] - fptype* allNumerators, // tmp: multichannel numerators[nevt], running_sum_over_helicities - fptype* allDenominators, // tmp: multichannel denominators[nevt], running_sum_over_helicities - unsigned int* allDiagramIdsOut, // output: multichannel channelIds[nevt] (1 to #diagrams) - bool mulChannelWeight, // if true, multiply channel weight to ME output - const int nevt ); // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#endif /* clang-format on */ - + // sigmaKin, computeDependentCouplings and friends are backend-owned: + // see backend/{cpu,simd,gpu}/SigmaKin.h for their declarations. //-------------------------------------------------------------------------- } diff --git a/madmatrix/model_handling.py b/madmatrix/model_handling.py index d7edb941db..4a8e45a916 100644 --- a/madmatrix/model_handling.py +++ b/madmatrix/model_handling.py @@ -1530,15 +1530,17 @@ def get_process_class_definitions(self, write=True): return replace_dict # AV - replace export_cpp.OneProcessExporterCPP method (fix CPPProcess.cc) + # backend_separation: cIPD/cIPC/cIPF/bsmIndepParam storage now lives in + # backend/{cpu,simd,gpu}/SigmaKin.cc. This method still computes the local + # tIPD/tIPC/tIPF assignment text (genuinely process-specific: which SM + # parameters/couplings this process uses), but ends each with a call to + # the corresponding backend setter instead of a storage-declaration + # variant + direct memcpy/gpuMemcpyToSymbol. def get_process_function_definitions(self, write=True): """The complete class definition for the process""" replace_dict = super().get_process_function_definitions(write=False) # defines replace_dict['initProc_lines'] replace_dict['hardcoded_initProc_lines'] = replace_dict['initProc_lines'].replace( 'm_pars->', 'Parameters::') - couplings2order_indep = [] - ###replace_dict['ncouplings'] = len(self.couplings2order) - ###replace_dict['ncouplingstimes2'] = 2 * replace_dict['ncouplings'] replace_dict['nparams'] = len(self.params2order) - ###replace_dict['nmodels'] = replace_dict['nparams'] + replace_dict['ncouplings'] # AV unused??? replace_dict['coupling_list'] = ' ' replace_dict['hel_amps_cc'] = '#include \"HelAmps_%s.cc\"' % self.model_name # AV coupling = [''] * len(self.couplings2order) @@ -1556,59 +1558,44 @@ def get_process_function_definitions(self, write=True): if "aS" in key and coup in coup_list: keep = False if keep: coupling_indep.append( coup ) # AV only indep! replace_dict['ncouplings'] = len(coupling_indep) # AV only indep! - replace_dict['nipc'] = len(coupling_indep) + + # dependent (running-alphas, event-by-event) flavor couplings -> for ProcessTables.h (Step 3). + flv_couplings_dep = [''] * len(self.couporderflv_dep) + for flv_coup, pos in self.couporderflv_dep.items(): + flv_couplings_dep[pos] = flv_coup + + # Cache counts for edit_processdata()/edit_processtables(), which run + # after generate_process_files() has populated couplings2order etc. + self._nipc = len(coupling_indep) + self._nipd = len(params) + self._nipf = len(flv_couplings) + self._ndpf = len(flv_couplings_dep) + if len(coupling_indep) > 0: - replace_dict['cipcassign'] = 'const cxtype tIPC[nIPC] = { cxmake( m_pars->%s ) };'\ + replace_dict['cipcassign'] = 'const cxtype tIPC[nIPC] = { cxmake( m_pars->%s ) };\n setIndependentCouplings( tIPC );'\ % ( ' ), cxmake( m_pars->'.join(coupling_indep) ) # AV only indep! - replace_dict['cipcdevice'] = '__device__ __constant__ fptype cIPC[nIPC * 2];' - replace_dict['cipcstatic'] = 'static fptype cIPC[nIPC * 2];' - replace_dict['cipc2tipcSym'] = 'gpuMemcpyToSymbol( cIPC, tIPC, nIPC * sizeof( cxtype ) );' - replace_dict['cipc2tipc'] = 'memcpy( cIPC, tIPC, nIPC * sizeof( cxtype ) );' - replace_dict['cipcdump'] = '\n //for ( int i=0; i'.join(flv_couplings) ) - replace_dict['cipfdevice'] = """__device__ __constant__ int cIPF_partner1[nMF * nIPF]; - __device__ __constant__ int cIPF_partner2[nMF * nIPF]; - __device__ __constant__ fptype cIPF_value[nMF * nIPF * 2];""" - replace_dict['cipfstatic'] = """static int cIPF_partner1[nMF * nIPF]; - static int cIPF_partner2[nMF * nIPF]; - static fptype cIPF_value[nMF * nIPF * 2];""" - replace_dict['cipf2tipfSym'] = """gpuMemcpyToSymbol( cIPF_partner1, tIPF_partner1, nMF * nIPF * sizeof( int ) ); - gpuMemcpyToSymbol( cIPF_partner2, tIPF_partner2, nMF * nIPF * sizeof( int ) ); - gpuMemcpyToSymbol( cIPF_value , tIPF_value , nMF * nIPF * sizeof( cxtype ) );""" - replace_dict['cipf2tipf'] = """memcpy( cIPF_partner1, tIPF_partner1, nMF * nIPF * sizeof( int ) ); - memcpy( cIPF_partner2, tIPF_partner2, nMF * nIPF * sizeof( int ) ); - memcpy( cIPF_value , tIPF_value , nMF * nIPF * sizeof( cxtype ) );""" - replace_dict['cipfdump'] = ''' - //for ( int i=0; i < nIPD; i++ ) { - // std::cout << std::setprecision(17) << "tIPF[i].partner1 = { "; - // for ( int j=0; j < nMF-1; j++ ) std::cout << std::setprecision(17) << tIPF[i].partner1[j] << ", "; - // std::cout << std::setprecision(17) << tIPF[i].partner1[nMF-1] << " }" << std::endl; - // std::cout << std::setprecision(17) << "tIPF[i].partner2 = { "; - // for ( int j=0; j < nMF-1; j++ ) std::cout << std::setprecision(17) << tIPF[i].partner2[j] << ", "; - // std::cout << std::setprecision(17) << tIPF[i].partner2[nMF-1] << " }" << std::endl; - // std::cout << std::setprecision(17) << "tIPF[i].value = { "; - // for ( int j=0; j < nMF-1; j++ ) std::cout << std::setprecision(17) << tIPF[i].value[j] << ", "; - // std::cout << std::setprecision(17) << tIPF[i].value[nMF-1] << " }" << std::endl; - //} -''' - coup_str_hrd_partner1 = '__device__ const int cIPF_partner1[nMF * nIPF] = { ' - coup_str_hrd_partner2 = '__device__ const int cIPF_partner2[nMF * nIPF] = { ' - coup_str_hrd_value = '__device__ const fptype cIPF_value[nMF * nIPF * 2] = { ' - for flv_coup in flv_couplings: - coup_str_hrd_partner1 += ( ('Parameters_%(model_name)s::%(coup)s.param1' % {"model_name": self.model_name, "coup": flv_coup} + '[%d], ') * nMF) % ( *range(nMF), ) - coup_str_hrd_partner2 += ( ('Parameters_%(model_name)s::%(coup)s.param2' % {"model_name": self.model_name, "coup": flv_coup} + '[%d], ') * nMF) % ( *range(nMF), ) - # Guard against null value[] slots: flavor combinations with no - # coupling are left null by the FLV_COUPLING constructor, so the - # hardcoded cIPF_value read must not dereference an uninitialised - # pointer. Mirrors the runtime path (value[j] ? *value[j] : 0). - value_base = 'Parameters_%(model_name)s::%(coup)s.value' % {"model_name": self.model_name, "coup": flv_coup} - for i in range(nMF): - coup_str_hrd_value += '(fptype)( %(b)s[%(i)d] ? %(b)s[%(i)d]->real() : 0. ), ' % {'b': value_base, 'i': i} - coup_str_hrd_value += '(fptype)( %(b)s[%(i)d] ? %(b)s[%(i)d]->imag() : 0. ), ' % {'b': value_base, 'i': i} - coup_str_hrd_partner1 = coup_str_hrd_partner1[:-2] + ' };' - coup_str_hrd_partner2 = coup_str_hrd_partner2[:-2] + ' };' - coup_str_hrd_value = coup_str_hrd_value[:-2] + ' };' - replace_dict['cipfhrdcod'] = '%s\n %s\n %s' % (coup_str_hrd_partner1, coup_str_hrd_partner2, coup_str_hrd_value) + } + setFlavorCouplings( tIPF_partner1, tIPF_partner2, tIPF_value );""" % ( ', m_pars->'.join(flv_couplings) ) + # Hardcoded variant: same shape, values come from Parameters:: instead of m_pars-> + hrd_lines = ['int tIPF_partner1[nMF * nIPF];', ' int tIPF_partner2[nMF * nIPF];', ' cxtype tIPF_value[nMF * nIPF];'] + for i, flv_coup in enumerate(flv_couplings): + base = 'Parameters_%s::%s' % (self.model_name, flv_coup) + for j in range(nMF): + hrd_lines.append(' tIPF_partner1[%d] = %s.param1[%d];' % (i * nMF + j, base, j)) + hrd_lines.append(' tIPF_partner2[%d] = %s.param2[%d];' % (i * nMF + j, base, j)) + hrd_lines.append(' tIPF_value[%d] = %s.value[%d] ? *%s.value[%d] : cxtype{};' % (i * nMF + j, base, j, base, j)) + hrd_lines.append(' setFlavorCouplings( tIPF_partner1, tIPF_partner2, tIPF_value );') + replace_dict['cipfhrdassign'] = '\n '.join(hrd_lines) else: replace_dict['cipfassign'] = '' - replace_dict['cipfdevice'] = """__device__ __constant__ int* cIPF_partner1 = nullptr; // unused as nIPF=0' - __device__ __constant__ int* cIPF_partner2 = nullptr; // unused as nIPF=0' - __device__ __constant__ fptype* cIPF_value = nullptr; // unused as nIPF=0'""" - replace_dict['cipfstatic'] = """static int* cIPF_partner1 = nullptr; // unused as nIPF=0' - static int* cIPF_partner2 = nullptr; // unused as nIPF=0' - static fptype* cIPF_value = nullptr; // unused as nIPF=0'""" - replace_dict['cipf2tipfSym'] = '' - replace_dict['cipf2tipf'] = '' - replace_dict['cipfdump'] = '' - replace_dict['cipfhrdcod'] = """__device__ const int* cIPF_partner1 = nullptr; // unused as nIPF=0' - __device__ const int* cIPF_partner2 = nullptr; // unused as nIPF=0' - __device__ const fptype* cIPF_value = nullptr; // unused as nIPF=0'""" - - # dependent (running-alphas, event-by-event) flavor couplings -> cDPF_* (Step 3). - # Unlike cIPF, these have NO baked-in value array: partner1/partner2 and the - # per-flavor idcoup (the index of the underlying dependent coupling in the - # event-by-event allcouplings buffer) are pure codegen constants. The actual - # complex values are gathered per event page in calculate_jamps (see - # super_get_matrix_element_calls). The single-leg serialization mirrors the - # Fortran side / write_flv_couplings (the unmerged partner has flavor index 1). - flv_couplings_dep = [''] * len(self.couporderflv_dep) - for flv_coup, pos in self.couporderflv_dep.items(): - flv_couplings_dep[pos] = flv_coup - replace_dict['ndpf'] = len(flv_couplings_dep) - if len(flv_couplings_dep): - nMF = max(len(ids) for ids in self.model['merged_particles'].values()) - flv_map = self.helas_call_writer.flv_couplings_map - partner1_vals, partner2_vals, idcoup_vals = [], [], [] - for name in flv_couplings_dep: - coupl = flv_map[name] - p1 = [-1] * nMF - p2 = [-1] * nMF - idc = ['-1'] * nMF - for key, gc in coupl.flavors.items(): - nonzero = [i for i in key if i != 0] - if len(nonzero) == 2: - k1, k2 = nonzero - else: - # single merged leg: unmerged partner has flavor index 1 - k1 = nonzero[0]; k2 = 1 - p1[k1-1] = k2-1 - p2[k2-1] = k1-1 - # symbolic idcoup: resolves to the position of this dependent coupling - # in the event-by-event allcouplings buffer (== COUPs index), defined in - # Parameters_dependentCouplings (Parameters_.h) - idc[k1-1] = '(int)Parameters_dependentCouplings::idcoup_%s' % gc - partner1_vals += [str(v) for v in p1] - partner2_vals += [str(v) for v in p2] - idcoup_vals += idc - cdpfdecl = '__device__ const int cDPF_partner1[nMF * nDPF] = { %s };\n' % ', '.join(partner1_vals) - cdpfdecl += ' __device__ const int cDPF_partner2[nMF * nDPF] = { %s };\n' % ', '.join(partner2_vals) - cdpfdecl += ' __device__ const int cDPF_idcoup[nMF * nDPF] = { %s };' % ', '.join(idcoup_vals) - replace_dict['cdpfdecl'] = cdpfdecl - else: - replace_dict['cdpfdecl'] = """__device__ const int* cDPF_partner1 = nullptr; // unused as nDPF=0 - __device__ const int* cDPF_partner2 = nullptr; // unused as nDPF=0 - __device__ const int* cDPF_idcoup = nullptr; // unused as nDPF=0""" + replace_dict['cipfhrdassign'] = '' + + # mdl_bsmIndepParam only exists as a symbol at all when the model has + # BSM params (see MGONGPUCPP_NBSMINDEPPARAM_GT_0, PR #625) - this is a + # pre-existing macro, not new, and must stay a #ifdef (not a runtime + # check) since the symbol itself may not exist to name-lookup. + replace_dict['bsmassign'] = '''#ifdef MGONGPUCPP_NBSMINDEPPARAM_GT_0 + if( Parameters::nBsmIndepParam > 0 ) setBsmIndepParam( m_pars->mdl_bsmIndepParam, Parameters::nBsmIndepParam ); +#endif''' + replace_dict['bsmhrdassign'] = '''#ifdef MGONGPUCPP_NBSMINDEPPARAM_GT_0 + if( Parameters::nBsmIndepParam > 0 ) setBsmIndepParam( Parameters::mdl_bsmIndepParam, Parameters::nBsmIndepParam ); +#endif''' - # FIXME! Here there should be different code generated depending on MGONGPUCPP_NBSMINDEPPARAM_GT_0 (issue #827) replace_dict['all_helicities'] = self.get_helicity_matrix(self.matrix_elements[0]) replace_dict['all_helicities'] = replace_dict['all_helicities'] .replace('helicities', 'tHel') replace_dict['all_flavors'] = self.get_flavor_matrix(self.matrix_elements[0]) replace_dict['all_flavors'] = replace_dict['all_flavors'].replace('flavors', 'tFlavors') - color_amplitudes = [me.get_color_amplitudes() for me in self.matrix_elements] # as in OneProcessExporterCPP.get_process_function_definitions - replace_dict['ncolor'] = len(color_amplitudes[0]) - # broken_symmetry_factor function: use the shared decay-aware symmetry - # data (same as the Fortran / standalone_cpp exporters) instead of the - # old simple PID-count version, so identical-particle and decay-chain - # symmetry factors match across backends. - _, nincoming = self.matrix_elements[0].get_nexternal_ninitial() - replace_dict['nincoming'] = nincoming - process = self.matrix_elements[0].get('processes')[0] - sym_data = export_v4.ProcessExporterFortran._get_broken_symmetry_data( - process, nincoming) - export_v4.ProcessExporterFortran._fill_broken_sym_replace_dict( - replace_dict, sym_data) file = self.read_template_file(self.process_definition_template) % replace_dict # HACK! ignore write=False case if len(params) == 0: # remove cIPD from OpenMP pragma (issue #349) @@ -1779,158 +1675,47 @@ def get_sigmaKin_lines(self, color_amplitudes, write=True): return replace_dict # AV - modify export_cpp.OneProcessExporterCPP method (fix CPPProcess.cc) + # backend_separation: calculate_jamps' prologue (signature, memory-access + # typedefs) and epilogue (color-choice bookkeeping, jamp output copy - was + # process_matrix.inc) are backend-conditional but process-independent, so + # they now live as real files in backend/{cpu,simd,gpu}/CalculateJamps.cc. + # Only the diagram/vertex-call sequence (helas_calls) is process-specific; + # it is written here to EvaluateDiagrams.inc, which that file #includes. def get_all_sigmaKin_lines(self, color_amplitudes, class_name): - """Get sigmaKin_process for all subprocesses for CPPProcess.cc""" - ret_lines = [] + """Write EvaluateDiagrams.inc for CPPProcess.cc""" if self.single_helicities: - ###misc.sprint(type(self.helas_call_writer)) - ###misc.sprint( 'before get_matrix_element_calls', self.matrix_elements[0].get_number_of_wavefunctions() ) # WRONG value of nwf, eg 7 for gg_tt helas_calls = self.helas_call_writer.get_matrix_element_calls(\ self.matrix_elements[0], color_amplitudes[0], multi_channel_map = self.multi_channel_map ) - ###misc.sprint( 'after get_matrix_element_calls', self.matrix_elements[0].get_number_of_wavefunctions() ) # CORRECT value of nwf, eg 5 for gg_tt assert len(self.matrix_elements) == 1 or len(self.matrix_elements) == 2 # how to handle if this is not true? self.couplings2order = self.helas_call_writer.couplings2order self.couporderflv = self.helas_call_writer.couporderflv self.couporderflv_dep = self.helas_call_writer.couporderflv_dep self.params2order = self.helas_call_writer.params2order - ret_lines.append(""" - // Evaluate QCD partial amplitudes jamps for this given helicity from Feynman diagrams - // Also compute running sums over helicities adding jamp2, numerator, denominator - // (NB: this function no longer handles matrix elements as the color sum has now been moved to a separate function/kernel) - // In CUDA, this function processes a single event - // ** NB1: NEW Nov2024! In CUDA this is now a kernel function (it used to be a device function) - // ** NB2: NEW Nov2024! in CUDA this now takes a channelId array as input (it used to take a scalar channelId as input) - // In C++, this function processes a single event "page" or SIMD vector (or for two in "mixed" precision mode, nParity=2) - // *** NB: in C++, calculate_jamps accepts a SCALAR channelId because it is GUARANTEED that all events in a SIMD vector have the same channelId #898 - __global__ void /* clang-format off */ - calculate_jamps( int ihel, - const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: indices of the flavor combinations -#ifdef MGONGPUCPP_GPUIMPL - fptype* allJamps, // output: jamp[2*ncolor*nevt] buffer for one helicity _within a super-buffer for dcNGoodHel helicities_ - bool storeChannelWeights, - fptype* allNumerators, // input/output: multichannel numerators[nevt], add helicity ihel - fptype* allDenominators, // input/output: multichannel denominators[nevt], add helicity ihel - fptype* colAllJamp2s, // output: allJamp2s[ncolor][nevt] super-buffer, sum over col/hel (nullptr to disable) - const int nevt, // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) - const bool processAllHelicities // input: if true, use blockIdx.y to index helicities -#else - cxtype_sv* allJamp_sv, // output: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for this helicity - bool storeChannelWeights, - fptype* allNumerators, // input/output: multichannel numerators[nevt], add helicity ihel - fptype* allDenominators, // input/output: multichannel denominators[nevt], add helicity ihel - fptype_sv* jamp2_sv, // output: jamp2[nParity][ncolor][neppV] for color choice (nullptr if disabled) - const int ievt00 // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) -#endif - ) - //ALWAYS_INLINE // attributes are not permitted in a function definition - { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; - using M_ACCESS = DeviceAccessMomenta; // non-trivial access: buffer includes all events - using W_ACCESS = DeviceAccessWavefunctions; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event - using A_ACCESS = DeviceAccessAmplitudes; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event - using CD_ACCESS = DeviceAccessCouplings; // non-trivial access (dependent couplings): buffer includes all events - using CI_ACCESS = DeviceAccessCouplingsFixed; // TRIVIAL access (independent couplings): buffer for one event - using F_ACCESS = DeviceAccessIflavorVec; // non-trivial access: buffer includes all events - using NUM_ACCESS = DeviceAccessNumerators; // non-trivial access: buffer includes all events - using DEN_ACCESS = DeviceAccessDenominators; // non-trivial access: buffer includes all events -#else - using namespace mg5amcCpu; - using M_ACCESS = HostAccessMomenta; // non-trivial access: buffer includes all events - using W_ACCESS = HostAccessWavefunctions; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event - using A_ACCESS = HostAccessAmplitudes; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event - using CD_ACCESS = HostAccessCouplings; // non-trivial access (dependent couplings): buffer includes all events - using CI_ACCESS = HostAccessCouplingsFixed; // TRIVIAL access (independent couplings): buffer for one event - using F_ACCESS = HostAccessIflavorVec; // non-trivial access: buffer includes all events - using NUM_ACCESS = HostAccessNumerators; // non-trivial access: buffer includes all events - using DEN_ACCESS = HostAccessDenominators; // non-trivial access: buffer includes all events -#endif - mgDebug( 0, __FUNCTION__ ); - //bool debug = true; -#ifndef MGONGPUCPP_GPUIMPL - //debug = ( ievt00 >= 64 && ievt00 < 80 && ihel == 3 ); // example: debug #831 - //if( debug ) printf( \"calculate_jamps: ievt00=%d ihel=%2d\\n\", ievt00, ihel ); -#else - //const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - //debug = ( ievt == 0 ); - //if( debug ) printf( \"calculate_jamps: ievt=%6d ihel=%2d\\n\", ievt, ihel ); - if (processAllHelicities) { - int ighel = blockIdx.y; - ihel = dcGoodHel[ighel]; - allJamps = allJamps + ighel * nevt; - allNumerators = allNumerators + ighel * nevt * processConfig::ndiagrams; - allDenominators = allDenominators + ighel * nevt; - } -#endif /* clang-format on */""") - nwavefuncs = self.matrix_elements[0].get_number_of_wavefunctions() - ret_lines.append(""" - // The variable nwf (which is specific to each P1 subdirectory, #644) is only used here - // It is hardcoded here because various attempts to hardcode it in CPPProcess.h at generation time gave the wrong result... - static const int nwf = %i; // #wavefunctions = #external (npar) + #internal: e.g. 5 for e+ e- -> mu+ mu- (1 internal is gamma or Z)"""%nwavefuncs ) - ret_lines.append(""" - // Local TEMPORARY variables for a subset of Feynman diagrams in the given CUDA event (ievt) or C++ event page (ipagV) - // [NB these variables are reused several times (and re-initialised each time) within the same event or event page] - // ** NB: in other words, amplitudes and wavefunctions still have TRIVIAL ACCESS: there is currently no need - // ** NB: to have large memory structurs for wavefunctions/amplitudes in all events (no kernel splitting yet)! - //MemoryBufferWavefunctions w_buffer[nwf]{ neppV }; - // Create memory for both momenta and wavefunctions separately, and later wrap them in ALOHAOBJ - fptype_sv pvec_sv[nwf][np4]; - cxtype_sv w_sv[nwf][nw6]; // particle wavefunctions within Feynman diagrams (nw6 is 4: spin wavefunctions, momenta are no more included, see before) - cxtype_sv amp_sv[1]; // invariant amplitude for one given Feynman diagram - - // Wrap the memory into ALOHAOBJ - ALOHAOBJ aloha_obj[nwf]; - for( int iwf = 0; iwf < nwf; iwf++ ) aloha_obj[iwf] = ALOHAOBJ{pvec_sv[iwf], w_sv[iwf]}; - fptype* amp_fp; - amp_fp = reinterpret_cast( amp_sv );""") + content = [] if fd_gauge: - ret_lines.append(""" + content.append(""" // special temporary ALOHAOBJ to hold F/Vtmp values in the combined vertex functions while using the FD gauge fptype_sv pvec_sv_tmp[1][np4]; - cxtype_sv w_sv_tmp[1][nw6]; + cxtype_sv w_sv_tmp[1][nw6]; ALOHAOBJ aloha_obj_tmp[1]; aloha_obj_tmp[0] = ALOHAOBJ{pvec_sv_tmp[0], w_sv_tmp[0]}; - + // special one value to hold tmp vertex value inside the combined vertex functions while using the FD gauge cxtype_sv amp_tmp_sv[1]; //to ensure proper aligment for vector instructions fptype* amp_tmp_fp; amp_tmp_fp = reinterpret_cast( amp_tmp_sv ); """) - ret_lines.append(""" - // Local variables for the given CUDA event (ievt) or C++ event page (ipagV) - // [jamp: sum (for one event or event page) of the invariant amplitudes for all Feynman diagrams in a given color combination] - cxtype_sv jamp_sv[ncolor] = {}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxtype is NOT, if "= {}" is missing!) - - // === Calculate wavefunctions and amplitudes for all diagrams in all processes === - // === (for one event in CUDA, for one - or two in mixed mode - SIMD event pages in C++ === - - // START LOOP ON IPARITY - for( int iParity = 0; iParity < nParity; ++iParity ) - { -#ifndef MGONGPUCPP_GPUIMPL - const int ievt0 = ievt00 + iParity * neppV; -#endif""") - ret_lines += helas_calls + content += helas_calls else: - ret_lines.extend([self.get_sigmaKin_single_process(i, me) \ - for i, me in enumerate(self.matrix_elements)]) - #ret_lines.extend([self.get_matrix_single_process(i, me, - # color_amplitudes[i], - # class_name) \ - # for i, me in enumerate(self.matrix_elements)]) - file_extend = [] - for i, me in enumerate(self.matrix_elements): - file = self.get_matrix_single_process( i, me, color_amplitudes[i], class_name ) - file = strip_banner(file, banner_mark = "!") # skip first 8 lines in process_matrix.inc (copyright) - file_extend.append( file ) - assert i == 0, "more than one ME in get_all_sigmaKin_lines" # AV sanity check (added for color_sum.cc but valid independently) - ret_lines.extend( file_extend ) - return '\n'.join(ret_lines) + content = [self.get_sigmaKin_single_process(i, me) \ + for i, me in enumerate(self.matrix_elements)] + ff = open(pjoin(self.path, 'EvaluateDiagrams.inc'), 'w') + ff.write('\n'.join(content)) + ff.close() + return '' # AV - modify export_cpp.OneProcessExporterCPP method (replace '# Process' by '// Process') def get_process_info_lines(self, matrix_element): @@ -1954,6 +1739,7 @@ def generate_process_files(self): super().generate_process_files() # needs to be after get_matrix_element_calls to have nwf ready self.edit_processdata() + self.edit_processtables() # NB: symlink of cudacpp.mk to makefile is overwritten by madevent makefile if this exists (#480) # NB: this relies on the assumption that cudacpp code is generated before madevent code files.ln(pjoin(self.path, "..", "makefile"), self.path, "makefile") @@ -1977,10 +1763,77 @@ def edit_processdata(self): replace_dict['proc_id'] = self.proc_id if self.proc_id > 0 else 1 den_factors = [str(m.get_denominator_factor()) for m in self.matrix_elements] replace_dict['den_factors'] = ",".join(den_factors) + # cached by get_process_function_definitions(), which runs earlier in + # super().generate_process_files() + replace_dict['nipd'] = self._nipd + replace_dict['nipc'] = self._nipc + replace_dict['nipf'] = self._nipf + replace_dict['ndpf'] = self._ndpf ff = open(pjoin(self.path, 'ProcessData.h'), 'w') ff.write(template % replace_dict) ff.close() + # backend_separation: process-specific compile-time DATA (arrays, not + # scalars) that backend-owned code needs but can't take as a runtime + # parameter without losing constexpr-ness (see ProcessTables.h). + def edit_processtables(self): + """Generate ProcessTables.h""" + template = open(pjoin(self.template_path, 'madmatrix', 'ProcessTables.h'), 'r').read() + replace_dict = {} + + # Dependent (event-by-event, running-alphas) flavor couplings: partner + # indices and the per-flavor idcoup are pure compile-time constants + # (the complex values are gathered per event page in calculate_jamps). + flv_couplings_dep = [''] * len(self.couporderflv_dep) + for flv_coup, pos in self.couporderflv_dep.items(): + flv_couplings_dep[pos] = flv_coup + if len(flv_couplings_dep): + nMF = max(len(ids) for ids in self.model['merged_particles'].values()) + flv_map = self.helas_call_writer.flv_couplings_map + partner1_vals, partner2_vals, idcoup_vals = [], [], [] + for name in flv_couplings_dep: + coupl = flv_map[name] + p1 = [-1] * nMF + p2 = [-1] * nMF + idc = ['-1'] * nMF + for key, gc in coupl.flavors.items(): + nonzero = [i for i in key if i != 0] + if len(nonzero) == 2: + k1, k2 = nonzero + else: + # single merged leg: unmerged partner has flavor index 1 + k1 = nonzero[0]; k2 = 1 + p1[k1-1] = k2-1 + p2[k2-1] = k1-1 + # symbolic idcoup: resolves to the position of this dependent coupling + # in the event-by-event allcouplings buffer (== COUPs index), defined in + # Parameters_dependentCouplings (Parameters_.h) + idc[k1-1] = '(int)Parameters_dependentCouplings::idcoup_%s' % gc + partner1_vals += [str(v) for v in p1] + partner2_vals += [str(v) for v in p2] + idcoup_vals += idc + cdpfdecl = '__device__ constexpr int cDPF_partner1[nMF * nDPF] = { %s };\n' % ', '.join(partner1_vals) + cdpfdecl += ' __device__ constexpr int cDPF_partner2[nMF * nDPF] = { %s };\n' % ', '.join(partner2_vals) + cdpfdecl += ' __device__ constexpr int cDPF_idcoup[nMF * nDPF] = { %s };' % ', '.join(idcoup_vals) + replace_dict['cdpfdecl'] = cdpfdecl + else: + replace_dict['cdpfdecl'] = """__device__ constexpr const int* cDPF_partner1 = nullptr; // unused as nDPF=0 + __device__ constexpr const int* cDPF_partner2 = nullptr; // unused as nDPF=0 + __device__ constexpr const int* cDPF_idcoup = nullptr; // unused as nDPF=0""" + + # broken_symmetry_factor data: same shared decay-aware symmetry data + # as the Fortran / standalone_cpp exporters. + _, nincoming = self.matrix_elements[0].get_nexternal_ninitial() + process = self.matrix_elements[0].get('processes')[0] + sym_data = export_v4.ProcessExporterFortran._get_broken_symmetry_data( + process, nincoming) + export_v4.ProcessExporterFortran._fill_broken_sym_replace_dict( + replace_dict, sym_data) + + ff = open(pjoin(self.path, 'ProcessTables.h'), 'w') + ff.write(template % replace_dict) + ff.close() + # AV - replace the export_cpp.OneProcessExporterCPP method (add debug printouts and multichannel handling #473) def edit_mgonGPU(self): """Generate mgOnGpuConfig.h""" From 2c3fcfe2db9d8d543a2d94f2bdcede16299186f6 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 18:15:30 +0200 Subject: [PATCH 05/25] BugFix not committed ProcessData and ProcessTables due to .gitignore --- .gitignore | 2 + .../template_files/madmatrix/ProcessData.h | 37 ++++++++++++++ .../template_files/madmatrix/ProcessTables.h | 51 +++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 madgraph/iolibs/template_files/madmatrix/ProcessData.h create mode 100644 madgraph/iolibs/template_files/madmatrix/ProcessTables.h diff --git a/.gitignore b/.gitignore index e601b64224..ccfce81acb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ __pycache__ *.mod *.a PROC* +!madgraph/iolibs/template_files/madmatrix/ProcessData.h +!madgraph/iolibs/template_files/madmatrix/ProcessTables.h cmd* ptest* *.pkl diff --git a/madgraph/iolibs/template_files/madmatrix/ProcessData.h b/madgraph/iolibs/template_files/madmatrix/ProcessData.h new file mode 100644 index 0000000000..be4bd937f0 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/ProcessData.h @@ -0,0 +1,37 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Process-specific compile-time constants, generated once per subprocess. +// This is the single source of truth for these values: CPPProcess (P1-generated) +// and the backend-owned driver code (backend/{cpu,simd,gpu}/) both read from here +// instead of duplicating the literals. + +#ifndef PROCESSDATA_H +#define PROCESSDATA_H 1 + +namespace ProcessData +{ + constexpr int np4 = 4; // dimensions of a 4-momentum (E,px,py,pz) + constexpr int nw6 = %(nwave)d; // dimensions of each wavefunction (HELAS KEK 91-11) + constexpr int npari = %(nincoming)d; + constexpr int nparf = %(noutcoming)d; + constexpr int npar = npari + nparf; + constexpr int ncomb = %(nbhel)d; // #helicity combinations + constexpr int ndiagrams = %(ndiagrams)d; + constexpr int ncolor = %(ncolor)d; + constexpr int nmaxflavor = %(nmaxflavor)d; + constexpr int nwf = %(nwf)d; // #wavefunctions = #external (npar) + #internal (see #644) + constexpr int nproc = %(nproc)d; // 2 if this process has a mirror process, else 1 + constexpr int proc_id = %(proc_id)d; + constexpr int helcolDenominators[1] = { %(den_factors)s }; // spin/color/identical-particle denominators + + // SM independent parameters/couplings/flavor-couplings used by this process + // (see #823: nIPC/nIPD/nIPF may vary per P1, unlike nicoup which is model-wide) + constexpr int nIPD = %(nipd)d; + constexpr int nIPC = %(nipc)d; + constexpr int nIPF = %(nipf)d; + constexpr int nDPF = %(ndpf)d; +} + +#endif // PROCESSDATA_H diff --git a/madgraph/iolibs/template_files/madmatrix/ProcessTables.h b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h new file mode 100644 index 0000000000..10378fb62a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h @@ -0,0 +1,51 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Process-specific compile-time data tables, generated once per subprocess, +// for backend-owned code (backend/{cpu,simd,gpu}/SigmaKin.cc) that can't take +// this data as a runtime parameter without losing constexpr-ness. Unlike +// ProcessData.h these are arrays, not scalars, and unlike ColorMatrixData.h +// there's no backend-conditional algorithm consuming them directly - it's +// pulled in via ProcessTables::name from backend-owned function bodies. +// +// Namespace-wrapped (unlike ProcessData.h) because it needs FLV_COUPLING, +// which is itself backend-namespaced (mg5amcCpu::/mg5amcGpu::, see Parameters.h). + +#ifndef PROCESSTABLES_H +#define PROCESSTABLES_H 1 + +#include "mgOnGpuConfig.h" // for __device__ +#include "ProcessData.h" +#include "Parameters.h" // for FLV_COUPLING::max_flavor + +#ifdef MGONGPUCPP_GPUIMPL +namespace mg5amcGpu +#else +namespace mg5amcCpu +#endif +{ + namespace ProcessTables + { + using ProcessData::nDPF; + constexpr int nMF = FLV_COUPLING::max_flavor; // max #merged flavors for any merged particle in the model + + // Dependent (event-by-event, running-alphas) flavor couplings: partner + // indices and the per-flavor idcoup are pure compile-time constants (the + // complex values are gathered per event page in calculate_jamps). +%(cdpfdecl)s + + // Decay-aware identical-particle (broken-)symmetry factor data, shared with + // the Fortran / standalone_cpp exporters (_get_broken_symmetry_data). + constexpr int broken_sym_ncomponents = %(broken_sym_ncomponents)d; + constexpr int broken_sym_nentries = %(broken_sym_nentries)d; + __device__ constexpr int broken_sym_component_starts[broken_sym_ncomponents] = { %(broken_sym_component_starts)s }; + __device__ constexpr int broken_sym_component_ends[broken_sym_ncomponents] = { %(broken_sym_component_ends)s }; + __device__ constexpr int broken_sym_component_old_factors[broken_sym_ncomponents] = { %(broken_sym_component_old_factors)s }; + __device__ constexpr int broken_sym_pid_list[broken_sym_nentries] = { %(broken_sym_pid_list)s }; + __device__ constexpr int broken_sym_block_starts[broken_sym_nentries] = { %(broken_sym_block_starts)s }; + __device__ constexpr int broken_sym_block_lengths[broken_sym_nentries] = { %(broken_sym_block_lengths)s }; + } +} + +#endif // PROCESSTABLES_H From b7a43d73d6b487b2a75a5bcac0c87c89b7c13886 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 18:25:16 +0200 Subject: [PATCH 06/25] BugFix: gpu includes --- madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h index 6ba1ec9097..c6254c7184 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h @@ -13,6 +13,8 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuCxtypes.h" // for cxtype +#include "mgOnGpuVectors.h" // for fptype_sv +#include "GpuAbstraction.h" // for gpuBlasHandle_t, gpuStream_t namespace mg5amcGpu { From 8571c2ea692901479e8c40dc0e3a0d4376bb362c Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 18:42:55 +0200 Subject: [PATCH 07/25] Makefile and stray file deletion Include right backend and copy right files. --- .../madmatrix/backend/cpu/coloramps.h | 58 --- .../madmatrix/backend/cpu/cpp_hel_amps_h.inc | 90 ---- .../backend/cpu/cpp_model_parameters_cc.inc | 101 ----- .../backend/cpu/cpp_model_parameters_h.inc | 264 ------------ .../madmatrix/backend/cpu/testxxx.cc | 384 ----------------- .../madmatrix/backend/gpu/coloramps.h | 60 --- .../madmatrix/backend/gpu/cpp_hel_amps_h.inc | 90 ---- .../backend/gpu/cpp_model_parameters_cc.inc | 101 ----- .../backend/gpu/cpp_model_parameters_h.inc | 264 ------------ .../madmatrix/backend/gpu/testxxx.cc | 385 ----------------- .../madmatrix/backend/simd/coloramps.h | 58 --- .../madmatrix/backend/simd/cpp_hel_amps_h.inc | 90 ---- .../backend/simd/cpp_model_parameters_cc.inc | 101 ----- .../backend/simd/cpp_model_parameters_h.inc | 264 ------------ .../madmatrix/backend/simd/testxxx.cc | 408 ------------------ .../template_files/madmatrix/madmatrix.mk | 20 +- madmatrix/output.py | 21 +- 17 files changed, 23 insertions(+), 2736 deletions(-) delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_hel_amps_h.inc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_cc.inc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_h.inc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_hel_amps_h.inc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_cc.inc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_h.inc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_hel_amps_h.inc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_cc.inc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_h.inc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h deleted file mode 100644 index 01ca72ac96..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/coloramps.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: O. Mattelaer, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef COLORAMPS_H -#define COLORAMPS_H 1 - -#include "ProcessData.h" - - -namespace mgOnGpu -{ - // Summary of numbering and indexing conventions for the relevant concepts (see issue #826 and PR #852) - // - Diagram number (no variable) in [0, N_diagrams-1]: all values are allowed (N_diagrams distinct values) - // It follows the same C-style indexing of MadSpace - // => this number is displayed for information before each block of code in CPPProcess.cc - // - Channel number ("channelId" in C, CHANNEL_ID in F) in [1, N_channels]: not all values are allowed (N_config <= N_channels <= N_diagrams distinct values) - // *** NB channelId is a diagram number: but ALL diagrams > N_channels, and also some < N_channels, do not have an associated SDE config number (#919) *** - // => this number (with F indexing as in ps/pdf output) is passed around as an API argument between cudacpp functions - // Note: the old API passes around a single CHANNEL_ID (and uses CHANNEL_ID=0 to indicate no-multichannel mode, but this is not used in coloramps.h), - // while the new API passes around an array of CHANNEL_ID's (and uses a NULL array pointer to indicate no-multichannel mode) - // - Channel number in C indexing: "channelID - 1" - // => this number (with C indexing) is used as the index of the channel2iconfig array below - // - Config number ("iconfig" in C, ICONFIG in F) in [1, N_config]: all values are allowed (N_config <= N_channels <= N_diagrams distinct values) - // - Config number in C indexing: "iconfig - 1" - // => this number (with C indexing) is used as the index of the icolamp array below - - // The number of channels in the channel2iconfig array below - // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** - constexpr unsigned int nchannels = %(nb_diag)i; - static_assert( nchannels <= ProcessData::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 - - // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) - // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) - // This array has N_diagrams elements, but only N_config <= N_diagrams valid values (iconfig>0) - // (NB: this array is created on the host in C++ code and on the device in GPU code, but a host copy is also needed in runTest #917) - __device__ constexpr int channel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed -%(channelc2iconfig_lines)s - }; - - // Host copy of the channel2iconfig array (this is needed in runTest #917) - constexpr const int* hostChannel2iconfig = channel2iconfig; - - // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) - constexpr unsigned int nconfigSDE = %(nb_channel)s; - - // Map iconfig to the mask of allowed colors (e.g. "colormask = icolamp[iconfig - 1]": input index uses C indexing) - // This array has N_config <= N_diagrams elements - // (NB: this array is created on the host in C++ code and on the device in GPU code) - __device__ constexpr bool icolamp[%(nb_channel)s][%(nb_color)s] = { // note: a trailing comma in the initializer list is allowed -%(is_LC)s - }; - -} - -#endif // COLORAMPS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_hel_amps_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_hel_amps_h.inc deleted file mode 100644 index 8dde0fff4b..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_hel_amps_h.inc +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (C) 2010 The ALOHA Development team and Contributors. -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Sep 2010) for the MG5aMC backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== -// This file has been automatically generated for %(output_name)s by -%(info_lines)s -//========================================================================== - -#ifndef HelAmps_%(model_name)s_H -#define HelAmps_%(model_name)s_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuVectors.h" - -#include "Parameters.h" - -#include -//#include -//#include -//#include -//#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - - // ALOHA-style object for easy flavor consolidation and non-template API - struct ALOHAOBJ { - - static constexpr int np4 = 4; // dimensions of 4-momenta (E,px,py,pz) - static constexpr int nw6 = %(nwave)d; // dimensions of each wavefunction (notice, this is +1 in case of FD gauge) - fptype_sv * pvec; - fptype * w; - int flv_index; - - __host__ __device__ ALOHAOBJ() {} - __host__ __device__ ALOHAOBJ(fptype_sv * pvec_sv, cxtype_sv * w_sv, int flv = -1) - : pvec(pvec_sv), w(reinterpret_cast(w_sv)), flv_index(flv) {} - }; - - struct FLV_COUPLING_VIEW { - - const int* const partner1; - const int* const partner2; - const fptype* const value; - - __host__ __device__ - FLV_COUPLING_VIEW(const int* p1, const int* p2, const fptype* v) - : partner1(p1), partner2(p2), value(v) {} - }; - - // FSTRIDE is the number of fptype's used to store one flavor slot of the value buffer: - // - independent (fixed) flavored couplings: FSTRIDE = nx2 = 2 (a single scalar complex, broadcast across the SIMD vector) - // - dependent (event-by-event, running-alphas) flavored couplings: FSTRIDE = nx2*neppC (an AOSOA SIMD record) - // It must match C_ACCESS::flv_stride of the access type the consuming vertex routine is instantiated with. - template - class FLV_COUPLING_ARRAY { - - static_assert(SIZE >= 0, "flvCOUPs SIZE must be non-negative"); - static_assert(STRIDE > 0, "flvCOUPs STRIDE must be positive"); - static_assert(FSTRIDE > 0, "flvCOUPs FSTRIDE must be positive"); - const int* const partner1; - const int* const partner2; - const fptype* const value; - - public: - __host__ __device__ - FLV_COUPLING_ARRAY(const int* p1, const int* p2, const fptype* v) - : partner1(p1), partner2(p2), value(v) {} - - __host__ __device__ - FLV_COUPLING_VIEW operator[](const int i) const { - return FLV_COUPLING_VIEW{ - partner1 + i*STRIDE, - partner2 + i*STRIDE, - value + i*FSTRIDE*STRIDE - }; - } - }; -%(function_declarations)s diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_cc.inc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_cc.inc deleted file mode 100644 index 4650fc76c8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_cc.inc +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== -// This file has been automatically generated for CUDA/C++ standalone by -%(info_lines)s -//========================================================================== - -#include "Parameters.h" - -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else -using namespace mg5amcCpu; -#endif - -#ifndef MGONGPU_HARDCODE_PARAM - -// Initialize static instance -Parameters* Parameters::instance = 0; - -// Function to get static instance - only one instance per program -Parameters* -Parameters::getInstance() -{ - if( instance == 0 ) - instance = new Parameters(); - return instance; -} - -void -Parameters::setIndependentParameters( SLHAReader& slha ) -{ - zero = 0; // define "zero" - ZERO = 0; // define "zero" - std::vector indices( 2, 0 ); // prepare a vector for indices - %(set_independent_parameters)s -} - -void -Parameters::setIndependentCouplings() -{ - %(set_independent_couplings)s - %(set_flv_couplings)s -} - -/* -void -Parameters::setDependentParameters() // now computed event-by-event (running alphas #373) -{ - %(set_dependent_parameters)s -} - -void -Parameters::setDependentCouplings() // now computed event-by-event (running alphas #373) -{ - %(set_dependent_couplings)s -} -*/ - -#endif - -// Routines for printing out parameters -void -Parameters::printIndependentParameters() -{ - std::cout << "model parameters independent of event kinematics:" << std::endl; - std::cout << "(Warning: aS in the runcard is ignored because event-by-event Gs are hardcoded or retrieved from Fortran)" << std::endl; - %(print_independent_parameters)s -} - -void -Parameters::printIndependentCouplings() -{ - std::cout << "model couplings independent of event kinematics:" << std::endl; - %(print_independent_couplings)s -} - -/* -void -Parameters::printDependentParameters() // now computed event-by-event (running alphas #373) -{ - std::cout << "model parameters dependent on event kinematics:" << std::endl; - %(print_dependent_parameters)s -} - -void -Parameters::printDependentCouplings() // now computed event-by-event (running alphas #373) -{ - std::cout << "model couplings dependent on event kinematics:" << std::endl; - %(print_dependent_couplings)s -} -*/ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_h.inc deleted file mode 100644 index c13f8ebe2f..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/cpp_model_parameters_h.inc +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. -// ========================================================================== -// This file has been automatically generated for CUDA/C++ standalone by -%(info_lines)s -//========================================================================== - -#ifndef Parameters_H -#define Parameters_H - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuCxtypes.h" -#include "mgOnGpuVectors.h" - -#include "constexpr_math.h" - -//========================================================================== - -// AV Jan 2024 (PR #625): this ugly #define was the only way I found to avoid creating arrays[nBsm] in CPPProcess.cc if nBsm is 0 -// The problem is that nBsm is determined when generating Parameters.h, which happens after CPPProcess.cc has already been generated -// For simplicity, keep this code hardcoded also for SM processes (a nullptr is needed as in the case nBsm == 0) -%(bsmdefine)s - -#ifndef MGONGPU_HARDCODE_PARAM%(eftwarn0)s - -#include "read_slha.h" - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - // Define FLV_COUPLING struct - struct FLV_COUPLING { - static constexpr int max_flavor = %(max_flavor)d; - - int partner1[max_flavor]; - int partner2[max_flavor]; - cxtype* value[max_flavor]; // because it will refer already calculated couplings - - FLV_COUPLING() { - for (int i = 0; i < max_flavor; ++i) { - partner1[i] = -1; - partner2[i] = -1; - value[i] = nullptr; // flavor slots with no coupling stay null so the - // cIPF_value setup (value[j] ? *value[j] : 0) does - // not dereference an uninitialised pointer - } - } - }; - - class Parameters - { - public: - - static Parameters* getInstance(); - - // Define "zero" - double zero, ZERO; - - %(independent_parameters)s - - %(independent_couplings)s - - %(dependent_parameters)s - - %(dependent_couplings)s - - %(flavor_independent_couplings)s - - %(flavor_dependent_couplings)s - - // Set parameters that are unchanged during the run - void setIndependentParameters( SLHAReader& slha ); - - // Set couplings that are unchanged during the run - void setIndependentCouplings(); - - // Set parameters that are changed event by event - //void setDependentParameters(); // now computed event-by-event (running alphas #373) - - // Set couplings that are changed event by event - //void setDependentCouplings(); // now computed event-by-event (running alphas #373) - - // Print parameters that are unchanged during the run - void printIndependentParameters(); - - // Print couplings that are unchanged during the run - void printIndependentCouplings(); - - // Print parameters that are changed event by event - //void printDependentParameters(); // now computed event-by-event (running alphas #373) - - // Print couplings that are changed event by event - //void printDependentCouplings(); // now computed event-by-event (running alphas #373) - - // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; - static constexpr int nBsmIndepParam = %(nbsmip)i; - %(hasbsmip)sdouble mdl_bsmIndepParam[nBsmIndepParam]; - - private: - - static Parameters* instance; - }; - -} // end namespace mg5amcGpu/mg5amcCpu - -#else%(eftwarn1)s - -#include -#include - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - // Hardcoded constexpr physics parameters - namespace Parameters // keep the same name rather than HardcodedParameters for simplicity - { - // Model parameters independent of aS - constexpr double zero = 0; - constexpr double ZERO = 0; - %(hardcoded_independent_parameters)s - // Model couplings independent of aS - %(hardcoded_independent_couplings)s - // Model parameters dependent on aS - %(hardcoded_dependent_parameters)s - // Model couplings dependent on aS - %(hardcoded_dependent_couplings)s - // Print parameters that are unchanged during the run - void printIndependentParameters(); - - // Print couplings that are unchanged during the run - void printIndependentCouplings(); - - // Print parameters that are changed event by event - //void printDependentParameters(); // now computed event-by-event (running alphas #373) - - // Print couplings that are changed event by event - //void printDependentCouplings(); // now computed event-by-event (running alphas #373) - - // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; - constexpr int nBsmIndepParam = %(nbsmip)i; - %(hasbsmip)s__device__ constexpr double mdl_bsmIndepParam[nBsmIndepParam] = { %(bsmip)s }; - } - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif - -//========================================================================== - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - namespace Parameters_dependentCouplings - { - constexpr size_t ndcoup = %(ndcoup)d; // #couplings that vary event by event because they depend on the running alphas QCD -%(idcoup)s - struct DependentCouplings_sv - { -%(dcoupdecl)s - }; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" // e.g. <> -#pragma GCC diagnostic ignored "-Wunused-variable" // e.g. <> -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> -#ifdef MGONGPUCPP_GPUIMPL -#pragma nv_diagnostic push -#pragma nv_diag_suppress 177 // e.g. <> -#endif - __host__ __device__ inline const DependentCouplings_sv computeDependentCouplings_fromG( const fptype_sv& G_sv, const double* bsmIndepParamPtr ) - { -#ifdef MGONGPU_HARDCODE_PARAM - using namespace Parameters; -#else%(eftspecial0)s -#endif - // NB: hardcode cxtype cI(0,1) instead of cxtype (or hardcoded cxsmpl) mdl_complexi (which exists in Parameters) because: - // (1) mdl_complexi is always (0,1); (2) mdl_complexi is undefined in device code; (3) need cxsmpl conversion to cxtype in code below - const cxtype cI( 0., 1. ); - DependentCouplings_sv out; -#if not( defined MGONGPU_CPPSIMD && defined MGONGPU_FPTYPE_FLOAT ) - // Couplings are (scalar, or vector of) doubles, or scalar floats - default implementation - { - const fptype_sv& G = G_sv; - // Model parameters dependent on aS -%(dcoupsetdpar)s - // Model couplings dependent on aS -%(dcoupsetdcoup)s - } -#else - // Couplings are VECTORS OF FLOATS: #439 special handling is needed (variable Gs are vector floats, fixed parameters are scalar doubles) - // Use an explicit loop to avoid <> - // Problems may come e.g. in EFTs from multiplying a vector float (related to aS-dependent G) by a scalar double (aS-independent parameters) - // (NB in pure SM processes this special handling is not needed, but we keep it here for simplicity, see PR #824)%(dcoupoutfptypev2)s - for( int i = 0; i < neppV; i++ ) - { - const fptype& G = G_sv[i]; - // Model parameters dependent on aS -%(dcoupsetdpar2)s - // Model couplings dependent on aS - %(dcoupsetdcoup2)s - }%(dcoupoutdcoup2)s -#endif - return out; - } -#ifdef MGONGPUCPP_GPUIMPL -#pragma GCC diagnostic pop -#pragma nv_diagnostic pop -#endif - } - - //========================================================================== - - namespace Parameters_independentCouplings - { - constexpr size_t nicoup = %(nicoup)d; // #couplings that are fixed for all events because they do not depend on the running alphas QCD -%(iicoup)s - } - - //========================================================================== - -#pragma GCC diagnostic push -#ifndef __clang__ -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> -#endif - // Compute the output couplings (e.g. gc10 and gc11) from the input gs - template - __device__ inline void - G2COUP( const fptype gs[], - fptype couplings[], - const double* bsmIndepParamPtr ) - { - mgDebug( 0, __FUNCTION__ ); - using namespace Parameters_dependentCouplings; - const fptype_sv& gs_sv = G_ACCESS::kernelAccessConst( gs ); - DependentCouplings_sv couplings_sv = computeDependentCouplings_fromG( gs_sv, bsmIndepParamPtr ); -%(dcoupaccessbuffer)s%(dcoupkernelaccess)s%(dcoupcompute)s - mgDebug( 1, __FUNCTION__ ); - return; - } -#pragma GCC diagnostic pop - -} // end namespace mg5amcGpu/mg5amcCpu - -//========================================================================== - -#endif // Parameters_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc deleted file mode 100644 index 7fe572468a..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testxxx.cc +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. -// ---------------------------------------------------------------------------- -// Use ./runTest.exe --gtest_filter=*xxx to run only testxxx.cc tests -// ---------------------------------------------------------------------------- - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "HelAmps_%(model_name)s.h" -#include "MemoryAccessMomenta.h" -#include "MemoryAccessWavefunctions.h" -#include "MemoryBuffers.h" -#include "epoch_process_id.h" - -#include - -#include -#include -#include // for signal and SIGFPE (see https://stackoverflow.com/a/17473528) -#include -#include -#include -#include -#define TESTID( s ) s##_CPU_XXX - -#define XTESTID( s ) TESTID( s ) - -namespace mg5amcCpu -{ - std::string fpeHandlerMessage = "unknown"; - int fpeHandlerIevt = -1; - inline void fpeHandlerTestxxx( int /*sig*/ ) - { - std::cerr << "Floating Point Exception (CPU neppV=" << neppV << "): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; - exit( 1 ); - } -} - -TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) -{ - using namespace mg5amcCpu; -#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) - auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); -#endif - constexpr bool dumpEvents = false; // dump the expected output of the test? - constexpr bool testEvents = !dumpEvents; // run the test? - constexpr fptype toleranceXXXs = std::is_same::value ? 1.E-15 : 1.E-5; - // Constant parameters - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - constexpr int np4 = CPPProcess::np4; - const int nevt = 32; // 12 independent tests plus 20 duplicates (need a multiple of 16 for floats '512z') - assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM - assert( nevt % neppV == 0 ); // nevt must be a multiple of neppV - // Fill in the input momenta - mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] - // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! - const fptype par0[np4 * nevt] = // AOS[nevt][np4] - { - 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) - 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) - 500, 180, 240, 400, // #3 (m=0 pT>0 pz>0) - 500, 180, 240, -400, // #4 (m=0 pT>0 pz<0) - 500, 0, 0, 500, // #5 DUPLICATE == #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, -500, // #6 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) - 500, 300, 400, 0, // #7 DUPLICATE == #2 (m=0 pT>0 pz=0) - 500, 180, 240, 400, // #8 DUPLICATE == #3 (m=0 pT>0 pz>0) - 500, 180, 240, -400, // #9 DUPLICATE == #4 (m=0 pT>0 pz<0) - 500, 0, 0, 500, // #10 DUPLICATE == #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, -500, // #11 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) - 500, 300, 400, 0, // #12 DUPLICATE == #2 (m=0 pT>0 pz=0) - 500, 180, 240, 400, // #13 DUPLICATE == #3 (m=0 pT>0 pz>0) - 500, 180, 240, -400, // #14 DUPLICATE == #4 (m=0 pT>0 pz<0) - 500, 0, 0, 500, // #15 DUPLICATE == #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, 0, // #16 (m=50>0 pT=0 pz=0) - 500, 0, 0, 300, // #17 (m=40>0 pT=0 pz>0) - 500, 0, 0, -300, // #18 (m=40>0 pT=0 pz<0) - 500, 180, 240, 0, // #19 (m=40>0 pT>0 pz=0) - 500, -240, -180, 0, // #20 (m=40>0 pT>0 pz=0) - 500, 180, 192, 144, // #21 (m=40>0 pT>0 pz>0) - 500, 180, 192, -144, // #22 (m=40>0 pT>0 pz<0) - 500, 0, 0, 0, // #23 DUPLICATE == #16 (m=50>0 pT=0 pz=0) - 500, 0, 0, 300, // #24 DUPLICATE == #17 (m=40>0 pT=0 pz>0) - 500, 0, 0, -300, // #25 DUPLICATE == #18 (m=40>0 pT=0 pz<0) - 500, 180, 240, 0, // #26 DUPLICATE == #19 (m=40>0 pT>0 pz=0) - 500, -240, -180, 0, // #27 DUPLICATE == #20 (m=40>0 pT>0 pz=0) - 500, 180, 192, 144, // #28 DUPLICATE == #21 (m=40>0 pT>0 pz>0) - 500, 180, 192, -144, // #29 DUPLICATE == #22 (m=40>0 pT>0 pz<0) - 500, 0, 0, 0, // #30 DUPLICATE == #16 (m=50>0 pT=0 pz=0) - 500, 0, 0, 300 // #31 DUPLICATE == #17 (m=40>0 pT=0 pz>0) - }; /* clang-format on */ - // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) - // See https://en.cppreference.com/w/c/language/array_initialization#Notes - fptype mass0[nevt] = {}; - bool ispzgt0[nevt] = {}; - bool ispzlt0[nevt] = {}; - bool isptgt0[nevt] = {}; - for( int ievt = 0; ievt < nevt; ievt++ ) - { - const fptype p0 = par0[ievt * np4 + 0]; - const fptype p1 = par0[ievt * np4 + 1]; - const fptype p2 = par0[ievt * np4 + 2]; - const fptype p3 = par0[ievt * np4 + 3]; - volatile fptype m2 = fpmax( p0 * p0 - p1 * p1 - p2 * p2 - p3 * p3, 0 ); // see #736 - if( m2 > 0 ) - mass0[ievt] = fpsqrt( (fptype)m2 ); - else - mass0[ievt] = 0; - ispzgt0[ievt] = ( p3 > 0 ); - ispzlt0[ievt] = ( p3 < 0 ); - isptgt0[ievt] = ( p1 != 0 ) || ( p2 != 0 ); - } - const int ipar0 = 0; // use only particle0 for this test - for( int ievt = 0; ievt < nevt; ievt++ ) - { - for( int ip4 = 0; ip4 < np4; ip4++ ) - { - MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA - } - } - // Expected output wavefunctions - std::vector> expwfs; -#include "testxxx_cc_ref.txt" // expwfs.push_back( {...} ); - std::string dumpFileName = "testxxx_cc_ref.txt.new"; - // Compute the output wavefunctions - // Dump new reference file if requested - constexpr int nw6 = CPPProcess::nw6; // dimensions of each wavefunction (HELAS KEK 91-11): e.g. 6 for e+ e- -> mu+ mu- (fermions and vectors) - int itest = 0; // index on the expected output vector - std::ofstream dumpFile; - if( dumpEvents ) - { - dumpFile.open( dumpFileName, std::ios::trunc ); - dumpFile << " // Copyright (C) 2020-2024 CERN and UCLouvain." << std::endl - << " // Licensed under the GNU Lesser General Public License (version 3 or later)." << std::endl - << " // Created by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin." << std::endl - << " // Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin." << std::endl; - } - // Lambda function for dumping wavefunctions - auto dumpwf6 = [&]( std::ostream& out, const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) - { - out << std::setprecision( 15 ) << std::scientific; - out << " expwfs.push_back( {"; - out << " // ---------" << std::endl; - for( int iw6 = 0; iw6 < nw6; iw6++ ) - { - out << std::setw( 26 ) << wf[iw6].real(); - out << ", " << std::setw( 22 ) << wf[iw6].imag(); - if( iw6 < nw6 - 1 ) - out << ", "; - else - out << " } );"; - out << " // itest=" << itest << ": " << xxx << "#" << ievt; - out << " nsp=" << nsp << " mass=" << (int)mass << std::endl; - } - out << std::defaultfloat; - }; - // Lambda function for testing wavefunctions (1) - auto testwf6 = [&]( const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) - { - if( dumpEvents ) dumpwf6( dumpFile, wf, xxx, ievt, nsp, mass ); - if( testEvents ) - { - std::array& expwf = expwfs[itest]; - //std::cout << "Testing " << std::setw(3) << itest << ": " << xxx << " #" << ievt << std::endl; - ////for ( int iw6 = 0; iw60) can be used without triggering FPEs (#701) - // This is done by filling the full SIMD vector with the value of ievt, which was already tested to respect the relevant assumptions - for( int jevt = 0; jevt < nevt; jevt++ ) - for( int ip4 = 0; ip4 < np4; ip4++ ) - MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), jevt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA - } - }; - // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) - // See https://en.cppreference.com/w/c/language/array_initialization#Notes - cxtype_sv outwfI[6] = {}; // last result of ixxxxx (mass==0) - cxtype_sv outwfO[6] = {}; // last result of oxxxxx (mass==0) - cxtype_sv outwf[6] = {}; - cxtype_sv outwf3[6] = {}; // NB: only 3 are filled by sxxxxx, but 6 are compared! - fptype* fp_outwfI = reinterpret_cast( outwfI ); // proof of concept for using fptype* in the interface - fptype* fp_outwfO = reinterpret_cast( outwfO ); // proof of concept for using fptype* in the interface - fptype* fp_outwf = reinterpret_cast( outwf ); // proof of concept for using fptype* in the interface - fptype* fp_outwf3 = reinterpret_cast( outwf3 ); // proof of concept for using fptype* in the interface - const int nhel = 1; - // *** START OF TESTING LOOP - for( auto nsp: { -1, +1 } ) // antifermion/fermion (or initial/final for scalar and vector) - { - for( int ievt = 0; ievt < nevt; ievt++ ) - { - using namespace mg5amcCpu; - if( debug ) - { - std::cout << std::endl; - std::cout << "nsp=" << nsp << " ievt=" << ievt << ": "; - for( int ip4 = 0; ip4 < np4; ip4++ ) std::cout << par0[ievt * np4 + ip4] << ", "; - std::cout << std::endl; - } - const int ipagV = ievt / neppV; // #event vector in this iteration - const fptype* ievt0Momenta = MemoryAccessMomenta::ieventAccessRecordConst( hstMomenta.data(), ipagV * neppV ); - // Test ixxxxx - NO ASSUMPTIONS - { - prepareTest( "ixxxxx", ievt ); - const fptype fmass = mass0[ievt]; - ixxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfI, ipar0 ); - testwf6( outwfI, "ixxxxx", ievt, nsp, fmass ); - ixxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfI, ipar0 ); - testwf6( outwfI, "ixxxxx", ievt, nsp, -fmass ); - } - // Test ipzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) - { - prepareTest( "ipzxxx", ievt ); - ipzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfI, "ipzxxx", ievt ); - testwf6( outwf, "ipzxxx", ievt, nsp, 0 ); - } - // Test imzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) - { - prepareTest( "imzxxx", ievt ); - imzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfI, "imzxxx", ievt ); - testwf6( outwf, "imzxxx", ievt, nsp, 0 ); - } - // Test ixzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) - if( mass0[ievt] == 0 && isptgt0[ievt] ) - { - prepareTest( "ixzxxx", ievt ); - ixzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfI, "ixzxxx", ievt ); - testwf6( outwf, "ixzxxx", ievt, nsp, 0 ); - } - // Test vxxxxx - NO ASSUMPTIONS - { - prepareTest( "vxxxxx", ievt ); - const fptype vmass = mass0[ievt]; - vxxxxx( ievt0Momenta, vmass, nhel, nsp, fp_outwf, ipar0 ); - testwf6( outwf, "vxxxxx", ievt, nsp, vmass ); - vxxxxx( ievt0Momenta, -vmass, nhel, nsp, fp_outwf, ipar0 ); - testwf6( outwf, "vxxxxx", ievt, nsp, -vmass ); - } - // Test sxxxxx - NO ASSUMPTIONS - { - prepareTest( "sxxxxx", ievt ); - const fptype smass = mass0[ievt]; - sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass>0") - testwf6( outwf3, "sxxxxx", ievt, nsp, smass ); - sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass<0") - testwf6( outwf3, "sxxxxx", ievt, nsp, -smass ); - } - // Test oxxxxx - NO ASSUMPTIONS - { - prepareTest( "oxxxxx", ievt ); - const fptype fmass = mass0[ievt]; - oxxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfO, ipar0 ); - testwf6( outwfO, "oxxxxx", ievt, nsp, fmass ); - oxxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfO, ipar0 ); - testwf6( outwfO, "oxxxxx", ievt, nsp, -fmass ); - } - // Test opzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) - { - prepareTest( "opzxxx", ievt ); - opzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfO, "opzxxx", ievt ); - testwf6( outwf, "opzxxx", ievt, nsp, 0 ); - } - // Test omzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) - { - prepareTest( "omzxxx", ievt ); - omzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfO, "omzxxx", ievt ); - testwf6( outwf, "omzxxx", ievt, nsp, 0 ); - } - // Test oxzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) - if( mass0[ievt] == 0 && isptgt0[ievt] ) - { - prepareTest( "oxzxxx", ievt ); - oxzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfO, "oxzxxx", ievt ); - testwf6( outwf, "oxzxxx", ievt, nsp, 0 ); - } - } - } - // *** END OF TESTING LOOP - if( dumpEvents ) - { - dumpFile.close(); - std::cout << "INFO: New reference data dumped to file '" << dumpFileName << "'" << std::endl; - } -#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) - signal( SIGFPE, fpeHandlerDefault ); -#endif -} - -//========================================================================== - -// Reset the GPU after ALL tests have gone out of scope -// (This was needed to avoid leaks in profilers, but compute-sanitizer reports no leaks, is it STILL needed?) -// ========= NB: resetting the GPU too early causes segfaults that are very difficult to debug #907 ========= -// Try to use atexit (https://stackoverflow.com/a/14610501) but this still crashes! -// ********* FIXME? avoid CUDA API calls in destructors? (see https://stackoverflow.com/a/16982503) ********* -void -myexit() -{ -} - -// Main function (see https://google.github.io/googletest/primer.html#writing-the-main-function) -// (NB: the test executables are now separate for C++ and CUDA, therefore main must be included all the time) -// (NB: previously, '#ifndef MGONGPUCPP_GPUIMPL' was ensuring that main was only included once while linking both C++ and CUDA tests) -int -main( int argc, char** argv ) -{ - atexit( myexit ); - testing::InitGoogleTest( &argc, argv ); - int status = RUN_ALL_TESTS(); - return status; -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h deleted file mode 100644 index 72dc0d4b8a..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/coloramps.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: O. Mattelaer, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef COLORAMPS_H -#define COLORAMPS_H 1 - -#include "ProcessData.h" - - -namespace mgOnGpu -{ - // Summary of numbering and indexing conventions for the relevant concepts (see issue #826 and PR #852) - // - Diagram number (no variable) in [0, N_diagrams-1]: all values are allowed (N_diagrams distinct values) - // It follows the same C-style indexing of MadSpace - // => this number is displayed for information before each block of code in CPPProcess.cc - // - Channel number ("channelId" in C, CHANNEL_ID in F) in [1, N_channels]: not all values are allowed (N_config <= N_channels <= N_diagrams distinct values) - // *** NB channelId is a diagram number: but ALL diagrams > N_channels, and also some < N_channels, do not have an associated SDE config number (#919) *** - // => this number (with F indexing as in ps/pdf output) is passed around as an API argument between cudacpp functions - // Note: the old API passes around a single CHANNEL_ID (and uses CHANNEL_ID=0 to indicate no-multichannel mode, but this is not used in coloramps.h), - // while the new API passes around an array of CHANNEL_ID's (and uses a NULL array pointer to indicate no-multichannel mode) - // - Channel number in C indexing: "channelID - 1" - // => this number (with C indexing) is used as the index of the channel2iconfig array below - // - Config number ("iconfig" in C, ICONFIG in F) in [1, N_config]: all values are allowed (N_config <= N_channels <= N_diagrams distinct values) - // - Config number in C indexing: "iconfig - 1" - // => this number (with C indexing) is used as the index of the icolamp array below - - // The number of channels in the channel2iconfig array below - // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** - constexpr unsigned int nchannels = %(nb_diag)i; - static_assert( nchannels <= ProcessData::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 - - // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) - // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) - // This array has N_diagrams elements, but only N_config <= N_diagrams valid values (iconfig>0) - // (NB: this array is created on the host in C++ code and on the device in GPU code, but a host copy is also needed in runTest #917) - __device__ constexpr int channel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed -%(channelc2iconfig_lines)s - }; - - // Host copy of the channel2iconfig array (this is needed in runTest #917) - constexpr int hostChannel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed -%(channelc2iconfig_lines)s - }; - - // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) - constexpr unsigned int nconfigSDE = %(nb_channel)s; - - // Map iconfig to the mask of allowed colors (e.g. "colormask = icolamp[iconfig - 1]": input index uses C indexing) - // This array has N_config <= N_diagrams elements - // (NB: this array is created on the host in C++ code and on the device in GPU code) - __device__ constexpr bool icolamp[%(nb_channel)s][%(nb_color)s] = { // note: a trailing comma in the initializer list is allowed -%(is_LC)s - }; - -} - -#endif // COLORAMPS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_hel_amps_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_hel_amps_h.inc deleted file mode 100644 index 8dde0fff4b..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_hel_amps_h.inc +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (C) 2010 The ALOHA Development team and Contributors. -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Sep 2010) for the MG5aMC backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== -// This file has been automatically generated for %(output_name)s by -%(info_lines)s -//========================================================================== - -#ifndef HelAmps_%(model_name)s_H -#define HelAmps_%(model_name)s_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuVectors.h" - -#include "Parameters.h" - -#include -//#include -//#include -//#include -//#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - - // ALOHA-style object for easy flavor consolidation and non-template API - struct ALOHAOBJ { - - static constexpr int np4 = 4; // dimensions of 4-momenta (E,px,py,pz) - static constexpr int nw6 = %(nwave)d; // dimensions of each wavefunction (notice, this is +1 in case of FD gauge) - fptype_sv * pvec; - fptype * w; - int flv_index; - - __host__ __device__ ALOHAOBJ() {} - __host__ __device__ ALOHAOBJ(fptype_sv * pvec_sv, cxtype_sv * w_sv, int flv = -1) - : pvec(pvec_sv), w(reinterpret_cast(w_sv)), flv_index(flv) {} - }; - - struct FLV_COUPLING_VIEW { - - const int* const partner1; - const int* const partner2; - const fptype* const value; - - __host__ __device__ - FLV_COUPLING_VIEW(const int* p1, const int* p2, const fptype* v) - : partner1(p1), partner2(p2), value(v) {} - }; - - // FSTRIDE is the number of fptype's used to store one flavor slot of the value buffer: - // - independent (fixed) flavored couplings: FSTRIDE = nx2 = 2 (a single scalar complex, broadcast across the SIMD vector) - // - dependent (event-by-event, running-alphas) flavored couplings: FSTRIDE = nx2*neppC (an AOSOA SIMD record) - // It must match C_ACCESS::flv_stride of the access type the consuming vertex routine is instantiated with. - template - class FLV_COUPLING_ARRAY { - - static_assert(SIZE >= 0, "flvCOUPs SIZE must be non-negative"); - static_assert(STRIDE > 0, "flvCOUPs STRIDE must be positive"); - static_assert(FSTRIDE > 0, "flvCOUPs FSTRIDE must be positive"); - const int* const partner1; - const int* const partner2; - const fptype* const value; - - public: - __host__ __device__ - FLV_COUPLING_ARRAY(const int* p1, const int* p2, const fptype* v) - : partner1(p1), partner2(p2), value(v) {} - - __host__ __device__ - FLV_COUPLING_VIEW operator[](const int i) const { - return FLV_COUPLING_VIEW{ - partner1 + i*STRIDE, - partner2 + i*STRIDE, - value + i*FSTRIDE*STRIDE - }; - } - }; -%(function_declarations)s diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_cc.inc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_cc.inc deleted file mode 100644 index 4650fc76c8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_cc.inc +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== -// This file has been automatically generated for CUDA/C++ standalone by -%(info_lines)s -//========================================================================== - -#include "Parameters.h" - -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else -using namespace mg5amcCpu; -#endif - -#ifndef MGONGPU_HARDCODE_PARAM - -// Initialize static instance -Parameters* Parameters::instance = 0; - -// Function to get static instance - only one instance per program -Parameters* -Parameters::getInstance() -{ - if( instance == 0 ) - instance = new Parameters(); - return instance; -} - -void -Parameters::setIndependentParameters( SLHAReader& slha ) -{ - zero = 0; // define "zero" - ZERO = 0; // define "zero" - std::vector indices( 2, 0 ); // prepare a vector for indices - %(set_independent_parameters)s -} - -void -Parameters::setIndependentCouplings() -{ - %(set_independent_couplings)s - %(set_flv_couplings)s -} - -/* -void -Parameters::setDependentParameters() // now computed event-by-event (running alphas #373) -{ - %(set_dependent_parameters)s -} - -void -Parameters::setDependentCouplings() // now computed event-by-event (running alphas #373) -{ - %(set_dependent_couplings)s -} -*/ - -#endif - -// Routines for printing out parameters -void -Parameters::printIndependentParameters() -{ - std::cout << "model parameters independent of event kinematics:" << std::endl; - std::cout << "(Warning: aS in the runcard is ignored because event-by-event Gs are hardcoded or retrieved from Fortran)" << std::endl; - %(print_independent_parameters)s -} - -void -Parameters::printIndependentCouplings() -{ - std::cout << "model couplings independent of event kinematics:" << std::endl; - %(print_independent_couplings)s -} - -/* -void -Parameters::printDependentParameters() // now computed event-by-event (running alphas #373) -{ - std::cout << "model parameters dependent on event kinematics:" << std::endl; - %(print_dependent_parameters)s -} - -void -Parameters::printDependentCouplings() // now computed event-by-event (running alphas #373) -{ - std::cout << "model couplings dependent on event kinematics:" << std::endl; - %(print_dependent_couplings)s -} -*/ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_h.inc deleted file mode 100644 index c13f8ebe2f..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/cpp_model_parameters_h.inc +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. -// ========================================================================== -// This file has been automatically generated for CUDA/C++ standalone by -%(info_lines)s -//========================================================================== - -#ifndef Parameters_H -#define Parameters_H - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuCxtypes.h" -#include "mgOnGpuVectors.h" - -#include "constexpr_math.h" - -//========================================================================== - -// AV Jan 2024 (PR #625): this ugly #define was the only way I found to avoid creating arrays[nBsm] in CPPProcess.cc if nBsm is 0 -// The problem is that nBsm is determined when generating Parameters.h, which happens after CPPProcess.cc has already been generated -// For simplicity, keep this code hardcoded also for SM processes (a nullptr is needed as in the case nBsm == 0) -%(bsmdefine)s - -#ifndef MGONGPU_HARDCODE_PARAM%(eftwarn0)s - -#include "read_slha.h" - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - // Define FLV_COUPLING struct - struct FLV_COUPLING { - static constexpr int max_flavor = %(max_flavor)d; - - int partner1[max_flavor]; - int partner2[max_flavor]; - cxtype* value[max_flavor]; // because it will refer already calculated couplings - - FLV_COUPLING() { - for (int i = 0; i < max_flavor; ++i) { - partner1[i] = -1; - partner2[i] = -1; - value[i] = nullptr; // flavor slots with no coupling stay null so the - // cIPF_value setup (value[j] ? *value[j] : 0) does - // not dereference an uninitialised pointer - } - } - }; - - class Parameters - { - public: - - static Parameters* getInstance(); - - // Define "zero" - double zero, ZERO; - - %(independent_parameters)s - - %(independent_couplings)s - - %(dependent_parameters)s - - %(dependent_couplings)s - - %(flavor_independent_couplings)s - - %(flavor_dependent_couplings)s - - // Set parameters that are unchanged during the run - void setIndependentParameters( SLHAReader& slha ); - - // Set couplings that are unchanged during the run - void setIndependentCouplings(); - - // Set parameters that are changed event by event - //void setDependentParameters(); // now computed event-by-event (running alphas #373) - - // Set couplings that are changed event by event - //void setDependentCouplings(); // now computed event-by-event (running alphas #373) - - // Print parameters that are unchanged during the run - void printIndependentParameters(); - - // Print couplings that are unchanged during the run - void printIndependentCouplings(); - - // Print parameters that are changed event by event - //void printDependentParameters(); // now computed event-by-event (running alphas #373) - - // Print couplings that are changed event by event - //void printDependentCouplings(); // now computed event-by-event (running alphas #373) - - // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; - static constexpr int nBsmIndepParam = %(nbsmip)i; - %(hasbsmip)sdouble mdl_bsmIndepParam[nBsmIndepParam]; - - private: - - static Parameters* instance; - }; - -} // end namespace mg5amcGpu/mg5amcCpu - -#else%(eftwarn1)s - -#include -#include - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - // Hardcoded constexpr physics parameters - namespace Parameters // keep the same name rather than HardcodedParameters for simplicity - { - // Model parameters independent of aS - constexpr double zero = 0; - constexpr double ZERO = 0; - %(hardcoded_independent_parameters)s - // Model couplings independent of aS - %(hardcoded_independent_couplings)s - // Model parameters dependent on aS - %(hardcoded_dependent_parameters)s - // Model couplings dependent on aS - %(hardcoded_dependent_couplings)s - // Print parameters that are unchanged during the run - void printIndependentParameters(); - - // Print couplings that are unchanged during the run - void printIndependentCouplings(); - - // Print parameters that are changed event by event - //void printDependentParameters(); // now computed event-by-event (running alphas #373) - - // Print couplings that are changed event by event - //void printDependentCouplings(); // now computed event-by-event (running alphas #373) - - // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; - constexpr int nBsmIndepParam = %(nbsmip)i; - %(hasbsmip)s__device__ constexpr double mdl_bsmIndepParam[nBsmIndepParam] = { %(bsmip)s }; - } - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif - -//========================================================================== - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - namespace Parameters_dependentCouplings - { - constexpr size_t ndcoup = %(ndcoup)d; // #couplings that vary event by event because they depend on the running alphas QCD -%(idcoup)s - struct DependentCouplings_sv - { -%(dcoupdecl)s - }; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" // e.g. <> -#pragma GCC diagnostic ignored "-Wunused-variable" // e.g. <> -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> -#ifdef MGONGPUCPP_GPUIMPL -#pragma nv_diagnostic push -#pragma nv_diag_suppress 177 // e.g. <> -#endif - __host__ __device__ inline const DependentCouplings_sv computeDependentCouplings_fromG( const fptype_sv& G_sv, const double* bsmIndepParamPtr ) - { -#ifdef MGONGPU_HARDCODE_PARAM - using namespace Parameters; -#else%(eftspecial0)s -#endif - // NB: hardcode cxtype cI(0,1) instead of cxtype (or hardcoded cxsmpl) mdl_complexi (which exists in Parameters) because: - // (1) mdl_complexi is always (0,1); (2) mdl_complexi is undefined in device code; (3) need cxsmpl conversion to cxtype in code below - const cxtype cI( 0., 1. ); - DependentCouplings_sv out; -#if not( defined MGONGPU_CPPSIMD && defined MGONGPU_FPTYPE_FLOAT ) - // Couplings are (scalar, or vector of) doubles, or scalar floats - default implementation - { - const fptype_sv& G = G_sv; - // Model parameters dependent on aS -%(dcoupsetdpar)s - // Model couplings dependent on aS -%(dcoupsetdcoup)s - } -#else - // Couplings are VECTORS OF FLOATS: #439 special handling is needed (variable Gs are vector floats, fixed parameters are scalar doubles) - // Use an explicit loop to avoid <> - // Problems may come e.g. in EFTs from multiplying a vector float (related to aS-dependent G) by a scalar double (aS-independent parameters) - // (NB in pure SM processes this special handling is not needed, but we keep it here for simplicity, see PR #824)%(dcoupoutfptypev2)s - for( int i = 0; i < neppV; i++ ) - { - const fptype& G = G_sv[i]; - // Model parameters dependent on aS -%(dcoupsetdpar2)s - // Model couplings dependent on aS - %(dcoupsetdcoup2)s - }%(dcoupoutdcoup2)s -#endif - return out; - } -#ifdef MGONGPUCPP_GPUIMPL -#pragma GCC diagnostic pop -#pragma nv_diagnostic pop -#endif - } - - //========================================================================== - - namespace Parameters_independentCouplings - { - constexpr size_t nicoup = %(nicoup)d; // #couplings that are fixed for all events because they do not depend on the running alphas QCD -%(iicoup)s - } - - //========================================================================== - -#pragma GCC diagnostic push -#ifndef __clang__ -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> -#endif - // Compute the output couplings (e.g. gc10 and gc11) from the input gs - template - __device__ inline void - G2COUP( const fptype gs[], - fptype couplings[], - const double* bsmIndepParamPtr ) - { - mgDebug( 0, __FUNCTION__ ); - using namespace Parameters_dependentCouplings; - const fptype_sv& gs_sv = G_ACCESS::kernelAccessConst( gs ); - DependentCouplings_sv couplings_sv = computeDependentCouplings_fromG( gs_sv, bsmIndepParamPtr ); -%(dcoupaccessbuffer)s%(dcoupkernelaccess)s%(dcoupcompute)s - mgDebug( 1, __FUNCTION__ ); - return; - } -#pragma GCC diagnostic pop - -} // end namespace mg5amcGpu/mg5amcCpu - -//========================================================================== - -#endif // Parameters_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc deleted file mode 100644 index fc2c912b5d..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testxxx.cc +++ /dev/null @@ -1,385 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. -// ---------------------------------------------------------------------------- -// Use ./runTest.exe --gtest_filter=*xxx to run only testxxx.cc tests -// ---------------------------------------------------------------------------- - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "HelAmps_%(model_name)s.h" -#include "MemoryAccessMomenta.h" -#include "MemoryAccessWavefunctions.h" -#include "MemoryBuffers.h" -#include "epoch_process_id.h" - -#include - -#include -#include -#include // for signal and SIGFPE (see https://stackoverflow.com/a/17473528) -#include -#include -#include -#include -#define TESTID( s ) s##_GPU_XXX - -#define XTESTID( s ) TESTID( s ) - -namespace mg5amcGpu -{ - std::string fpeHandlerMessage = "unknown"; - int fpeHandlerIevt = -1; - inline void fpeHandlerTestxxx( int /*sig*/ ) - { - std::cerr << "Floating Point Exception (GPU): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; - exit( 1 ); - } -} - -TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) -{ - using namespace mg5amcGpu; -#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) - auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); -#endif - constexpr bool dumpEvents = false; // dump the expected output of the test? - constexpr bool testEvents = !dumpEvents; // run the test? - constexpr fptype toleranceXXXs = std::is_same::value ? 1.E-15 : 1.E-5; - // Constant parameters - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - constexpr int np4 = CPPProcess::np4; - const int nevt = 32; // 12 independent tests plus 20 duplicates (need a multiple of 16 for floats '512z') - assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM - assert( nevt % neppV == 0 ); // nevt must be a multiple of neppV - // Fill in the input momenta - mg5amcGpu::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] - // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! - const fptype par0[np4 * nevt] = // AOS[nevt][np4] - { - 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) - 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) - 500, 180, 240, 400, // #3 (m=0 pT>0 pz>0) - 500, 180, 240, -400, // #4 (m=0 pT>0 pz<0) - 500, 0, 0, 500, // #5 DUPLICATE == #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, -500, // #6 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) - 500, 300, 400, 0, // #7 DUPLICATE == #2 (m=0 pT>0 pz=0) - 500, 180, 240, 400, // #8 DUPLICATE == #3 (m=0 pT>0 pz>0) - 500, 180, 240, -400, // #9 DUPLICATE == #4 (m=0 pT>0 pz<0) - 500, 0, 0, 500, // #10 DUPLICATE == #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, -500, // #11 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) - 500, 300, 400, 0, // #12 DUPLICATE == #2 (m=0 pT>0 pz=0) - 500, 180, 240, 400, // #13 DUPLICATE == #3 (m=0 pT>0 pz>0) - 500, 180, 240, -400, // #14 DUPLICATE == #4 (m=0 pT>0 pz<0) - 500, 0, 0, 500, // #15 DUPLICATE == #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, 0, // #16 (m=50>0 pT=0 pz=0) - 500, 0, 0, 300, // #17 (m=40>0 pT=0 pz>0) - 500, 0, 0, -300, // #18 (m=40>0 pT=0 pz<0) - 500, 180, 240, 0, // #19 (m=40>0 pT>0 pz=0) - 500, -240, -180, 0, // #20 (m=40>0 pT>0 pz=0) - 500, 180, 192, 144, // #21 (m=40>0 pT>0 pz>0) - 500, 180, 192, -144, // #22 (m=40>0 pT>0 pz<0) - 500, 0, 0, 0, // #23 DUPLICATE == #16 (m=50>0 pT=0 pz=0) - 500, 0, 0, 300, // #24 DUPLICATE == #17 (m=40>0 pT=0 pz>0) - 500, 0, 0, -300, // #25 DUPLICATE == #18 (m=40>0 pT=0 pz<0) - 500, 180, 240, 0, // #26 DUPLICATE == #19 (m=40>0 pT>0 pz=0) - 500, -240, -180, 0, // #27 DUPLICATE == #20 (m=40>0 pT>0 pz=0) - 500, 180, 192, 144, // #28 DUPLICATE == #21 (m=40>0 pT>0 pz>0) - 500, 180, 192, -144, // #29 DUPLICATE == #22 (m=40>0 pT>0 pz<0) - 500, 0, 0, 0, // #30 DUPLICATE == #16 (m=50>0 pT=0 pz=0) - 500, 0, 0, 300 // #31 DUPLICATE == #17 (m=40>0 pT=0 pz>0) - }; /* clang-format on */ - // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) - // See https://en.cppreference.com/w/c/language/array_initialization#Notes - fptype mass0[nevt] = {}; - bool ispzgt0[nevt] = {}; - bool ispzlt0[nevt] = {}; - bool isptgt0[nevt] = {}; - for( int ievt = 0; ievt < nevt; ievt++ ) - { - const fptype p0 = par0[ievt * np4 + 0]; - const fptype p1 = par0[ievt * np4 + 1]; - const fptype p2 = par0[ievt * np4 + 2]; - const fptype p3 = par0[ievt * np4 + 3]; - volatile fptype m2 = fpmax( p0 * p0 - p1 * p1 - p2 * p2 - p3 * p3, 0 ); // see #736 - if( m2 > 0 ) - mass0[ievt] = fpsqrt( (fptype)m2 ); - else - mass0[ievt] = 0; - ispzgt0[ievt] = ( p3 > 0 ); - ispzlt0[ievt] = ( p3 < 0 ); - isptgt0[ievt] = ( p1 != 0 ) || ( p2 != 0 ); - } - const int ipar0 = 0; // use only particle0 for this test - for( int ievt = 0; ievt < nevt; ievt++ ) - { - for( int ip4 = 0; ip4 < np4; ip4++ ) - { - MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA - } - } - // Expected output wavefunctions - std::vector> expwfs; -#include "testxxx_cc_ref.txt" // expwfs.push_back( {...} ); - std::string dumpFileName = "testxxx_cc_ref.txt.new"; - // Compute the output wavefunctions - // Dump new reference file if requested - constexpr int nw6 = CPPProcess::nw6; // dimensions of each wavefunction (HELAS KEK 91-11): e.g. 6 for e+ e- -> mu+ mu- (fermions and vectors) - int itest = 0; // index on the expected output vector - std::ofstream dumpFile; - if( dumpEvents ) - { - dumpFile.open( dumpFileName, std::ios::trunc ); - dumpFile << " // Copyright (C) 2020-2024 CERN and UCLouvain." << std::endl - << " // Licensed under the GNU Lesser General Public License (version 3 or later)." << std::endl - << " // Created by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin." << std::endl - << " // Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin." << std::endl; - } - // Lambda function for dumping wavefunctions - auto dumpwf6 = [&]( std::ostream& out, const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) - { - out << std::setprecision( 15 ) << std::scientific; - out << " expwfs.push_back( {"; - out << " // ---------" << std::endl; - for( int iw6 = 0; iw6 < nw6; iw6++ ) - { - out << std::setw( 26 ) << wf[iw6].real(); - out << ", " << std::setw( 22 ) << wf[iw6].imag(); - if( iw6 < nw6 - 1 ) - out << ", "; - else - out << " } );"; - out << " // itest=" << itest << ": " << xxx << "#" << ievt; - out << " nsp=" << nsp << " mass=" << (int)mass << std::endl; - } - out << std::defaultfloat; - }; - // Lambda function for testing wavefunctions (1) - auto testwf6 = [&]( const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) - { - if( dumpEvents ) dumpwf6( dumpFile, wf, xxx, ievt, nsp, mass ); - if( testEvents ) - { - std::array& expwf = expwfs[itest]; - //std::cout << "Testing " << std::setw(3) << itest << ": " << xxx << " #" << ievt << std::endl; - ////for ( int iw6 = 0; iw60) can be used without triggering FPEs (#701) - // This is done by filling the full SIMD vector with the value of ievt, which was already tested to respect the relevant assumptions - for( int jevt = 0; jevt < nevt; jevt++ ) - for( int ip4 = 0; ip4 < np4; ip4++ ) - MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), jevt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA - } - }; - // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) - // See https://en.cppreference.com/w/c/language/array_initialization#Notes - cxtype_sv outwfI[6] = {}; // last result of ixxxxx (mass==0) - cxtype_sv outwfO[6] = {}; // last result of oxxxxx (mass==0) - cxtype_sv outwf[6] = {}; - cxtype_sv outwf3[6] = {}; // NB: only 3 are filled by sxxxxx, but 6 are compared! - fptype* fp_outwfI = reinterpret_cast( outwfI ); // proof of concept for using fptype* in the interface - fptype* fp_outwfO = reinterpret_cast( outwfO ); // proof of concept for using fptype* in the interface - fptype* fp_outwf = reinterpret_cast( outwf ); // proof of concept for using fptype* in the interface - fptype* fp_outwf3 = reinterpret_cast( outwf3 ); // proof of concept for using fptype* in the interface - const int nhel = 1; - // *** START OF TESTING LOOP - for( auto nsp: { -1, +1 } ) // antifermion/fermion (or initial/final for scalar and vector) - { - for( int ievt = 0; ievt < nevt; ievt++ ) - { - using namespace mg5amcGpu; - if( debug ) - { - std::cout << std::endl; - std::cout << "nsp=" << nsp << " ievt=" << ievt << ": "; - for( int ip4 = 0; ip4 < np4; ip4++ ) std::cout << par0[ievt * np4 + ip4] << ", "; - std::cout << std::endl; - } - const int ipagV = ievt / neppV; // #event vector in this iteration - const fptype* ievt0Momenta = MemoryAccessMomenta::ieventAccessRecordConst( hstMomenta.data(), ipagV * neppV ); - // Test ixxxxx - NO ASSUMPTIONS - { - prepareTest( "ixxxxx", ievt ); - const fptype fmass = mass0[ievt]; - ixxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfI, ipar0 ); - testwf6( outwfI, "ixxxxx", ievt, nsp, fmass ); - ixxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfI, ipar0 ); - testwf6( outwfI, "ixxxxx", ievt, nsp, -fmass ); - } - // Test ipzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) - { - prepareTest( "ipzxxx", ievt ); - ipzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfI, "ipzxxx", ievt ); - testwf6( outwf, "ipzxxx", ievt, nsp, 0 ); - } - // Test imzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) - { - prepareTest( "imzxxx", ievt ); - imzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfI, "imzxxx", ievt ); - testwf6( outwf, "imzxxx", ievt, nsp, 0 ); - } - // Test ixzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) - if( mass0[ievt] == 0 && isptgt0[ievt] ) - { - prepareTest( "ixzxxx", ievt ); - ixzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfI, "ixzxxx", ievt ); - testwf6( outwf, "ixzxxx", ievt, nsp, 0 ); - } - // Test vxxxxx - NO ASSUMPTIONS - { - prepareTest( "vxxxxx", ievt ); - const fptype vmass = mass0[ievt]; - vxxxxx( ievt0Momenta, vmass, nhel, nsp, fp_outwf, ipar0 ); - testwf6( outwf, "vxxxxx", ievt, nsp, vmass ); - vxxxxx( ievt0Momenta, -vmass, nhel, nsp, fp_outwf, ipar0 ); - testwf6( outwf, "vxxxxx", ievt, nsp, -vmass ); - } - // Test sxxxxx - NO ASSUMPTIONS - { - prepareTest( "sxxxxx", ievt ); - const fptype smass = mass0[ievt]; - sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass>0") - testwf6( outwf3, "sxxxxx", ievt, nsp, smass ); - sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass<0") - testwf6( outwf3, "sxxxxx", ievt, nsp, -smass ); - } - // Test oxxxxx - NO ASSUMPTIONS - { - prepareTest( "oxxxxx", ievt ); - const fptype fmass = mass0[ievt]; - oxxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfO, ipar0 ); - testwf6( outwfO, "oxxxxx", ievt, nsp, fmass ); - oxxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfO, ipar0 ); - testwf6( outwfO, "oxxxxx", ievt, nsp, -fmass ); - } - // Test opzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) - { - prepareTest( "opzxxx", ievt ); - opzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfO, "opzxxx", ievt ); - testwf6( outwf, "opzxxx", ievt, nsp, 0 ); - } - // Test omzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) - { - prepareTest( "omzxxx", ievt ); - omzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfO, "omzxxx", ievt ); - testwf6( outwf, "omzxxx", ievt, nsp, 0 ); - } - // Test oxzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) - if( mass0[ievt] == 0 && isptgt0[ievt] ) - { - prepareTest( "oxzxxx", ievt ); - oxzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfO, "oxzxxx", ievt ); - testwf6( outwf, "oxzxxx", ievt, nsp, 0 ); - } - } - } - // *** END OF TESTING LOOP - if( dumpEvents ) - { - dumpFile.close(); - std::cout << "INFO: New reference data dumped to file '" << dumpFileName << "'" << std::endl; - } -#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) - signal( SIGFPE, fpeHandlerDefault ); -#endif -} - -//========================================================================== - -// Reset the GPU after ALL tests have gone out of scope -// (This was needed to avoid leaks in profilers, but compute-sanitizer reports no leaks, is it STILL needed?) -// ========= NB: resetting the GPU too early causes segfaults that are very difficult to debug #907 ========= -// Try to use atexit (https://stackoverflow.com/a/14610501) but this still crashes! -// ********* FIXME? avoid CUDA API calls in destructors? (see https://stackoverflow.com/a/16982503) ********* -void -myexit() -{ - //checkGpu( gpuDeviceReset() ); // FIXME??? this still crashes! should systematically avoid CUDA calls in all destructors? -} - -// Main function (see https://google.github.io/googletest/primer.html#writing-the-main-function) -// (NB: the test executables are now separate for C++ and CUDA, therefore main must be included all the time) -// (NB: previously, '#ifndef MGONGPUCPP_GPUIMPL' was ensuring that main was only included once while linking both C++ and CUDA tests) -int -main( int argc, char** argv ) -{ - atexit( myexit ); - testing::InitGoogleTest( &argc, argv ); - int status = RUN_ALL_TESTS(); - return status; -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h deleted file mode 100644 index 01ca72ac96..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/coloramps.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: O. Mattelaer, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef COLORAMPS_H -#define COLORAMPS_H 1 - -#include "ProcessData.h" - - -namespace mgOnGpu -{ - // Summary of numbering and indexing conventions for the relevant concepts (see issue #826 and PR #852) - // - Diagram number (no variable) in [0, N_diagrams-1]: all values are allowed (N_diagrams distinct values) - // It follows the same C-style indexing of MadSpace - // => this number is displayed for information before each block of code in CPPProcess.cc - // - Channel number ("channelId" in C, CHANNEL_ID in F) in [1, N_channels]: not all values are allowed (N_config <= N_channels <= N_diagrams distinct values) - // *** NB channelId is a diagram number: but ALL diagrams > N_channels, and also some < N_channels, do not have an associated SDE config number (#919) *** - // => this number (with F indexing as in ps/pdf output) is passed around as an API argument between cudacpp functions - // Note: the old API passes around a single CHANNEL_ID (and uses CHANNEL_ID=0 to indicate no-multichannel mode, but this is not used in coloramps.h), - // while the new API passes around an array of CHANNEL_ID's (and uses a NULL array pointer to indicate no-multichannel mode) - // - Channel number in C indexing: "channelID - 1" - // => this number (with C indexing) is used as the index of the channel2iconfig array below - // - Config number ("iconfig" in C, ICONFIG in F) in [1, N_config]: all values are allowed (N_config <= N_channels <= N_diagrams distinct values) - // - Config number in C indexing: "iconfig - 1" - // => this number (with C indexing) is used as the index of the icolamp array below - - // The number of channels in the channel2iconfig array below - // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** - constexpr unsigned int nchannels = %(nb_diag)i; - static_assert( nchannels <= ProcessData::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 - - // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) - // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) - // This array has N_diagrams elements, but only N_config <= N_diagrams valid values (iconfig>0) - // (NB: this array is created on the host in C++ code and on the device in GPU code, but a host copy is also needed in runTest #917) - __device__ constexpr int channel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed -%(channelc2iconfig_lines)s - }; - - // Host copy of the channel2iconfig array (this is needed in runTest #917) - constexpr const int* hostChannel2iconfig = channel2iconfig; - - // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) - constexpr unsigned int nconfigSDE = %(nb_channel)s; - - // Map iconfig to the mask of allowed colors (e.g. "colormask = icolamp[iconfig - 1]": input index uses C indexing) - // This array has N_config <= N_diagrams elements - // (NB: this array is created on the host in C++ code and on the device in GPU code) - __device__ constexpr bool icolamp[%(nb_channel)s][%(nb_color)s] = { // note: a trailing comma in the initializer list is allowed -%(is_LC)s - }; - -} - -#endif // COLORAMPS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_hel_amps_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_hel_amps_h.inc deleted file mode 100644 index 8dde0fff4b..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_hel_amps_h.inc +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (C) 2010 The ALOHA Development team and Contributors. -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Sep 2010) for the MG5aMC backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== -// This file has been automatically generated for %(output_name)s by -%(info_lines)s -//========================================================================== - -#ifndef HelAmps_%(model_name)s_H -#define HelAmps_%(model_name)s_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuVectors.h" - -#include "Parameters.h" - -#include -//#include -//#include -//#include -//#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - - // ALOHA-style object for easy flavor consolidation and non-template API - struct ALOHAOBJ { - - static constexpr int np4 = 4; // dimensions of 4-momenta (E,px,py,pz) - static constexpr int nw6 = %(nwave)d; // dimensions of each wavefunction (notice, this is +1 in case of FD gauge) - fptype_sv * pvec; - fptype * w; - int flv_index; - - __host__ __device__ ALOHAOBJ() {} - __host__ __device__ ALOHAOBJ(fptype_sv * pvec_sv, cxtype_sv * w_sv, int flv = -1) - : pvec(pvec_sv), w(reinterpret_cast(w_sv)), flv_index(flv) {} - }; - - struct FLV_COUPLING_VIEW { - - const int* const partner1; - const int* const partner2; - const fptype* const value; - - __host__ __device__ - FLV_COUPLING_VIEW(const int* p1, const int* p2, const fptype* v) - : partner1(p1), partner2(p2), value(v) {} - }; - - // FSTRIDE is the number of fptype's used to store one flavor slot of the value buffer: - // - independent (fixed) flavored couplings: FSTRIDE = nx2 = 2 (a single scalar complex, broadcast across the SIMD vector) - // - dependent (event-by-event, running-alphas) flavored couplings: FSTRIDE = nx2*neppC (an AOSOA SIMD record) - // It must match C_ACCESS::flv_stride of the access type the consuming vertex routine is instantiated with. - template - class FLV_COUPLING_ARRAY { - - static_assert(SIZE >= 0, "flvCOUPs SIZE must be non-negative"); - static_assert(STRIDE > 0, "flvCOUPs STRIDE must be positive"); - static_assert(FSTRIDE > 0, "flvCOUPs FSTRIDE must be positive"); - const int* const partner1; - const int* const partner2; - const fptype* const value; - - public: - __host__ __device__ - FLV_COUPLING_ARRAY(const int* p1, const int* p2, const fptype* v) - : partner1(p1), partner2(p2), value(v) {} - - __host__ __device__ - FLV_COUPLING_VIEW operator[](const int i) const { - return FLV_COUPLING_VIEW{ - partner1 + i*STRIDE, - partner2 + i*STRIDE, - value + i*FSTRIDE*STRIDE - }; - } - }; -%(function_declarations)s diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_cc.inc b/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_cc.inc deleted file mode 100644 index 4650fc76c8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_cc.inc +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== -// This file has been automatically generated for CUDA/C++ standalone by -%(info_lines)s -//========================================================================== - -#include "Parameters.h" - -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else -using namespace mg5amcCpu; -#endif - -#ifndef MGONGPU_HARDCODE_PARAM - -// Initialize static instance -Parameters* Parameters::instance = 0; - -// Function to get static instance - only one instance per program -Parameters* -Parameters::getInstance() -{ - if( instance == 0 ) - instance = new Parameters(); - return instance; -} - -void -Parameters::setIndependentParameters( SLHAReader& slha ) -{ - zero = 0; // define "zero" - ZERO = 0; // define "zero" - std::vector indices( 2, 0 ); // prepare a vector for indices - %(set_independent_parameters)s -} - -void -Parameters::setIndependentCouplings() -{ - %(set_independent_couplings)s - %(set_flv_couplings)s -} - -/* -void -Parameters::setDependentParameters() // now computed event-by-event (running alphas #373) -{ - %(set_dependent_parameters)s -} - -void -Parameters::setDependentCouplings() // now computed event-by-event (running alphas #373) -{ - %(set_dependent_couplings)s -} -*/ - -#endif - -// Routines for printing out parameters -void -Parameters::printIndependentParameters() -{ - std::cout << "model parameters independent of event kinematics:" << std::endl; - std::cout << "(Warning: aS in the runcard is ignored because event-by-event Gs are hardcoded or retrieved from Fortran)" << std::endl; - %(print_independent_parameters)s -} - -void -Parameters::printIndependentCouplings() -{ - std::cout << "model couplings independent of event kinematics:" << std::endl; - %(print_independent_couplings)s -} - -/* -void -Parameters::printDependentParameters() // now computed event-by-event (running alphas #373) -{ - std::cout << "model parameters dependent on event kinematics:" << std::endl; - %(print_dependent_parameters)s -} - -void -Parameters::printDependentCouplings() // now computed event-by-event (running alphas #373) -{ - std::cout << "model couplings dependent on event kinematics:" << std::endl; - %(print_dependent_couplings)s -} -*/ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_h.inc deleted file mode 100644 index c13f8ebe2f..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/cpp_model_parameters_h.inc +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: A. Valassi (Sep 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. -// ========================================================================== -// This file has been automatically generated for CUDA/C++ standalone by -%(info_lines)s -//========================================================================== - -#ifndef Parameters_H -#define Parameters_H - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuCxtypes.h" -#include "mgOnGpuVectors.h" - -#include "constexpr_math.h" - -//========================================================================== - -// AV Jan 2024 (PR #625): this ugly #define was the only way I found to avoid creating arrays[nBsm] in CPPProcess.cc if nBsm is 0 -// The problem is that nBsm is determined when generating Parameters.h, which happens after CPPProcess.cc has already been generated -// For simplicity, keep this code hardcoded also for SM processes (a nullptr is needed as in the case nBsm == 0) -%(bsmdefine)s - -#ifndef MGONGPU_HARDCODE_PARAM%(eftwarn0)s - -#include "read_slha.h" - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - // Define FLV_COUPLING struct - struct FLV_COUPLING { - static constexpr int max_flavor = %(max_flavor)d; - - int partner1[max_flavor]; - int partner2[max_flavor]; - cxtype* value[max_flavor]; // because it will refer already calculated couplings - - FLV_COUPLING() { - for (int i = 0; i < max_flavor; ++i) { - partner1[i] = -1; - partner2[i] = -1; - value[i] = nullptr; // flavor slots with no coupling stay null so the - // cIPF_value setup (value[j] ? *value[j] : 0) does - // not dereference an uninitialised pointer - } - } - }; - - class Parameters - { - public: - - static Parameters* getInstance(); - - // Define "zero" - double zero, ZERO; - - %(independent_parameters)s - - %(independent_couplings)s - - %(dependent_parameters)s - - %(dependent_couplings)s - - %(flavor_independent_couplings)s - - %(flavor_dependent_couplings)s - - // Set parameters that are unchanged during the run - void setIndependentParameters( SLHAReader& slha ); - - // Set couplings that are unchanged during the run - void setIndependentCouplings(); - - // Set parameters that are changed event by event - //void setDependentParameters(); // now computed event-by-event (running alphas #373) - - // Set couplings that are changed event by event - //void setDependentCouplings(); // now computed event-by-event (running alphas #373) - - // Print parameters that are unchanged during the run - void printIndependentParameters(); - - // Print couplings that are unchanged during the run - void printIndependentCouplings(); - - // Print parameters that are changed event by event - //void printDependentParameters(); // now computed event-by-event (running alphas #373) - - // Print couplings that are changed event by event - //void printDependentCouplings(); // now computed event-by-event (running alphas #373) - - // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; - static constexpr int nBsmIndepParam = %(nbsmip)i; - %(hasbsmip)sdouble mdl_bsmIndepParam[nBsmIndepParam]; - - private: - - static Parameters* instance; - }; - -} // end namespace mg5amcGpu/mg5amcCpu - -#else%(eftwarn1)s - -#include -#include - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - // Hardcoded constexpr physics parameters - namespace Parameters // keep the same name rather than HardcodedParameters for simplicity - { - // Model parameters independent of aS - constexpr double zero = 0; - constexpr double ZERO = 0; - %(hardcoded_independent_parameters)s - // Model couplings independent of aS - %(hardcoded_independent_couplings)s - // Model parameters dependent on aS - %(hardcoded_dependent_parameters)s - // Model couplings dependent on aS - %(hardcoded_dependent_couplings)s - // Print parameters that are unchanged during the run - void printIndependentParameters(); - - // Print couplings that are unchanged during the run - void printIndependentCouplings(); - - // Print parameters that are changed event by event - //void printDependentParameters(); // now computed event-by-event (running alphas #373) - - // Print couplings that are changed event by event - //void printDependentCouplings(); // now computed event-by-event (running alphas #373) - - // BSM parameters that do not depend on alphaS but are needed in the computation of alphaS-dependent couplings; - constexpr int nBsmIndepParam = %(nbsmip)i; - %(hasbsmip)s__device__ constexpr double mdl_bsmIndepParam[nBsmIndepParam] = { %(bsmip)s }; - } - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif - -//========================================================================== - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - namespace Parameters_dependentCouplings - { - constexpr size_t ndcoup = %(ndcoup)d; // #couplings that vary event by event because they depend on the running alphas QCD -%(idcoup)s - struct DependentCouplings_sv - { -%(dcoupdecl)s - }; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" // e.g. <> -#pragma GCC diagnostic ignored "-Wunused-variable" // e.g. <> -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> -#ifdef MGONGPUCPP_GPUIMPL -#pragma nv_diagnostic push -#pragma nv_diag_suppress 177 // e.g. <> -#endif - __host__ __device__ inline const DependentCouplings_sv computeDependentCouplings_fromG( const fptype_sv& G_sv, const double* bsmIndepParamPtr ) - { -#ifdef MGONGPU_HARDCODE_PARAM - using namespace Parameters; -#else%(eftspecial0)s -#endif - // NB: hardcode cxtype cI(0,1) instead of cxtype (or hardcoded cxsmpl) mdl_complexi (which exists in Parameters) because: - // (1) mdl_complexi is always (0,1); (2) mdl_complexi is undefined in device code; (3) need cxsmpl conversion to cxtype in code below - const cxtype cI( 0., 1. ); - DependentCouplings_sv out; -#if not( defined MGONGPU_CPPSIMD && defined MGONGPU_FPTYPE_FLOAT ) - // Couplings are (scalar, or vector of) doubles, or scalar floats - default implementation - { - const fptype_sv& G = G_sv; - // Model parameters dependent on aS -%(dcoupsetdpar)s - // Model couplings dependent on aS -%(dcoupsetdcoup)s - } -#else - // Couplings are VECTORS OF FLOATS: #439 special handling is needed (variable Gs are vector floats, fixed parameters are scalar doubles) - // Use an explicit loop to avoid <> - // Problems may come e.g. in EFTs from multiplying a vector float (related to aS-dependent G) by a scalar double (aS-independent parameters) - // (NB in pure SM processes this special handling is not needed, but we keep it here for simplicity, see PR #824)%(dcoupoutfptypev2)s - for( int i = 0; i < neppV; i++ ) - { - const fptype& G = G_sv[i]; - // Model parameters dependent on aS -%(dcoupsetdpar2)s - // Model couplings dependent on aS - %(dcoupsetdcoup2)s - }%(dcoupoutdcoup2)s -#endif - return out; - } -#ifdef MGONGPUCPP_GPUIMPL -#pragma GCC diagnostic pop -#pragma nv_diagnostic pop -#endif - } - - //========================================================================== - - namespace Parameters_independentCouplings - { - constexpr size_t nicoup = %(nicoup)d; // #couplings that are fixed for all events because they do not depend on the running alphas QCD -%(iicoup)s - } - - //========================================================================== - -#pragma GCC diagnostic push -#ifndef __clang__ -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" // e.g. <> -#endif - // Compute the output couplings (e.g. gc10 and gc11) from the input gs - template - __device__ inline void - G2COUP( const fptype gs[], - fptype couplings[], - const double* bsmIndepParamPtr ) - { - mgDebug( 0, __FUNCTION__ ); - using namespace Parameters_dependentCouplings; - const fptype_sv& gs_sv = G_ACCESS::kernelAccessConst( gs ); - DependentCouplings_sv couplings_sv = computeDependentCouplings_fromG( gs_sv, bsmIndepParamPtr ); -%(dcoupaccessbuffer)s%(dcoupkernelaccess)s%(dcoupcompute)s - mgDebug( 1, __FUNCTION__ ); - return; - } -#pragma GCC diagnostic pop - -} // end namespace mg5amcGpu/mg5amcCpu - -//========================================================================== - -#endif // Parameters_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc deleted file mode 100644 index 9d5963fb58..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/testxxx.cc +++ /dev/null @@ -1,408 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. -// ---------------------------------------------------------------------------- -// Use ./runTest.exe --gtest_filter=*xxx to run only testxxx.cc tests -// ---------------------------------------------------------------------------- - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "HelAmps_%(model_name)s.h" -#include "MemoryAccessMomenta.h" -#include "MemoryAccessWavefunctions.h" -#include "MemoryBuffers.h" -#include "epoch_process_id.h" - -#include - -#include -#include -#include // for signal and SIGFPE (see https://stackoverflow.com/a/17473528) -#include -#include -#include -#include -#define TESTID( s ) s##_CPU_XXX - -#define XTESTID( s ) TESTID( s ) - -namespace mg5amcCpu -{ - std::string fpeHandlerMessage = "unknown"; - int fpeHandlerIevt = -1; - inline void fpeHandlerTestxxx( int /*sig*/ ) - { - std::cerr << "Floating Point Exception (CPU neppV=" << neppV << "): '" << fpeHandlerMessage << "' ievt=" << fpeHandlerIevt << std::endl; - exit( 1 ); - } -} - -TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) -{ - using namespace mg5amcCpu; -#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) - auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); -#endif - constexpr bool dumpEvents = false; // dump the expected output of the test? - constexpr bool testEvents = !dumpEvents; // run the test? - constexpr fptype toleranceXXXs = std::is_same::value ? 1.E-15 : 1.E-5; - // Constant parameters - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - constexpr int np4 = CPPProcess::np4; - const int nevt = 32; // 12 independent tests plus 20 duplicates (need a multiple of 16 for floats '512z') - assert( nevt % neppM == 0 ); // nevt must be a multiple of neppM - assert( nevt % neppV == 0 ); // nevt must be a multiple of neppV - // Fill in the input momenta - mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] - // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! - const fptype par0[np4 * nevt] = // AOS[nevt][np4] - { - 500, 0, 0, 500, // #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, -500, // #1 (m=0 pT=0 -E=pz<0) - 500, 300, 400, 0, // #2 (m=0 pT>0 pz=0) - 500, 180, 240, 400, // #3 (m=0 pT>0 pz>0) - 500, 180, 240, -400, // #4 (m=0 pT>0 pz<0) - 500, 0, 0, 500, // #5 DUPLICATE == #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, -500, // #6 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) - 500, 300, 400, 0, // #7 DUPLICATE == #2 (m=0 pT>0 pz=0) - 500, 180, 240, 400, // #8 DUPLICATE == #3 (m=0 pT>0 pz>0) - 500, 180, 240, -400, // #9 DUPLICATE == #4 (m=0 pT>0 pz<0) - 500, 0, 0, 500, // #10 DUPLICATE == #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, -500, // #11 DUPLICATE == #1 (m=0 pT=0 -E=pz<0) - 500, 300, 400, 0, // #12 DUPLICATE == #2 (m=0 pT>0 pz=0) - 500, 180, 240, 400, // #13 DUPLICATE == #3 (m=0 pT>0 pz>0) - 500, 180, 240, -400, // #14 DUPLICATE == #4 (m=0 pT>0 pz<0) - 500, 0, 0, 500, // #15 DUPLICATE == #0 (m=0 pT=0 E=pz>0) - 500, 0, 0, 0, // #16 (m=50>0 pT=0 pz=0) - 500, 0, 0, 300, // #17 (m=40>0 pT=0 pz>0) - 500, 0, 0, -300, // #18 (m=40>0 pT=0 pz<0) - 500, 180, 240, 0, // #19 (m=40>0 pT>0 pz=0) - 500, -240, -180, 0, // #20 (m=40>0 pT>0 pz=0) - 500, 180, 192, 144, // #21 (m=40>0 pT>0 pz>0) - 500, 180, 192, -144, // #22 (m=40>0 pT>0 pz<0) - 500, 0, 0, 0, // #23 DUPLICATE == #16 (m=50>0 pT=0 pz=0) - 500, 0, 0, 300, // #24 DUPLICATE == #17 (m=40>0 pT=0 pz>0) - 500, 0, 0, -300, // #25 DUPLICATE == #18 (m=40>0 pT=0 pz<0) - 500, 180, 240, 0, // #26 DUPLICATE == #19 (m=40>0 pT>0 pz=0) - 500, -240, -180, 0, // #27 DUPLICATE == #20 (m=40>0 pT>0 pz=0) - 500, 180, 192, 144, // #28 DUPLICATE == #21 (m=40>0 pT>0 pz>0) - 500, 180, 192, -144, // #29 DUPLICATE == #22 (m=40>0 pT>0 pz<0) - 500, 0, 0, 0, // #30 DUPLICATE == #16 (m=50>0 pT=0 pz=0) - 500, 0, 0, 300 // #31 DUPLICATE == #17 (m=40>0 pT=0 pz>0) - }; /* clang-format on */ - // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) - // See https://en.cppreference.com/w/c/language/array_initialization#Notes - fptype mass0[nevt] = {}; - bool ispzgt0[nevt] = {}; - bool ispzlt0[nevt] = {}; - bool isptgt0[nevt] = {}; - for( int ievt = 0; ievt < nevt; ievt++ ) - { - const fptype p0 = par0[ievt * np4 + 0]; - const fptype p1 = par0[ievt * np4 + 1]; - const fptype p2 = par0[ievt * np4 + 2]; - const fptype p3 = par0[ievt * np4 + 3]; - volatile fptype m2 = fpmax( p0 * p0 - p1 * p1 - p2 * p2 - p3 * p3, 0 ); // see #736 - if( m2 > 0 ) - mass0[ievt] = fpsqrt( (fptype)m2 ); - else - mass0[ievt] = 0; - ispzgt0[ievt] = ( p3 > 0 ); - ispzlt0[ievt] = ( p3 < 0 ); - isptgt0[ievt] = ( p1 != 0 ) || ( p2 != 0 ); - } - const int ipar0 = 0; // use only particle0 for this test - for( int ievt = 0; ievt < nevt; ievt++ ) - { - for( int ip4 = 0; ip4 < np4; ip4++ ) - { - MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA - } - } - // Expected output wavefunctions - std::vector> expwfs; -#include "testxxx_cc_ref.txt" // expwfs.push_back( {...} ); - std::string dumpFileName = "testxxx_cc_ref.txt.new"; - // Compute the output wavefunctions - // Dump new reference file if requested - constexpr int nw6 = CPPProcess::nw6; // dimensions of each wavefunction (HELAS KEK 91-11): e.g. 6 for e+ e- -> mu+ mu- (fermions and vectors) - int itest = 0; // index on the expected output vector - std::ofstream dumpFile; - if( dumpEvents ) - { - dumpFile.open( dumpFileName, std::ios::trunc ); - dumpFile << " // Copyright (C) 2020-2024 CERN and UCLouvain." << std::endl - << " // Licensed under the GNU Lesser General Public License (version 3 or later)." << std::endl - << " // Created by: A. Valassi (Apr 2021) for the MG5aMC CUDACPP plugin." << std::endl - << " // Further modified by: A. Valassi (2021-2024) for the MG5aMC CUDACPP plugin." << std::endl; - } - // Lambda function for dumping wavefunctions - auto dumpwf6 = [&]( std::ostream& out, const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) - { - out << std::setprecision( 15 ) << std::scientific; - out << " expwfs.push_back( {"; - out << " // ---------" << std::endl; - for( int iw6 = 0; iw6 < nw6; iw6++ ) - { - const int ieppV = ievt % neppV; // #event in the current event vector in this iteration -#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK - out << std::setw( 26 ) << cxreal( wf[iw6][ieppV] ) << ", "; - out << std::setw( 22 ) << cximag( wf[iw6][ieppV] ); -#else - out << std::setw( 26 ) << wf[iw6].real()[ieppV] << ", "; - out << std::setw( 22 ) << wf[iw6].imag()[ieppV]; -#endif - if( iw6 < nw6 - 1 ) - out << ", "; - else - out << " } );"; - out << " // itest=" << itest << ": " << xxx << "#" << ievt; - out << " nsp=" << nsp << " mass=" << (int)mass << std::endl; - } - out << std::defaultfloat; - }; - // Lambda function for testing wavefunctions (1) - auto testwf6 = [&]( const cxtype_sv wf[6], const char* xxx, int ievt, int nsp, fptype mass ) - { - if( dumpEvents ) dumpwf6( dumpFile, wf, xxx, ievt, nsp, mass ); - if( testEvents ) - { - std::array& expwf = expwfs[itest]; - //std::cout << "Testing " << std::setw(3) << itest << ": " << xxx << " #" << ievt << std::endl; - ////for ( int iw6 = 0; iw60) can be used without triggering FPEs (#701) - // This is done by filling the full SIMD vector with the value of ievt, which was already tested to respect the relevant assumptions - for( int jevt = 0; jevt < nevt; jevt++ ) - for( int ip4 = 0; ip4 < np4; ip4++ ) - MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), jevt, ip4, ipar0 ) = par0[ievt * np4 + ip4]; // AOS to AOSOA - } - }; - // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) - // See https://en.cppreference.com/w/c/language/array_initialization#Notes - cxtype_sv outwfI[6] = {}; // last result of ixxxxx (mass==0) - cxtype_sv outwfO[6] = {}; // last result of oxxxxx (mass==0) - cxtype_sv outwf[6] = {}; - cxtype_sv outwf3[6] = {}; // NB: only 3 are filled by sxxxxx, but 6 are compared! - fptype* fp_outwfI = reinterpret_cast( outwfI ); // proof of concept for using fptype* in the interface - fptype* fp_outwfO = reinterpret_cast( outwfO ); // proof of concept for using fptype* in the interface - fptype* fp_outwf = reinterpret_cast( outwf ); // proof of concept for using fptype* in the interface - fptype* fp_outwf3 = reinterpret_cast( outwf3 ); // proof of concept for using fptype* in the interface - const int nhel = 1; - // *** START OF TESTING LOOP - for( auto nsp: { -1, +1 } ) // antifermion/fermion (or initial/final for scalar and vector) - { - for( int ievt = 0; ievt < nevt; ievt++ ) - { - using namespace mg5amcCpu; - if( debug ) - { - std::cout << std::endl; - std::cout << "nsp=" << nsp << " ievt=" << ievt << ": "; - for( int ip4 = 0; ip4 < np4; ip4++ ) std::cout << par0[ievt * np4 + ip4] << ", "; - std::cout << std::endl; - } - const int ipagV = ievt / neppV; // #event vector in this iteration - const fptype* ievt0Momenta = MemoryAccessMomenta::ieventAccessRecordConst( hstMomenta.data(), ipagV * neppV ); - // Test ixxxxx - NO ASSUMPTIONS - { - prepareTest( "ixxxxx", ievt ); - const fptype fmass = mass0[ievt]; - ixxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfI, ipar0 ); - testwf6( outwfI, "ixxxxx", ievt, nsp, fmass ); - ixxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfI, ipar0 ); - testwf6( outwfI, "ixxxxx", ievt, nsp, -fmass ); - } - // Test ipzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) - { - prepareTest( "ipzxxx", ievt ); - ipzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfI, "ipzxxx", ievt ); - testwf6( outwf, "ipzxxx", ievt, nsp, 0 ); - } - // Test imzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) - { - prepareTest( "imzxxx", ievt ); - imzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfI, "imzxxx", ievt ); - testwf6( outwf, "imzxxx", ievt, nsp, 0 ); - } - // Test ixzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) - if( mass0[ievt] == 0 && isptgt0[ievt] ) - { - prepareTest( "ixzxxx", ievt ); - ixzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfI, "ixzxxx", ievt ); - testwf6( outwf, "ixzxxx", ievt, nsp, 0 ); - } - // Test vxxxxx - NO ASSUMPTIONS - { - prepareTest( "vxxxxx", ievt ); - const fptype vmass = mass0[ievt]; - vxxxxx( ievt0Momenta, vmass, nhel, nsp, fp_outwf, ipar0 ); - testwf6( outwf, "vxxxxx", ievt, nsp, vmass ); - vxxxxx( ievt0Momenta, -vmass, nhel, nsp, fp_outwf, ipar0 ); - testwf6( outwf, "vxxxxx", ievt, nsp, -vmass ); - } - // Test sxxxxx - NO ASSUMPTIONS - { - prepareTest( "sxxxxx", ievt ); - const fptype smass = mass0[ievt]; - sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass>0") - testwf6( outwf3, "sxxxxx", ievt, nsp, smass ); - sxxxxx( ievt0Momenta, nsp, fp_outwf3, ipar0 ); // no mass, no helicity (was "smass<0") - testwf6( outwf3, "sxxxxx", ievt, nsp, -smass ); - } - // Test oxxxxx - NO ASSUMPTIONS - { - prepareTest( "oxxxxx", ievt ); - const fptype fmass = mass0[ievt]; - oxxxxx( ievt0Momenta, fmass, nhel, nsp, fp_outwfO, ipar0 ); - testwf6( outwfO, "oxxxxx", ievt, nsp, fmass ); - oxxxxx( ievt0Momenta, -fmass, nhel, nsp, fp_outwfO, ipar0 ); - testwf6( outwfO, "oxxxxx", ievt, nsp, -fmass ); - } - // Test opzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == +PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzgt0[ievt] ) - { - prepareTest( "opzxxx", ievt ); - opzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfO, "opzxxx", ievt ); - testwf6( outwf, "opzxxx", ievt, nsp, 0 ); - } - // Test omzxxx - ASSUMPTIONS: (FMASS == 0) and (PX == PY == 0 and E == -PZ > 0) - if( mass0[ievt] == 0 && !isptgt0[ievt] && ispzlt0[ievt] ) - { - prepareTest( "omzxxx", ievt ); - omzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfO, "omzxxx", ievt ); - testwf6( outwf, "omzxxx", ievt, nsp, 0 ); - } - // Test oxzxxx - ASSUMPTIONS: (FMASS == 0) and (PT > 0) - if( mass0[ievt] == 0 && isptgt0[ievt] ) - { - prepareTest( "oxzxxx", ievt ); - oxzxxx( ievt0Momenta, nhel, nsp, fp_outwf, ipar0 ); - testwf6two( outwf, outwfO, "oxzxxx", ievt ); - testwf6( outwf, "oxzxxx", ievt, nsp, 0 ); - } - } - } - // *** END OF TESTING LOOP - if( dumpEvents ) - { - dumpFile.close(); - std::cout << "INFO: New reference data dumped to file '" << dumpFileName << "'" << std::endl; - } -#ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) - signal( SIGFPE, fpeHandlerDefault ); -#endif -} - -//========================================================================== - -// Reset the GPU after ALL tests have gone out of scope -// (This was needed to avoid leaks in profilers, but compute-sanitizer reports no leaks, is it STILL needed?) -// ========= NB: resetting the GPU too early causes segfaults that are very difficult to debug #907 ========= -// Try to use atexit (https://stackoverflow.com/a/14610501) but this still crashes! -// ********* FIXME? avoid CUDA API calls in destructors? (see https://stackoverflow.com/a/16982503) ********* -void -myexit() -{ -} - -// Main function (see https://google.github.io/googletest/primer.html#writing-the-main-function) -// (NB: the test executables are now separate for C++ and CUDA, therefore main must be included all the time) -// (NB: previously, '#ifndef MGONGPUCPP_GPUIMPL' was ensuring that main was only included once while linking both C++ and CUDA tests) -int -main( int argc, char** argv ) -{ - atexit( myexit ); - testing::InitGoogleTest( &argc, argv ); - int status = RUN_ALL_TESTS(); - return status; -} diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk index c82234690e..185e37137e 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk @@ -378,6 +378,15 @@ export GPUSUFFIX # Export BACKEND (resolved from cppauto above if needed; used e.g. to name the common library) export BACKEND +# Map BACKEND to its backend/ source subdirectory +ifneq ($(GPUCC),) + override BACKENDDIR = gpu +else ifeq ($(BACKEND),cppnone) + override BACKENDDIR = cpu +else + override BACKENDDIR = simd +endif + #------------------------------------------------------------------------------- #=== Configure ccache for C++ and CUDA/HIP builds @@ -399,7 +408,7 @@ endif #=== Configure common compiler flags for C++ and CUDA/HIP -INCFLAGS = -I. +INCFLAGS = -I. -Ibackend/$(BACKENDDIR) OPTFLAGS = -O3 # HIP requires -O2 to avoid "Memory access fault" in gq_ttq (#806) @@ -704,7 +713,10 @@ processid_short=$(shell basename $(CURDIR)) ###$(info processid_short=$(processid_short)) MADMATRIX_LIB = madmatrix_$(processid_short)_$(BACKEND) -objects_lib=$(BUILDDIR)/CPPProcess.o $(BUILDDIR)/color_sum.o $(BUILDDIR)/MatrixElementKernels.o $(BUILDDIR)/CrossSectionKernels.o $(BUILDDIR)/umami.o +objects_lib=$(BUILDDIR)/CPPProcess.o $(BUILDDIR)/color_sum.o $(BUILDDIR)/MatrixElementKernels.o $(BUILDDIR)/CrossSectionKernels.o $(BUILDDIR)/umami.o $(BUILDDIR)/SigmaKin.o + +# Backend-owned sources +vpath %%.cc backend/$(BACKENDDIR) # Explicitly define the default goal (this is not necessary as it is the first target, which is implicitly the default goal) .DEFAULT_GOAL := all.$(TAG) @@ -751,11 +763,11 @@ endif # incompatible backends (different BACKEND, FPTYPE, etc.) in the same directory. # Use USEBUILDDIR=1 to build for multiple backends simultaneously without cleaning. ifeq ($(GPUCC),) -$(BUILDDIR)/%%.o : %%.cc *.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) +$(BUILDDIR)/%%.o : %%.cc *.h backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(INCFLAGS) $(CXXFLAGS) -c $< -o $@ else -$(BUILDDIR)/%%.o : %%.cc *.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) +$(BUILDDIR)/%%.o : %%.cc *.h backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi $(GPUCC) $(CPPFLAGS) $(INCFLAGS) $(GPUFLAGS) -c -x $(GPULANGUAGE) $< -o $@ endif diff --git a/madmatrix/output.py b/madmatrix/output.py index d6218cd7ea..1252520a77 100644 --- a/madmatrix/output.py +++ b/madmatrix/output.py @@ -124,20 +124,13 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): sorted(os.listdir(pjoin(backend_template_dir, _backend_variant)))) del _backend_variant - to_link_in_P = ['nvtx.h', 'GpuRuntime.h', 'GpuAbstraction.h', 'color_sum.h', - 'MemoryAccessHelpers.h', 'MemoryAccessVectors.h', - 'MemoryAccessMatrixElements.h', 'MemoryAccessMomenta.h', - 'MemoryAccessRandomNumbers.h', 'MemoryAccessWeights.h', - 'MemoryAccessAmplitudes.h', 'MemoryAccessWavefunctions.h', - 'MemoryAccessGs.h', 'MemoryAccessCouplingsFixed.h', - 'MemoryAccessNumerators.h', 'MemoryAccessDenominators.h', - 'MemoryAccessChannelIds.h', 'MemoryAccessIflavorVec.h', - 'CrossSectionKernels.cc', 'CrossSectionKernels.h', - 'MatrixElementKernels.cc', 'MatrixElementKernels.h', - 'EventStatistics.h', - 'MemoryBuffers.h', # this is generated from a template in Subprocesses but we still link it in P1 - 'MemoryAccessCouplings.h', # this is generated from a template in Subprocesses but we still link it in P1 - 'umami.h', 'umami.cc', 'rambo.h'] + # Backend-owned skeleton files (GpuRuntime.h, color_sum.{h,cc}, the + # MemoryAccess*.h family, MatrixElementKernels/CrossSectionKernels/umami.cc, + # etc.) are NOT linked flat into P* any more: they are only available via + # backend// (see _link_backend_dirs_in_P below), sourced by the + # Makefile's INCFLAGS/vpath (see BACKENDDIR in madmatrix.mk). Only files + # with no backend/ counterpart - genuinely backend-agnostic - stay here. + to_link_in_P = ['nvtx.h', 'umami.h', 'rambo.h'] template_src_make = pjoin(madmatrix_templates, 'madmatrix_src.mk') template_Sub_make = pjoin(madmatrix_templates, 'madmatrix.mk') From a1eadfe9fe5c0d139ba5b75bbd27ee8d0188bdd6 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 18:57:35 +0200 Subject: [PATCH 08/25] Redundant template file delete --- .../madmatrix/CrossSectionKernels.cc | 238 ------- .../madmatrix/CrossSectionKernels.h | 139 ---- .../madmatrix/EventStatistics.h | 175 ----- .../template_files/madmatrix/GpuAbstraction.h | 168 ----- .../template_files/madmatrix/GpuRuntime.h | 102 --- .../madmatrix/MatrixElementKernels.cc | 509 ------------- .../madmatrix/MatrixElementKernels.h | 250 ------- .../madmatrix/MemoryAccessAmplitudes.h | 165 ----- .../madmatrix/MemoryAccessChannelIds.h | 126 ---- .../madmatrix/MemoryAccessCouplings.h | 276 -------- .../madmatrix/MemoryAccessCouplingsFixed.h | 89 --- .../madmatrix/MemoryAccessDenominators.h | 31 - .../template_files/madmatrix/MemoryAccessGs.h | 171 ----- .../madmatrix/MemoryAccessHelpers.h | 158 ----- .../madmatrix/MemoryAccessIflavorVec.h | 125 ---- .../madmatrix/MemoryAccessMatrixElements.h | 147 ---- .../madmatrix/MemoryAccessMomenta.h | 276 -------- .../madmatrix/MemoryAccessNumerators.h | 31 - .../madmatrix/MemoryAccessRandomNumbers.h | 145 ---- .../madmatrix/MemoryAccessVectors.h | 138 ---- .../madmatrix/MemoryAccessWavefunctions.h | 170 ----- .../madmatrix/MemoryAccessWeights.h | 150 ---- .../template_files/madmatrix/MemoryBuffers.h | 603 ---------------- .../template_files/madmatrix/color_sum.cc | 434 ------------ .../template_files/madmatrix/color_sum.h | 105 --- .../template_files/madmatrix/madmatrix.mk | 10 +- .../iolibs/template_files/madmatrix/umami.cc | 667 ------------------ madmatrix/model_handling.py | 24 - madmatrix/output.py | 40 +- 29 files changed, 14 insertions(+), 5648 deletions(-) delete mode 100644 madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/EventStatistics.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/GpuAbstraction.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/GpuRuntime.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessAmplitudes.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessChannelIds.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplings.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplingsFixed.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessDenominators.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessGs.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessHelpers.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessIflavorVec.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessMatrixElements.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessMomenta.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessNumerators.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessRandomNumbers.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessVectors.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessWavefunctions.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryAccessWeights.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/MemoryBuffers.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/color_sum.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/color_sum.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/umami.cc diff --git a/madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.cc deleted file mode 100644 index 7a3e0e0519..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.cc +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "CrossSectionKernels.h" - -#include "GpuAbstraction.h" -#include "MemoryAccessMatrixElements.h" -#include "MemoryAccessWeights.h" -#include "MemoryBuffers.h" - -#include - -// ****************************************************************************************** -// *** NB: Disabling fast math is essential here, otherwise results are undefined *** -// *** NB: This file CrossSectionKernels.cc IS BUILT WITH -fno-fast-math in the Makefile! *** -// *** NB: Attempts with __attribute__((optimize("-fno-fast-math"))) were unsatisfactory *** -// ****************************************************************************************** - -inline bool -fp_is_nan( const fptype& fp ) -{ - //#pragma clang diagnostic push - //#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) - return std::isnan( fp ); // always false for clang in fast math mode (tautological compare)? - //#pragma clang diagnostic pop -} - -inline bool -fp_is_abnormal( const fptype& fp ) -{ - if( fp_is_nan( fp ) ) return true; - if( fp != fp ) return true; - return false; -} - -inline bool -fp_is_zero( const fptype& fp ) -{ - if( fp == 0 ) return true; - return false; -} - -// See https://en.cppreference.com/w/cpp/numeric/math/FP_categories -inline const char* -fp_show_class( const fptype& fp ) -{ - switch( std::fpclassify( fp ) ) - { - case FP_INFINITE: return "Inf"; - case FP_NAN: return "NaN"; - case FP_NORMAL: return "normal"; - case FP_SUBNORMAL: return "subnormal"; - case FP_ZERO: return "zero"; - default: return "unknown"; - } -} - -inline void -debug_me_is_abnormal( const fptype& me, size_t ievtALL ) -{ - std::cout << "DEBUG[" << ievtALL << "]" - << " ME=" << me - << " fpisabnormal=" << fp_is_abnormal( me ) - << " fpclass=" << fp_show_class( me ) - << " (me==me)=" << ( me == me ) - << " (me==me+1)=" << ( me == me + 1 ) - << " isnan=" << fp_is_nan( me ) - << " isfinite=" << std::isfinite( me ) - << " isnormal=" << std::isnormal( me ) - << " is0=" << ( me == 0 ) - << " is1=" << ( me == 1 ) - << " abs(ME)=" << std::abs( me ) - << " isnan=" << fp_is_nan( std::abs( me ) ) - << std::endl; -} - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ) - { - for( unsigned int ievt = 0; ievt < nevt; ievt++ ) - { - if( fp_is_abnormal( hstMEs[ievt] ) ) - { - std::cout << "WARNING! flagging abnormal ME for ievt=" << ievt << std::endl; - hstMEs[ievt] = std::sqrt( -1. ); - } - } - } - - //-------------------------------------------------------------------------- - - CrossSectionKernelHost::CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats, // output: event statistics - const size_t nevt ) - : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) - , NumberOfEvents( nevt ) - { - if( m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: samplingWeights must be a host array" ); - if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: matrixElements must be a host array" ); - if( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with samplingWeights" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with matrixElements" ); - } - - //-------------------------------------------------------------------------- - - void CrossSectionKernelHost::updateEventStatistics( const bool debug ) - { - EventStatistics stats; // new statistics for the new nevt events - // FIRST PASS: COUNT ALL/ABN/ZERO EVENTS, COMPUTE MIN/MAX, COMPUTE REFS AS MEANS OF SIMPLE SUMS - for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration - { - const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); - const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); - const size_t ievtALL = m_iter * nevt() + ievt; - // The following events are abnormal in a run with "-p 2048 256 12 -d" - // - check.exe/commonrand: ME[310744,451171,3007871,3163868,4471038,5473927] with fast math - // - check.exe/curand: ME[578162,1725762,2163579,5407629,5435532,6014690] with fast math - // - gcheck.exe/curand: ME[596016,1446938] with fast math - // Debug NaN/abnormal issues - //if ( ievtALL == 310744 ) // this ME is abnormal both with and without fast math - // debug_me_is_abnormal( me, ievtALL ); - //if ( ievtALL == 5473927 ) // this ME is abnormal only with fast math - // debug_me_is_abnormal( me, ievtALL ); - stats.nevtALL++; - if( fp_is_abnormal( me ) ) - { - if( debug ) // only printed out with "-p -d" (matrixelementALL is not filled without -p) - std::cout << "WARNING! ME[" << ievtALL << "] is NaN/abnormal" << std::endl; - stats.nevtABN++; - continue; - } - if( fp_is_zero( me ) ) stats.nevtZERO++; - stats.minME = std::min( stats.minME, (double)me ); - stats.maxME = std::max( stats.maxME, (double)me ); - stats.minWG = std::min( stats.minWG, (double)wg ); - stats.maxWG = std::max( stats.maxWG, (double)wg ); - stats.sumMEdiff += me; // NB stats.refME is 0 here - stats.sumWGdiff += wg; // NB stats.refWG is 0 here - } - stats.refME = stats.meanME(); // draft ref - stats.refWG = stats.meanWG(); // draft ref - stats.sumMEdiff = 0; - stats.sumWGdiff = 0; - // SECOND PASS: IMPROVE MEANS FROM SUMS OF DIFFS TO PREVIOUS REF, UPDATE REF - for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration - { - const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); - const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); - if( fp_is_abnormal( me ) ) continue; - stats.sumMEdiff += ( me - stats.refME ); - stats.sumWGdiff += ( wg - stats.refWG ); - } - stats.refME = stats.meanME(); // final ref - stats.refWG = stats.meanWG(); // final ref - stats.sumMEdiff = 0; - stats.sumWGdiff = 0; - // THIRD PASS: COMPUTE STDDEV FROM SQUARED SUMS OF DIFFS TO REF - for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration - { - const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); - const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); - if( fp_is_abnormal( me ) ) continue; - stats.sqsMEdiff += std::pow( me - stats.refME, 2 ); - stats.sqsWGdiff += std::pow( wg - stats.refWG, 2 ); - } - // FOURTH PASS: UPDATE THE OVERALL STATS BY ADDING THE NEW STATS - m_stats += stats; - // Increment the iterations counter - m_iter++; - } - - //-------------------------------------------------------------------------- -} - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - /* - //-------------------------------------------------------------------------- - - CrossSectionKernelDevice::CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats, // output: event statistics - const size_t gpublocks, - const size_t gputhreads ) - : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) - , NumberOfEvents( gpublocks*gputhreads ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if ( ! m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: samplingWeights must be a device array" ); - if ( ! m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelDevice: matrixElements must be a device array" ); - if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0" ); - if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0" ); - if ( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with samplingWeights" ); - if ( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch with matrixElements" ); - } - - //-------------------------------------------------------------------------- - - void CrossSectionKernelDevice::setGrid( const size_t gpublocks, const size_t gputhreads ) - { - if ( m_gpublocks == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gpublocks must be > 0 in setGrid" ); - if ( m_gputhreads == 0 ) throw std::runtime_error( "CrossSectionKernelDevice: gputhreads must be > 0 in setGrid" ); - if ( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "CrossSectionKernelDevice: nevt mismatch in setGrid" ); - } - - //-------------------------------------------------------------------------- - - void CrossSectionKernelDevice::updateEventStatistics( const bool debug ) - { - // Increment the iterations counter - m_iter++; - } - - //-------------------------------------------------------------------------- - */ - -} -#endif - -//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.h deleted file mode 100644 index d1e29b7999..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.h +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef CROSSSECTIONKERNELS_H -#define CROSSSECTIONKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "EventStatistics.h" -#include "MemoryBuffers.h" - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - // Helper function for Bridge.h: must be compiled without fast math - // Iterate through all output MEs and replace any NaN/abnormal ones by sqrt(-1) - void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ); - - //-------------------------------------------------------------------------- - - // A base class encapsulating the calculation of event statistics on a CPU host or on a GPU device - class CrossSectionKernelBase //: virtual public ICrossSectionKernel - { - protected: - - // Constructor from existing input and output buffers - CrossSectionKernelBase( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats ) // output: event statistics - : m_samplingWeights( samplingWeights ) - , m_matrixElements( matrixElements ) - , m_stats( stats ) - , m_iter( 0 ) - { - // NB: do not initialise EventStatistics (you may be asked to update an existing result) - } - - public: - - // Destructor - virtual ~CrossSectionKernelBase() {} - - // Update event statistics - virtual void updateEventStatistics( const bool debug = false ) = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - protected: - - // The buffer for the sampling weights - const BufferWeights& m_samplingWeights; - - // The buffer for the output matrix elements - const BufferMatrixElements& m_matrixElements; - - // The event statistics - EventStatistics& m_stats; - - // The number of iterations processed so far - size_t m_iter; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating the calculation of event statistics on a CPU host - class CrossSectionKernelHost final : public CrossSectionKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats, // output: event statistics - const size_t nevt ); - - // Destructor - virtual ~CrossSectionKernelHost() {} - - // Update event statistics - void updateEventStatistics( const bool debug = false ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - }; - - //-------------------------------------------------------------------------- - - /* -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating the calculation of event statistics on a GPU device - class CrossSectionKernelDevice : public CrossSectionKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats, // output: event statistics - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~CrossSectionKernelDevice(){} - - // Reset gpublocks and gputhreads - void setGrid( const size_t gpublocks, const size_t gputhreads ); - - // Update event statistics - void updateEventStatistics( const bool debug=false ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - - }; -#endif - */ - - //-------------------------------------------------------------------------- -} -#endif // CROSSSECTIONKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/EventStatistics.h deleted file mode 100644 index 4a1fd8464d..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/EventStatistics.h +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef EventStatistics_H -#define EventStatistics_H 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" // for npar (meGeVexponent) - -#include -#include -#include -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - // The EventStatistics struct is used to accumulate running aggregates of event statistics. - // This will eventually include the process cross section and the process maximum weight: - // one important case of EventStatistics will then be the "gridpack" result set, which is - // the output of the "integration" step and the input to "unweighted event generation" step. - // The current implementation only includes statistics for matrix elements (ME) and sampling weights (WG); - // in first approximation, the process cross section and maximum weight are just the mean ME and maximum ME, - // but eventually the sampling weights WG (e.g. from Rambo) must also be taken into account in the calculation. - // The implementation uses differences to reference values to improve numerical precision. - struct EventStatistics - { - public: - size_t nevtALL; // total number of events used - size_t nevtABN; // number of events used, where ME is abnormal (nevtABN <= nevtALL) - size_t nevtZERO; // number of not-abnormal events used, where ME is zero (nevtZERO <= nevtOK) - double minME; // minimum matrix element - double maxME; // maximum matrix element - double minWG; // minimum sampling weight - double maxWG; // maximum sampling weight - double refME; // "reference" matrix element (normally the current mean) - double refWG; // "reference" sampling weight (normally the current mean) - double sumMEdiff; // sum of diff to ref for matrix element - double sumWGdiff; // sum of diff to ref for sampling weight - double sqsMEdiff; // squared sum of diff to ref for matrix element - double sqsWGdiff; // squared sum of diff to ref for sampling weight - std::string tag; // a text tag for printouts - // Number of events used, where ME is not abnormal - size_t nevtOK() const { return nevtALL - nevtABN; } - // Mean matrix element - // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] - double meanME() const - { - return refME + ( nevtOK() > 0 ? sumMEdiff / nevtOK() : 0 ); - } - // Mean sampling weight - // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] - double meanWG() const - { - return refWG + ( nevtOK() > 0 ? sumWGdiff / nevtOK() : 0 ); - } - // Variance matrix element - // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] - double varME() const { return ( sqsMEdiff - std::pow( sumMEdiff, 2 ) / nevtOK() ) / nevtOK(); } - // Variance sampling weight - // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] - double varWG() const { return ( sqsWGdiff - std::pow( sumWGdiff, 2 ) / nevtOK() ) / nevtOK(); } - // Standard deviation matrix element - double stdME() const { return std::sqrt( varME() ); } - // Standard deviation sampling weight - double stdWG() const { return std::sqrt( varWG() ); } - // Update reference matrix element - void updateRefME( const double newRef ) - { - const double deltaRef = refME - newRef; - sqsMEdiff += deltaRef * ( 2 * sumMEdiff + nevtOK() * deltaRef ); - sumMEdiff += deltaRef * nevtOK(); - refME = newRef; - } - // Update reference sampling weight - void updateRefWG( const double newRef ) - { - const double deltaRef = refWG - newRef; - sqsWGdiff += deltaRef * ( 2 * sumWGdiff + nevtOK() * deltaRef ); - sumWGdiff += deltaRef * nevtOK(); - refWG = newRef; - } - // Constructor - EventStatistics() - : nevtALL( 0 ) - , nevtABN( 0 ) - , nevtZERO( 0 ) - , minME( std::numeric_limits::max() ) - , maxME( std::numeric_limits::lowest() ) - , minWG( std::numeric_limits::max() ) - , maxWG( std::numeric_limits::lowest() ) - , refME( 0 ) - , refWG( 0 ) - , sumMEdiff( 0 ) - , sumWGdiff( 0 ) - , sqsMEdiff( 0 ) - , sqsWGdiff( 0 ) - , tag( "" ) {} - // Combine two EventStatistics -#ifdef __clang__ - // Disable optimizations for this function in HIP (work around FPE crash #1003: originally using #if __HIP_CLANG_ONLY__) - // Disable optimizations for this function in clang tout court (work around FPE crash #1005: now using #ifdef __clang__) - // See https://clang.llvm.org/docs/LanguageExtensions.html#extensions-for-selectively-disabling-optimization - __attribute__( ( optnone ) ) -#endif - EventStatistics& - operator+=( const EventStatistics& stats ) - { - EventStatistics s1 = *this; // temporary copy - EventStatistics s2 = stats; // temporary copy - EventStatistics& sum = *this; - sum.nevtALL = s1.nevtALL + s2.nevtALL; - sum.nevtABN = s1.nevtABN + s2.nevtABN; - sum.nevtZERO = s1.nevtZERO + s2.nevtZERO; - sum.minME = std::min( s1.minME, s2.minME ); - sum.maxME = std::max( s1.maxME, s2.maxME ); - sum.minWG = std::min( s1.minWG, s2.minWG ); - sum.maxWG = std::max( s1.maxWG, s2.maxWG ); - sum.refME = ( s1.meanME() * s1.nevtOK() + s2.meanME() * s2.nevtOK() ) / sum.nevtOK(); // new mean ME - s1.updateRefME( sum.refME ); - s2.updateRefME( sum.refME ); - sum.sumMEdiff = s1.sumMEdiff + s2.sumMEdiff; - sum.sqsMEdiff = s1.sqsMEdiff + s2.sqsMEdiff; - sum.refWG = ( s1.meanWG() * s1.nevtOK() + s2.meanWG() * s2.nevtOK() ) / sum.nevtOK(); // new mean WG - s1.updateRefWG( sum.refWG ); - s2.updateRefWG( sum.refWG ); - sum.sumWGdiff = s1.sumWGdiff + s2.sumWGdiff; - sum.sqsWGdiff = s1.sqsWGdiff + s2.sqsWGdiff; - return sum; - } - // Printout - void printout( std::ostream& out ) const - { - const EventStatistics& s = *this; - constexpr int meGeVexponent = -( 2 * CPPProcess::npar - 8 ); - out << s.tag << "NumMatrixElems(notAbnormal) = " << s.nevtOK() << std::endl - << std::scientific // fixed format: affects all floats (default precision: 6) - << s.tag << "MeanMatrixElemValue = ( " << s.meanME() - << " +- " << s.stdME() / std::sqrt( s.nevtOK() ) << " ) GeV^" << meGeVexponent << std::endl // standard error - << s.tag << "[Min,Max]MatrixElemValue = [ " << s.minME - << " , " << s.maxME << " ] GeV^" << meGeVexponent << std::endl - << s.tag << "StdDevMatrixElemValue = ( " << s.stdME() - << std::string( 16, ' ' ) << " ) GeV^" << meGeVexponent << std::endl - << s.tag << "MeanWeight = ( " << s.meanWG() - << " +- " << s.stdWG() / std::sqrt( s.nevtOK() ) << std::endl // standard error - << s.tag << "[Min,Max]Weight = [ " << s.minWG - << " , " << s.maxWG << " ]" << std::endl - << s.tag << "StdDevWeight = ( " << s.stdWG() - << std::string( 16, ' ' ) << " )" << std::endl - << std::defaultfloat; // default format: affects all floats - } - }; - - //-------------------------------------------------------------------------- - - inline std::ostream& operator<<( std::ostream& out, const EventStatistics& s ) - { - s.printout( out ); - return out; - } - - //-------------------------------------------------------------------------- -} - -#endif // EventStatistics_H diff --git a/madgraph/iolibs/template_files/madmatrix/GpuAbstraction.h b/madgraph/iolibs/template_files/madmatrix/GpuAbstraction.h deleted file mode 100644 index 6f709dfe9a..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/GpuAbstraction.h +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: J. Teig (Jul 2023) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2020-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MG5AMC_GPUABSTRACTION_H -#define MG5AMC_GPUABSTRACTION_H 1 - -#include "mgOnGpuConfig.h" - -#include - -//-------------------------------------------------------------------------- - -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) - -#ifndef MGONGPU_HAS_NO_BLAS -#include "cublas_v2.h" -#endif - -#define gpuError_t cudaError_t -#define gpuPeekAtLastError cudaPeekAtLastError -#define gpuGetErrorString cudaGetErrorString -#define gpuSuccess cudaSuccess - -#define gpuMallocHost( ptr, size ) checkGpu( cudaMallocHost( ptr, size ) ) -#define gpuMalloc( ptr, size ) checkGpu( cudaMalloc( ptr, size ) ) - -#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( cudaMemcpy( dstData, srcData, srcBytes, func ) ) -#define gpuMemset( data, value, bytes ) checkGpu( cudaMemset( data, value, bytes ) ) -#define gpuMemcpyHostToDevice cudaMemcpyHostToDevice -#define gpuMemcpyDeviceToHost cudaMemcpyDeviceToHost -#define gpuMemcpyDeviceToDevice cudaMemcpyDeviceToDevice -#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( cudaMemcpyToSymbol( type1, type2, size ) ) - -#define gpuFree( ptr ) checkGpu( cudaFree( ptr ) ) -#define gpuFreeHost( ptr ) checkGpu( cudaFreeHost( ptr ) ) - -#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( cudaGetSymbolAddress( devPtr, symbol ) ) - -#define gpuSetDevice cudaSetDevice -#define gpuDeviceSynchronize cudaDeviceSynchronize -#define gpuDeviceReset cudaDeviceReset - -#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) -//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> -#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) -#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) - -#define gpuStream_t cudaStream_t -#define gpuStreamCreate( pStream ) checkGpu( cudaStreamCreate( pStream ) ) -#define gpuStreamDestroy( stream ) checkGpu( cudaStreamDestroy( stream ) ) -#define gpuMallocAsync( ptr, size, stream ) checkGpu( cudaMallocAsync( ptr, size, stream ) ) -#define gpuFreeAsync( ptr, stream ) checkGpu( cudaFreeAsync( ptr, stream ) ) -#define gpuStreamSynchronize( stream ) checkGpu( cudaStreamSynchronize( stream ) ) - -#define gpuBlasStatus_t cublasStatus_t -#define GPUBLAS_STATUS_SUCCESS CUBLAS_STATUS_SUCCESS -#ifndef MGONGPU_HAS_NO_BLAS -#define gpuBlasHandle_t cublasHandle_t -#else -#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds -#endif -#define gpuBlasCreate cublasCreate -#define gpuBlasDestroy cublasDestroy -#define gpuBlasSetStream cublasSetStream - -#define gpuBlasSaxpy cublasSaxpy -#define gpuBlasSdot cublasSdot -#define gpuBlasSgemv cublasSgemv -#define gpuBlasSgemm cublasSgemm -#define gpuBlasSgemmStridedBatched cublasSgemmStridedBatched -#define gpuBlasDaxpy cublasDaxpy -#define gpuBlasDdot cublasDdot -#define gpuBlasDgemv cublasDgemv -#define gpuBlasDgemm cublasDgemm -#define gpuBlasDgemmStridedBatched cublasDgemmStridedBatched -#define GPUBLAS_OP_N CUBLAS_OP_N -#define GPUBLAS_OP_T CUBLAS_OP_T - -//-------------------------------------------------------------------------- - -#elif defined __HIPCC__ - -#ifndef MGONGPU_HAS_NO_BLAS -#include "hipblas/hipblas.h" -#endif - -#define gpuError_t hipError_t -#define gpuPeekAtLastError hipPeekAtLastError -#define gpuGetErrorString hipGetErrorString -#define gpuSuccess hipSuccess - -#define gpuMallocHost( ptr, size ) checkGpu( hipHostMalloc( ptr, size ) ) // HostMalloc better -#define gpuMalloc( ptr, size ) checkGpu( hipMalloc( ptr, size ) ) - -#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( hipMemcpy( dstData, srcData, srcBytes, func ) ) -#define gpuMemset( data, value, bytes ) checkGpu( hipMemset( data, value, bytes ) ) -#define gpuMemcpyHostToDevice hipMemcpyHostToDevice -#define gpuMemcpyDeviceToHost hipMemcpyDeviceToHost -#define gpuMemcpyDeviceToDevice hipMemcpyDeviceToDevice -#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( hipMemcpyToSymbol( type1, type2, size ) ) - -#define gpuFree( ptr ) checkGpu( hipFree( ptr ) ) -#define gpuFreeHost( ptr ) checkGpu( hipHostFree( ptr ) ) - -#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( hipGetSymbolAddress( devPtr, symbol ) ) - -#define gpuSetDevice hipSetDevice -#define gpuDeviceSynchronize hipDeviceSynchronize -#define gpuDeviceReset hipDeviceReset - -#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) -//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> -#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) -#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) - -#define gpuStream_t hipStream_t -#define gpuStreamCreate( pStream ) checkGpu( hipStreamCreate( pStream ) ) -#define gpuStreamDestroy( stream ) checkGpu( hipStreamDestroy( stream ) ) -#define gpuMallocAsync( ptr, size, stream ) checkGpu( hipMallocAsync( ptr, size, stream ) ) -#define gpuFreeAsync( ptr, stream ) checkGpu( hipFreeAsync( ptr, stream ) ) -#define gpuStreamSynchronize( stream ) checkGpu( hipStreamSynchronize( stream ) ) - -#define gpuBlasStatus_t hipblasStatus_t -#define GPUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS -#ifndef MGONGPU_HAS_NO_BLAS -#define gpuBlasHandle_t hipblasHandle_t -#else -#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds -#endif -#define gpuBlasCreate hipblasCreate -#define gpuBlasDestroy hipblasDestroy -#define gpuBlasSetStream hipblasSetStream - -#define gpuBlasSaxpy hipblasSaxpy -#define gpuBlasSdot hipblasSdot -#define gpuBlasSgemv hipblasSgemv -#define gpuBlasSgemm hipblasSgemm -#define gpuBlasSgemmStridedBatched hipblasSgemmStridedBatched -#define gpuBlasDaxpy hipblasDaxpy -#define gpuBlasDdot hipblasDdot -#define gpuBlasDgemv hipblasDgemv -#define gpuBlasDgemm hipblasDgemm -#define gpuBlasDgemmStridedBatched hipblasDgemmStridedBatched -#define GPUBLAS_OP_N HIPBLAS_OP_N -#define GPUBLAS_OP_T HIPBLAS_OP_T - -#endif - -//-------------------------------------------------------------------------- - -#ifdef MGONGPU_FPTYPE2_FLOAT -#define gpuBlasTaxpy gpuBlasSaxpy -#define gpuBlasTdot gpuBlasSdot -#define gpuBlasTgemv gpuBlasSgemv -#define gpuBlasTgemm gpuBlasSgemm -#define gpuBlasTgemmStridedBatched gpuBlasSgemmStridedBatched -#else -#define gpuBlasTaxpy gpuBlasDaxpy -#define gpuBlasTdot gpuBlasDdot -#define gpuBlasTgemv gpuBlasDgemv -#define gpuBlasTgemm gpuBlasDgemm -#define gpuBlasTgemmStridedBatched gpuBlasDgemmStridedBatched -#endif - -#endif // MG5AMC_GPUABSTRACTION_H diff --git a/madgraph/iolibs/template_files/madmatrix/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/GpuRuntime.h deleted file mode 100644 index 2c8ae1a804..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/GpuRuntime.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: J. Teig (Jun 2023, based on earlier work by S. Roiser) for the MG5aMC CUDACPP plugin. -// Further modified by: O. Mattelaer, S. Roiser, J. Teig, A. Valassi, Z. Wettersten (2020-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MG5AMC_GPURUNTIME_H -#define MG5AMC_GPURUNTIME_H 1 - -// MG5AMC on GPU uses the CUDA runtime API, not the lower level CUDA driver API -// See https://docs.nvidia.com/cuda/cuda-runtime-api/driver-vs-runtime-api.html#driver-vs-runtime-api - -#include "GpuAbstraction.h" - -#include - -//-------------------------------------------------------------------------- - -// See https://stackoverflow.com/a/14038590 -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ -#define checkGpu( code ) { assertGpu( code, __FILE__, __LINE__ ); } -inline void assertGpu( gpuError_t code, const char* file, int line, bool abort = true ) -{ - if( code != gpuSuccess ) - { - printf( "ERROR! assertGpu: '%s' (%d) in %s:%d\n", gpuGetErrorString( code ), code, file, line ); - if( abort ) assert( code == gpuSuccess ); - } -} -#endif /* clang-format on */ - -//-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ -#ifndef MGONGPU_HAS_NO_BLAS -#define checkGpuBlas( code ){ assertGpuBlas( code, __FILE__, __LINE__ ); } -inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, bool abort = true ) -{ - if ( code != GPUBLAS_STATUS_SUCCESS ) - { - printf( "ERROR! assertGpuBlas: '%d' in %s:%d\n", code, file, line ); - if( abort ) assert( code == GPUBLAS_STATUS_SUCCESS ); - } -} -#endif -#endif /* clang-format on */ - -//-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - // Instantiate a GpuRuntime at the beginnining of the application's main to - // invoke gpuSetDevice(0) in the constructor and book a gpuDeviceReset() call in the destructor - // *** FIXME! This will all need to be designed differently when going to multi-GPU nodes! *** - struct GpuRuntime final - { - GpuRuntime( const bool debug = true ) - : m_debug( debug ) { setUp( m_debug ); } - ~GpuRuntime() { tearDown( m_debug ); } - GpuRuntime( const GpuRuntime& ) = delete; - GpuRuntime( GpuRuntime&& ) = delete; - GpuRuntime& operator=( const GpuRuntime& ) = delete; - GpuRuntime& operator=( GpuRuntime&& ) = delete; - bool m_debug; - - // Set up CUDA application - // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** - // Calling cudaSetDevice on startup is useful to properly book-keep the time spent in CUDA initialization - static void setUp( const bool debug = false ) // ZW: changed debug default to false - { - // ** NB: it is useful to call cudaSetDevice, or cudaFree, to properly book-keep the time spent in CUDA initialization - // ** NB: otherwise, the first CUDA operation (eg a cudaMemcpyToSymbol in CPPProcess ctor) appears to take much longer! - /* - // [We initially added cudaFree(0) to "ease profile analysis" only because it shows up as a big recognizable block!] - // No explicit initialization is needed: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#initialization - // It is not clear what cudaFree(0) does at all: https://stackoverflow.com/questions/69967813/ - if ( debug ) std::cout << "__CudaRuntime: calling cudaFree(0)" << std::endl; - checkCuda( cudaFree( 0 ) ); // SLOW! - */ - // Replace cudaFree(0) by cudaSetDevice(0), even if it is not really needed either - // (but see https://developer.nvidia.com/blog/cuda-pro-tip-always-set-current-device-avoid-multithreading-bugs) - if( debug ) std::cout << "__GpuRuntime: calling GpuSetDevice(0)" << std::endl; - checkGpu( gpuSetDevice( 0 ) ); // SLOW! - } - - // Tear down CUDA application (call cudaDeviceReset) - // ** NB: strictly speaking this is not needed when using the CUDA runtime API ** - // Calling cudaDeviceReset on shutdown is only needed for checking memory leaks in cuda-memcheck - // See https://docs.nvidia.com/cuda/cuda-memcheck/index.html#leak-checking - static void tearDown( const bool debug = false ) // ZW: changed debug default to false - { - if( debug ) std::cout << "__GpuRuntime: calling GpuDeviceReset()" << std::endl; - checkGpu( gpuDeviceReset() ); - } - }; -} -#endif - -//-------------------------------------------------------------------------- - -#endif // MG5AMC_GPURUNTIME_H diff --git a/madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.cc deleted file mode 100644 index 872e4795e3..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.cc +++ /dev/null @@ -1,509 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "MatrixElementKernels.h" - -#include "CPPProcess.h" -#include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation -#include "MemoryAccessMomenta.h" -#include "MemoryBuffers.h" - -#include // for fetestexcept -#include -#include - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - MatrixElementKernelBase::MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol ) // output: color selection - : m_momenta( momenta ) - , m_gs( gs ) - , m_iflavorVec( iflavorVec ) - , m_rndhel( rndhel ) - , m_rndcol( rndcol ) - , m_channelIds( channelIds ) - , m_matrixElements( matrixElements ) - , m_selhel( selhel ) - , m_selcol( selcol ) -#ifdef MGONGPU_CHANNELID_DEBUG - , m_nevtProcessedByChannel() - , m_tag() -#endif - { - //std::cout << "DEBUG: MatrixElementKernelBase ctor " << this << std::endl; -#ifdef MGONGPU_CHANNELID_DEBUG - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) - m_nevtProcessedByChannel[channelId] = 0; -#endif - } - - //-------------------------------------------------------------------------- - - MatrixElementKernelBase::~MatrixElementKernelBase() - { - //std::cout << "DEBUG: MatrixElementKernelBase dtor " << this << std::endl; -#ifdef MGONGPU_CHANNELID_DEBUG - MatrixElementKernelBase::dumpNevtProcessedByChannel(); -#endif -#ifdef MGONGPUCPP_VERBOSE - MatrixElementKernelBase::dumpSignallingFPEs(); -#endif - } - - //-------------------------------------------------------------------------- - -#ifdef MGONGPU_CHANNELID_DEBUG - void MatrixElementKernelBase::updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ) - { - if( pHstChannelIds != nullptr ) - { - //std::cout << "DEBUG " << this << ": not nullptr " << nevt << std::endl; - for( unsigned int ievt = 0; ievt < nevt; ievt++ ) - { - const size_t channelId = pHstChannelIds[ievt]; // Fortran indexing - //assert( channelId > 0 ); - //assert( channelId < CPPProcess::ndiagrams ); - m_nevtProcessedByChannel[channelId]++; - } - } - else - { - //std::cout << "DEBUG " << this << ": nullptr " << std::endl; - m_nevtProcessedByChannel[0] += nevt; - } - } -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPU_CHANNELID_DEBUG - void MatrixElementKernelBase::dumpNevtProcessedByChannel() - { - size_t nevtProcessed = 0; - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) - nevtProcessed += m_nevtProcessedByChannel[channelId]; - std::ostringstream sstr; - sstr << " {"; - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) - { - if( m_nevtProcessedByChannel[channelId] > 0 ) - { - if( sstr.str() != " {" ) sstr << ","; - if( channelId == 0 ) - sstr << " no-multichannel"; - else - sstr << " " << channelId; - sstr << " : " << m_nevtProcessedByChannel[channelId]; - } - } - sstr << " }"; - std::cout << "DEBUG: MEK " << this; - if( m_tag != "" ) std::cout << " " << m_tag; - std::cout << " processed " << nevtProcessed << " events across " << CPPProcess::ndiagrams << " channels" << sstr.str() << std::endl; - } -#endif - - //-------------------------------------------------------------------------- - - void MatrixElementKernelBase::dumpSignallingFPEs() - { - // New strategy for issue #831: add a final report of FPEs - // Note: normally only underflow will be reported here (inexact is switched off because it would almost always signal; - // divbyzero, invalid and overflow are configured by feenablexcept to send a SIGFPE signal, and are normally fixed in the code) - // Note: this is now called in the individual destructors of MEK classes rather than in that of MatrixElementKernelBase(#837) - std::string fpes; - if( std::fetestexcept( FE_DIVBYZERO ) ) fpes += " FE_DIVBYZERO"; - if( std::fetestexcept( FE_INVALID ) ) fpes += " FE_INVALID"; - if( std::fetestexcept( FE_OVERFLOW ) ) fpes += " FE_OVERFLOW"; - if( std::fetestexcept( FE_UNDERFLOW ) ) fpes += " FE_UNDERFLOW"; - //if( std::fetestexcept( FE_INEXACT ) ) fpes += " FE_INEXACT"; // do not print this out: this would almost always signal! - if( fpes == "" ) - std::cout << "INFO: No Floating Point Exceptions have been reported" << std::endl; - else - std::cerr << "INFO: The following Floating Point Exceptions have been reported:" << fpes << std::endl; - } - - //-------------------------------------------------------------------------- -} - -//============================================================================ - -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu -{ - - //-------------------------------------------------------------------------- - - MatrixElementKernelHost::MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt ) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( nevt ) - , m_couplings( nevt ) - , m_numerators( nevt * CPPProcess::ndiagrams ) - , m_denominators( nevt ) - { - //std::cout << "DEBUG: MatrixElementKernelHost::ctor " << this << std::endl; - if( m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: momenta must be a host array" ); - if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: matrixElements must be a host array" ); - if( m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: channelIds must be a device array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with matrixElements" ); - if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with channelIds" ); - if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with iflavorVec" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( nevt % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MatrixElementKernelHost: nevt should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Fail gently and avoid "Illegal instruction (core dumped)" if the host does not support the SIMD used in the ME calculation - // Note: this prevents a crash on pmpe04 but not on some github CI nodes? - // [NB: SIMD vectorization in mg5amc C++ code is only used in the ME calculation below MatrixElementKernelHost!] - if( !MatrixElementKernelHost::hostSupportsSIMD() ) - throw std::runtime_error( "Host does not support the SIMD implementation of MatrixElementKernelsHost" ); - } - - //-------------------------------------------------------------------------- - - MatrixElementKernelHost::~MatrixElementKernelHost() - { - //std::cout << "DEBUG: MatrixElementKernelBase::dtor " << this << std::endl; - } - - //-------------------------------------------------------------------------- - - int MatrixElementKernelHost::computeGoodHelicities() - { - HostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); - // ... 0d1. Compute good helicity mask on the host - computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); - sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_numerators.data(), m_denominators.data(), hstIsGoodHel.data(), nevt() ); - // ... 0d2. Copy good helicity list to static memory on the host - // [FIXME! REMOVE THIS STATIC THAT BREAKS MULTITHREADING?] - return sigmaKin_setGoodHel( hstIsGoodHel.data() ); - } - - //-------------------------------------------------------------------------- - - void MatrixElementKernelHost::computeMatrixElements( const bool useChannelIds ) - { - computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_numerators.data(), m_denominators.data(), nullptr, true, nevt() ); -#ifdef MGONGPU_CHANNELID_DEBUG - //std::cout << "DEBUG: MatrixElementKernelHost::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; - MatrixElementKernelBase::updateNevtProcessedByChannel( pChannelIds, nevt() ); -#endif - } - - //-------------------------------------------------------------------------- - - // Does this host system support the SIMD used in the matrix element calculation? - bool MatrixElementKernelHost::hostSupportsSIMD( const bool verbose ) - { -#if defined __AVX512VL__ - bool known = true; - bool ok = __builtin_cpu_supports( "avx512vl" ); - const std::string tag = "skylake-avx512 (AVX512VL)"; -#elif defined __AVX2__ - bool known = true; - bool ok = __builtin_cpu_supports( "avx2" ); - const std::string tag = "haswell (AVX2)"; -#elif defined __SSE4_2__ -#ifdef __PPC__ - // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html - bool known = true; - bool ok = __builtin_cpu_supports( "vsx" ); - const std::string tag = "powerpc vsx (128bit as in SSE4.2)"; -#elif defined( __x86_64__ ) || defined( __i386__ ) - bool known = true; - bool ok = __builtin_cpu_supports( "sse4.2" ); - const std::string tag = "nehalem (SSE4.2)"; -#else // AV FIXME! Added by OM for Mac, should identify the correct __xxx__ flag that should be targeted - // DM now we have an explicit NEON target for ARM - bool known = false; // __builtin_cpu_supports is not supported - bool ok = true; // this is just an assumption! - const std::string tag = "simd arch not defined"; -#endif -#elif defined __ARM_NEON // consider using __BUILTIN_CPU_SUPPORTS__ - bool known = false; // __builtin_cpu_supports is not supported - // See https://stackoverflow.com/q/62783908 - // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu - bool ok = true; // this is just an assumption! - const std::string tag = "arm neon (128bit as in SSE4.2)"; -#else - bool known = true; - bool ok = true; - const std::string tag = "none"; -#endif - if( verbose ) - { - if( tag == "none" ) - std::cout << "INFO: The application does not require the host to support any AVX feature" << std::endl; - else if( ok && known ) - std::cout << "INFO: The application is built for " << tag << " and the host supports it" << std::endl; - else if( ok ) - std::cout << "WARNING: The application is built for " << tag << " but it is unknown if the host supports it" << std::endl; - else - std::cout << "ERROR! The application is built for " << tag << " but the host does not support it" << std::endl; - } - return ok; - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - //-------------------------------------------------------------------------- - - MatrixElementKernelDevice::MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( gpublocks * gputhreads ) - , m_couplings( this->nevt() ) - , m_pHelMEs() - , m_pHelJamps() - , m_pHelNumerators() - , m_pHelDenominators() - , m_colJamp2s( CPPProcess::ncolor * this->nevt() ) -#ifdef MGONGPU_CHANNELID_DEBUG - , m_hstChannelIds( this->nevt() ) -#endif -#ifndef MGONGPU_HAS_NO_BLAS - , m_blasColorSum( false ) - , m_blasTf32Tensor( false ) - , m_pHelBlasTmp() - , m_blasHandle() -#endif - , m_helStreams() - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - //std::cout << "DEBUG: MatrixElementKernelDevice::ctor " << this << std::endl; - if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: momenta must be a device array" ); - if( !m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: matrixElements must be a device array" ); - if( !m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: channelIds must be a device array" ); // FIXME?! - if( !m_iflavorVec.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelDevice: iflavorVec must be a device array" ); - if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with matrixElements" ); - if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with channelIds" ); - if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch with iflavorVec" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( m_gputhreads % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MatrixElementKernelHost: gputhreads should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Create the "one-helicity" jamp buffer that will be used for helicity filtering - m_pHelJamps.reset( new DeviceBufferSimple( CPPProcess::ncolor * mgOnGpu::nx2 * this->nevt() ) ); - // Create the "one-helicity" numerator and denominator buffers that will be used for helicity filtering - m_pHelNumerators.reset( new DeviceBufferSimple( this->nevt() * CPPProcess::ndiagrams ) ); - m_pHelDenominators.reset( new DeviceBufferSimple( this->nevt() ) ); - // Decide at runtime whether to use BLAS for color sums - // Decide at runtime whether TF32TENSOR math should be used in cuBLAS - static bool first = true; - if( first ) - { - first = false; - // Analyse environment variable CUDACPP_RUNTIME_BLASCOLORSUM - const char* blasEnv = getenv( "CUDACPP_RUNTIME_BLASCOLORSUM" ); - if( blasEnv && std::string( blasEnv ) != "" ) - { -#ifndef MGONGPU_HAS_NO_BLAS - m_blasColorSum = true; // fixme? eventually set default=true and decode "Y" and "N" choices? - std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty: enable BLAS" << std::endl; -#else - throw std::runtime_error( "Env variable CUDACPP_RUNTIME_BLASCOLORSUM is set and non-empty, but BLAS was disabled at build time" ); -#endif - } - else - { -#ifndef MGONGPU_HAS_NO_BLAS - std::cout << "INFO: Env variable CUDACPP_RUNTIME_BLASCOLORSUM is empty or not set: disable BLAS" << std::endl; -#else - std::cout << "INFO: BLAS was disabled at build time" << std::endl; -#endif - } -#ifndef MGONGPU_HAS_NO_BLAS -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) - // Analyse environment variable CUDACPP_RUNTIME_CUBLASTF32TENSOR - const char* blasEnv2 = getenv( "CUDACPP_RUNTIME_CUBLASTF32TENSOR" ); - if( blasEnv2 && std::string( blasEnv2 ) != "" ) - { - if( m_blasColorSum ) - { -#ifdef MGONGPU_FPTYPE2_FLOAT - m_blasTf32Tensor = true; - std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty: enable CUBLAS_TF32_TENSOR_OP_MATH" << std::endl; -#else - std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but color sums use FP64" << std::endl; -#endif - } - else - std::cout << "WARNING! Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is set and non-empty, but BLAS was disabled at runtime" << std::endl; - } -#ifdef MGONGPU_FPTYPE2_FLOAT - else - { - if( m_blasColorSum ) - std::cout << "INFO: Env variable CUDACPP_RUNTIME_CUBLASTF32TENSOR is empty or not set: keep cuBLAS math defaults" << std::endl; - } -#endif -#endif -#endif - } - } - - //-------------------------------------------------------------------------- - - MatrixElementKernelDevice::~MatrixElementKernelDevice() - { - //std::cout << "DEBUG: MatrixElementKernelDevice::dtor " << this << std::endl; -#ifndef MGONGPU_HAS_NO_BLAS - if( m_blasHandle ) gpuBlasDestroy( m_blasHandle ); -#endif - for( int ihel = 0; ihel < CPPProcess::ncomb; ihel++ ) - { - if( m_helStreams[ihel] ) gpuStreamDestroy( m_helStreams[ihel] ); // do not destroy if nullptr - } - } - - //-------------------------------------------------------------------------- - - // FIXME! The relevance of this function should be reassessed (#543 and #902) - void MatrixElementKernelDevice::setGrid( const int /*gpublocks*/, const int /*gputhreads*/ ) - { - if( m_gpublocks == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gpublocks must be > 0 in setGrid" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "MatrixElementKernelDevice: gputhreads must be > 0 in setGrid" ); - if( this->nevt() != m_gpublocks * m_gputhreads ) throw std::runtime_error( "MatrixElementKernelDevice: nevt mismatch in setGrid" ); - } - - //-------------------------------------------------------------------------- - - int MatrixElementKernelDevice::computeGoodHelicities() - { - PinnedHostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); - // ... 0d1. Compute good helicity mask (a host variable) on the device - gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); - const int nevt = m_gpublocks * m_gputhreads; - sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_pHelJamps->data(), m_pHelNumerators->data(), m_pHelDenominators->data(), hstIsGoodHel.data(), nevt ); - // ... 0d3. Set good helicity list in host static memory - int nGoodHel = sigmaKin_setGoodHel( hstIsGoodHel.data() ); - assert( nGoodHel > 0 ); // SANITY CHECK: there should be at least one good helicity - // Create one GPU stream for each good helicity - for( int ighel = 0; ighel < nGoodHel; ighel++ ) - gpuStreamCreate( &m_helStreams[ighel] ); -#ifndef MGONGPU_HAS_NO_BLAS - // Create one cuBLAS/hipBLAS handle for each good helicity (attached to the default stream) - if( m_blasColorSum ) - { - checkGpuBlas( gpuBlasCreate( &m_blasHandle ) ); -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) - if( m_blasTf32Tensor ) - checkGpuBlas( cublasSetMathMode( m_blasHandle, CUBLAS_TF32_TENSOR_OP_MATH ) ); // enable TF32 tensor cores -#endif - } -#endif - // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) - m_pHelMEs.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); - // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) - // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) - m_pHelJamps.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); - // ... Create the "many-helicity" super-buffers of nGoodHel numerator and denominator buffers (dynamically allocated) - // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) - m_pHelNumerators.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ndiagrams * nevt ) ); - m_pHelDenominators.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); -#ifndef MGONGPU_HAS_NO_BLAS - // Create the "many-helicity" super-buffers of real/imag ncolor*nevt temporary buffers for cuBLAS/hipBLAS intermediate results in color_sum_blas -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Mixed precision mode: need two fptype2[ncolor*2*nevt] buffers and one fptype2[nevt] buffer per good helicity - if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ( 2 * CPPProcess::ncolor * mgOnGpu::nx2 + 1 ) * nevt ) ); -#else - // Standard single/double precision mode: need one fptype2[ncolor*2*nevt] buffer per good helicity - if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); -#endif -#endif - // Return the number of good helicities - return nGoodHel; - } - - //-------------------------------------------------------------------------- - - void MatrixElementKernelDevice::computeMatrixElements( const bool useChannelIds ) - { - gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); -#ifndef MGONGPU_HAS_NO_BLAS - fptype2* ghelAllBlasTmp = ( m_blasColorSum ? m_pHelBlasTmp->data() : nullptr ); - gpuBlasHandle_t* pBlasHandle = ( m_blasColorSum ? &m_blasHandle : nullptr ); -#else - fptype2* ghelAllBlasTmp = nullptr; - gpuBlasHandle_t* pBlasHandle = nullptr; -#endif - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_colJamp2s.data(), m_pHelNumerators->data(), m_pHelDenominators->data(), nullptr, true, m_pHelMEs->data(), m_pHelJamps->data(), ghelAllBlasTmp, pBlasHandle, m_helStreams, false, m_gpublocks, m_gputhreads ); -#ifdef MGONGPU_CHANNELID_DEBUG - //std::cout << "DEBUG: MatrixElementKernelDevice::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; - copyHostFromDevice( m_hstChannelIds, m_channelIds ); // FIXME?! - const unsigned int* pHstChannelIds = ( useChannelIds ? m_hstChannelIds.data() : nullptr ); - MatrixElementKernelBase::updateNevtProcessedByChannel( pHstChannelIds, nevt() ); -#endif - checkGpu( gpuPeekAtLastError() ); // is this needed? - checkGpu( gpuDeviceSynchronize() ); // probably not needed? but it avoids errors in sigmaKin above from appearing later on in random places... - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.h deleted file mode 100644 index 01041e43b7..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.h +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MATRIXELEMENTKERNELS_H -#define MATRIXELEMENTKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "GpuAbstraction.h" -#include "MemoryBuffers.h" - -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - // A base class encapsulating matrix element calculations on a CPU host or on a GPU device - class MatrixElementKernelBase //: virtual public IMatrixElementKernel - { - protected: - - // Constructor from existing input and output buffers - MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol); // output: color selection - - public: - - // Destructor - virtual ~MatrixElementKernelBase(); - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - virtual int computeGoodHelicities() = 0; - - // Compute matrix elements - virtual void computeMatrixElements( const bool useChannelIds ) = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - // Dump signalling FPEs (#831 and #837) - static void dumpSignallingFPEs(); - -#ifdef MGONGPU_CHANNELID_DEBUG - // Add a MEK identifier for the channelId debug printout - void setTagForNevtProcessedByChannel( const std::string& tag ) { m_tag = tag; } - - protected: - // Update number of events processed by channel - void updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ); - - // Dump number of events processed by channel - void dumpNevtProcessedByChannel(); -#endif - - protected: - - // The buffer for the input momenta - const BufferMomenta& m_momenta; - - // The buffer for the gs to calculate the alphaS values - const BufferGs& m_gs; - - // The buffer for the flavor indices for the flavor combination - const BufferIflavorVec& m_iflavorVec; - - // The buffer for the random numbers for helicity selection - const BufferRndNumHelicity& m_rndhel; - - // The buffer for the random numbers for color selection - const BufferRndNumColor& m_rndcol; - - // The buffer for the channel ids for single-diagram enhancement - const BufferChannelIds& m_channelIds; - - // The buffer for the output matrix elements - BufferMatrixElements& m_matrixElements; - - // The buffer for the output helicity selection - BufferSelectedHelicity& m_selhel; - - // The buffer for the output color selection - BufferSelectedColor& m_selcol; - -#ifdef MGONGPU_CHANNELID_DEBUG - // The events-per-channel counter for debugging - std::map m_nevtProcessedByChannel; - - // The tag for events-per-channel debugging - std::string m_tag; -#endif - }; - - //-------------------------------------------------------------------------- - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating matrix element calculations on a CPU host - class MatrixElementKernelHost final : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~MatrixElementKernelHost(); - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // Does this host system support the SIMD used in the matrix element calculation? - // [NB: this is private, SIMD vectorization in mg5amc C++ code is currently only used in the ME calculations below MatrixElementKernelHost!] - static bool hostSupportsSIMD( const bool verbose = false ); // ZW: default verbose false - - private: - - // The buffer for the event-by-event couplings that depends on alphas QCD - HostBufferCouplings m_couplings; - - // The buffer for the event-by-event numerators of multichannel factors - HostBufferNumerators m_numerators; - - // The buffer for the event-by-event denominators of multichannel factors - HostBufferDenominators m_denominators; - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating matrix element calculations on a GPU device - class MatrixElementKernelDevice : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MatrixElementKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~MatrixElementKernelDevice(); - - // Reset gpublocks and gputhreads - void setGrid( const int gpublocks, const int gputhreads ); - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The buffer for the event-by-event couplings that depends on alphas QCD - DeviceBufferCouplings m_couplings; - - // The super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelMEs; - - // The super-buffer of nGoodHel jamp buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelJamps; - - // The super-buffer of nGoodHel numerator buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelNumerators; - - // The super-buffer of nGoodHel denominator buffers (dynamically allocated because nGoodHel is determined at runtime) - std::unique_ptr m_pHelDenominators; - - // The super-buffer of ncolor jamp2 buffers - DeviceBufferSimple m_colJamp2s; - -#ifdef MGONGPU_CHANNELID_DEBUG - // The **host** buffer for the channelId array - // FIXME? MEKD should accept a host buffer as an argument instead of a device buffer, so that a second copy can be avoided? - PinnedHostBufferChannelIds m_hstChannelIds; -#endif - -#ifndef MGONGPU_HAS_NO_BLAS - // Decide at runtime whether to use BLAS for color sums - bool m_blasColorSum; - - // Decide at runtime whether TF32TENSOR math should be used in cuBLAS - bool m_blasTf32Tensor; - - // The super-buffer of nGoodHel cuBLAS/hipBLAS temporary buffers - std::unique_ptr m_pHelBlasTmp; - - // The cuBLAS/hipBLAS handle (a single one for all good helicities) - gpuBlasHandle_t m_blasHandle; -#endif - - // The array of GPU streams (one for each good helicity) - gpuStream_t m_helStreams[CPPProcess::ncomb]; // reserve ncomb streams (but only nGoodHel <= ncomb will be used) - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; -#endif - - //-------------------------------------------------------------------------- -} -#endif // MATRIXELEMENTKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessAmplitudes.h deleted file mode 100644 index f41e40bf82..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessAmplitudes.h +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessAmplitudes_H -#define MemoryAccessAmplitudes_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuCxtypes.h" - -#include "MemoryAccessHelpers.h" - -#define MGONGPU_TRIVIAL_AMPLITUDES 1 - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - -#ifndef MGONGPU_TRIVIAL_AMPLITUDES - - // A class describing the internal layout of memory buffers for amplitudes - // This implementation uses an AOSOA[npagA][nx2][neppA] where nevt=npagA*neppA - // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] - class MemoryAccessAmplitudesBase //_AOSOAv1 - { - public: - - // Number of Events Per Page in the amplitude AOSOA memory buffer layout - static constexpr int neppA = 1; // AOS (just a test...) - - private: - - friend class MemoryAccessHelper; - friend class KernelAccessHelper; - friend class KernelAccessHelper; - - // The number of floating point components of a complex number - static constexpr int nx2 = mgOnGpu::nx2; - - //-------------------------------------------------------------------------- - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static __host__ __device__ inline fptype* - ieventAccessRecord( fptype* buffer, - const int ievt ) - { - const int ipagA = ievt / neppA; // #event "A-page" - const int ieppA = ievt % neppA; // #event in the current event A-page - constexpr int ix2 = 0; - return &( buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA] ); // AOSOA[ipagA][ix2][ieppA] - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] - // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] - static __host__ __device__ inline fptype& - decodeRecord( fptype* buffer, - const int ix2 ) - { - constexpr int ipagA = 0; - constexpr int ieppA = 0; - return buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA]; // AOSOA[ipagA][ix2][ieppA] - } - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on explicit event numbers - // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations - class MemoryAccessAmplitudes : public MemoryAccessAmplitudesBase - { - public: - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] - static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] - static constexpr auto decodeRecordIx2Const = - MemoryAccessHelper::template decodeRecordConst; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] - static constexpr auto ieventAccessIx2 = - MemoryAccessHelper::template ieventAccessField; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] - static constexpr auto ieventAccessIx2Const = - MemoryAccessHelper::template ieventAccessFieldConst; - }; - -#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on implicit kernel rules - // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations - template - class KernelAccessAmplitudes - { - public: - -#ifndef MGONGPU_TRIVIAL_AMPLITUDES - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non-const) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] - static constexpr auto kernelAccessIx2 = - KernelAccessHelper::template kernelAccessField; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] - static constexpr auto kernelAccessIx2Const = - KernelAccessHelper::template kernelAccessFieldConst; - -#else - - static __host__ __device__ inline cxtype_sv* - kernelAccess( fptype* buffer ) - { - return reinterpret_cast( buffer ); - } - - static __host__ __device__ inline const cxtype_sv* - kernelAccessConst( const fptype* buffer ) - { - return reinterpret_cast( buffer ); - } - -#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES - }; - - //---------------------------------------------------------------------------- - - typedef KernelAccessAmplitudes HostAccessAmplitudes; - typedef KernelAccessAmplitudes DeviceAccessAmplitudes; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessChannelIds.h deleted file mode 100644 index 331e7f12c3..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessChannelIds.h +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Dec 2023, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessChannelIds_H -#define MemoryAccessChannelIds_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryAccessHelpers.h" -#include "MemoryAccessVectors.h" -#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - - // A class describing the internal layout of memory buffers for channel ids - // This implementation uses a plain ARRAY[nevt] - // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] - class MemoryAccessChannelIdsBase //_ARRAYv1 - { - private: - - friend class MemoryAccessHelper; - friend class KernelAccessHelper; - friend class KernelAccessHelper; - - //-------------------------------------------------------------------------- - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] - static __host__ __device__ inline unsigned int* - ieventAccessRecord( unsigned int* buffer, - const int ievt ) - { - return &( buffer[ievt] ); // ARRAY[nevt] - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] - // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] - static __host__ __device__ inline unsigned int& - decodeRecord( unsigned int* buffer ) - { - constexpr int ievt = 0; - return buffer[ievt]; // ARRAY[nevt] - } - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on explicit event numbers - // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations - class MemoryAccessChannelIds : public MemoryAccessChannelIdsBase - { - public: - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] - static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] - static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on implicit kernel rules - // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations - template - class KernelAccessChannelIds - { - public: - - // Expose selected functions from MemoryAccessChannelIds - static constexpr auto ieventAccessRecordConst = MemoryAccessChannelIds::ieventAccessRecordConst; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] - static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) - // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] - static __host__ __device__ inline const uint_sv& - kernelAccessConst( const unsigned int* buffer ) - { - const unsigned int& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif - } - }; - - //---------------------------------------------------------------------------- - - typedef KernelAccessChannelIds HostAccessChannelIds; - typedef KernelAccessChannelIds DeviceAccessChannelIds; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplings.h deleted file mode 100644 index fd3fb80c65..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplings.h +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Roiser, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessCouplings_H -#define MemoryAccessCouplings_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuCxtypes.h" - -#include "MemoryAccessHelpers.h" -#include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM -#include "MemoryBuffers.h" // for HostBufferCouplings::isaligned - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - - // A class describing the internal layout of memory buffers for couplings - // This implementation uses an AOSOA[npagC][ndcoup][nx2][neppC] "super-buffer" where nevt=npagC*neppC - // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling - // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] - class MemoryAccessCouplingsBase //_AOSOAv1 - { - public: - - // Number of Events Per Page in the coupling AOSOA memory buffer layout - static constexpr int neppC = MemoryAccessMomentaBase::neppM; // use the same AOSOA striding as for momenta - - // SANITY CHECK: check that neppC is a power of two - static_assert( ispoweroftwo( neppC ), "neppC is not a power of 2" ); - - //-------------------------------------------------------------------------- - // ** NB! A single super-buffer AOSOA[npagC][ndcoup][nx2][neppC] includes data for ndcoup different couplings ** - // ** NB! The ieventAccessRecord and kernelAccess functions refer to the buffer for one individual coupling ** - // ** NB! Use idcoupAccessBuffer to add a fixed offset and locate the buffer for one given individual coupling ** - //-------------------------------------------------------------------------- - - // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) - // [Signature (non-const) ===> fptype* idcoupAccessBuffer( fptype* buffer, const int idcoup ) <===] - // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? - static __host__ __device__ inline fptype* - idcoupAccessBuffer( fptype* buffer, // input "super-buffer" - const int idcoup ) - { - constexpr int ipagC = 0; - constexpr int ieppC = 0; - constexpr int ix2 = 0; - // NB! this effectively adds an offset "idcoup * nx2 * neppC" - return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] - } - - // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) - // [Signature (const) ===> const fptype* idcoupAccessBufferConst( const fptype* buffer, const int idcoup ) <===] - // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? - static __host__ __device__ inline const fptype* - idcoupAccessBufferConst( const fptype* buffer, // input "super-buffer" - const int idcoup ) - { - return idcoupAccessBuffer( const_cast( buffer ), idcoup ); - } - - private: - - friend class MemoryAccessHelper; - friend class KernelAccessHelper; - friend class KernelAccessHelper; - - // The number of couplings that dependent on the running alphas QCD in this specific process - static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; - - // The number of floating point components of a complex number - static constexpr int nx2 = mgOnGpu::nx2; - - //-------------------------------------------------------------------------- - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static __host__ __device__ inline fptype* - ieventAccessRecord( fptype* buffer, - const int ievt ) - { - const int ipagC = ievt / neppC; // #event "C-page" - const int ieppC = ievt %% neppC; // #event in the current event C-page - constexpr int idcoup = 0; - constexpr int ix2 = 0; - return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] - // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] - static __host__ __device__ inline fptype& - decodeRecord( fptype* buffer, - const int ix2 ) - { - constexpr int ipagC = 0; - constexpr int ieppC = 0; - // NB! the offset "idcoup * nx2 * neppC" has been added in idcoupAccessBuffer - constexpr int idcoup = 0; - return buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC]; // AOSOA[ipagC][idcoup][ix2][ieppC] - } - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on explicit event numbers - // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations - class MemoryAccessCouplings : public MemoryAccessCouplingsBase - { - public: - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] - static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] - static constexpr auto decodeRecordIx2Const = - MemoryAccessHelper::template decodeRecordConst; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] - static constexpr auto ieventAccessIx2 = - MemoryAccessHelper::template ieventAccessField; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] - static constexpr auto ieventAccessIx2Const = - MemoryAccessHelper::template ieventAccessFieldConst; - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on implicit kernel rules - // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations - template - class KernelAccessCouplings - { - public: - - // Expose selected functions from MemoryAccessCouplingsBase - static constexpr auto idcoupAccessBuffer = MemoryAccessCouplingsBase::idcoupAccessBuffer; - static constexpr auto idcoupAccessBufferConst = MemoryAccessCouplingsBase::idcoupAccessBufferConst; - - // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. - // For dependent (event-by-event, running-alphas) couplings the value is an AOSOA record [nx2][neppC] - // (real and imaginary SIMD lanes), so consecutive flavor slots are nx2*neppC fptype's apart. - static constexpr int flv_stride = MemoryAccessCouplingsBase::neppC * mgOnGpu::nx2; - - // Expose selected functions from MemoryAccessCouplings - static constexpr auto ieventAccessRecordConst = MemoryAccessCouplings::ieventAccessRecordConst; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] - static constexpr auto kernelAccessIx2_s = - KernelAccessHelper::template kernelAccessField; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] - static constexpr auto kernelAccessIx2Const_s = - KernelAccessHelper::template kernelAccessFieldConst; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] - static __host__ __device__ inline fptype_sv& - kernelAccessIx2( fptype* buffer, - const int ix2 ) - { - fptype& out = kernelAccessIx2_s( buffer, ix2 ); -#ifndef MGONGPU_CPPSIMD - return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays - constexpr int neppC = MemoryAccessCouplingsBase::neppC; - static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif - } - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] - static __host__ __device__ inline const fptype_sv& - kernelAccessIx2Const( const fptype* buffer, - const int ix2 ) - { - return kernelAccessIx2( const_cast( buffer ), ix2 ); - } - - /* - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] - static __host__ __device__ inline const fptype_sv& - kernelAccessIx2Const( const fptype* buffer, - const int ix2 ) - { - const fptype& out = kernelAccessIx2Const_s( buffer, ix2 ); -#ifndef MGONGPU_CPPSIMD - return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays - constexpr int neppC = MemoryAccessCouplingsBase::neppC; - static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif - } - */ - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non const, SCALAR OR VECTOR) ===> cxtype_sv_ref kernelAccess( fptype* buffer ) <===] - static __host__ __device__ inline cxtype_sv_ref - kernelAccess( fptype* buffer ) - { - /* - fptype_sv& real = kernelAccessIx2( buffer, 0 ); - fptype_sv& imag = kernelAccessIx2( buffer, 1 ); - printf( "C_ACCESS::kernelAccess: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); - return cxtype_sv_ref( real, imag ); - */ - return cxtype_sv_ref( kernelAccessIx2( buffer, 0 ), - kernelAccessIx2( buffer, 1 ) ); - } - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] - static __host__ __device__ inline cxtype_sv - kernelAccessConst( const fptype* buffer ) - { - /* - const fptype_sv& real = kernelAccessIx2Const( buffer, 0 ); - const fptype_sv& imag = kernelAccessIx2Const( buffer, 1 ); - printf( "C_ACCESS::kernelAccessConst: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); - return cxtype_sv( real, imag ); - */ - return cxtype_sv( kernelAccessIx2Const( buffer, 0 ), - kernelAccessIx2Const( buffer, 1 ) ); - } - }; - - //---------------------------------------------------------------------------- - - typedef KernelAccessCouplings HostAccessCouplings; - typedef KernelAccessCouplings DeviceAccessCouplings; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplingsFixed.h deleted file mode 100644 index 757de7b6f2..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplingsFixed.h +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Apr 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessCouplingsFixed_H -#define MemoryAccessCouplingsFixed_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuCxtypes.h" -#include "mgOnGpuVectors.h" - -//#include "MemoryAccessHelpers.h" - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - - // A class describing the internal layout of memory buffers for fixed couplings - // This implementation uses a STRUCT[ndcoup][nx2] "super-buffer" layout: in practice, the cIPC global array - // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling - // [If many implementations are used, a suffix _Sv1 should be appended to the class name] - class MemoryAccessCouplingsFixedBase //_Sv1 - { - public: - - // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) - // [Signature (const) ===> const fptype* iicoupAccessBufferConst( const fptype* buffer, const int iicoup ) <===] - static __host__ __device__ inline const fptype* - iicoupAccessBufferConst( const fptype* buffer, // input "super-buffer": in practice, the cIPC global array - const int iicoup ) - { - constexpr int ix2 = 0; - // NB! this effectively adds an offset "iicoup * nx2" - return &( buffer[iicoup * nx2 + ix2] ); // STRUCT[idcoup][ix2] - } - - private: - - // The number of floating point components of a complex number - static constexpr int nx2 = mgOnGpu::nx2; - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on implicit kernel rules - // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations - template - class KernelAccessCouplingsFixed - { - public: - - // Expose selected functions from MemoryAccessCouplingsFixedBase - static constexpr auto iicoupAccessBufferConst = MemoryAccessCouplingsFixedBase::iicoupAccessBufferConst; - - // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. - // For fixed (independent) couplings the value is a single scalar complex (real,imag): nx2 fptype's, broadcast across the SIMD vector. - static constexpr int flv_stride = mgOnGpu::nx2; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] - static __host__ __device__ inline const cxtype_sv - kernelAccessConst( const fptype* buffer ) - { - // TRIVIAL ACCESS to fixed-couplings buffers! - //return cxmake( fptype_sv{ buffer[0] }, fptype_sv{ buffer[1] } ); // NO! BUG #339! - const fptype_sv r_sv = fptype_sv{ 0 } + buffer[0]; - const fptype_sv i_sv = fptype_sv{ 0 } + buffer[1]; - return cxmake( r_sv, i_sv ); // ugly but effective - } - }; - - //---------------------------------------------------------------------------- - - typedef KernelAccessCouplingsFixed HostAccessCouplingsFixed; - typedef KernelAccessCouplingsFixed DeviceAccessCouplingsFixed; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessDenominators.h deleted file mode 100644 index 205952e514..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessDenominators.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessDenominators_H -#define MemoryAccessDenominators_H 1 - -#include "MemoryAccessGs.h" - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - - // A class describing the internal layout of memory buffers for denominators - // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs - - typedef KernelAccessGs HostAccessDenominators; - typedef KernelAccessGs DeviceAccessDenominators; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessGs.h deleted file mode 100644 index bfcb3e3222..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessGs.h +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Roiser, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessGs_H -#define MemoryAccessGs_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryAccessHelpers.h" -#include "MemoryAccessVectors.h" -#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - - // A class describing the internal layout of memory buffers for Gs - // This implementation uses a plain ARRAY[nevt] - // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] - class MemoryAccessGsBase //_ARRAYv1 - { - private: - - friend class MemoryAccessHelper; - friend class KernelAccessHelper; - friend class KernelAccessHelper; - - //-------------------------------------------------------------------------- - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static __host__ __device__ inline fptype* - ieventAccessRecord( fptype* buffer, - const int ievt ) - { - return &( buffer[ievt] ); // ARRAY[nevt] - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] - // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] - static __host__ __device__ inline fptype& - decodeRecord( fptype* buffer ) - { - constexpr int ievt = 0; - return buffer[ievt]; // ARRAY[nevt] - } - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on explicit event numbers - // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations - class MemoryAccessGs : public MemoryAccessGsBase - { - public: - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] - static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] - static constexpr auto decodeRecordConst = - MemoryAccessHelper::template decodeRecordConst<>; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] - static constexpr auto ieventAccess = - MemoryAccessHelper::template ieventAccessField<>; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] - static constexpr auto ieventAccessConst = - MemoryAccessHelper::template ieventAccessFieldConst<>; - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on implicit kernel rules - // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations - template - class KernelAccessGs - { - public: - - // Expose selected functions from MemoryAccessGs - static constexpr auto ieventAccessRecord = MemoryAccessGs::ieventAccessRecord; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non-const, SCALAR) ===> fptype& kernelAccess( fptype* buffer ) <===] - static constexpr auto kernelAccess_s = - KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) - // [Signature (non-const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( fptype* buffer ) <===] - static __host__ __device__ inline fptype_sv& - kernelAccess( fptype* buffer ) - { - fptype& out = kernelAccess_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif - } - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) - // [Signature (SCALAR OR VECTOR) ===> fptype_sv* kernelAccess( fptype* buffer ) <===] - static __host__ __device__ inline fptype_sv* - kernelAccessP( fptype* buffer ) - { - return reinterpret_cast( buffer ); - } - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] - static constexpr auto kernelAccessConst_s = - KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) - // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccess( const fptype* buffer ) <===] - static __host__ __device__ inline const fptype_sv& - kernelAccessConst( const fptype* buffer ) - { - const fptype& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif - } - }; - - //---------------------------------------------------------------------------- - - typedef KernelAccessGs HostAccessGs; - typedef KernelAccessGs DeviceAccessGs; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessHelpers.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessHelpers.h deleted file mode 100644 index 8b5bd27f4d..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessHelpers.h +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessHelpers_H -#define MemoryAccessHelpers_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuFptypes.h" - -//---------------------------------------------------------------------------- - -// A templated helper class that includes the boilerplate code for MemoryAccess classes -template -class MemoryAccessHelper -{ -public: - - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecord = T::ieventAccessRecord; - - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] - static __host__ __device__ inline const FT* - ieventAccessRecordConst( const FT* buffer, - const int ievt ) - { - return ieventAccessRecord( const_cast( buffer ), ievt ); - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] - static constexpr auto decodeRecord = T::decodeRecord; - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, Ts... args ) <===] - template - static __host__ __device__ inline const FT& - decodeRecordConst( const FT* buffer, - Ts... args ) // variadic template - { - return T::decodeRecord( const_cast( buffer ), args... ); - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (non-const) ===> fptype& ieventAccessField( fptype* buffer, const ievt, Ts... args ) <===] - template - static __host__ __device__ inline FT& - ieventAccessField( FT* buffer, - const int ievt, - Ts... args ) // variadic template - { - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - return T::decodeRecord( T::ieventAccessRecord( buffer, ievt ), args... ); - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const fptype& ieventAccessFieldConst( const fptype* buffer, const ievt, Ts... args ) <===] - template - static __host__ __device__ inline const FT& - ieventAccessFieldConst( const FT* buffer, - const int ievt, - Ts... args ) // variadic template - { - return ieventAccessField( const_cast( buffer ), ievt, args... ); - } -}; - -//---------------------------------------------------------------------------- - -// A templated helper class that includes the boilerplate code for KernelAccess classes -template -class KernelAccessHelper : public MemoryAccessHelper -{ -public: - - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) - // [Signature (non-const) ===> fptype* kernelAccessRecord( fptype* buffer ) <===] - static __host__ __device__ inline FT* - kernelAccessRecord( FT* buffer ) - { - if constexpr( !onDevice ) // requires c++17 also in CUDA (#333) - { - // FIXME #436: clarify that buffer includes all events on device, and only the record for an event subset on host! - // FIXME #436: am I not assuming that the following line is always identical to buffer for all access classes T? - return T::ieventAccessRecord( buffer, 0 ); - } - else - { -#ifdef MGONGPUCPP_GPUIMPL - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid - //printf( "kernelAccessRecord: ievt=%d threadId=%d\n", ievt, threadIdx.x ); - return T::ieventAccessRecord( buffer, ievt ); // NB fptype and fptype_sv coincide for CUDA -#else - throw std::runtime_error( "kernelAccessRecord on device is only implemented in CUDA" ); -#endif - } - } - - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) - // [Signature (const) ===> const fptype* kernelAccessRecordConst( const fptype* buffer ) <===] - static __host__ __device__ inline const FT* - kernelAccessRecordConst( const FT* buffer ) - { - return kernelAccessRecord( const_cast( buffer ) ); - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non-const) ===> fptype& kernelAccessField( fptype* buffer, Ts... args ) <===] - template - static __host__ __device__ inline FT& - kernelAccessField( FT* buffer, - Ts... args ) // variadic template - { - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - return T::decodeRecord( kernelAccessRecord( buffer ), args... ); - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const) ===> const fptype& kernelAccessFieldConst( const fptype* buffer, Ts... args ) <===] - template - static __host__ __device__ inline const FT& - kernelAccessFieldConst( const FT* buffer, - Ts... args ) // variadic template - { - return kernelAccessField( const_cast( buffer ), args... ); - } - - //-------------------------------------------------------------------------- -}; - -#endif // MemoryAccessHelpers_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessIflavorVec.h deleted file mode 100644 index ffc74b0f0c..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessIflavorVec.h +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: D. Massaro (Jan 2026, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessIflavorVec_H -#define MemoryAccessIflavorVec_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryAccessHelpers.h" -#include "MemoryAccessVectors.h" -#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - - // A class describing the internal layout of memory buffers for channel ids - // This implementation uses a plain ARRAY[nevt] - // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] - class MemoryAccessIflavorVecBase //_ARRAYv1 - { - private: - - friend class MemoryAccessHelper; - friend class KernelAccessHelper; - friend class KernelAccessHelper; - - //-------------------------------------------------------------------------- - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] - static __host__ __device__ inline unsigned int* - ieventAccessRecord( unsigned int* buffer, - const int ievt ) - { - return &( buffer[ievt] ); // ARRAY[nevt] - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] - // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] - static __host__ __device__ inline unsigned int& - decodeRecord( unsigned int* buffer ) - { - constexpr int ievt = 0; - return buffer[ievt]; // ARRAY[nevt] - } - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on explicit event numbers - // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations - class MemoryAccessIflavorVec : public MemoryAccessIflavorVecBase - { - public: - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] - static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] - static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on implicit kernel rules - // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations - template - class KernelAccessIflavorVec - { - public: - - // Expose selected functions from MemoryAccessIflavorVec - static constexpr auto ieventAccessRecordConst = MemoryAccessIflavorVec::ieventAccessRecordConst; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] - static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) - // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] - static __host__ __device__ inline const uint_sv& - kernelAccessConst( const unsigned int* buffer ) - { - const unsigned int& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif - } - }; - - //---------------------------------------------------------------------------- - - typedef KernelAccessIflavorVec HostAccessIflavorVec; - typedef KernelAccessIflavorVec DeviceAccessIflavorVec; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessMatrixElements.h deleted file mode 100644 index 4236e20602..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessMatrixElements.h +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessMatrixElements_H -#define MemoryAccessMatrixElements_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryAccessHelpers.h" -#include "MemoryAccessVectors.h" -#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - - // A class describing the internal layout of memory buffers for matrix elements - // This implementation uses a plain ARRAY[nevt] - // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] - class MemoryAccessMatrixElementsBase //_ARRAYv1 - { - private: - - friend class MemoryAccessHelper; - friend class KernelAccessHelper; - friend class KernelAccessHelper; - - //-------------------------------------------------------------------------- - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static __host__ __device__ inline fptype* - ieventAccessRecord( fptype* buffer, - const int ievt ) - { - return &( buffer[ievt] ); // ARRAY[nevt] - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] - // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] - static __host__ __device__ inline fptype& - decodeRecord( fptype* buffer ) - { - constexpr int ievt = 0; - return buffer[ievt]; // ARRAY[nevt] - } - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on explicit event numbers - // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations - class MemoryAccessMatrixElements : public MemoryAccessMatrixElementsBase - { - public: - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] - static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] - static constexpr auto decodeRecordConst = - MemoryAccessHelper::template decodeRecordConst<>; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] - static constexpr auto ieventAccess = - MemoryAccessHelper::template ieventAccessField<>; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] - static constexpr auto ieventAccessConst = - MemoryAccessHelper::template ieventAccessFieldConst<>; - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on implicit kernel rules - // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations - template - class KernelAccessMatrixElements - { - public: - - // Expose selected functions from MemoryAccessMatrixElements - static constexpr auto ieventAccessRecord = MemoryAccessMatrixElements::ieventAccessRecord; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non-const, SCALAR) ===> fptype& kernelAccess_s( fptype* buffer ) <===] - static constexpr auto kernelAccess_s = - KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) - // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( const fptype* buffer ) <===] - static __host__ __device__ inline fptype_sv& - kernelAccess( fptype* buffer ) - { - fptype& out = kernelAccess_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif - } - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] - static constexpr auto kernelAccessConst = - KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 - }; - - //---------------------------------------------------------------------------- - - typedef KernelAccessMatrixElements HostAccessMatrixElements; - typedef KernelAccessMatrixElements DeviceAccessMatrixElements; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessMomenta.h deleted file mode 100644 index 3ef4d76fbd..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessMomenta.h +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessMomenta_H -#define MemoryAccessMomenta_H 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "MemoryAccessHelpers.h" -#include "MemoryAccessVectors.h" - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - - // A class describing the internal layout of memory buffers for momenta - // This implementation uses an AOSOA[npagM][npar][np4][neppM] where nevt=npagM*neppM - // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] - class MemoryAccessMomentaBase //_AOSOAv1 - { - public: - - // Number of Events Per Page in the momenta AOSOA memory buffer layout - // (these are all best kept as a compile-time constants: see issue #23) -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - // ----------------------------------------------------------------------------------------------- - // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline - // --- This is relevant to ensure coalesced access to momenta in global memory - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------------- - //static constexpr int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) - static constexpr int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) - //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 1.03E9 instead of 1.11E9 in eemumu) -#else - // ----------------------------------------------------------------------------------------------- - // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register - // --- This is relevant to ensure faster access to momenta from C++ memory cache lines - // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded - // --- In practice, neppR, neppM and neppV could now (in principle) all be different - // ----------------------------------------------------------------------------------------------- -#ifdef MGONGPU_CPPSIMD - static constexpr int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance - //static constexpr int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) - //static constexpr int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) - //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 4.66E6 instead of 5.09E9 in eemumu) - //static constexpr int neppM = MGONGPU_CPPSIMD*2; // FOR TESTS -#else - static constexpr int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) -#endif -#endif /* clang-format on */ - - // SANITY CHECK: check that neppM is a power of two - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - - private: - - friend class MemoryAccessHelper; - friend class KernelAccessHelper; - friend class KernelAccessHelper; - - // The number of components of a 4-momentum - static constexpr int np4 = CPPProcess::np4; - - // The number of particles in this physics process - static constexpr int npar = CPPProcess::npar; - - //-------------------------------------------------------------------------- - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static __host__ __device__ inline fptype* - ieventAccessRecord( fptype* buffer, - const int ievt ) - { - const int ipagM = ievt / neppM; // #event "M-page" - const int ieppM = ievt % neppM; // #event in the current event M-page - constexpr int ip4 = 0; - constexpr int ipar = 0; - return &( buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM] ); // AOSOA[ipagM][ipar][ip4][ieppM] - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] - // [NB: expand variadic template "Ts... args" to "const int ip4, const int ipar" and rename "Field" as "Ip4Ipar"] - static __host__ __device__ inline fptype& - decodeRecord( fptype* buffer, - const int ip4, - const int ipar ) - { - constexpr int ipagM = 0; - constexpr int ieppM = 0; - return buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] - } - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on explicit event numbers - // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations - class MemoryAccessMomenta : public MemoryAccessMomentaBase - { - public: - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int ipar ) <===] - static constexpr auto decodeRecordIp4Ipar = MemoryAccessHelper::decodeRecord; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int ipar ) <===] - static constexpr auto decodeRecordIp4IparConst = - MemoryAccessHelper::template decodeRecordConst; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (non-const) ===> fptype& ieventAccessIp4Ipar( fptype* buffer, const ievt, const int ipar, const int ipar ) <===] - static constexpr auto ieventAccessIp4Ipar = - MemoryAccessHelper::template ieventAccessField; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] - // DEFAULT VERSION - static constexpr auto ieventAccessIp4IparConst = - MemoryAccessHelper::template ieventAccessFieldConst; - - /* - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] - // DEBUG VERSION WITH PRINTOUTS - static __host__ __device__ inline const fptype& - ieventAccessIp4IparConst( const fptype* buffer, - const int ievt, - const int ip4, - const int ipar ) - { - const fptype& out = MemoryAccessHelper::template ieventAccessFieldConst( buffer, ievt, ip4, ipar ); - printf( "ipar=%2d ip4=%2d ievt=%8d out=%8.3f\n", ipar, ip4, ievt, out ); - return out; - } - */ - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on implicit kernel rules - // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations - template - class KernelAccessMomenta - { - public: - - // Expose selected functions from MemoryAccessMomenta - static constexpr auto ieventAccessRecordConst = MemoryAccessMomenta::ieventAccessRecordConst; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIp4Ipar( fptype* buffer, const int ipar, const int ipar ) <===] - static constexpr auto kernelAccessIp4Ipar = - KernelAccessHelper::template kernelAccessField; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] - // DEFAULT VERSION - static constexpr auto kernelAccessIp4IparConst_s = - KernelAccessHelper::template kernelAccessFieldConst; - - /* - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] - // DEBUG VERSION WITH PRINTOUTS - static __host__ __device__ inline const fptype& - kernelAccessIp4IparConst_s( const fptype* buffer, - const int ip4, - const int ipar ) - { - const fptype& out = KernelAccessHelper::template kernelAccessFieldConst( buffer, ip4, ipar ); - printf( "ipar=%2d ip4=%2d ievt='kernel' out=%8.3f\n", ipar, ip4, out ); - return out; - } - */ - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR OR VECTOR) ===> fptype_sv kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] - // FIXME? Eventually return by const reference and support aligned arrays only? - // FIXME? Currently return by value to support also unaligned and arbitrary arrays - static __host__ __device__ inline fptype_sv - kernelAccessIp4IparConst( const fptype* buffer, - const int ip4, - const int ipar ) - { - const fptype& out = kernelAccessIp4IparConst_s( buffer, ip4, ipar ); -#ifndef MGONGPU_CPPSIMD - return out; -#else - constexpr int neppM = MemoryAccessMomentaBase::neppM; - constexpr bool useContiguousEventsIfPossible = true; // DEFAULT - //constexpr bool useContiguousEventsIfPossible = false; // FOR PERFORMANCE TESTS (treat as arbitrary array even if it is an AOSOA) - // Use c++17 "if constexpr": compile-time branching - if constexpr( useContiguousEventsIfPossible && ( neppM >= neppV ) && ( neppM % neppV == 0 ) ) - { - //constexpr bool skipAlignmentCheck = true; // FASTEST (SEGFAULTS IF MISALIGNED ACCESS, NEEDS A SANITY CHECK ELSEWHERE!) - constexpr bool skipAlignmentCheck = false; // DEFAULT: A BIT SLOWER BUT SAFER [ALLOWS MISALIGNED ACCESS] - if constexpr( skipAlignmentCheck ) - { - //static bool first=true; if( first ){ std::cout << "WARNING! assume aligned AOSOA, skip check" << std::endl; first=false; } // SLOWER (5.06E6) - // FASTEST? (5.09E6 in eemumu 512y) - // This assumes alignment for momenta1d without checking - causes segmentation fault in reinterpret_cast if not aligned! - return mg5amcCpu::fptypevFromAlignedArray( out ); // use reinterpret_cast - } - else if( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ) - { - //static bool first=true; if( first ){ std::cout << "WARNING! aligned AOSOA, reinterpret cast" << std::endl; first=false; } // SLOWER (5.00E6) - // DEFAULT! A tiny bit (<1%) slower because of the alignment check (5.07E6 in eemumu 512y) - // This explicitly checks buffer alignment to avoid segmentation faults in reinterpret_cast - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast - } - else - { - //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWER (4.93E6) - // A bit (1%) slower (5.05E6 in eemumu 512y) - // This does not require buffer alignment, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 - return mg5amcCpu::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) - } - } - else - { - //static bool first=true; if( first ){ std::cout << "WARNING! arbitrary array" << std::endl; first=false; } // SLOWER (5.08E6) - // ?!Used to be much slower, now a tiny bit faster for AOSOA?! (5.11E6 for AOSOA, 4.64E6 for AOS in eemumu 512y) - // This does not even require AOSOA with neppM>=neppV and neppM%neppV==0 (e.g. can be used with AOS neppM==1) - constexpr int ievt0 = 0; // just make it explicit in the code that buffer refers to a given ievt0 and decoderIeppV fetches event ievt0+ieppV - auto decoderIeppv = [buffer, ip4, ipar]( int ieppV ) - -> const fptype& - { return MemoryAccessMomenta::ieventAccessIp4IparConst( buffer, ievt0 + ieppV, ip4, ipar ); }; - return mg5amcCpu::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) - } -#endif - } - - // Is this a HostAccess or DeviceAccess class? - // [this is only needed for a warning printout in rambo.h for nparf==1 #358] - static __host__ __device__ inline constexpr bool - isOnDevice() - { - return onDevice; - } - }; - - //---------------------------------------------------------------------------- - - typedef KernelAccessMomenta HostAccessMomenta; - typedef KernelAccessMomenta DeviceAccessMomenta; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessNumerators.h deleted file mode 100644 index c8d0343b7e..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessNumerators.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessNumerators_H -#define MemoryAccessNumerators_H 1 - -#include "MemoryAccessGs.h" - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - - // A class describing the internal layout of memory buffers for numerators - // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs - - typedef KernelAccessGs HostAccessNumerators; - typedef KernelAccessGs DeviceAccessNumerators; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessRandomNumbers.h deleted file mode 100644 index 57f1d02081..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessRandomNumbers.h +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessRandomNumbers_H -#define MemoryAccessRandomNumbers_H 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "MemoryAccessHelpers.h" - -#ifdef MGONGPUCPP_GPUIMPL -using mg5amcGpu::CPPProcess; -#else -using mg5amcCpu::CPPProcess; -#endif - -//---------------------------------------------------------------------------- - -// A class describing the internal layout of memory buffers for random numbers -// This implementation uses an AOSOA[npagR][nparf][np4][neppR] where nevt=npagR*neppR -// [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] -class MemoryAccessRandomNumbersBase //_AOSOAv1 -{ -public: /* clang-format off */ - - // Number of Events Per Page in the random number AOSOA memory buffer layout - // *** NB Different values of neppR lead to different physics results: the *** - // *** same 1d array is generated, but it is interpreted in different ways *** - static constexpr int neppR = 8; // HARDCODED TO GIVE ALWAYS THE SAME PHYSICS RESULTS! - //static constexpr int neppR = 1; // AOS (tests of sectors/requests) - -private: /* clang-format on */ - - friend class MemoryAccessHelper; - friend class KernelAccessHelper; - friend class KernelAccessHelper; - - // The number of components of a 4-momentum - static constexpr int np4 = CPPProcess::np4; - - // The number of final state particles in this physics process - static constexpr int nparf = CPPProcess::nparf; - - //-------------------------------------------------------------------------- - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static __host__ __device__ inline fptype* - ieventAccessRecord( fptype* buffer, - const int ievt ) - { - const int ipagR = ievt / neppR; // #event "R-page" - const int ieppR = ievt % neppR; // #event in the current event R-page - constexpr int ip4 = 0; - constexpr int iparf = 0; - return &( buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR] ); // AOSOA[ipagR][iparf][ip4][ieppR] - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] - // [NB: expand variadic template "Ts... args" to "const int ip4, const int iparf" and rename "Field" as "Ip4Iparf"] - static __host__ __device__ inline fptype& - decodeRecord( fptype* buffer, - const int ip4, - const int iparf ) - { - constexpr int ipagR = 0; - constexpr int ieppR = 0; - return buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR]; // AOSOA[ipagR][iparf][ip4][ieppR] - } -}; - -//---------------------------------------------------------------------------- - -// A class providing access to memory buffers for a given event, based on explicit event numbers -// Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations -class MemoryAccessRandomNumbers : public MemoryAccessRandomNumbersBase -{ -public: - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int iparf ) <===] - static constexpr auto decodeRecordIp4Iparf = MemoryAccessHelper::decodeRecord; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int iparf ) <===] - static constexpr auto decodeRecordIp4IparfConst = - MemoryAccessHelper::template decodeRecordConst; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (non-const) ===> fptype& ieventAccessIp4Iparf( fptype* buffer, const ievt, const int ipar, const int iparf ) <===] - static constexpr auto ieventAccessIp4Iparf = - MemoryAccessHelper::template ieventAccessField; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const fptype& ieventAccessIp4IparfConst( const fptype* buffer, const ievt, const int ipar, const int iparf ) <===] - static constexpr auto ieventAccessIp4IparfConst = - MemoryAccessHelper::template ieventAccessFieldConst; -}; - -//---------------------------------------------------------------------------- - -// A class providing access to memory buffers for a given event, based on implicit kernel rules -// Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations -template -class KernelAccessRandomNumbers -{ -public: - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non-const) ===> fptype& kernelAccessIp4Iparf( fptype* buffer, const int ipar, const int iparf ) <===] - static constexpr auto kernelAccessIp4Iparf = - KernelAccessHelper::template kernelAccessField; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const) ===> const fptype& kernelAccessIp4IparfConst( const fptype* buffer, const int ipar, const int iparf ) <===] - static constexpr auto kernelAccessIp4IparfConst = - KernelAccessHelper::template kernelAccessFieldConst; -}; - -//---------------------------------------------------------------------------- - -typedef KernelAccessRandomNumbers HostAccessRandomNumbers; -typedef KernelAccessRandomNumbers DeviceAccessRandomNumbers; - -//---------------------------------------------------------------------------- - -#endif // MemoryAccessRandomNumbers_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessVectors.h deleted file mode 100644 index 8563d8f2f7..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessVectors.h +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessVectors_H -#define MemoryAccessVectors_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuVectors.h" - -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu // this is only needed for CPU SIMD vectorization -{ - -#ifdef MGONGPU_CPPSIMD - //-------------------------------------------------------------------------- - - // Cast one non-const fptype_v reference (one vector of neppV fptype values) from one non-const fptype reference (#435), - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned - inline fptype_v& fptypevFromAlignedArray( fptype& ref ) - { - return *reinterpret_cast( &ref ); - } - - inline uint_v& uintvFromAlignedArray( unsigned int& ref ) - { - return *reinterpret_cast( &ref ); - } - - // Cast one const fptype_v reference (one vector of neppV fptype values) from one const fptype reference, - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned - inline const fptype_v& fptypevFromAlignedArray( const fptype& ref ) - { - return *reinterpret_cast( &ref ); - } - - inline const uint_v& uintvFromAlignedArray( const unsigned int& ref ) - { - return *reinterpret_cast( &ref ); - } - - // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", but that the arrays are not aligned - inline fptype_v fptypevFromUnalignedArray( const fptype& ref ) - { -#if MGONGPU_CPPSIMD == 2 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (2) - *( &ref + 1 ) }; -#elif MGONGPU_CPPSIMD == 4 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (4) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ) }; -#elif MGONGPU_CPPSIMD == 8 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (8) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ), - *( &ref + 4 ), - *( &ref + 5 ), - *( &ref + 6 ), - *( &ref + 7 ) }; -#elif MGONGPU_CPPSIMD == 16 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (16) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ), - *( &ref + 4 ), - *( &ref + 5 ), - *( &ref + 6 ), - *( &ref + 7 ), - *( &ref + 8 ), - *( &ref + 9 ), - *( &ref + 10 ), - *( &ref + 11 ), - *( &ref + 12 ), - *( &ref + 13 ), - *( &ref + 14 ), - *( &ref + 15 ) }; -#else -#error Internal error! Unknown MGONGPU_CPPSIMD value -#endif - } - - // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, - // with no a priori assumption on how the input fptype array should be decoded - template - inline fptype_v fptypevFromArbitraryArray( Functor decoderIeppv ) - { -#if MGONGPU_CPPSIMD == 2 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (2) - decoderIeppv( 1 ) }; -#elif MGONGPU_CPPSIMD == 4 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (4) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ) }; -#elif MGONGPU_CPPSIMD == 8 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (8) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ), - decoderIeppv( 4 ), - decoderIeppv( 5 ), - decoderIeppv( 6 ), - decoderIeppv( 7 ) }; -#elif MGONGPU_CPPSIMD == 16 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (16) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ), - decoderIeppv( 4 ), - decoderIeppv( 5 ), - decoderIeppv( 6 ), - decoderIeppv( 7 ), - decoderIeppv( 8 ), - decoderIeppv( 9 ), - decoderIeppv( 10 ), - decoderIeppv( 11 ), - decoderIeppv( 12 ), - decoderIeppv( 13 ), - decoderIeppv( 14 ), - decoderIeppv( 15 ) }; -#else -#error Internal error! Unknown MGONGPU_CPPSIMD value -#endif - } - - //-------------------------------------------------------------------------- -#endif - -} // end namespace -#endif - -#endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessWavefunctions.h deleted file mode 100644 index 14a37c7fea..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessWavefunctions.h +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessWavefunctions_H -#define MemoryAccessWavefunctions_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuCxtypes.h" - -#include "MemoryAccessHelpers.h" - -#define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - -#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS - - // A class describing the internal layout of memory buffers for wavefunctions - // This implementation uses an AOSOA[npagW][nw6][nx2][neppW] where nevt=npagW*neppW - // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] - class MemoryAccessWavefunctionsBase //_AOSOAv1 - { - public: - - // Number of Events Per Page in the wavefunction AOSOA memory buffer layout - static constexpr int neppW = 1; // AOS (just a test...) - - private: - - friend class MemoryAccessHelper; - friend class KernelAccessHelper; - friend class KernelAccessHelper; - - // The number of components of a (fermion or vector) wavefunction - static constexpr int nw6 = mgOnGpu::nw6; - - // The number of floating point components of a complex number - static constexpr int nx2 = mgOnGpu::nx2; - - //-------------------------------------------------------------------------- - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static __host__ __device__ inline fptype* - ieventAccessRecord( fptype* buffer, - const int ievt ) - { - const int ipagW = ievt / neppW; // #event "W-page" - const int ieppW = ievt % neppW; // #event in the current event W-page - constexpr int iw6 = 0; - constexpr int ix2 = 0; - return &( buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW] ); // AOSOA[ipagW][iw6][ix2][ieppW] - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] - // [NB: expand variadic template "Ts... args" to "const int iw6, const int ix2" and rename "Field" as "Iw6Ix2"] - static __host__ __device__ inline fptype& - decodeRecord( fptype* buffer, - const int iw6, - const int ix2 ) - { - constexpr int ipagW = 0; - constexpr int ieppW = 0; - return buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW]; // AOSOA[ipagW][iw6][ix2][ieppW] - } - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on explicit event numbers - // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations - class MemoryAccessWavefunctions : public MemoryAccessWavefunctionsBase - { - public: - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int iw6, const int ix2 ) <===] - static constexpr auto decodeRecordIw6Ix2 = MemoryAccessHelper::decodeRecord; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int iw6, const int ix2 ) <===] - static constexpr auto decodeRecordIw6Ix2Const = - MemoryAccessHelper::template decodeRecordConst; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (non-const) ===> fptype& ieventAccessIw6Ix2( fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] - static constexpr auto ieventAccessIw6Ix2 = - MemoryAccessHelper::template ieventAccessField; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const fptype& ieventAccessIw6Ix2Const( const fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] - static constexpr auto ieventAccessIw6Ix2Const = - MemoryAccessHelper::template ieventAccessFieldConst; - }; - -#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on implicit kernel rules - // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations - template - class KernelAccessWavefunctions - { - public: - -#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non-const) ===> fptype& kernelAccessIw6Ix2( fptype* buffer, const int iw6, const int ix2 ) <===] - static constexpr auto kernelAccessIw6Ix2 = - KernelAccessHelper::template kernelAccessField; - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const) ===> const fptype& kernelAccessIw6Ix2Const( const fptype* buffer, const int iw6, const int ix2 ) <===] - static constexpr auto kernelAccessIw6Ix2Const = - KernelAccessHelper::template kernelAccessFieldConst; - -#else - - static __host__ __device__ inline cxtype_sv* - kernelAccess( fptype* buffer ) - { - return reinterpret_cast( buffer ); - } - - static __host__ __device__ inline const cxtype_sv* - kernelAccessConst( const fptype* buffer ) - { - return reinterpret_cast( buffer ); - } - -#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS - }; - - //---------------------------------------------------------------------------- - - typedef KernelAccessWavefunctions HostAccessWavefunctions; - typedef KernelAccessWavefunctions DeviceAccessWavefunctions; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessWeights.h deleted file mode 100644 index c56527f581..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessWeights.h +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessWeights_H -#define MemoryAccessWeights_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryAccessHelpers.h" - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //---------------------------------------------------------------------------- - - // A class describing the internal layout of memory buffers for weights - // This implementation uses a plain ARRAY[nevt] - // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] - class MemoryAccessWeightsBase //_ARRAYv1 - { - private: - - friend class MemoryAccessHelper; - friend class KernelAccessHelper; - friend class KernelAccessHelper; - - //-------------------------------------------------------------------------- - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - // (in other words: first locate the event record for a given event, then locate an element in that record) - //-------------------------------------------------------------------------- - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static __host__ __device__ inline fptype* - ieventAccessRecord( fptype* buffer, - const int ievt ) - { - return &( buffer[ievt] ); // ARRAY[nevt] - } - - //-------------------------------------------------------------------------- - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] - // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] - static __host__ __device__ inline fptype& - decodeRecord( fptype* buffer ) - { - constexpr int ievt = 0; - return buffer[ievt]; // ARRAY[nevt] - } - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on explicit event numbers - // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations - class MemoryAccessWeights : public MemoryAccessWeightsBase - { - public: - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; - - // Locate an event record (output) in a memory buffer (input) from the given event number (input) - // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] - static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] - static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; - - // Locate a field (output) of an event record (input) from the given field indexes (input) - // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] - static constexpr auto decodeRecordConst = - MemoryAccessHelper::template decodeRecordConst<>; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] - static constexpr auto ieventAccess = - MemoryAccessHelper::template ieventAccessField<>; - - // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) - // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] - static constexpr auto ieventAccessConst = - MemoryAccessHelper::template ieventAccessFieldConst<>; - }; - - //---------------------------------------------------------------------------- - - // A class providing access to memory buffers for a given event, based on implicit kernel rules - // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations - template - class KernelAccessWeights - { - public: - - /* - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] - // FINAL IMPLEMENTATION FOR CUDA 11.4 - static constexpr auto kernelAccess = - KernelAccessHelper::template kernelAccessField<>; - */ - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] - // TEMPORARY HACK FOR CUDA 11.1 - static __host__ __device__ inline fptype& - kernelAccess( fptype* buffer ) - { - return KernelAccessHelper::template kernelAccessField<>( buffer ); - } - - /* - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] - // FINAL IMPLEMENTATION FOR CUDA 11.4 - static constexpr auto kernelAccessConst = - KernelAccessHelper::template kernelAccessFieldConst<>; - */ - - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] - // TEMPORARY HACK FOR CUDA 11.1 - static __host__ __device__ inline const fptype& - kernelAccessConst( const fptype* buffer ) - { - return KernelAccessHelper::template kernelAccessFieldConst<>( buffer ); - } - }; - - //---------------------------------------------------------------------------- - - typedef KernelAccessWeights HostAccessWeights; - typedef KernelAccessWeights DeviceAccessWeights; - - //---------------------------------------------------------------------------- - -} // end namespace mg5amcGpu/mg5amcCpu - -#endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/MemoryBuffers.h deleted file mode 100644 index d259aa7456..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryBuffers.h +++ /dev/null @@ -1,603 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021, based on earlier work by S. Hageboeck) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Roiser, J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryBuffers_H -#define MemoryBuffers_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuCxtypes.h" - -#include "CPPProcess.h" -#include "GpuRuntime.h" -#include "Parameters.h" -#include "processConfig.h" - -#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - namespace MemoryBuffers - { - // Process-independent compile-time constants - static constexpr size_t np4 = CPPProcess::np4; - static constexpr size_t nw6 = CPPProcess::nw6; - static constexpr size_t nx2 = mgOnGpu::nx2; - // Process-dependent compile-time constants - static constexpr size_t nparf = CPPProcess::nparf; - static constexpr size_t npar = CPPProcess::npar; - static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; - static constexpr size_t ncolor = CPPProcess::ncolor; - } - - //-------------------------------------------------------------------------- - - // An abstract interface encapsulating a given number of events - class INumberOfEvents - { - public: - virtual ~INumberOfEvents() {} - virtual size_t nevt() const = 0; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating a given number of events - class NumberOfEvents : virtual public INumberOfEvents - { - public: - NumberOfEvents( const size_t nevt ) - : m_nevt( nevt ) {} - virtual ~NumberOfEvents() {} - virtual size_t nevt() const override { return m_nevt; } - private: - const size_t m_nevt; - }; - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer (not necessarily an event buffer) - template - class BufferBase : virtual public INumberOfEvents - { - protected: - BufferBase( const size_t size, const bool onDevice ) - : m_size( size ), m_data( nullptr ), m_isOnDevice( onDevice ) {} - public: - virtual ~BufferBase() {} - T* data() { return m_data; } - const T* data() const { return m_data; } - T& operator[]( const size_t index ) { return m_data[index]; } - const T& operator[]( const size_t index ) const { return m_data[index]; } - size_t size() const { return m_size; } - size_t bytes() const { return m_size * sizeof( T ); } - bool isOnDevice() const { return m_isOnDevice; } - virtual size_t nevt() const override { throw std::runtime_error( "This BufferBase is not an event buffer" ); } - protected: - const size_t m_size; - T* m_data; - const bool m_isOnDevice; - }; - - //-------------------------------------------------------------------------- - -#ifndef MGONGPUCPP_GPUIMPL - constexpr bool HostBufferALIGNED = false; // ismisaligned=false - constexpr bool HostBufferMISALIGNED = true; // ismisaligned=true - - // A class encapsulating a C++ host buffer - template - class HostBufferBase : public BufferBase - { - public: - HostBufferBase( const size_t size ) - : BufferBase( size, false ) - { - if constexpr( !ismisaligned ) - this->m_data = new( std::align_val_t( cppAlign ) ) T[size](); - else - this->m_data = new( std::align_val_t( cppAlign ) ) T[size + 1]() + 1; // TEST MISALIGNMENT! - } - virtual ~HostBufferBase() - { - if constexpr( !ismisaligned ) - ::operator delete[]( this->m_data, std::align_val_t( cppAlign ) ); - else - ::operator delete[]( ( this->m_data ) - 1, std::align_val_t( cppAlign ) ); // TEST MISALIGNMENT! - } - static constexpr bool isaligned() { return !ismisaligned; } - public: - static constexpr size_t cppAlign = mgOnGpu::cppAlign; - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA pinned host buffer - template - class PinnedHostBufferBase : public BufferBase - { - public: - PinnedHostBufferBase( const size_t size ) - : BufferBase( size, false ) - { - gpuMallocHost( &( this->m_data ), this->bytes() ); - } - virtual ~PinnedHostBufferBase() - { - gpuFreeHost( this->m_data ); - } - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA device buffer - template - class DeviceBufferBase : public BufferBase - { - public: - DeviceBufferBase( const size_t size ) - : BufferBase( size, true ) - { - gpuMalloc( &( this->m_data ), this->bytes() ); - } - virtual ~DeviceBufferBase() - { - gpuFree( this->m_data ); - } - }; -#endif - - //-------------------------------------------------------------------------- - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for a given number of events - template - class HostBuffer : public HostBufferBase, virtual private NumberOfEvents - { - public: - HostBuffer( const size_t nevt ) - : NumberOfEvents( nevt ) - , HostBufferBase( sizePerEvent * nevt ) - { - //std::cout << "HostBuffer::ctor " << this << " " << nevt << std::endl; - } - virtual ~HostBuffer() - { - //std::cout << "HostBuffer::dtor " << this << std::endl; - } - virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA pinned host buffer for a given number of events - template - class PinnedHostBuffer : public PinnedHostBufferBase, virtual private NumberOfEvents - { - public: - PinnedHostBuffer( const size_t nevt ) - : NumberOfEvents( nevt ) - , PinnedHostBufferBase( sizePerEvent * nevt ) {} - virtual ~PinnedHostBuffer() {} - virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a CUDA device buffer for a given number of events - template - class DeviceBuffer : public DeviceBufferBase, virtual protected NumberOfEvents - { - public: - DeviceBuffer( const size_t nevt ) - : NumberOfEvents( nevt ) - , DeviceBufferBase( sizePerEvent * nevt ) - { - //std::cout << "DeviceBuffer::ctor " << this << " " << nevt << std::endl; - } - virtual ~DeviceBuffer() - { - //std::cout << "DeviceBuffer::dtor " << this << std::endl; - } - virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating a simple CUDA device buffer managed on an ad-hoc basis - typedef DeviceBuffer DeviceBufferSimple; - typedef DeviceBuffer DeviceBufferSimple2; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for momenta random numbers - typedef BufferBase BufferRndNumMomenta; - - // The size (number of elements) per event in a memory buffer for momenta random numbers - constexpr size_t sizePerEventRndNumMomenta = MemoryBuffers::np4 * MemoryBuffers::nparf; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for momenta random numbers - typedef HostBuffer HostBufferRndNumMomenta; -#else - // A class encapsulating a CUDA pinned host buffer for momenta random numbers - typedef PinnedHostBuffer PinnedHostBufferRndNumMomenta; - // A class encapsulating a CUDA device buffer for momenta random numbers - typedef DeviceBuffer DeviceBufferRndNumMomenta; -#endif - - //-------------------------------------------------------------------------- - - /* - // A base class encapsulating a memory buffer with ONE fptype per event - typedef BufferBase BufferOneFp; - - // The size (number of elements) per event in a memory buffer with ONE fptype per event - constexpr size_t sizePerEventOneFp = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer with ONE fptype per event - typedef HostBuffer HostBufferOneFp; -#else - // A class encapsulating a CUDA pinned host buffer for gs - typedef PinnedHostBuffer PinnedHostBufferOneFp; - // A class encapsulating a CUDA device buffer for gs - typedef DeviceBuffer DeviceBufferOneFp; -#endif - - // Memory buffers for Gs (related to the event-by-event strength of running coupling constant alphas QCD) - typedef BufferOneFp BufferGs; - typedef HostBufferOneFp HostBufferGs; - typedef PinnedHostBufferOneFp PinnedHostBufferGs; - typedef DeviceBufferOneFp DeviceBufferGs; - */ - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for Gs (related to the event-by-event strength of running coupling constant alphas QCD) - typedef BufferBase BufferGs; - - // The size (number of elements) per event in a memory buffer for Gs - constexpr size_t sizePerEventGs = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for gs - typedef HostBuffer HostBufferGs; -#else - // A class encapsulating a CUDA pinned host buffer for gs - typedef PinnedHostBuffer PinnedHostBufferGs; - // A class encapsulating a CUDA device buffer for gs - typedef DeviceBuffer DeviceBufferGs; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for numerators (of the multichannel single-diagram enhancement factors) - typedef BufferBase BufferNumerators; - - // The size (number of elements) per event in a memory buffer for numerators - // (should be equal to the number of diagrams in the process) - constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for numerators - typedef HostBuffer HostBufferNumerators; -#else - // A class encapsulating a CUDA pinned host buffer for numerators - typedef PinnedHostBuffer PinnedHostBufferNumerators; - // A class encapsulating a CUDA device buffer for numerators - typedef DeviceBuffer DeviceBufferNumerators; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for denominators (of the multichannel single-diagram enhancement factors) - typedef BufferBase BufferDenominators; - - // The size (number of elements) per event in a memory buffer for denominators - constexpr size_t sizePerEventDenominators = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for denominators - typedef HostBuffer HostBufferDenominators; -#else - // A class encapsulating a CUDA pinned host buffer for denominators - typedef PinnedHostBuffer PinnedHostBufferDenominators; - // A class encapsulating a CUDA device buffer for denominators - typedef DeviceBuffer DeviceBufferDenominators; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for couplings that depend on the event-by-event running coupling constant alphas QCD - typedef BufferBase BufferCouplings; - - // The size (number of elements) per event in a memory buffer for random numbers - constexpr size_t sizePerEventCouplings = MemoryBuffers::ndcoup * MemoryBuffers::nx2; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for couplings - typedef HostBuffer HostBufferCouplings; -#else - // A class encapsulating a CUDA pinned host buffer for couplings - typedef PinnedHostBuffer PinnedHostBufferCouplings; - // A class encapsulating a CUDA device buffer for couplings - typedef DeviceBuffer DeviceBufferCouplings; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for momenta - typedef BufferBase BufferMomenta; - - // The size (number of elements) per event in a memory buffer for momenta - constexpr size_t sizePerEventMomenta = MemoryBuffers::np4 * MemoryBuffers::npar; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for momenta - typedef HostBuffer HostBufferMomenta; - //typedef HostBuffer HostBufferMomenta; // TEST MISALIGNMENT! -#else - // A class encapsulating a CUDA pinned host buffer for momenta - typedef PinnedHostBuffer PinnedHostBufferMomenta; - // A class encapsulating a CUDA device buffer for momenta - typedef DeviceBuffer DeviceBufferMomenta; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for sampling weights - typedef BufferBase BufferWeights; - - // The size (number of elements) per event in a memory buffer for sampling weights - constexpr size_t sizePerEventWeights = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for sampling weights - typedef HostBuffer HostBufferWeights; -#else - // A class encapsulating a CUDA pinned host buffer for sampling weights - typedef PinnedHostBuffer PinnedHostBufferWeights; - // A class encapsulating a CUDA device buffer for sampling weights - typedef DeviceBuffer DeviceBufferWeights; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for matrix elements - typedef BufferBase BufferMatrixElements; - - // The size (number of elements) per event in a memory buffer for matrix elements - constexpr size_t sizePerEventMatrixElements = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for matrix elements - typedef HostBuffer HostBufferMatrixElements; -#else - // A class encapsulating a CUDA pinned host buffer for matrix elements - typedef PinnedHostBuffer PinnedHostBufferMatrixElements; - // A class encapsulating a CUDA device buffer for matrix elements - typedef DeviceBuffer DeviceBufferMatrixElements; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for the helicity mask - typedef BufferBase BufferHelicityMask; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for the helicity mask - typedef HostBufferBase HostBufferHelicityMask; -#else - // A class encapsulating a CUDA pinned host buffer for the helicity mask - typedef PinnedHostBufferBase PinnedHostBufferHelicityMask; - // A class encapsulating a CUDA device buffer for the helicity mask - typedef DeviceBufferBase DeviceBufferHelicityMask; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for wavefunctions - typedef BufferBase BufferWavefunctions; - - // The size (number of elements) per event in a memory buffer for wavefunctions - constexpr size_t sizePerEventWavefunctions = MemoryBuffers::nw6 * MemoryBuffers::nx2; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for wavefunctions - typedef HostBuffer HostBufferWavefunctions; -#else - // A class encapsulating a CUDA pinned host buffer for wavefunctions - typedef PinnedHostBuffer PinnedHostBufferWavefunctions; - // A class encapsulating a CUDA device buffer for wavefunctions - typedef DeviceBuffer DeviceBufferWavefunctions; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for helicity random numbers - typedef BufferBase BufferRndNumHelicity; - - // The size (number of elements) per event in a memory buffer for helicity random numbers - constexpr size_t sizePerEventRndNumHelicity = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for helicity random numbers - typedef HostBuffer HostBufferRndNumHelicity; -#else - // A class encapsulating a CUDA pinned host buffer for helicity random numbers - typedef PinnedHostBuffer PinnedHostBufferRndNumHelicity; - // A class encapsulating a CUDA device buffer for helicity random numbers - typedef DeviceBuffer DeviceBufferRndNumHelicity; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for color random numbers - typedef BufferBase BufferRndNumColor; - - // The size (number of elements) per event in a memory buffer for color random numbers - constexpr size_t sizePerEventRndNumColor = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for color random numbers - typedef HostBuffer HostBufferRndNumColor; -#else - // A class encapsulating a CUDA pinned host buffer for color random numbers - typedef PinnedHostBuffer PinnedHostBufferRndNumColor; - // A class encapsulating a CUDA device buffer for color random numbers - typedef DeviceBuffer DeviceBufferRndNumColor; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for channel ids - typedef BufferBase BufferChannelIds; - - // The size (number of elements) per event in a memory buffer for channel ids - constexpr size_t sizePerEventChannelId = 1; - -#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) - // A class encapsulating a C++ host buffer for channel ids - typedef HostBuffer HostBufferChannelIds; -#else - // A class encapsulating a CUDA pinned host buffer for channel ids - typedef PinnedHostBuffer PinnedHostBufferChannelIds; - // A class encapsulating a CUDA device buffer for channel ids - typedef DeviceBuffer DeviceBufferChannelIds; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for channel ids - typedef BufferBase BufferIflavorVec; - - // The size (number of elements) per event in a memory buffer for channel ids - constexpr size_t sizePerEventIflavorVec = 1; - -#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) - // A class encapsulating a C++ host buffer for channel ids - typedef HostBuffer HostBufferIflavorVec; -#else - // A class encapsulating a CUDA pinned host buffer for channel ids - typedef PinnedHostBuffer PinnedHostBufferIflavorVec; - // A class encapsulating a CUDA device buffer for channel ids - typedef DeviceBuffer DeviceBufferIflavorVec; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for helicity selection - typedef BufferBase BufferSelectedHelicity; - - // The size (number of elements) per event in a memory buffer for helicity selection - constexpr size_t sizePerEventSelectedHelicity = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for helicity selection - typedef HostBuffer HostBufferSelectedHelicity; -#else - // A class encapsulating a CUDA pinned host buffer for helicity selection - typedef PinnedHostBuffer PinnedHostBufferSelectedHelicity; - // A class encapsulating a CUDA device buffer for helicity selection - typedef DeviceBuffer DeviceBufferSelectedHelicity; -#endif - - //-------------------------------------------------------------------------- - - // A base class encapsulating a memory buffer for color selection - typedef BufferBase BufferSelectedColor; - - // The size (number of elements) per event in a memory buffer for color selection - constexpr size_t sizePerEventSelectedColor = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for color selection - typedef HostBuffer HostBufferSelectedColor; -#else - // A class encapsulating a CUDA pinned host buffer for color selection - typedef PinnedHostBuffer PinnedHostBufferSelectedColor; - // A class encapsulating a CUDA device buffer for color selection - typedef DeviceBuffer DeviceBufferSelectedColor; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // The size (number of elements) per event in a memory buffer for jamps - constexpr size_t sizePerEventJamps = MemoryBuffers::ncolor * MemoryBuffers::nx2; - - // A class encapsulating a CUDA device buffer for color selection - typedef DeviceBuffer DeviceBufferJamps; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - template - void copyDeviceFromHost( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy - { - if( dst.size() != src.size() ) - { - std::ostringstream sstr; - sstr << "Size (#elements) mismatch in copyDeviceFromHost: dst=" << dst.size() << ", src=" << src.size(); - throw std::runtime_error( sstr.str() ); - } - if( dst.bytes() != src.bytes() ) - { - std::ostringstream sstr; - sstr << "Size (#bytes) mismatch in copyDeviceFromHost: dst=" << dst.bytes() << ", src=" << src.bytes(); - throw std::runtime_error( sstr.str() ); - } - // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array - gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyHostToDevice ); - } -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - template - void copyHostFromDevice( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy - { - if( dst.size() != src.size() ) - { - std::ostringstream sstr; - sstr << "Size (#elements) mismatch in copyHostFromDevice: dst=" << dst.size() << ", src=" << src.size(); - throw std::runtime_error( sstr.str() ); - } - if( dst.bytes() != src.bytes() ) - { - std::ostringstream sstr; - sstr << "Size (#bytes) mismatch in copyHostFromDevice: dst=" << dst.bytes() << ", src=" << src.bytes(); - throw std::runtime_error( sstr.str() ); - } - // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array - gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyDeviceToHost ); - } -#endif - - //-------------------------------------------------------------------------- -} - -#endif // MemoryBuffers_H diff --git a/madgraph/iolibs/template_files/madmatrix/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/color_sum.cc deleted file mode 100644 index 30c6799932..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/color_sum.cc +++ /dev/null @@ -1,434 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "color_sum.h" - -#include "mgOnGpuConfig.h" - -#include "MemoryAccessMatrixElements.h" - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors - - //-------------------------------------------------------------------------- - - // *** COLOR MATRIX BELOW *** -%(color_matrix_lines)s - -#ifdef MGONGPUCPP_GPUIMPL - // The normalized color matrix (divide each column by denom) - template - struct NormalizedColorMatrix - { - constexpr __host__ __device__ NormalizedColorMatrix() - : value() - { - for( int icol = 0; icol < ncolor; icol++ ) - for( int jcol = 0; jcol < ncolor; jcol++ ) - value[icol * ncolor + jcol] = colorMatrix[icol][jcol] / colorDenom[icol]; - } - T value[ncolor * ncolor]; - }; - // The fptype2 version is the default used by kernels (supporting mixed floating point mode also in blas) - static __device__ fptype2 s_pNormalizedColorMatrix2[ncolor * ncolor]; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - void createNormalizedColorMatrix() - { - static bool first = true; - if( first ) - { - first = false; - constexpr NormalizedColorMatrix normalizedColorMatrix2; - gpuMemcpyToSymbol( s_pNormalizedColorMatrix2, normalizedColorMatrix2.value, ncolor * ncolor * sizeof( fptype2 ) ); - } - } -#endif - - //-------------------------------------------------------------------------- - -#ifndef MGONGPUCPP_GPUIMPL - void - color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity - const int ievt0 ) // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) - { - // Pre-compute a constexpr triangular color matrix properly normalized #475 - struct TriangularNormalizedColorMatrix - { - // See https://stackoverflow.com/a/34465458 - __host__ __device__ constexpr TriangularNormalizedColorMatrix() - : value() - { - for( int icol = 0; icol < ncolor; icol++ ) - { - // Diagonal terms - value[icol][icol] = colorMatrix[icol][icol] / colorDenom[icol]; - // Off-diagonal terms - for( int jcol = icol + 1; jcol < ncolor; jcol++ ) - value[icol][jcol] = 2 * colorMatrix[icol][jcol] / colorDenom[icol]; - } - } - fptype2 value[ncolor][ncolor]; - }; - static constexpr auto cf2 = TriangularNormalizedColorMatrix(); - // Use the property that M is a real matrix (see #475): - // we can rewrite the quadratic form (A-iB)(M)(A+iB) as AMA - iBMA + iBMA + BMB = AMA + BMB - // In addition, on C++ use the property that M is symmetric (see #475), - // and also use constexpr to compute "2*" and "/colorDenom[icol]" once and for all at compile time: - // we gain (not a factor 2...) in speed here as we only loop over the up diagonal part of the matrix. - // Strangely, CUDA is slower instead, so keep the old implementation for the moment. - fptype_sv deltaMEs = { 0 }; -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype_sv deltaMEs_next = { 0 }; - // Mixed mode: merge two neppV vectors into one neppV2 vector - fptype2_sv jampR_sv[ncolor]; - fptype2_sv jampI_sv[ncolor]; - for( int icol = 0; icol < ncolor; icol++ ) - { - jampR_sv[icol] = fpvmerge( cxreal( allJamp_sv[icol] ), cxreal( allJamp_sv[ncolor + icol] ) ); - jampI_sv[icol] = fpvmerge( cximag( allJamp_sv[icol] ), cximag( allJamp_sv[ncolor + icol] ) ); - } -#else - const cxtype_sv* jamp_sv = allJamp_sv; -#endif - // Loop over icol - for( int icol = 0; icol < ncolor; icol++ ) - { - // Diagonal terms -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype2_sv& jampRi_sv = jampR_sv[icol]; - fptype2_sv& jampIi_sv = jampI_sv[icol]; -#else - fptype2_sv jampRi_sv = (fptype2_sv)( cxreal( jamp_sv[icol] ) ); - fptype2_sv jampIi_sv = (fptype2_sv)( cximag( jamp_sv[icol] ) ); -#endif - fptype2_sv ztempR_sv = cf2.value[icol][icol] * jampRi_sv; - fptype2_sv ztempI_sv = cf2.value[icol][icol] * jampIi_sv; - // Loop over jcol - for( int jcol = icol + 1; jcol < ncolor; jcol++ ) - { - // Off-diagonal terms -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype2_sv& jampRj_sv = jampR_sv[jcol]; - fptype2_sv& jampIj_sv = jampI_sv[jcol]; -#else - fptype2_sv jampRj_sv = (fptype2_sv)( cxreal( jamp_sv[jcol] ) ); - fptype2_sv jampIj_sv = (fptype2_sv)( cximag( jamp_sv[jcol] ) ); -#endif - ztempR_sv += cf2.value[icol][jcol] * jampRj_sv; - ztempI_sv += cf2.value[icol][jcol] * jampIj_sv; - } - fptype2_sv deltaMEs2 = ( jampRi_sv * ztempR_sv + jampIi_sv * ztempI_sv ); // may underflow #831 -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - deltaMEs += fpvsplit0( deltaMEs2 ); - deltaMEs_next += fpvsplit1( deltaMEs2 ); -#else - deltaMEs += deltaMEs2; -#endif - } - // *** STORE THE RESULTS *** - using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events - fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); - // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) - fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); - MEs_sv += deltaMEs; // fix #435 -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype* MEs_next = E_ACCESS::ieventAccessRecord( allMEs, ievt0 + neppV ); - fptype_sv& MEs_sv_next = E_ACCESS::kernelAccess( MEs_next ); - MEs_sv_next += deltaMEs_next; -#endif - } -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity - const int nGoodHel, // input: number of good helicities - const int nevtIfAllHelicities ) // input: zero in single-helicity mode, number of events in multi-helicity mode - { - if (nevtIfAllHelicities) { - int ighel = blockIdx.y; - allMEs = allMEs + ighel * nevtIfAllHelicities; // MEs for one specific helicity ighel - allJamps = allJamps + ighel * nevtIfAllHelicities; // Jamps for one specific helicity ighel - } - using J_ACCESS = DeviceAccessJamp; - fptype jampR[ncolor]; - fptype jampI[ncolor]; - for( int icol = 0; icol < ncolor; icol++ ) - { - constexpr int ihel0 = 0; // the input buffer allJamps already points to a specific helicity - cxtype jamp = J_ACCESS::kernelAccessIcolIhelNhelConst( allJamps, icol, ihel0, nGoodHel ); - jampR[icol] = jamp.real(); - jampI[icol] = jamp.imag(); - } - // Loop over icol - fptype deltaMEs = { 0 }; - for( int icol = 0; icol < ncolor; icol++ ) - { - fptype2 ztempR = { 0 }; - fptype2 ztempI = { 0 }; - fptype2 jampRi = jampR[icol]; - fptype2 jampIi = jampI[icol]; - // OLD IMPLEMENTATION (ihel3: symmetric square matrix) - Loop over all jcol - //for( int jcol = 0; jcol < ncolor; jcol++ ) - //{ - // fptype2 jampRj = jampR[jcol]; - // fptype2 jampIj = jampI[jcol]; - // ztempR += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix - // ztempI += s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix - //} - // NEW IMPLEMENTATION #475 (ihel3p1: triangular lower diagonal matrix) - Loop over jcol < icol - ztempR += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampRi; // use fptype2 version of color matrix - ztempI += s_pNormalizedColorMatrix2[icol * ncolor + icol] * jampIi; // use fptype2 version of color matrix - for( int jcol = 0; jcol < icol; jcol++ ) - { - fptype2 jampRj = jampR[jcol]; - fptype2 jampIj = jampI[jcol]; - ztempR += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampRj; // use fptype2 version of color matrix - ztempI += 2 * s_pNormalizedColorMatrix2[icol * ncolor + jcol] * jampIj; // use fptype2 version of color matrix - } - deltaMEs += ztempR * jampRi; - deltaMEs += ztempI * jampIi; - } - // *** STORE THE RESULTS *** - using E_ACCESS = DeviceAccessMatrixElements; // non-trivial access: buffer includes all events - // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) - E_ACCESS::kernelAccess( allMEs ) += deltaMEs; // fix #435 - } -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL -#ifndef MGONGPU_HAS_NO_BLAS -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - __global__ void - convertD2F_Jamps( fptype2* allJampsFpt2, // output: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel - const fptype* allJamps, // input: jamp[2][ncolor][ihel][nevt] for one specific helicity ihel - const int nhel ) // input: number of good helicities nGoodHel - { - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - constexpr int ihel = 0; // the input buffer allJamps already points to a specific helicity - // NB! From a functional point of view, any striding will be ok here as long as ncolor*2*nevt elements are all correctly copied! - // NB! Just in case this may be better for performance reasons, however, the same striding as in compute_jamps and cuBLAS is used here - for( int ix2 = 0; ix2 < mgOnGpu::nx2; ix2++ ) - for( int icol = 0; icol < ncolor; icol++ ) - allJampsFpt2[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] = - allJamps[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt]; - } -#endif -#endif -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL -#ifndef MGONGPU_HAS_NO_BLAS -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - __global__ void - convertF2D_MEs( fptype* allMEs, // output: allMEs[nevt] for one specific helicity - const fptype2* allMEsFpt2 ) // input: allMEs[nevt] for one specific helicity - { - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - allMEs[ievt] = allMEsFpt2[ievt]; - } -#endif -#endif -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ -#ifndef MGONGPU_HAS_NO_BLAS - void - color_sum_blas( fptype* ghelAllMEs, // output: allMEs super-buffer[nhel][nevt], add |M|^2 separately for each helicity - const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nhel][nevt] for nhel good helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nhel good helicities - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) -#else - gpuStream_t* /*ghelStreams*/, // input: cuda streams (index is ighel: only the first nhel <= ncomb are non-null) -#endif - const int nhel, // input: number of good helicities (nhel == nGoodHel) - const int gpublocks, // input: cuda gpublocks - const int gputhreads ) // input: cuda gputhreads - { - const int nevt = gpublocks * gputhreads; - - // Get the address associated with the normalized color matrix in device memory - static fptype2* devNormColMat = nullptr; - if( !devNormColMat ) gpuGetSymbolAddress( (void**)&devNormColMat, s_pNormalizedColorMatrix2 ); - -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Mixed precision mode: need two fptype2[2*ncolor*nhel*nevt] buffers and one fptype2[nhel*nevt] buffers for the nhel helicities - fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of first fptype2[ncolor*2*nhel*nevt] buffer - fptype2* ghelAllJampsFpt2 = ghelAllBlasTmp + ncolor * mgOnGpu::nx2 * nhel * nevt; // start of second fptype2[ncolor*2*nhel*nevt] buffer - fptype2* ghelAllMEsFpt2 = ghelAllBlasTmp + 2 * ncolor * mgOnGpu::nx2 * nhel * nevt; // start of fptype2[nhel*nevt] buffer - // Convert jamps from double to float - for( int ighel = 0; ighel < nhel; ighel++ ) - { - const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // jamps for a single helicity ihel - fptype2* hAllJampsFpt2 = ghelAllJampsFpt2 + ighel * nevt; // jamps for a single helicity ihel - gpuLaunchKernelStream( convertD2F_Jamps, gpublocks, gputhreads, ghelStreams[ighel], hAllJampsFpt2, hAllJamps, nhel ); - } - // Real and imaginary components - const fptype2* ghelAllJampsReal = ghelAllJampsFpt2; - const fptype2* ghelAllJampsImag = ghelAllJampsFpt2 + ncolor * nhel * nevt; -#else - // Standard single or double precision mode: need one fptype2[ncolor*2*nhel*nevt] buffer - static_assert( std::is_same::value ); - fptype2* ghelAllZtempBoth = ghelAllBlasTmp; // start of fptype2[ncolor*2*nhel*nevt] buffer - fptype2* ghelAllMEsFpt2 = ghelAllMEs; - // Real and imaginary components - const fptype2* ghelAllJampsReal = ghelAllJamps; // this is not a cast (the two types are identical) - const fptype2* ghelAllJampsImag = ghelAllJamps + ncolor * nhel * nevt; // this is not a cast (the two types are identical) -#endif - // Real and imaginary components - fptype2* ghelAllZtempReal = ghelAllZtempBoth; - fptype2* ghelAllZtempImag = ghelAllZtempBoth + ncolor * nhel * nevt; - - // Note: striding for cuBLAS from DeviceAccessJamp: - // - ghelAllJamps(icol,ihel,ievt).real is ghelAllJamps[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] - // - ghelAllJamps(icol,ihel,ievt).imag is ghelAllJamps[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] - - // Step 1: Compute Ztemp[ncolor][nhel*nevt] = ColorMatrix[ncolor][ncolor] * JampsVector[ncolor][nhel*nevt] for both real and imag - // In this case alpha=1 and beta=0: the operation is Ztemp = alpha * ColorMatrix * JampsVector + beta * Ztemp - fptype2 alpha1 = 1; - fptype2 beta1 = 0; - const int ncolorM = ncolor; - const int nevtN = nhel*nevt; - const int ncolorK = ncolor; - checkGpuBlas( gpuBlasTgemm( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose ColMat - GPUBLAS_OP_T, // transpose JampsV (new1) - ncolorM, nevtN, ncolorK, - &alpha1, - devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK - ghelAllJampsReal, nevtN, // JampsV is nevtN x ncolorK - &beta1, - ghelAllZtempReal, ncolorM ) ); // Ztemp is ncolorM x nevtN - checkGpuBlas( gpuBlasTgemm( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose ColMat - GPUBLAS_OP_T, // transpose JampsV (new1) - ncolorM, nevtN, ncolorK, - &alpha1, - devNormColMat, ncolorM, // ColMat is ncolorM x ncolorK - ghelAllJampsImag, nevtN, // JampsV is nevtN x ncolorK (new1) - &beta1, - ghelAllZtempImag, ncolorM ) ); // Ztemp is ncolorM x nevtN - - // Step 2: For each ievt, compute the dot product of JampsVector[ncolor][ievt] dot tmp[ncolor][ievt] - // In this case alpha=1 and beta=1: the operation is ME = alpha * ( Tmp dot JampsVector ) + beta * ME - // Use cublasSgemmStridedBatched to perform these batched dot products in one call - fptype2 alpha2 = 1; - fptype2 beta2 = 1; - checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose JampsV (new1) - GPUBLAS_OP_N, // do not transpose Tmp - 1, 1, ncolor, // result is 1x1 (dot product) - &alpha2, - ghelAllJampsReal, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column - ghelAllZtempReal, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column - &beta2, - ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) - nevtN ) ); // there are nevtN (nhel*nevt) "batches" - checkGpuBlas( gpuBlasTgemmStridedBatched( *pBlasHandle, - GPUBLAS_OP_N, // do not transpose JampsV (new1) - GPUBLAS_OP_N, // do not transpose Tmp - 1, 1, ncolor, // result is 1x1 (dot product) - &alpha2, - ghelAllJampsImag, nevtN, 1, // allJamps is nevtN x ncolor, stride 1 for each ievt column (new1) - ghelAllZtempImag, ncolor, ncolor, // allZtemp is ncolor x nevtN, with stride ncolor for each ievt column - &beta2, - ghelAllMEsFpt2, 1, 1, // output is a 1x1 result for each "batch" (i.e. for each ievt) - nevtN ) ); // there are nevt (nhel*nevt) "batches" - -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Convert MEs from float to double - for( int ighel = 0; ighel < nhel; ighel++ ) - { - fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for a single helicity ihel - fptype2* hAllMEsFpt2 = ghelAllMEsFpt2 + ighel * nevt; // MEs for a single helicity ihel - gpuLaunchKernelStream( convertF2D_MEs, gpublocks, gputhreads, ghelStreams[ighel], hAllMEs, hAllMEsFpt2 ); - } -#endif - } -#endif /* clang-format on */ -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - void - color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) - const int nGoodHel, // input: number of good helicities - const int gpublocks, // input: cuda gpublocks - const int gputhreads, // input: cuda gputhreads - const bool processAllHelicities ) // input: if true, use blockIdx.y to index helicities - { - const int nevt = gpublocks * gputhreads; - // CASE 1: KERNEL - if( !pBlasHandle ) - { - assert( ghelAllBlasTmp == nullptr ); // sanity check for HASBLAS=hasNoBlas or CUDACPP_RUNTIME_BLASCOLORSUM not set - if (processAllHelicities) { - gpuLaunchKernel2D( color_sum_kernel, gpublocks, nGoodHel, gputhreads, ghelStreams[0], ghelAllMEs, ghelAllJamps, nGoodHel, nevt ); - } else { - // Loop over helicities - for( int ighel = 0; ighel < nGoodHel; ighel++ ) - { - fptype* hAllMEs = ghelAllMEs + ighel * nevt; // MEs for one specific helicity ighel - const fptype* hAllJamps = ghelAllJamps + ighel * nevt; // Jamps for one specific helicity ighel - gpuStream_t hStream = ghelStreams[ighel]; - gpuLaunchKernelStream( color_sum_kernel, gpublocks, gputhreads, hStream, hAllMEs, hAllJamps, nGoodHel, 0 ); - } - } - } - // CASE 2: BLAS - else - { -#ifdef MGONGPU_HAS_NO_BLAS - assert( false ); // sanity check: no path to this statement for HASBLAS=hasNoBlas -#else - if (processAllHelicities) { - assert( false ); // BLAS in async mode not supported for now - } else { - checkGpu( gpuDeviceSynchronize() ); // do not start the BLAS color sum for all helicities until the loop over helicities has completed - // Reset the tmp buffer -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( 2 * ncolor * mgOnGpu::nx2 + 1 ) * sizeof( fptype2 ) ); -#else - gpuMemset( ghelAllBlasTmp, 0, nGoodHel * nevt * ( ncolor * mgOnGpu::nx2 ) * sizeof( fptype2 ) ); -#endif - // Delegate the color sum to BLAS for - color_sum_blas( ghelAllMEs, ghelAllJamps, ghelAllBlasTmp, pBlasHandle, ghelStreams, nGoodHel, gpublocks, gputhreads ); - } -#endif - } - } -#endif - - //-------------------------------------------------------------------------- - -} // end namespace diff --git a/madgraph/iolibs/template_files/madmatrix/color_sum.h b/madgraph/iolibs/template_files/madmatrix/color_sum.h deleted file mode 100644 index 347184c4e1..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/color_sum.h +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef COLOR_SUM_H -#define COLOR_SUM_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuVectors.h" - -#include "CPPProcess.h" -#include "GpuAbstraction.h" - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - class DeviceAccessJamp - { - public: - static __device__ inline cxtype_ref - kernelAccessIcolIhelNhel( fptype* buffer, const int icol, const int ihel, const int nhel ) - { - const int ncolor = CPPProcess::ncolor; // the number of leading colors - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) - //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" - // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) - // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS - //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" - // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) - return cxtype_ref( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], - buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); - } - static __device__ inline const cxtype - kernelAccessIcolIhelNhelConst( const fptype* buffer, const int icol, const int ihel, const int nhel ) - { - const int ncolor = CPPProcess::ncolor; // the number of leading colors - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) - //return cxtype_ref( buffer[icol * 2 * nevt + ievt], buffer[icol * 2 * nevt + nevt + ievt] ); // "old" - // (ONE HELICITY) New "new1" striding for cuBLAS: two separate ncolor*nevt matrices for each of real and imag (ievt last) - // The "new1" striding was used for both HASBLAS=hasBlas and hasNoBlas builds and for both CUDA kernels and cuBLAS - //return cxtype_ref( buffer[0 * ncolor * nevt + icol * nevt + ievt], buffer[1 * ncolor * nevt + icol * nevt + ievt] ); // "new1" - // (ALL HELICITIES) New striding for cuBLAS: two separate ncolor*nhel*nevt matrices for each of real and imag (ievt last) - return cxtype( buffer[0 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt], - buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); - } - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - void createNormalizedColorMatrix(); -#endif - - //-------------------------------------------------------------------------- - -#ifndef MGONGPUCPP_GPUIMPL - void - color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity - const int ievt0 ); // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - void - color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) - const int nGoodHel, // input: number of good helicities - const int gpublocks, // input: cuda gpublocks - const int gputhreads, // input: cuda gputhreads - const bool processAllHelicities); // input: if true, use blockIdx.y to index helicities -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity - const int nGoodHel, // input: number of good helicities - const int nevtIfAllHelicities); // input: zero in single-helicity mode, number of events in multi-helicity mode -#endif - - //-------------------------------------------------------------------------- -} - -#endif // COLOR_SUM_H diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk index 185e37137e..3664dcde4c 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk @@ -408,7 +408,7 @@ endif #=== Configure common compiler flags for C++ and CUDA/HIP -INCFLAGS = -I. -Ibackend/$(BACKENDDIR) +INCFLAGS = -I. -I../../backend/$(BACKENDDIR) OPTFLAGS = -O3 # HIP requires -O2 to avoid "Memory access fault" in gq_ttq (#806) @@ -715,8 +715,8 @@ processid_short=$(shell basename $(CURDIR)) MADMATRIX_LIB = madmatrix_$(processid_short)_$(BACKEND) objects_lib=$(BUILDDIR)/CPPProcess.o $(BUILDDIR)/color_sum.o $(BUILDDIR)/MatrixElementKernels.o $(BUILDDIR)/CrossSectionKernels.o $(BUILDDIR)/umami.o $(BUILDDIR)/SigmaKin.o -# Backend-owned sources -vpath %%.cc backend/$(BACKENDDIR) +# Backend-owned sources +vpath %%.cc ../../backend/$(BACKENDDIR) # Explicitly define the default goal (this is not necessary as it is the first target, which is implicitly the default goal) .DEFAULT_GOAL := all.$(TAG) @@ -763,11 +763,11 @@ endif # incompatible backends (different BACKEND, FPTYPE, etc.) in the same directory. # Use USEBUILDDIR=1 to build for multiple backends simultaneously without cleaning. ifeq ($(GPUCC),) -$(BUILDDIR)/%%.o : %%.cc *.h backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) +$(BUILDDIR)/%%.o : %%.cc *.h ../../backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(INCFLAGS) $(CXXFLAGS) -c $< -o $@ else -$(BUILDDIR)/%%.o : %%.cc *.h backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) +$(BUILDDIR)/%%.o : %%.cc *.h ../../backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi $(GPUCC) $(CPPFLAGS) $(INCFLAGS) $(GPUFLAGS) -c -x $(GPULANGUAGE) $< -o $@ endif diff --git a/madgraph/iolibs/template_files/madmatrix/umami.cc b/madgraph/iolibs/template_files/madmatrix/umami.cc deleted file mode 100644 index d19c93bb9b..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/umami.cc +++ /dev/null @@ -1,667 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: T. Heimel (Nov 2025) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro (2026). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "umami.h" - -#include "CPPProcess.h" -#include "GpuRuntime.h" -#include "MemoryAccessMomenta.h" -#include "MemoryBuffers.h" - -#include -#include -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else -using namespace mg5amcCpu; -#endif - -namespace -{ - - void* initialize_impl( - const fptype* momenta, - const fptype* couplings, - const unsigned int* flavor_indices, - fptype* matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL - fptype* color_jamps, -#endif - fptype* numerators, - fptype* denominators, - std::size_t count ) - { - bool is_good_hel[CPPProcess::ncomb]; - sigmaKin_getGoodHel( - momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, -#ifdef MGONGPUCPP_GPUIMPL - color_jamps, -#endif - is_good_hel, - count ); - sigmaKin_setGoodHel( is_good_hel ); - return nullptr; - } - - void initialize( - const fptype* momenta, - const fptype* couplings, - const unsigned int* flavor_indices, - fptype* matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL - fptype* color_jamps, -#endif - fptype* numerators, - fptype* denominators, - std::size_t count ) - { - // static local initialization is called exactly once in a thread-safe way - static void* dummy = initialize_impl( momenta, couplings, flavor_indices, matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL - color_jamps, -#endif - numerators, - denominators, - count ); - } - -#ifdef MGONGPUCPP_GPUIMPL - __device__ -#endif - void - transpose_momenta( const double* momenta_in, fptype* momenta_out, std::size_t i_event_in, std::size_t i_event_out, std::size_t stride ) - { - std::size_t page_size = MemoryAccessMomentaBase::neppM; - std::size_t i_page = i_event_out / page_size; - std::size_t i_vector = i_event_out % page_size; - - for( std::size_t i_part = 0; i_part < CPPProcess::npar; ++i_part ) - { - for( std::size_t i_mom = 0; i_mom < 4; ++i_mom ) - { - momenta_out[i_page * CPPProcess::npar * 4 * page_size + - i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( CPPProcess::npar * i_mom + i_part ) + i_event_in]; - } - } - } - -#ifdef MGONGPUCPP_GPUIMPL - - __global__ void copy_inputs( - const double* momenta_in, - const double* helicity_random_in, - const double* color_random_in, - const double* diagram_random_in, - const double* alpha_s_in, - const unsigned int* flavor_indices_in, - fptype* momenta, - fptype* helicity_random, - fptype* color_random, - fptype* diagram_random, - fptype* g_s, - unsigned int* flavor_indices, - std::size_t count, - std::size_t stride, - std::size_t offset ) - { - std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; - if( i_event >= count ) return; - - transpose_momenta( &momenta_in[offset], momenta, i_event, i_event, stride ); - diagram_random[i_event] = diagram_random_in ? diagram_random_in[i_event + offset] : 0.5; - helicity_random[i_event] = helicity_random_in ? helicity_random_in[i_event + offset] : 0.5; - color_random[i_event] = color_random_in ? color_random_in[i_event + offset] : 0.5; - g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; - flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; - } - - __global__ void copy_outputs( - fptype* denominators, - fptype* numerators, - fptype* matrix_elements, - unsigned int* diagram_index, - int* color_index, - int* helicity_index, - double* m2_out, - double* amp2_out, - int* diagram_out, - int* color_out, - int* helicity_out, - std::size_t count, - std::size_t stride, - std::size_t offset ) - { - std::size_t i_event = blockDim.x * blockIdx.x + threadIdx.x; - if( i_event >= count ) return; - - if( m2_out ) m2_out[i_event + offset] = matrix_elements[i_event]; - if( amp2_out ) - { - double denominator = denominators[i_event]; - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) - { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_event * CPPProcess::ndiagrams + i_diag] / denominator; - } - } - if( diagram_out ) diagram_out[i_event + offset] = diagram_index[i_event] - 1; - if( color_out ) color_out[i_event + offset] = color_index[i_event] - 1; - if( helicity_out ) helicity_out[i_event + offset] = helicity_index[i_event] - 1; - } - -#endif // MGONGPUCPP_GPUIMPL - - struct InterfaceInstance - { - bool initialized = false; - }; - - std::vector g_externalMasses; - -} - -extern "C" -{ - UmamiStatus umami_get_meta( UmamiMetaKey meta_key, void* result ) - { - switch( meta_key ) - { - case UMAMI_META_DEVICE: - { - UmamiDevice& device = *static_cast( result ); -#ifdef MGONGPUCPP_GPUIMPL -#ifdef __CUDACC__ - device = UMAMI_DEVICE_CUDA; -#elif defined( __HIPCC__ ) - device = UMAMI_DEVICE_HIP; -#endif -#else - device = UMAMI_DEVICE_CPU; -#endif - break; - } - case UMAMI_META_PARTICLE_COUNT: - *static_cast( result ) = CPPProcess::npar; - break; - case UMAMI_META_DIAGRAM_COUNT: - *static_cast( result ) = CPPProcess::ndiagrams; - break; - case UMAMI_META_HELICITY_COUNT: - *static_cast( result ) = CPPProcess::ncomb; - break; - case UMAMI_META_COLOR_COUNT: - return UMAMI_ERROR_UNSUPPORTED_META; - case UMAMI_META_MASSES: - { - if( g_externalMasses.size() != (size_t)CPPProcess::npar ) return UMAMI_ERROR_UNINITIALIZED_META; - - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - static_cast( result )[ipar] = g_externalMasses[ipar]; - break; - } - default: - return UMAMI_ERROR_UNSUPPORTED_META; - } - return UMAMI_SUCCESS; - } - - UmamiStatus umami_supported_inputs( bool const** supported, int* count ) - { - // MOMENTA, ALPHA_S, FLAVOR_INDEX, RANDOM_COLOR, RANDOM_HELICITY, RANDOM_DIAGRAM, - // HELICITY_INDEX=false, DIAGRAM_INDEX=true, CHANNEL_INDEX=false - static const bool data[UMAMI_INPUT_KEY_COUNT] = { true, true, true, true, true, true, false, true }; - *supported = data; - *count = UMAMI_INPUT_KEY_COUNT; - return UMAMI_SUCCESS; - } - - UmamiStatus umami_required_inputs( bool const** required, int* count ) - { - static const bool data[UMAMI_INPUT_KEY_COUNT] = { true }; // MOMENTA only - *required = data; - *count = UMAMI_INPUT_KEY_COUNT; - return UMAMI_SUCCESS; - } - - UmamiStatus umami_supported_outputs( bool const** supported, int* count ) - { - // MATRIX_ELEMENT, DIAGRAM_AMP2, COLOR_INDEX, HELICITY_INDEX, DIAGRAM_INDEX, GPU_STREAM -#ifdef MGONGPUCPP_GPUIMPL - static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true, true }; -#else - static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true }; -#endif - *supported = data; - *count = UMAMI_OUTPUT_KEY_COUNT; - return UMAMI_SUCCESS; - } - - UmamiStatus umami_initialize( UmamiHandle* handle, char const* param_card_path ) - { - CPPProcess process; - process.initProc( param_card_path ); - - const std::vector& masses = process.getMasses(); - g_externalMasses.assign( masses.begin(), masses.end() ); - - auto instance = new InterfaceInstance(); - *handle = instance; - return UMAMI_SUCCESS; - } - - UmamiStatus umami_set_parameter( - [[maybe_unused]] UmamiHandle handle, - [[maybe_unused]] char const* name, - [[maybe_unused]] double parameter_real, - [[maybe_unused]] double parameter_imag ) - { - return UMAMI_ERROR_NOT_IMPLEMENTED; - } - - UmamiStatus umami_get_parameter( - [[maybe_unused]] UmamiHandle handle, - [[maybe_unused]] char const* name, - [[maybe_unused]] double* parameter_real, - [[maybe_unused]] double* parameter_imag ) - { - return UMAMI_ERROR_NOT_IMPLEMENTED; - } - - UmamiStatus umami_matrix_element( - UmamiHandle handle, - size_t count, - size_t stride, - size_t offset, - size_t input_count, - UmamiInputKey const* input_keys, - void const* const* inputs, - size_t output_count, - UmamiOutputKey const* output_keys, - void* const* outputs ) - { - const double* momenta_in = nullptr; - const double* alpha_s_in = nullptr; - const unsigned int* flavor_indices_in = nullptr; - const double* random_color_in = nullptr; - const double* random_helicity_in = nullptr; - const double* random_diagram_in = nullptr; - [[maybe_unused]] const int* diagram_in = nullptr; // TODO: unused - - for( std::size_t i = 0; i < input_count; ++i ) - { - const void* input = inputs[i]; - switch( input_keys[i] ) - { - case UMAMI_IN_MOMENTA: - momenta_in = static_cast( input ); - break; - case UMAMI_IN_ALPHA_S: - alpha_s_in = static_cast( input ); - break; - case UMAMI_IN_FLAVOR_INDEX: - flavor_indices_in = static_cast( input ); - break; - case UMAMI_IN_RANDOM_COLOR: - random_color_in = static_cast( input ); - break; - case UMAMI_IN_RANDOM_HELICITY: - random_helicity_in = static_cast( input ); - break; - case UMAMI_IN_RANDOM_DIAGRAM: - random_diagram_in = static_cast( input ); - break; - case UMAMI_IN_HELICITY_INDEX: - return UMAMI_ERROR_UNSUPPORTED_INPUT; - case UMAMI_IN_DIAGRAM_INDEX: - diagram_in = static_cast( input ); - break; - default: - return UMAMI_ERROR_UNSUPPORTED_INPUT; - } - } - if( !momenta_in ) return UMAMI_ERROR_MISSING_INPUT; - -#ifdef MGONGPUCPP_GPUIMPL - gpuStream_t gpu_stream = nullptr; -#endif - double* m2_out = nullptr; - double* amp2_out = nullptr; - int* diagram_out = nullptr; - int* color_out = nullptr; - int* helicity_out = nullptr; - for( std::size_t i = 0; i < output_count; ++i ) - { - void* output = outputs[i]; - switch( output_keys[i] ) - { - case UMAMI_OUT_MATRIX_ELEMENT: - m2_out = static_cast( output ); - break; - case UMAMI_OUT_DIAGRAM_AMP2: - amp2_out = static_cast( output ); - break; - case UMAMI_OUT_COLOR_INDEX: - color_out = static_cast( output ); - break; - case UMAMI_OUT_HELICITY_INDEX: - helicity_out = static_cast( output ); - break; - case UMAMI_OUT_DIAGRAM_INDEX: - diagram_out = static_cast( output ); - break; -#ifdef MGONGPUCPP_GPUIMPL - case UMAMI_OUT_GPU_STREAM: - gpu_stream = static_cast( output ); - break; -#endif - default: - return UMAMI_ERROR_UNSUPPORTED_OUTPUT; - } - } - -#ifdef MGONGPUCPP_GPUIMPL - std::size_t n_threads = 256; - std::size_t n_blocks = ( count + n_threads - 1 ) / n_threads; - std::size_t rounded_count = n_blocks * n_threads; - - fptype *momenta, *couplings, *g_s, *helicity_random, *color_random, *diagram_random, *color_jamps; - fptype *matrix_elements, *numerators, *denominators, *ghel_matrix_elements, *ghel_jamps; - int *helicity_index, *color_index; - unsigned int *flavor_indices, *diagram_index; - - std::size_t n_coup = mg5amcGpu::Parameters_dependentCouplings::ndcoup; - std::array, 16> ptrs_and_sizes = {{ - {reinterpret_cast(&momenta), rounded_count * CPPProcess::npar * 4 * sizeof( fptype )}, - {reinterpret_cast(&couplings), rounded_count * n_coup * 2 * sizeof( fptype )}, - {reinterpret_cast(&g_s), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&flavor_indices), rounded_count * sizeof( unsigned int )}, - {reinterpret_cast(&helicity_random), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&color_random), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&diagram_random), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&matrix_elements), rounded_count * sizeof( fptype )}, - {reinterpret_cast(&diagram_index), rounded_count * sizeof( unsigned int )}, - {reinterpret_cast(&color_jamps), rounded_count * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, - {reinterpret_cast(&numerators), rounded_count * CPPProcess::ndiagrams * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&denominators), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&helicity_index), rounded_count * sizeof( int )}, - {reinterpret_cast(&color_index), rounded_count * sizeof( int )}, - {reinterpret_cast(&ghel_matrix_elements), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&ghel_jamps), rounded_count * CPPProcess::ncomb * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, - }}; - std::size_t total_size = 0; - constexpr std::size_t MAX_SIZE = std::max(sizeof(fptype), sizeof(int)); - for (auto [ptr, size] : ptrs_and_sizes) { - std::size_t aligned_size = (size + MAX_SIZE - 1) / MAX_SIZE * MAX_SIZE; - total_size += aligned_size; - } - uint8_t* buffer; - // we can consider caching this between matrix element calls - gpuMallocAsync( &buffer, total_size, gpu_stream ); - std::size_t buf_offset = 0; - for (auto [ptr, size] : ptrs_and_sizes) { - std::size_t aligned_size = (size + 7) / 8 * 8; - *ptr = buffer + buf_offset; - buf_offset += aligned_size; - } - - copy_inputs<<>>( - momenta_in, - random_helicity_in, - random_color_in, - random_diagram_in, - alpha_s_in, - flavor_indices_in, - momenta, - helicity_random, - color_random, - diagram_random, - g_s, - flavor_indices, - count, - stride, - offset ); - computeDependentCouplings<<>>( g_s, couplings ); - checkGpu( gpuPeekAtLastError() ); - - InterfaceInstance* instance = static_cast( handle ); - if( !instance->initialized ) - { - initialize( - momenta, couplings, flavor_indices, matrix_elements, color_jamps, numerators, denominators, rounded_count ); - instance->initialized = true; - } - - sigmaKin( - momenta, - couplings, - flavor_indices, - helicity_random, - color_random, - nullptr, - diagram_random, - matrix_elements, - helicity_index, - color_index, - color_jamps, - numerators, - denominators, - diagram_index, - false, - ghel_matrix_elements, - ghel_jamps, - nullptr, - nullptr, - &gpu_stream, - true, - n_blocks, - n_threads ); - - copy_outputs<<>>( - denominators, - numerators, - matrix_elements, - diagram_index, - color_index, - helicity_index, - m2_out, - amp2_out, - diagram_out, - color_out, - helicity_out, - count, - stride, - offset ); - checkGpu( gpuPeekAtLastError() ); - - gpuFreeAsync( buffer, gpu_stream ); -#else // MGONGPUCPP_GPUIMPL - constexpr std::size_t vector_size = MemoryAccessMomentaBase::neppM; - // need to round to round to double page size for some reason - constexpr std::size_t page_size2 = 2 * vector_size; - std::vector permutation; - std::size_t rounded_count; - - constexpr std::size_t flavor_count = CPPProcess::nmaxflavor; - HostBufferBase flavor_indices( ((count + page_size2 - 1) / page_size2 + flavor_count) * page_size2 ); - bool sort_flavors = vector_size > 1 && flavor_count > 1 && flavor_indices_in; - if ( sort_flavors ) - { - permutation.resize(count); - std::size_t voffset = 0; - std::size_t vector_indices[flavor_count] = {}; - std::size_t vector_counts[flavor_count] = {}; - // determine permutation of inputs such that all entries in a SIMD vector - // have the same flavor index - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - unsigned int flav = flavor_indices_in[i_event + offset]; - auto& vcount = vector_counts[flav]; - auto& vindex = vector_indices[flav]; - if ( vcount == 0 ) - { - vindex = voffset * page_size2; - for ( std::size_t i = 0; i < page_size2; ++i) { - flavor_indices[voffset * page_size2 + i] = flav; - } - voffset += 1; - } - permutation[i_event] = vindex + vcount; - vcount = (vcount + 1) % page_size2; - } - rounded_count = voffset * page_size2; - } else { - rounded_count = ( count + page_size2 - 1 ) / page_size2 * page_size2; - } - - HostBufferBase momenta( rounded_count * CPPProcess::npar * 4 ); - HostBufferBase couplings( rounded_count * mg5amcCpu::Parameters_dependentCouplings::ndcoup * 2 ); - HostBufferBase g_s( rounded_count ); - HostBufferBase helicity_random( rounded_count ); - HostBufferBase color_random( rounded_count ); - HostBufferBase diagram_random( rounded_count ); - HostBufferBase matrix_elements( rounded_count ); - HostBufferBase diagram_index( rounded_count ); - HostBufferBase numerators( rounded_count * CPPProcess::ndiagrams ); - HostBufferBase denominators( rounded_count ); - HostBufferBase helicity_index( rounded_count ); - HostBufferBase color_index( rounded_count ); - if ( sort_flavors ) { - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - std::size_t i_sorted = permutation[i_event]; - transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_sorted, stride ); - helicity_random[i_sorted] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; - color_random[i_sorted] = random_color_in ? random_color_in[i_event + offset] : 0.5; - diagram_random[i_sorted] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; - g_s[i_sorted] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; - } - } else { - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_event, stride ); - helicity_random[i_event] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; - color_random[i_event] = random_color_in ? random_color_in[i_event + offset] : 0.5; - diagram_random[i_event] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; - g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; - flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; - } - for ( std::size_t i_event = count; i_event < rounded_count; ++i_event ) { - flavor_indices[i_event] = 0; - } - } - computeDependentCouplings( g_s.data(), couplings.data(), rounded_count ); - - InterfaceInstance* instance = static_cast( handle ); - if( !instance->initialized ) - { - initialize( - momenta.data(), - couplings.data(), - flavor_indices.data(), - matrix_elements.data(), - numerators.data(), - denominators.data(), - rounded_count ); - instance->initialized = true; - } - - sigmaKin( - momenta.data(), - couplings.data(), - flavor_indices.data(), - helicity_random.data(), - color_random.data(), - nullptr, - diagram_random.data(), - matrix_elements.data(), - helicity_index.data(), - color_index.data(), - numerators.data(), - denominators.data(), - diagram_index.data(), - false, - rounded_count ); - - if ( sort_flavors ) - { - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - std::size_t i_sorted = permutation[i_event]; - std::size_t page_size = MemoryAccessMomentaBase::neppM; - std::size_t i_page = i_sorted / page_size; - std::size_t i_vector = i_sorted % page_size; // vector lane - - double denominator = denominators[i_sorted]; - if( m2_out != nullptr ) - { - m2_out[i_event + offset] = matrix_elements[i_sorted]; - } - if( amp2_out != nullptr ) - { - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) - { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; - } - } - if( diagram_out != nullptr ) - { - diagram_out[i_event + offset] = diagram_index[i_sorted] - 1; - } - if( color_out != nullptr ) - { - color_out[i_event + offset] = color_index[i_sorted] - 1; - } - if( helicity_out != nullptr ) - { - helicity_out[i_event + offset] = helicity_index[i_sorted] - 1; - } - } - } else { - std::size_t page_size = MemoryAccessMomentaBase::neppM; - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - std::size_t i_page = i_event / page_size; - std::size_t i_vector = i_event % page_size; - - double denominator = denominators[i_event]; - if( m2_out != nullptr ) - { - m2_out[i_event + offset] = matrix_elements[i_event]; - } - if( amp2_out != nullptr ) - { - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) - { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; - } - } - if( diagram_out != nullptr ) - { - diagram_out[i_event + offset] = diagram_index[i_event] - 1; - } - if( color_out != nullptr ) - { - color_out[i_event + offset] = color_index[i_event] - 1; - } - if( helicity_out != nullptr ) - { - helicity_out[i_event + offset] = helicity_index[i_event] - 1; - } - } - } -#endif // MGONGPUCPP_GPUIMPL - return UMAMI_SUCCESS; - } - - UmamiStatus umami_free( UmamiHandle handle ) - { - InterfaceInstance* instance = static_cast( handle ); - delete instance; - return UMAMI_SUCCESS; - } -} diff --git a/madmatrix/model_handling.py b/madmatrix/model_handling.py index 4a8e45a916..6a49100439 100644 --- a/madmatrix/model_handling.py +++ b/madmatrix/model_handling.py @@ -1734,8 +1734,6 @@ def generate_process_files(self): self.edit_processConfig() # sub process specific, not to be symlinked from the Subprocesses directory self.edit_colorsum() # AV new file (NB this is Sigma-specific, should not be a symlink to Subprocesses) self.edit_coloramps() - self.edit_memorybuffers() # AV new file (NB this is generic in Subprocesses and then linked in Sigma-specific) - self.edit_memoryaccesscouplings() # AV new file (NB this is generic in Subprocesses and then linked in Sigma-specific) super().generate_process_files() # needs to be after get_matrix_element_calls to have nwf ready self.edit_processdata() @@ -1950,28 +1948,6 @@ def edit_coloramps(self): ff.write(template % replace_dict) ff.close() - # AV - new method - def edit_memorybuffers(self): - """Generate MemoryBuffers.h""" - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_memorybuffers') - template = open(pjoin(self.template_path,'madmatrix','MemoryBuffers.h'),'r').read() - replace_dict = {} - replace_dict['model_name'] = self.model_name - ff = open(pjoin(self.path, '..', 'MemoryBuffers.h'),'w') - ff.write(template % replace_dict) - ff.close() - - # AV - new method - def edit_memoryaccesscouplings(self): - """Generate MemoryAccessCouplings.h""" - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_memoryaccesscouplings') - template = open(pjoin(self.template_path,'madmatrix','MemoryAccessCouplings.h'),'r').read() - replace_dict = {} - replace_dict['model_name'] = self.model_name - ff = open(pjoin(self.path, '..', 'MemoryAccessCouplings.h'),'w') - ff.write(template % replace_dict) - ff.close() - # AV - overload the export_cpp.OneProcessExporterCPP method (add debug printout and truncate last \n) # [*NB export_cpp.UFOModelConverterGPU.write_process_h_file is not called!*] def write_process_h_file(self, writer): diff --git a/madmatrix/output.py b/madmatrix/output.py index 1252520a77..03885b813a 100644 --- a/madmatrix/output.py +++ b/madmatrix/output.py @@ -86,18 +86,10 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): 'mgOnGpuFptypes.h', 'mgOnGpuCxtypes.h', 'mgOnGpuVectors.h', 'constexpr_math.h', 'read_slha.h', 'read_slha.cc' ]), - 'SubProcesses': relative_path_list(madmatrix_templates, ['nvtx.h', 'GpuRuntime.h', 'GpuAbstraction.h', 'color_sum.h', 'color_sum.cc', - 'MemoryAccessHelpers.h', 'MemoryAccessVectors.h', - 'MemoryAccessMatrixElements.h', 'MemoryAccessMomenta.h', - 'MemoryAccessRandomNumbers.h', 'MemoryAccessWeights.h', - 'MemoryAccessAmplitudes.h', 'MemoryAccessWavefunctions.h', - 'MemoryAccessGs.h', 'MemoryAccessCouplingsFixed.h', - 'MemoryAccessNumerators.h', 'MemoryAccessDenominators.h', - 'MemoryAccessChannelIds.h', 'MemoryAccessIflavorVec.h', - 'CrossSectionKernels.cc', 'CrossSectionKernels.h', - 'MatrixElementKernels.cc', 'MatrixElementKernels.h', - 'EventStatistics.h', - 'umami.h', 'umami.cc', 'rambo.h']), + # Backend-owned skeleton files live only under backend// now + # (see backend_variants below); only genuinely backend-agnostic files + # (no backend/ counterpart) are copied flat into SubProcesses/. + 'SubProcesses': relative_path_list(madmatrix_templates, ['nvtx.h', 'umami.h', 'rambo.h']), # run_card.toml is generated in finalize() (ProcessExporterMG7.create_run_card) # from the template, not copied verbatim. # Default cards for the optional post-processing tools @@ -126,10 +118,10 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): # Backend-owned skeleton files (GpuRuntime.h, color_sum.{h,cc}, the # MemoryAccess*.h family, MatrixElementKernels/CrossSectionKernels/umami.cc, - # etc.) are NOT linked flat into P* any more: they are only available via - # backend// (see _link_backend_dirs_in_P below), sourced by the - # Makefile's INCFLAGS/vpath (see BACKENDDIR in madmatrix.mk). Only files - # with no backend/ counterpart - genuinely backend-agnostic - stay here. + # etc.) are NOT linked into P* at all: they are compiled straight from the + # single top-level backend// dir via the Makefile's INCFLAGS/vpath + # (see BACKENDDIR in madmatrix.mk). Only files with no backend/ counterpart + # - genuinely backend-agnostic - stay here. to_link_in_P = ['nvtx.h', 'umami.h', 'rambo.h'] template_src_make = pjoin(madmatrix_templates, 'madmatrix_src.mk') @@ -188,21 +180,7 @@ def generate_subprocess_directory(self, matrix_element, cpp_helas_call_writer, p # guarded wavefunction/amplitude calls. if cpp_helas_call_writer is not None: cpp_helas_call_writer.use_flavor_mask = self.use_flavor_mask - out = super().generate_subprocess_directory(matrix_element, cpp_helas_call_writer, proc_number) - self._link_backend_dirs_in_P(matrix_element) - return out - - # Symlink the top-level backend//* into this P*'s own backend//. - def _link_backend_dirs_in_P(self, matrix_element): - proc_dir_name = "P%s" % matrix_element.get('processes')[0].shell_string() - dirpath = pjoin(self.dir_path, 'SubProcesses', proc_dir_name) - with misc.chdir(dirpath): - for backend in self.backend_variants: - backend_rel = pjoin('backend', backend) - os.makedirs(backend_rel, exist_ok=True) - src_dir = pjoin('..', '..', 'backend', backend) - for fname in sorted(os.listdir(src_dir)): - files.ln(pjoin(src_dir, fname), starting_dir=backend_rel) + return super().generate_subprocess_directory(matrix_element, cpp_helas_call_writer, proc_number) # AV (default from OM's tutorial) - add a debug printout def convert_model(self, model, wanted_lorentz=[], wanted_couplings=[]): From 050f49ef280e96d3b437d7d4f04d202eef5726e7 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 19:19:00 +0200 Subject: [PATCH 09/25] More generated code into template files cleaning of the EvaluateDiagrams.inc --- .../madmatrix/backend/cpu/SigmaKin.cc | 50 ++++++++++++ .../madmatrix/backend/gpu/SigmaKin.cc | 52 ++++++++++++ .../madmatrix/backend/simd/SigmaKin.cc | 50 ++++++++++++ .../template_files/madmatrix/madmatrix.mk | 9 ++- madmatrix/model_handling.py | 81 ------------------- 5 files changed, 159 insertions(+), 83 deletions(-) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc index 9536d8abb6..f1dcb874e0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc @@ -160,6 +160,56 @@ namespace mg5amcCpu for( int iParity = 0; iParity < nParity; ++iParity ) { const int ievt0 = ievt00 + iParity * neppV; + + constexpr size_t nxcoup = ndcoup + nIPC; // both dependent and independent couplings + const fptype* allCOUPs[nxcoup]; + for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) + allCOUPs[idcoup] = CD_ACCESS::idcoupAccessBufferConst( allcouplings, idcoup ); // dependent couplings, vary event-by-event + for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) + allCOUPs[ndcoup + iicoup] = CI_ACCESS::iicoupAccessBufferConst( cIPC, iicoup ); // independent couplings, fixed for all events + // C++ kernels take input/output buffers with momenta/MEs for one specific event (the first in the current event page) + const fptype* momenta = M_ACCESS::ieventAccessRecordConst( allmomenta, ievt0 ); + const fptype* COUPs[nxcoup]; + for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) + COUPs[idcoup] = CD_ACCESS::ieventAccessRecordConst( allCOUPs[idcoup], ievt0 ); // dependent couplings, vary event-by-event + for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) + COUPs[ndcoup + iicoup] = allCOUPs[ndcoup + iicoup]; // independent couplings, fixed for all events + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + // Create an array of views over the Flavor Couplings + FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; + + // Dependent (event-by-event, running-alphas) flavor couplings (Step 3): the per-flavor + // values are NOT baked in (they run per event). Gather the current values of the + // underlying dependent couplings for this event page into an AOSOA buffer dpf_value + // (one nx2*neppC SIMD record per (coupling,flavor) slot, matching CD_ACCESS), then build + // an ordinary value-based view over it. The flavor index is constant across a SIMD lane + // (guaranteed by the phase-space integrator), so each lane gets its own running value + // while sharing the same flavor selection. This is the direct analogue of Fortran's + // FLV_xx%VAL(k)%P => GC_yyy(J). The vertex routines are instantiated with CD_ACCESS so + // get_coupling_def reads dpf_value with the right per-flavor stride (CD_ACCESS::flv_stride). + constexpr int ndpfbuf = ( nDPF > 0 ? nDPF * nMF * CD_ACCESS::flv_stride : 1 ); + alignas( mgOnGpu::cppAlign ) fptype dpf_value[ndpfbuf]{}; + for( int idpf = 0; idpf < nDPF; idpf++ ) + for( int imf = 0; imf < nMF; imf++ ) + { + const int idc = cDPF_idcoup[idpf * nMF + imf]; + if( idc >= 0 ) + CD_ACCESS::kernelAccess( dpf_value + ( idpf * nMF + imf ) * CD_ACCESS::flv_stride ) = + CD_ACCESS::kernelAccessConst( COUPs[idc] ); + } + FLV_COUPLING_ARRAY flvCOUPs_dep{ cDPF_partner1, cDPF_partner2, dpf_value }; + + // Reset color flows (reset jamp_sv) at the beginning of a new event or event page + for( int i = 0; i < ncolor; i++ ) { jamp_sv[i] = cxzero_sv(); } + + // Numerators and denominators for the current event (CUDA) or SIMD event page (C++) + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + // Scalar iflavor for the current event (constant across the SIMD vector) + const unsigned int* iflavor_rec = F_ACCESS::ieventAccessRecordConst( iflavorVec, ievt0 ); + const uint_sv iflavor_sv = F_ACCESS::kernelAccessConst( iflavor_rec ); + const unsigned int iflavor = reinterpret_cast(&iflavor_sv)[0]; #include "EvaluateDiagrams.inc" // *** COLOR CHOICE BELOW *** diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc index 605ff0b89a..ad271a29ad 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc @@ -179,6 +179,58 @@ namespace mg5amcGpu for( int iParity = 0; iParity < nParity; ++iParity ) { + constexpr size_t nxcoup = ndcoup + nIPC; // both dependent and independent couplings + const fptype* allCOUPs[nxcoup]; +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#pragma nv_diagnostic push +#pragma nv_diag_suppress 186 // e.g. <> +#endif + for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) + allCOUPs[idcoup] = CD_ACCESS::idcoupAccessBufferConst( allcouplings, idcoup ); // dependent couplings, vary event-by-event + for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) + allCOUPs[ndcoup + iicoup] = CI_ACCESS::iicoupAccessBufferConst( cIPC, iicoup ); // independent couplings, fixed for all events +#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#pragma nv_diagnostic pop +#endif + // CUDA kernels take input/output buffers with momenta/MEs for all events + const fptype* momenta = allmomenta; + const fptype* COUPs[nxcoup]; + for( size_t ixcoup = 0; ixcoup < nxcoup; ixcoup++ ) COUPs[ixcoup] = allCOUPs[ixcoup]; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid + fptype* numerators = &allNumerators[ievt * processConfig::ndiagrams]; + fptype* denominators = allDenominators; + // Create an array of views over the Flavor Couplings + FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; + + // Dependent (event-by-event, running-alphas) flavor couplings (Step 3): the per-flavor + // values are NOT baked in (they run per event). Gather the current values of the + // underlying dependent couplings for this event page into an AOSOA buffer dpf_value + // (one nx2*neppC SIMD record per (coupling,flavor) slot, matching CD_ACCESS), then build + // an ordinary value-based view over it. The flavor index is constant across a SIMD lane + // (guaranteed by the phase-space integrator), so each lane gets its own running value + // while sharing the same flavor selection. This is the direct analogue of Fortran's + // FLV_xx%VAL(k)%P => GC_yyy(J). The vertex routines are instantiated with CD_ACCESS so + // get_coupling_def reads dpf_value with the right per-flavor stride (CD_ACCESS::flv_stride). + constexpr int ndpfbuf = ( nDPF > 0 ? nDPF * nMF * CD_ACCESS::flv_stride : 1 ); + fptype dpf_value[ndpfbuf]{}; + for( int idpf = 0; idpf < nDPF; idpf++ ) + for( int imf = 0; imf < nMF; imf++ ) + { + const int idc = cDPF_idcoup[idpf * nMF + imf]; + if( idc >= 0 ) + CD_ACCESS::kernelAccess( dpf_value + ( idpf * nMF + imf ) * CD_ACCESS::flv_stride ) = + CD_ACCESS::kernelAccessConst( COUPs[idc] ); + } + FLV_COUPLING_ARRAY flvCOUPs_dep{ cDPF_partner1, cDPF_partner2, dpf_value }; + + // Reset color flows (reset jamp_sv) at the beginning of a new event or event page + for( int i = 0; i < ncolor; i++ ) { jamp_sv[i] = cxzero_sv(); } + + // Numerators and denominators for the current event (CUDA) + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + // Scalar iflavor for the current event + const unsigned int iflavor = F_ACCESS::kernelAccessConst( iflavorVec ); #include "EvaluateDiagrams.inc" // *** COLOR CHOICE BELOW *** diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc index 2d04ded182..0bd5c9652d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc @@ -180,6 +180,56 @@ namespace mg5amcCpu for( int iParity = 0; iParity < nParity; ++iParity ) { const int ievt0 = ievt00 + iParity * neppV; + + constexpr size_t nxcoup = ndcoup + nIPC; // both dependent and independent couplings + const fptype* allCOUPs[nxcoup]; + for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) + allCOUPs[idcoup] = CD_ACCESS::idcoupAccessBufferConst( allcouplings, idcoup ); // dependent couplings, vary event-by-event + for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) + allCOUPs[ndcoup + iicoup] = CI_ACCESS::iicoupAccessBufferConst( cIPC, iicoup ); // independent couplings, fixed for all events + // C++ kernels take input/output buffers with momenta/MEs for one specific event (the first in the current event page) + const fptype* momenta = M_ACCESS::ieventAccessRecordConst( allmomenta, ievt0 ); + const fptype* COUPs[nxcoup]; + for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) + COUPs[idcoup] = CD_ACCESS::ieventAccessRecordConst( allCOUPs[idcoup], ievt0 ); // dependent couplings, vary event-by-event + for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) + COUPs[ndcoup + iicoup] = allCOUPs[ndcoup + iicoup]; // independent couplings, fixed for all events + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + // Create an array of views over the Flavor Couplings + FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; + + // Dependent (event-by-event, running-alphas) flavor couplings (Step 3): the per-flavor + // values are NOT baked in (they run per event). Gather the current values of the + // underlying dependent couplings for this event page into an AOSOA buffer dpf_value + // (one nx2*neppC SIMD record per (coupling,flavor) slot, matching CD_ACCESS), then build + // an ordinary value-based view over it. The flavor index is constant across a SIMD lane + // (guaranteed by the phase-space integrator), so each lane gets its own running value + // while sharing the same flavor selection. This is the direct analogue of Fortran's + // FLV_xx%VAL(k)%P => GC_yyy(J). The vertex routines are instantiated with CD_ACCESS so + // get_coupling_def reads dpf_value with the right per-flavor stride (CD_ACCESS::flv_stride). + constexpr int ndpfbuf = ( nDPF > 0 ? nDPF * nMF * CD_ACCESS::flv_stride : 1 ); + alignas( mgOnGpu::cppAlign ) fptype dpf_value[ndpfbuf]{}; + for( int idpf = 0; idpf < nDPF; idpf++ ) + for( int imf = 0; imf < nMF; imf++ ) + { + const int idc = cDPF_idcoup[idpf * nMF + imf]; + if( idc >= 0 ) + CD_ACCESS::kernelAccess( dpf_value + ( idpf * nMF + imf ) * CD_ACCESS::flv_stride ) = + CD_ACCESS::kernelAccessConst( COUPs[idc] ); + } + FLV_COUPLING_ARRAY flvCOUPs_dep{ cDPF_partner1, cDPF_partner2, dpf_value }; + + // Reset color flows (reset jamp_sv) at the beginning of a new event or event page + for( int i = 0; i < ncolor; i++ ) { jamp_sv[i] = cxzero_sv(); } + + // Numerators and denominators for the current event (CUDA) or SIMD event page (C++) + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + // Scalar iflavor for the current event (constant across the SIMD vector) + const unsigned int* iflavor_rec = F_ACCESS::ieventAccessRecordConst( iflavorVec, ievt0 ); + const uint_sv iflavor_sv = F_ACCESS::kernelAccessConst( iflavor_rec ); + const unsigned int iflavor = reinterpret_cast(&iflavor_sv)[0]; #include "EvaluateDiagrams.inc" // *** COLOR CHOICE BELOW *** diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk index 3664dcde4c..70254f32cd 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk @@ -31,6 +31,11 @@ ifeq ($(HRDCOD),) override HRDCOD = 0 endif +# default USEBUILDDIR = 1 +ifeq ($(USEBUILDDIR),) + override USEBUILDDIR = 1 +endif + # Check that the user-defined choices of BACKEND, FPTYPE, HELINL, HRDCOD are supported # (NB: use 'filter' and 'words' instead of 'findstring' because they properly handle whitespace-separated words) override SUPPORTED_BACKENDS = cuda hip cppnone cppsse4 cppavx2 cpp512y cpp512z cppauto @@ -72,7 +77,7 @@ endif # Build directory "full" tag (used for build lockfiles to prevent mixing builds with different options) override DIRTAG := $(patsubst cpp%%,%%,$(BACKEND))_$(FPTYPE)_inl$(HELINL)_hrd$(HRDCOD) -# Build directory: current directory by default, or build. if USEBUILDDIR==1 +# Build directory: build. by default (USEBUILDDIR=1), or current directory if USEBUILDDIR=0 # NB: using '=' (not ':=') ensures BACKEND is evaluated lazily after potential cppauto resolution ifeq ($(USEBUILDDIR),1) override MADMATRIX_BUILDDIR = build.$(BACKEND) @@ -680,7 +685,7 @@ override TAG = $(patsubst cpp%%,%%,$(BACKEND))_$(FPTYPE)_inl$(HELINL)_hrd$(HRDCO # Export TAG (so that there is no need to check/define it again in src/Makefile) export TAG -# Build directory for object files: current directory by default, or build. if USEBUILDDIR==1 +# Build directory for object files: build. by default, or current directory if USEBUILDDIR=0 override BUILDDIR = $(MADMATRIX_BUILDDIR) ###override INCDIR = ../../include diff --git a/madmatrix/model_handling.py b/madmatrix/model_handling.py index 6a49100439..38eda7c593 100644 --- a/madmatrix/model_handling.py +++ b/madmatrix/model_handling.py @@ -2281,87 +2281,6 @@ def super_get_matrix_element_calls(self, matrix_element, color_amplitudes, multi ###misc.sprint(multi_channel_map) res = [] ###res.append('for(int i=0;i<%s;i++){jamp[i] = cxtype(0.,0.);}' % len(color_amplitudes)) - res.append("""//constexpr size_t nxcoup = ndcoup + nicoup; // both dependent and independent couplings (BUG #823) - constexpr size_t nxcoup = ndcoup + nIPC; // both dependent and independent couplings (FIX #823) - const fptype* allCOUPs[nxcoup]; -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#pragma nv_diagnostic push -#pragma nv_diag_suppress 186 // e.g. <> -#endif - for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) - allCOUPs[idcoup] = CD_ACCESS::idcoupAccessBufferConst( allcouplings, idcoup ); // dependent couplings, vary event-by-event - //for( size_t iicoup = 0; iicoup < nicoup; iicoup++ ) // BUG #823 - for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) // FIX #823 - allCOUPs[ndcoup + iicoup] = CI_ACCESS::iicoupAccessBufferConst( cIPC, iicoup ); // independent couplings, fixed for all events -#ifdef MGONGPUCPP_GPUIMPL -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#pragma nv_diagnostic pop -#endif - // CUDA kernels take input/output buffers with momenta/MEs for all events - const fptype* momenta = allmomenta; - const fptype* COUPs[nxcoup]; - for( size_t ixcoup = 0; ixcoup < nxcoup; ixcoup++ ) COUPs[ixcoup] = allCOUPs[ixcoup]; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid - fptype* numerators = &allNumerators[ievt * processConfig::ndiagrams]; - fptype* denominators = allDenominators; -#else - // C++ kernels take input/output buffers with momenta/MEs for one specific event (the first in the current event page) - const fptype* momenta = M_ACCESS::ieventAccessRecordConst( allmomenta, ievt0 ); - const fptype* COUPs[nxcoup]; - for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) - COUPs[idcoup] = CD_ACCESS::ieventAccessRecordConst( allCOUPs[idcoup], ievt0 ); // dependent couplings, vary event-by-event - //for( size_t iicoup = 0; iicoup < nicoup; iicoup++ ) // BUG #823 - for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) // FIX #823 - COUPs[ndcoup + iicoup] = allCOUPs[ndcoup + iicoup]; // independent couplings, fixed for all events - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); - fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); -#endif - // Create an array of views over the Flavor Couplings - FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; - - // Dependent (event-by-event, running-alphas) flavor couplings (Step 3): the per-flavor - // values are NOT baked in (they run per event). Gather the current values of the - // underlying dependent couplings for this event page into an AOSOA buffer dpf_value - // (one nx2*neppC SIMD record per (coupling,flavor) slot, matching CD_ACCESS), then build - // an ordinary value-based view over it. The flavor index is constant across a SIMD lane - // (guaranteed by the phase-space integrator), so each lane gets its own running value - // while sharing the same flavor selection. This is the direct analogue of Fortran's - // FLV_xx%VAL(k)%P => GC_yyy(J). The vertex routines are instantiated with CD_ACCESS so - // get_coupling_def reads dpf_value with the right per-flavor stride (CD_ACCESS::flv_stride). - constexpr int ndpfbuf = ( nDPF > 0 ? nDPF * nMF * CD_ACCESS::flv_stride : 1 ); -#ifndef MGONGPUCPP_GPUIMPL - // cppAlign is only defined for SIMD - alignas( mgOnGpu::cppAlign ) fptype dpf_value[ndpfbuf]{}; -#else - fptype dpf_value[ndpfbuf]{}; -#endif - for( int idpf = 0; idpf < nDPF; idpf++ ) - for( int imf = 0; imf < nMF; imf++ ) - { - const int idc = cDPF_idcoup[idpf * nMF + imf]; - if( idc >= 0 ) - CD_ACCESS::kernelAccess( dpf_value + ( idpf * nMF + imf ) * CD_ACCESS::flv_stride ) = - CD_ACCESS::kernelAccessConst( COUPs[idc] ); - } - FLV_COUPLING_ARRAY flvCOUPs_dep{ cDPF_partner1, cDPF_partner2, dpf_value }; - - // Reset color flows (reset jamp_sv) at the beginning of a new event or event page - for( int i = 0; i < ncolor; i++ ) { jamp_sv[i] = cxzero_sv(); } - - // Numerators and denominators for the current event (CUDA) or SIMD event page (C++) - fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); - fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); - // Scalar iflavor for the current event - // for GPU it is an int - // for SIMD it is also an int, since it is constant across the SIMD vector -#ifdef MGONGPUCPP_GPUIMPL - const unsigned int iflavor = F_ACCESS::kernelAccessConst( iflavorVec ); -#else - const unsigned int* iflavor_rec = F_ACCESS::ieventAccessRecordConst( iflavorVec, ievt0 ); - const uint_sv iflavor_sv = F_ACCESS::kernelAccessConst( iflavor_rec ); - const unsigned int iflavor = reinterpret_cast(&iflavor_sv)[0]; -#endif -""") diagrams = matrix_element.get('diagrams') diag_to_config = {} for config in sorted(multi_channel_map.keys()): From a72f55a8b2dbc718b5f408dc7897716d660077a8 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 23:10:21 +0200 Subject: [PATCH 10/25] Delete f* files and copy acc. output mode only in standalone_mg7 copy needed files - e.g. check_sa.cc delete them from /backend and delete the death code files --- .../template_files/madmatrix/ProcessData.h | 6 + .../backend/cpu/CommonRandomNumberKernel.cc | 35 - .../backend/cpu/CurandRandomNumberKernel.cc | 127 -- .../backend/cpu/HiprandRandomNumberKernel.cc | 137 --- .../madmatrix/backend/cpu/MadgraphTest.h | 335 ----- .../madmatrix/backend/cpu/MemoryBuffers.h | 3 +- .../backend/cpu/RamboSamplingKernels.cc | 168 --- .../backend/cpu/RamboSamplingKernels.h | 134 -- .../backend/cpu/RandomNumberKernels.h | 188 --- .../madmatrix/backend/cpu/SigmaKin.cc | 13 +- .../madmatrix/backend/cpu/check_sa.cc | 1020 ---------------- .../madmatrix/backend/cpu/fbridge.cc | 145 --- .../madmatrix/backend/cpu/fbridge.h | 48 - .../madmatrix/backend/cpu/fsampler.cc | 152 --- .../madmatrix/backend/cpu/massless_rambo.h | 184 --- .../madmatrix/backend/cpu/runTest.cc | 245 ---- .../madmatrix/backend/cpu/testmisc.cc | 437 ------- .../backend/gpu/CommonRandomNumberKernel.cc | 35 - .../backend/gpu/CurandRandomNumberKernel.cc | 128 -- .../backend/gpu/HiprandRandomNumberKernel.cc | 138 --- .../madmatrix/backend/gpu/MadgraphTest.h | 335 ----- .../madmatrix/backend/gpu/MemoryBuffers.h | 3 +- .../backend/gpu/RamboSamplingKernels.cc | 232 ---- .../backend/gpu/RamboSamplingKernels.h | 167 --- .../backend/gpu/RandomNumberKernels.h | 188 --- .../madmatrix/backend/gpu/SigmaKin.cc | 21 +- .../madmatrix/backend/gpu/check_sa.cc | 1081 ----------------- .../madmatrix/backend/gpu/fbridge.cc | 147 --- .../madmatrix/backend/gpu/fbridge.h | 48 - .../madmatrix/backend/gpu/fsampler.cc | 152 --- .../madmatrix/backend/gpu/massless_rambo.h | 180 --- .../madmatrix/backend/gpu/runTest.cc | 279 ----- .../madmatrix/backend/gpu/testmisc.cc | 437 ------- .../backend/simd/CommonRandomNumberKernel.cc | 35 - .../backend/simd/CurandRandomNumberKernel.cc | 127 -- .../backend/simd/HiprandRandomNumberKernel.cc | 137 --- .../madmatrix/backend/simd/MadgraphTest.h | 335 ----- .../madmatrix/backend/simd/MemoryBuffers.h | 3 +- .../backend/simd/RamboSamplingKernels.cc | 168 --- .../backend/simd/RamboSamplingKernels.h | 134 -- .../backend/simd/RandomNumberKernels.h | 188 --- .../madmatrix/backend/simd/SigmaKin.cc | 13 +- .../madmatrix/backend/simd/check_sa.cc | 1020 ---------------- .../madmatrix/backend/simd/fbridge.cc | 145 --- .../madmatrix/backend/simd/fbridge.h | 48 - .../madmatrix/backend/simd/fsampler.cc | 152 --- .../madmatrix/backend/simd/massless_rambo.h | 184 --- .../madmatrix/backend/simd/runTest.cc | 245 ---- .../madmatrix/backend/simd/testmisc.cc | 479 -------- .../template_files/madmatrix/check_sa.cc | 2 +- .../madmatrix/epoch_process_id.h | 17 - .../template_files/madmatrix/fbridge.cc | 161 --- .../iolibs/template_files/madmatrix/fbridge.h | 52 - .../template_files/madmatrix/fsampler.cc | 166 --- .../template_files/madmatrix/processConfig.h | 15 - .../template_files/madmatrix/process_cc.inc | 1 - .../madmatrix/process_sigmaKin_function.inc | 14 +- .../template_files/madmatrix/runTest.cc | 2 +- .../template_files/madmatrix/testmisc.cc | 2 +- .../template_files/madmatrix/testxxx.cc | 2 +- madmatrix/model_handling.py | 27 +- 61 files changed, 44 insertions(+), 10778 deletions(-) delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/runTest.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/runTest.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/runTest.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/epoch_process_id.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/fbridge.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/fbridge.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/fsampler.cc delete mode 100644 madgraph/iolibs/template_files/madmatrix/processConfig.h diff --git a/madgraph/iolibs/template_files/madmatrix/ProcessData.h b/madgraph/iolibs/template_files/madmatrix/ProcessData.h index be4bd937f0..d00063291b 100644 --- a/madgraph/iolibs/template_files/madmatrix/ProcessData.h +++ b/madgraph/iolibs/template_files/madmatrix/ProcessData.h @@ -34,4 +34,10 @@ namespace ProcessData constexpr int nDPF = %(ndpf)d; } +// Process identification for test/debug tooling. Must stay #define (not +// constexpr): used for macro token-pasting (test suite names) and +// stringification, e.g. TEST( XTESTID( MG_EPOCH_PROCESS_ID ), ... ). +#define MG_EPOCH_PROCESS_ID %(processid_uppercase)s +#define MG_EPOCH_REFERENCE_FILE_NAME "../../test/ref/dump_CPUTest.%(processid)s.txt" + #endif // PROCESSDATA_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc deleted file mode 100644 index 14c1ceeaf5..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CommonRandomNumberKernel.cc +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "CommonRandomNumbers.h" -#include "GpuAbstraction.h" -#include "MemoryBuffers.h" -#include "RandomNumberKernels.h" - -#include - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- - - CommonRandomNumberKernel::CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ) - : RandomNumberKernelBase( rnarray ) - , m_seed( 20211220 ) - { - if( m_rnarray.isOnDevice() ) - throw std::runtime_error( "CommonRandomNumberKernel on host with a device random number array" ); - } - - //-------------------------------------------------------------------------- - - void CommonRandomNumberKernel::generateRnarray() - { - std::vector rnd = CommonRandomNumbers::generate( m_rnarray.size(), m_seed ); // NB: generate as double (HARDCODED) - std::copy( rnd.begin(), rnd.end(), m_rnarray.data() ); // NB: copy may imply a double-to-float conversion - } - - //-------------------------------------------------------------------------- -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc deleted file mode 100644 index 24ee78772b..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CurandRandomNumberKernel.cc +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "GpuRuntime.h" -#include "MemoryBuffers.h" -#include "RandomNumberKernels.h" - -#include - -#ifndef MGONGPU_HAS_NO_CURAND /* clang-format off */ -// NB This must come AFTER mgOnGpuConfig.h which contains our definition of __global__ when MGONGPUCPP_GPUIMPL is not defined -#include "curand.h" -#define checkCurand( code ){ assertCurand( code, __FILE__, __LINE__ ); } -inline void assertCurand( curandStatus_t code, const char *file, int line, bool abort = true ) -{ - if ( code != CURAND_STATUS_SUCCESS ) - { - printf( "CurandAssert: %s:%d code=%d\n", file, line, code ); - if ( abort ) assert( code == CURAND_STATUS_SUCCESS ); - } -} -#endif /* clang-format on */ - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- -#ifndef MGONGPU_HAS_NO_CURAND - CurandRandomNumberKernel::CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) - : RandomNumberKernelBase( rnarray ) - , m_isOnDevice( onDevice ) - { - if( m_isOnDevice ) - { - throw std::runtime_error( "CurandRandomNumberKernel does not support CurandDevice on CPU host" ); - } - else - { - if( m_rnarray.isOnDevice() ) - throw std::runtime_error( "CurandRandomNumberKernel on host with a device random number array" ); - } - createGenerator(); - } - - //-------------------------------------------------------------------------- - - CurandRandomNumberKernel::~CurandRandomNumberKernel() - { - destroyGenerator(); - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::seedGenerator( const unsigned int seed ) - { - if( m_isOnDevice ) - { - destroyGenerator(); // workaround for #429 - createGenerator(); // workaround for #429 - } - //printf( "seedGenerator: seed %d\n", seed ); - checkCurand( curandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::createGenerator() - { - // [NB Timings are for GenRnGen host|device (cpp|cuda) generation of 256*32*1 events with nproc=1: rn(0) is host=0.0012s] - const curandRngType_t type = CURAND_RNG_PSEUDO_MTGP32; // 0.00082s | 0.00064s (FOR FAST TESTS) - //const curandRngType_t type = CURAND_RNG_PSEUDO_XORWOW; // 0.049s | 0.0016s - //const curandRngType_t type = CURAND_RNG_PSEUDO_MRG32K3A; // 0.71s | 0.0012s (better but slower, especially in c++) - //const curandRngType_t type = CURAND_RNG_PSEUDO_MT19937; // 21s | 0.021s - //const curandRngType_t type = CURAND_RNG_PSEUDO_PHILOX4_32_10; // 0.024s | 0.00026s (used to segfault?) - if( m_isOnDevice ) - { - checkCurand( curandCreateGenerator( &m_rnGen, type ) ); - } - else - { - checkCurand( curandCreateGeneratorHost( &m_rnGen, type ) ); - } - //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_LEGACY ) ); // fails with code=104 (see #429) - checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_BEST ) ); - //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_DYNAMIC ) ); // fails with code=104 (see #429) - //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_SEEDED ) ); // fails with code=104 (see #429) - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::destroyGenerator() - { - checkCurand( curandDestroyGenerator( m_rnGen ) ); - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::generateRnarray() - { -#if defined MGONGPU_FPTYPE_DOUBLE - checkCurand( curandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#elif defined MGONGPU_FPTYPE_FLOAT - checkCurand( curandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#endif - /* - printf( "\nCurandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); - fptype* data = m_rnarray.data(); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) - { - data = new fptype[m_rnarray.size()](); - checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); - } -#endif - for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) - printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) delete[] data; -#endif - */ - } - - //-------------------------------------------------------------------------- -#endif -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc deleted file mode 100644 index 2fd64a2dce..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/HiprandRandomNumberKernel.cc +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2024) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "mgOnGpuConfig.h" - -#include "GpuRuntime.h" -#include "MemoryBuffers.h" -#include "RandomNumberKernels.h" - -#include - -#ifndef MGONGPU_HAS_NO_HIPRAND /* clang-format off */ -#ifndef __HIP_PLATFORM_AMD__ -#define __HIP_PLATFORM_AMD__ 1 // enable hiprand for AMD (rocrand) -#endif -#include -#define checkHiprand( code ){ assertHiprand( code, __FILE__, __LINE__ ); } -inline void assertHiprand( hiprandStatus_t code, const char *file, int line, bool abort = true ) -{ - if ( code != HIPRAND_STATUS_SUCCESS ) - { - printf( "HiprandAssert: %s:%d code=%d\n", file, line, code ); - if ( abort ) assert( code == HIPRAND_STATUS_SUCCESS ); - } -} -#endif /* clang-format on */ - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- -#ifndef MGONGPU_HAS_NO_HIPRAND - HiprandRandomNumberKernel::HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) - : RandomNumberKernelBase( rnarray ) - , m_isOnDevice( onDevice ) - { - if( m_isOnDevice ) - { - throw std::runtime_error( "HiprandRandomNumberKernel does not support HiprandDevice on CPU host" ); - } - else - { - if( m_rnarray.isOnDevice() ) - throw std::runtime_error( "HiprandRandomNumberKernel on host with a device random number array" ); - } - createGenerator(); - } - - //-------------------------------------------------------------------------- - - HiprandRandomNumberKernel::~HiprandRandomNumberKernel() - { - destroyGenerator(); - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::seedGenerator( const unsigned int seed ) - { - if( m_isOnDevice ) - { - destroyGenerator(); // workaround for #429 - createGenerator(); // workaround for #429 - } - //printf( "seedGenerator: seed %d\n", seed ); - checkHiprand( hiprandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::createGenerator() - { - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_DEFAULT; - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_XORWOW; - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MRG32K3A; - const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MTGP32; // same as curand; not implemented yet (code=1000) in host code - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MT19937; - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_PHILOX4_32_10; - if( m_isOnDevice ) - { - checkHiprand( hiprandCreateGenerator( &m_rnGen, type ) ); - } - else - { - // See https://github.com/ROCm/hipRAND/issues/76 - throw std::runtime_error( "HiprandRandomNumberKernel on host is not supported yet (hiprandCreateGeneratorHost is not implemented yet)" ); - //checkHiprand( hiprandCreateGeneratorHost( &m_rnGen, type ) ); // ALWAYS FAILS WITH CODE=1000 - } - // FIXME: hiprand ordering is not implemented yet - // See https://github.com/ROCm/hipRAND/issues/75 - /* - //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_LEGACY ) ); - checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_BEST ) ); - //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_DYNAMIC ) ); - //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_SEEDED ) ); - */ - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::destroyGenerator() - { - checkHiprand( hiprandDestroyGenerator( m_rnGen ) ); - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::generateRnarray() - { -#if defined MGONGPU_FPTYPE_DOUBLE - checkHiprand( hiprandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#elif defined MGONGPU_FPTYPE_FLOAT - checkHiprand( hiprandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#endif - /* - printf( "\nHiprandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); - fptype* data = m_rnarray.data(); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) - { - data = new fptype[m_rnarray.size()](); - checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); - } -#endif - for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) - printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) delete[] data; -#endif - */ - } - - //-------------------------------------------------------------------------- -#endif -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h deleted file mode 100644 index 641917d8ff..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MadgraphTest.h +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Hageboeck (Dec 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, J. Teig, A. Valassi (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MADGRAPHTEST_H_ -#define MADGRAPHTEST_H_ 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" - -#include - -#include -#include -//#ifdef __HIPCC__ -//#include // see https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 -//#else -//#include // bypass this completely to ease portability on LUMI #803 -//#endif -#include -#include -#include -#include -#include -#include - -using mg5amcCpu::CPPProcess; - -namespace -{ - struct ReferenceData - { - std::vector>> momenta; - std::vector MEs; - std::vector ChanIds; - std::vector SelHels; - std::vector SelCols; - }; - - /// Read batches of reference data from a file and store them in a map. - std::map readReferenceData( const std::string& refFileName ) - { - std::cout << "INFO: Opening reference file " << refFileName << std::endl; - std::ifstream referenceFile( refFileName.c_str() ); - EXPECT_TRUE( referenceFile.is_open() ) << refFileName; - std::map referenceData; - unsigned int evtNo; - unsigned int batchNo; - for( std::string line; std::getline( referenceFile, line ); ) - { - std::stringstream lineStr( line ); - if( line.empty() || line[0] == '#' ) - { - continue; - } - else if( line.find( "Event" ) != std::string::npos ) - { - std::string dummy; - lineStr >> dummy >> evtNo >> dummy >> batchNo; - } - else if( line.find( "ME" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].MEs.size() ) - referenceData[batchNo].MEs.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].MEs[evtNo]; - } - else if( line.find( "ChanId" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].ChanIds.size() ) - referenceData[batchNo].ChanIds.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].ChanIds[evtNo]; - referenceData[batchNo].ChanIds[evtNo] = 0; // disable ChanId comparison if multichannel is not supported #976 - } - else if( line.find( "SelHel" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].SelHels.size() ) - referenceData[batchNo].SelHels.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].SelHels[evtNo]; - } - else if( line.find( "SelCol" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].SelCols.size() ) - referenceData[batchNo].SelCols.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].SelCols[evtNo]; - } - else - { - unsigned int particleIndex; - lineStr >> particleIndex; - if( evtNo <= referenceData[batchNo].momenta.size() ) - referenceData[batchNo].momenta.resize( evtNo + 1 ); - if( particleIndex <= referenceData[batchNo].momenta[evtNo].size() ) - referenceData[batchNo].momenta[evtNo].resize( particleIndex + 1 ); - auto& fourVec = referenceData[batchNo].momenta[evtNo][particleIndex]; - for( unsigned int i = 0; i < fourVec.size(); ++i ) - { - EXPECT_TRUE( lineStr.good() ); - lineStr >> fourVec[i]; - } - EXPECT_TRUE( lineStr.eof() ); - } - } - return referenceData; - } - -} - -/** - * Test driver providing a common interface for testing different implementations. - * Users need to implement: - * - Functions to retrieve matrix element and 4-momenta. These are used in the tests. - * - Driver functions that run the madgraph workflow. - */ -class TestDriverBase -{ - std::string m_refFileName; -public: - const unsigned int nparticle; - static constexpr unsigned int niter = 2; - static constexpr unsigned int gpublocks = 2; - static constexpr unsigned int gputhreads = 128; - static constexpr unsigned int nevt = gpublocks * gputhreads; - - TestDriverBase( unsigned int npart, const std::string& refFileName ) - : m_refFileName( refFileName ) - , nparticle( npart ) - { - } - TestDriverBase() = delete; - virtual ~TestDriverBase() {} - const std::string& getRefFileName() { return m_refFileName; } - - // ------------------------------------------------ - // Interface for retrieving info from madgraph - // ------------------------------------------------ - virtual fptype getMomentum( std::size_t evtNo, unsigned int particleNo, unsigned int component ) const = 0; - virtual fptype getMatrixElement( std::size_t evtNo ) const = 0; - virtual int getChannelId( std::size_t ievt ) const = 0; - virtual int getSelectedHelicity( std::size_t ievt ) const = 0; - virtual int getSelectedColor( std::size_t ievt ) const = 0; - - // ------------------------------------------------ - // Interface for steering madgraph run - // ------------------------------------------------ - virtual void prepareRandomNumbers( unsigned int iiter ) = 0; - virtual void prepareMomenta( fptype energy ) = 0; - virtual void runSigmaKin( std::size_t iiter ) = 0; - - /// Print the requested event into the stream. If the reference data has enough events, it will be printed as well. - void dumpParticles( std::ostream& stream, std::size_t ievt, unsigned int numParticles, unsigned int nDigit, const ReferenceData& referenceData ) const - { - const auto width = nDigit + 8; - for( unsigned int ipar = 0; ipar < numParticles; ipar++ ) - { - // NB: 'setw' affects only the next field (of any type) - stream << std::scientific // fixed format: affects all floats (default nDigit: 6) - << std::setprecision( nDigit ) - << std::setw( 4 ) << ipar - << std::setw( width ) << getMomentum( ievt, ipar, 0 ) - << std::setw( width ) << getMomentum( ievt, ipar, 1 ) - << std::setw( width ) << getMomentum( ievt, ipar, 2 ) - << std::setw( width ) << getMomentum( ievt, ipar, 3 ) - << "\n"; - if( ievt < referenceData.momenta.size() ) - { - stream << "ref" << ipar; - stream << std::setw( width ) << referenceData.momenta[ievt][ipar][0] - << std::setw( width ) << referenceData.momenta[ievt][ipar][1] - << std::setw( width ) << referenceData.momenta[ievt][ipar][2] - << std::setw( width ) << referenceData.momenta[ievt][ipar][3] - << "\n\n"; - } - stream << std::flush << std::defaultfloat; // default format: affects all floats - } - } -}; - -/** - * Test class that's defining all tests to run with a Madgraph workflow. - */ -class MadgraphTest -{ -public: - MadgraphTest( TestDriverBase& testDriverRef ) - : testDriver( &testDriverRef ) {} - ~MadgraphTest() {} - void CompareMomentaAndME( testing::Test& googleTest ) const; // NB: googleTest is ONLY needed for the HasFailure method... -private: - TestDriverBase* testDriver; // non-owning pointer -}; - -void -MadgraphTest::CompareMomentaAndME( testing::Test& googleTest ) const -{ - const fptype toleranceMomenta = std::is_same::value ? 1.E-10 : 4.E-2; // see #735 -#ifdef __APPLE__ - const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-2; // see #583 -#else - //const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 2.E-3; // fails smeft/hip #843 - const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-3; -#endif - constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) - // Dump events to a new reference file? - const char* dumpEventsC = getenv( "CUDACPP_RUNTEST_DUMPEVENTS" ); - const bool dumpEvents = ( dumpEventsC != 0 ) && ( std::string( dumpEventsC ) != "" ); - const std::string refFileName = testDriver->getRefFileName(); - /* -#ifdef __HIPCC__ - const std::string dumpFileName = std::experimental::filesystem::path( refFileName ).filename(); -#else - const std::string dumpFileName = std::filesystem::path( refFileName ).filename(); -#endif - */ - const std::string dumpFileName = refFileName; // bypass std::filesystem #803 - std::ofstream dumpFile; - if( dumpEvents ) - { - dumpFile.open( dumpFileName, std::ios::trunc ); - } - // Read reference data - std::map referenceData; - if( !dumpEvents ) - { - referenceData = readReferenceData( refFileName ); - } - ASSERT_FALSE( googleTest.HasFailure() ); // It doesn't make any sense to continue if we couldn't read the reference file. - // ************************************** - // *** START MAIN LOOP ON #ITERATIONS *** - // ************************************** - for( unsigned int iiter = 0; iiter < testDriver->niter; ++iiter ) - { - testDriver->prepareRandomNumbers( iiter ); - testDriver->prepareMomenta( energy ); - testDriver->runSigmaKin( iiter ); - // --- Run checks on all events produced in this iteration - for( std::size_t ievt = 0; ievt < testDriver->nevt && !googleTest.HasFailure(); ++ievt ) - { - if( dumpEvents ) - { - ASSERT_TRUE( dumpFile.is_open() ) << dumpFileName; - dumpFile << "Event " << std::setw( 8 ) << ievt << " " - << "Batch " << std::setw( 4 ) << iiter << "\n"; - testDriver->dumpParticles( dumpFile, ievt, testDriver->nparticle, 15, ReferenceData() ); - // Dump matrix element - dumpFile << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) - << testDriver->getMatrixElement( ievt ) << "\n" - << std::defaultfloat; - // Dump channelId - dumpFile << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n"; - // Dump selected helicity and color - dumpFile << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n"; - dumpFile << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" - << std::endl; // leave one line between events - continue; - } - // Check that we have the required reference data - ASSERT_GT( referenceData.size(), iiter ) - << "Don't have enough reference data for iteration " << iiter << ". Ref file:" << refFileName; - ASSERT_GT( referenceData[iiter].MEs.size(), ievt ) - << "Don't have enough reference MEs for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].ChanIds.size(), ievt ) - << "Don't have enough reference ChanIds for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].SelHels.size(), ievt ) - << "Don't have enough reference SelHels for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].SelCols.size(), ievt ) - << "Don't have enough reference SelCols for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].momenta.size(), ievt ) - << "Don't have enough reference momenta for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GE( referenceData[iiter].momenta[ievt].size(), testDriver->nparticle ) - << "Don't have enough reference particles for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - // This trace will help to understand the event that is being checked. - // It will only be printed in case of failures: - std::stringstream eventTrace; - eventTrace << "In comparing event " << ievt << " from iteration " << iiter << "\n"; - testDriver->dumpParticles( eventTrace, ievt, testDriver->nparticle, 15, referenceData[iiter] ); - eventTrace << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) - << testDriver->getMatrixElement( ievt ) << "\n" - << std::setw( 4 ) << "r.ME" << std::scientific << std::setw( 15 + 8 ) - << referenceData[iiter].MEs[ievt] << std::endl - << std::defaultfloat; - eventTrace << std::setw( 8 ) << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n" - << std::setw( 8 ) << "r.ChanId" << std::setw( 8 ) << referenceData[iiter].ChanIds[ievt] << std::endl; - eventTrace << std::setw( 8 ) << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n" - << std::setw( 8 ) << "r.SelHel" << std::setw( 8 ) << referenceData[iiter].SelHels[ievt] << std::endl; - eventTrace << std::setw( 8 ) << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" - << std::setw( 8 ) << "r.SelCol" << std::setw( 8 ) << referenceData[iiter].SelCols[ievt] << std::endl; - SCOPED_TRACE( eventTrace.str() ); - // Compare Momenta - for( unsigned int ipar = 0; ipar < testDriver->nparticle; ++ipar ) - { - std::stringstream momentumErrors; - for( unsigned int icomp = 0; icomp < CPPProcess::np4; ++icomp ) - { - const fptype pMadg = testDriver->getMomentum( ievt, ipar, icomp ); - const fptype pOrig = referenceData[iiter].momenta[ievt][ipar][icomp]; - //const fptype relDelta = fabs( ( pMadg - pOrig ) / pOrig ); // computing relDelta may lead to FPEs - const fptype delta = fabs( pMadg - pOrig ); - if( delta > toleranceMomenta * fabs( pOrig ) ) // better than "relDelta > toleranceMomenta" - { - momentumErrors << std::setprecision( 15 ) << std::scientific << "\nparticle " << ipar << "\tcomponent " << icomp - << "\n\t madGraph: " << std::setw( 22 ) << pMadg - << "\n\t reference: " << std::setw( 22 ) << pOrig - << "\n\t relative delta exceeds tolerance of " << toleranceMomenta; - } - } - ASSERT_TRUE( momentumErrors.str().empty() ) << momentumErrors.str(); - } - // Compare ME: - EXPECT_NEAR( testDriver->getMatrixElement( ievt ), - referenceData[iiter].MEs[ievt], - toleranceMEs * referenceData[iiter].MEs[ievt] ); - // Compare channelId - EXPECT_EQ( testDriver->getChannelId( ievt ), - referenceData[iiter].ChanIds[ievt] ); - // Compare selected helicity and color - EXPECT_EQ( testDriver->getSelectedHelicity( ievt ), - referenceData[iiter].SelHels[ievt] ); - EXPECT_EQ( testDriver->getSelectedColor( ievt ), - referenceData[iiter].SelCols[ievt] ); - } - } - if( dumpEvents ) - { - std::cout << "Event dump written to " << dumpFileName << std::endl; - } -} - -#endif /* MADGRAPHTEST_H_ */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h index 0dd7244541..4d7edc9f0d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h @@ -14,7 +14,6 @@ #include "ProcessData.h" #include "GpuRuntime.h" #include "Parameters.h" -#include "processConfig.h" #include @@ -204,7 +203,7 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for numerators // (should be equal to the number of diagrams in the process) - constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; + constexpr size_t sizePerEventNumerators = ProcessData::ndiagrams; // A class encapsulating a C++ host buffer for numerators typedef HostBuffer HostBufferNumerators; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc deleted file mode 100644 index e1e17d6247..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.cc +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "RamboSamplingKernels.h" - -#include "GpuRuntime.h" -#include "MemoryAccessMomenta.h" -#include "MemoryAccessRandomNumbers.h" -#include "MemoryAccessWeights.h" -#include "MemoryBuffers.h" -#include "rambo.h" // inline classic (massive) RAMBO, ported from standalone_cpp -#include "massless_rambo.h" // inline implementation of massless RAMBO algorithms and kernels - -#include - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- - - MasslessRamboSamplingKernelHost::MasslessRamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t nevt ) - : SamplingKernelBase( energy, rndmom, momenta, weights ) - , NumberOfEvents( nevt ) - { - if( m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: rndmom must be a host array" ); - if( m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: momenta must be a host array" ); - if( m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: weights must be a host array" ); - if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with rndmom" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with momenta" ); - if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with weights" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( nevt % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Sanity checks for memory access (random number buffer) - constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout - static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); - if( nevt % neppR != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppR=" << neppR; - throw std::runtime_error( sstr.str() ); - } - } - - //-------------------------------------------------------------------------- - - void - MasslessRamboSamplingKernelHost::getMomentaInitial() - { - constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; - // ** START LOOP ON IEVT ** - for( size_t ievt = 0; ievt < nevt(); ++ievt ) - { - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); - getMomentaInitial( m_energy, ievtMomenta ); - } - // ** END LOOP ON IEVT ** - } - - //-------------------------------------------------------------------------- - - void - MasslessRamboSamplingKernelHost::getMomentaFinal() - { - constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; - // ** START LOOP ON IEVT ** - for( size_t ievt = 0; ievt < nevt(); ++ievt ) - { - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - const fptype* ievtRndmom = MemoryAccessRandomNumbers::ieventAccessRecordConst( m_rndmom.data(), ievt ); - fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); - fptype* ievtWeights = MemoryAccessWeights::ieventAccessRecord( m_weights.data(), ievt ); - getMomentaFinal( m_energy, ievtRndmom, ievtMomenta, ievtWeights ); - } - // ** END LOOP ON IEVT ** - } - - //-------------------------------------------------------------------------- - - RamboSamplingKernelHost::RamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED - const std::vector& masses, // input: external-leg masses - const int ninitial, // input: #initial-state particles - const size_t nevt, // input: #events - BufferMomenta& momenta, // output: momenta - BufferWeights& weights ) // output: weights - : SamplingKernelBase( energy, rndmom, momenta, weights ) - , NumberOfEvents( nevt ) - , m_masses( masses.begin(), masses.end() ) - , m_ninitial( ninitial ) - { - if( m_momenta.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: momenta must be a host array" ); - if( m_weights.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: weights must be a host array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with momenta" ); - if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with weights" ); - - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( nevt % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "RamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - } - - //-------------------------------------------------------------------------- - - void - RamboSamplingKernelHost::getMomentaInitial() - { - // NOOP - } - - //-------------------------------------------------------------------------- - - void - RamboSamplingKernelHost::getMomentaFinal() - { - const int npar = (int)m_masses.size(); - // ** START LOOP ON IEVT ** - for( size_t ievt = 0; ievt < nevt(); ++ievt ) - { - // Clas. RAMBO returns [E,px,py,pz] vector per ex. particle - // own RNG, intial final once - // For reproducibility betwn fptype = FP32/FP64 generation in FP64 - double wgt = 0.; - const std::vector> point = - rambo::get_momenta( m_ninitial, (double)m_energy, m_masses, wgt ); - for( int ipar = 0; ipar < npar; ++ipar ) - for( int ip4 = 0; ip4 < 4; ++ip4 ) - MemoryAccessMomenta::ieventAccessIp4Ipar( m_momenta.data(), ievt, ip4, ipar ) = (fptype)point[ipar][ip4]; - MemoryAccessWeights::ieventAccess( m_weights.data(), ievt ) = (fptype)wgt; - } - // ** END LOOP ON IEVT ** - } - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h deleted file mode 100644 index e0aa2aea7f..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RamboSamplingKernels.h +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef RAMBOSAMPLINGKERNELS_H -#define RAMBOSAMPLINGKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryBuffers.h" - -#include - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- - - // A base class encapsulating phase space sampling on a CPU host or on a GPU device - class SamplingKernelBase //: virtual public ISamplingKernel - { - protected: - - // Constructor from existing input and output buffers - SamplingKernelBase( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights ) // output: weights - : m_energy( energy ) - , m_rndmom( rndmom ) - , m_momenta( momenta ) - , m_weights( weights ) - { - } - - public: - - // Destructor - virtual ~SamplingKernelBase() {} - - // Get momenta of initial state particles - virtual void getMomentaInitial() = 0; - - // Get momenta of final state particles and weights - virtual void getMomentaFinal() = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - protected: - - // The energy - const fptype m_energy; - - // The buffer for the input random numbers - const BufferRndNumMomenta& m_rndmom; - - // The buffer for the output momenta - BufferMomenta& m_momenta; - - // The buffer for the output weights - BufferWeights& m_weights; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating RAMBO phase space sampling on a CPU host - class MasslessRamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MasslessRamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t nevt ); - - // Destructor - virtual ~MasslessRamboSamplingKernelHost() {} - - // Get momenta of initial state particles - void getMomentaInitial() override final; - - // Get momenta of final state particles and weights - void getMomentaFinal() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - }; - - //-------------------------------------------------------------------------- - - // Compability port mirroring the massless momenta implementation - // For now own RNG internally (or keep to match Fortran) TODO - // rndmom just for interface (maybe delete later) TODO - class RamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents - { - public: - - RamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED - const std::vector& masses, // input: external-leg masses - const int ninitial, // input: #n initial-state particles - const size_t nevt, // input: #n events - BufferMomenta& momenta, // output: momenta - BufferWeights& weights); // output: weights - - virtual ~RamboSamplingKernelHost() {} - - // No-op, kept to match the massless - void getMomentaInitial() override final; - - // All the magic here - void getMomentaFinal() override final; - - bool isOnDevice() const override final { return false; } - - private: - - // The EXTERNAL masses - std::vector m_masses; - - // The number of inital particles - const int m_ninitial; - }; - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- -} -#endif // RAMBOSAMPLINGKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h deleted file mode 100644 index 480824a7e5..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/RandomNumberKernels.h +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef RANDOMNUMBERKERNELS_H -#define RANDOMNUMBERKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryBuffers.h" - -// Forward definition from curand.h (the full header is only needed in CurandRandomKernel.cc) -struct curandGenerator_st; - -// Forward definition from hiprand.h (the full header is only needed in HiprandRandomKernel.cc) -struct rocrand_generator_base_type; -typedef rocrand_generator_base_type hiprandGenerator_st; - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- - - /* - // An interface encapsulating random number generation on a CPU host or on a GPU device - class IRandomNumberKernel - { - public: - - // Destructor - virtual ~IRandomNumberKernel(){} - - // Seed the random number generator - virtual void seedGenerator( const unsigned int seed ) = 0; - - // Generate the random number array - virtual void generateRnarray() = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - }; - */ - - //-------------------------------------------------------------------------- - - // A base class encapsulating random number generation on a CPU host or on a GPU device - class RandomNumberKernelBase //: virtual public IRandomNumberKernel - { - - protected: - - // Constructor from an existing output buffer - RandomNumberKernelBase( BufferRndNumMomenta& rnarray ) - : m_rnarray( rnarray ) {} - - public: - - // Destructor - virtual ~RandomNumberKernelBase() {} - - // Seed the random number generator - virtual void seedGenerator( const unsigned int seed ) = 0; - - // Generate the random number array - virtual void generateRnarray() = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - protected: - - // The buffer for the output random numbers - BufferRndNumMomenta& m_rnarray; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating common random number generation on a CPU host - class CommonRandomNumberKernel final : public RandomNumberKernelBase - { - public: - - // Constructor from an existing output buffer - CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ); - - // Destructor - ~CommonRandomNumberKernel() {} - - // Seed the random number generator - void seedGenerator( const unsigned int seed ) override final { m_seed = seed; }; - - // Generate the random number array - void generateRnarray() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // The generator seed - unsigned int m_seed; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating CURAND random number generation on a CPU host or on a GPU device - class CurandRandomNumberKernel final : public RandomNumberKernelBase - { - public: - - // Constructor from an existing output buffer - CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); - - // Destructor - ~CurandRandomNumberKernel(); - - // Seed the random number generator - void seedGenerator( const unsigned int seed ) override final; - - // Generate the random number array - void generateRnarray() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return m_isOnDevice; } - - private: - - // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void createGenerator(); - - // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void destroyGenerator(); - - private: - - // Is this a host or device kernel? - const bool m_isOnDevice; - - // The curand generator - // (NB: curand.h defines typedef curandGenerator_t as a pointer to forward-defined 'struct curandGenerator_st') - curandGenerator_st* m_rnGen; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating HIPRAND random number generation on a CPU host or on a GPU device - class HiprandRandomNumberKernel final : public RandomNumberKernelBase - { - public: - - // Constructor from an existing output buffer - HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); - - // Destructor - ~HiprandRandomNumberKernel(); - - // Seed the random number generator - void seedGenerator( const unsigned int seed ) override final; - - // Generate the random number array - void generateRnarray() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return m_isOnDevice; } - - private: - - // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void createGenerator(); - - // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void destroyGenerator(); - - private: - - // Is this a host or device kernel? - const bool m_isOnDevice; - - // The hiprand generator - // (NB: hiprand.h defines typedef hiprandGenerator_t as a pointer to forward-defined 'struct hiprandGenerator_st') - hiprandGenerator_st* m_rnGen; - }; - - //-------------------------------------------------------------------------- -} -#endif // RANDOMNUMBERKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc index f1dcb874e0..9700f35338 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc @@ -25,7 +25,6 @@ #include "MemoryAccessWavefunctions.h" #include "color_sum.h" #include "coloramps.h" -#include "processConfig.h" namespace mg5amcCpu { @@ -174,7 +173,7 @@ namespace mg5amcCpu COUPs[idcoup] = CD_ACCESS::ieventAccessRecordConst( allCOUPs[idcoup], ievt0 ); // dependent couplings, vary event-by-event for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) COUPs[ndcoup + iicoup] = allCOUPs[ndcoup + iicoup]; // independent couplings, fixed for all events - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); // Create an array of views over the Flavor Couplings FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; @@ -399,11 +398,11 @@ namespace mg5amcCpu fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); MEs_sv = fptype_sv{ 0 }; - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); - for( int i = 0; i < processConfig::ndiagrams; ++i ) + for( int i = 0; i < ndiagrams; ++i ) { numerators_sv[i] = fptype_sv{ 0 }; } @@ -471,14 +470,14 @@ namespace mg5amcCpu for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) { if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - normalization += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + normalization += allNumerators[ievt / neppV * neppV * ndiagrams + ichan * neppV + ieppV % neppV]; } channelIdVec[ieppV] = mgOnGpu::nchannels; for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) { if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - numerator_sum += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + numerator_sum += allNumerators[ievt / neppV * neppV * ndiagrams + ichan * neppV + ieppV % neppV]; if( allrnddiagram[ievt] < numerator_sum / normalization ) { @@ -555,7 +554,7 @@ namespace mg5amcCpu if( mulChannelWeight && allChannelIds != nullptr ) // fix segfault #892 (not 'channelIds[0] != 0') { const unsigned int channelId = getChannelId( allChannelIds, ievt0, false ); - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc deleted file mode 100644 index ebce23a97b..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/check_sa.cc +++ /dev/null @@ -1,1020 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: O. Mattelaer (Nov 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, D. Massaro, O. Mattelaer, S. Roiser, J. Teig, A. Thete, A. Valassi (2020-2026). -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== -// -// Standalone script for MadGraph7 standalone mode. -// Generates phase-space points with RAMBO and evaluates the matrix element -// through the UMAMI interface (umami.h). -// -// Two run modes: -// * matrix (default): evaluates one phase-space point (generated with the -// classic standalone RAMBO, so identical to the one of -// the Fortran/C++ standalone 'check' drivers at the -// same energy) and prints it together with the matrix -// element of every flavor combination. -// * perf : runs nblocks*nthreads*niter events on a single flavor -// and prints performance counters. -// -//========================================================================== - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "GpuAbstraction.h" -#include "GpuRuntime.h" -#include "MemoryAccessMomenta.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" -#include "epoch_process_id.h" -#include "read_slha.h" -#include "timermap.h" -#include "umami.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define STRINGIFY( s ) #s -#define XSTRINGIFY( s ) STRINGIFY( s ) -#define SEP79 79 - -namespace -{ - using namespace mg5amcCpu; - - // Fixed physics inputs - fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV - constexpr unsigned long long kSeed = 20200805ULL; // reproducible RAMBO seed - - // Matrix-mode always runs 8 events on a single flavor index. - constexpr unsigned int kMatrixBlocks = 1; - constexpr unsigned int kMatrixThreads = 8; - - // Power of GeV of the matrix-element output; depends only on the number of external legs. - constexpr int kMEGeVExponent = -( 2 * CPPProcess::npar - 8 ); - - bool is_number( const char* s ) - { - const char* t = s; - while( *t != '\0' && isdigit( *t ) ) ++t; - return (int)strlen( s ) == t - s; - } - - // Accepts plain decimal numbers such as "1000" or "1000.0" (used for the - // optional energy argument of matrix mode). - bool is_float( const char* s ) - { - int ndots = 0; - const char* t = s; - while( *t != '\0' && ( isdigit( *t ) || ( *t == '.' && ndots++ == 0 ) ) ) ++t; - return t != s && (int)strlen( s ) == t - s; - } - - enum Mode { MODE_MATRIX, MODE_PERF }; - - enum RamboType { RAMBO_MASSIVE, RAMBO_MASSLESS }; - - // One external-particle list per LHE event, each particle stored as (E, px, py, pz). - using LheEvent = std::array, CPPProcess::npar>; - - bool read_lhe_events( const std::string& path, std::vector& events ) - { - constexpr int npar = CPPProcess::npar; - std::ifstream in( path ); - if( !in ) - { - std::cerr << "ERROR! cannot open LHE file '" << path << "'" << std::endl; - return false; - } - std::string line; - while( std::getline( in, line ) ) - { - if( line.find( "" ) == std::string::npos ) continue; - if( !std::getline( in, line ) ) break; - std::istringstream hdr( line ); - int nptcl = 0; - hdr >> nptcl; - if( nptcl != npar ) - { - std::cerr << "ERROR! LHE event has " << nptcl << " particles, expected " << npar << std::endl; - return false; - } - // particle lines: pdg status mother1 mother2 color1 color2 px py pz E m lifetime spin - LheEvent ev; - int ipar = 0; - while( ipar < npar && std::getline( in, line ) ) - { - if( line.empty() ) continue; - std::istringstream ls( line ); - long pdg; - int status, m1, m2, c1, c2; - double px, py, pz, E; - if( !( ls >> pdg >> status >> m1 >> m2 >> c1 >> c2 >> px >> py >> pz >> E ) ) - { - std::cerr << "ERROR! malformed LHE particle line: " << line << std::endl; - return false; - } - ev[ipar] = { E, px, py, pz }; - ++ipar; - } - if( ipar != npar ) - { - std::cerr << "ERROR! truncated LHE event (got " << ipar << " of " << npar << " particles)" << std::endl; - return false; - } - events.push_back( ev ); - } - if( events.empty() ) - { - std::cerr << "ERROR! no events found in '" << path << "'" << std::endl; - return false; - } - return true; - } - - int usage( const char* argv0, int ret = 1 ) - { - std::cout - << "Usage:\n" - << " " << argv0 << " [matrix] [-v|--verbose] []\n" - << " " << argv0 << " perf [-v|--verbose] [-f|--flavor ] [--rambo-massless]" - << " [-e|--events ] [<#blocksPerGrid> <#threadsPerBlock>] <#iterations>\n" - << " " << argv0 << " -p [opts] (legacy alias for `perf`)\n" - << "\n" - << "Subcommands:\n" - << " matrix (default) Evaluate one phase-space point (classic standalone\n" - << " RAMBO, identical to the Fortran/C++ 'check' drivers,\n" - << " Ecms = , default 1000 GeV) and print it with\n" - << " the matrix element for each flavor combination.\n" - << " With -v also prints backend/fptype/hardcodePARAM header.\n" - << " perf Run #blocks*#threads events over #iterations iterations\n" - << " on a single flavor index, then print performance counters.\n" - << " Always prints inputs + backend/fptype header.\n" - << " With -v also dumps every event's phase-space point and ME.\n" - << "\n" - << "Options:\n" - << " -e|--events (perf only) Read the external momenta from an LHE\n" - << " file instead of generating them with RAMBO. The events are\n" - << " processed in batches of #blocks*#threads; #iterations is\n" - << " ignored (derived from the number of events in the file).\n" - << "\n" - << "perf-mode defaults if positional args are omitted:\n" - << " #blocksPerGrid = 64, #threadsPerBlock = 256, #iterations = 1.\n"; - return ret; - } - - // AOSOA -> UMAMI SoA single-event helper. Layout reminder: - // AOSOA: aosoa[i_page * npar*4*neppM + ipar*4*neppM + ip4*neppM + i_vector] - // UMAMI: soa[ip4 * npar*nevt + ipar*nevt + ievt] - __host__ __device__ inline void - aosoa_to_umami_one( const fptype* aosoa, - double* soa, - std::size_t ievt, - std::size_t nevt ) - { - constexpr int npar = CPPProcess::npar; - for( int ipar = 0; ipar < npar; ++ipar ) - { - for( int ip4 = 0; ip4 < 4; ++ip4 ) - { - soa[(std::size_t)ip4 * npar * nevt + (std::size_t)ipar * nevt + ievt] = - (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, ip4, ipar ); - } - } - } - - - const char* backend_label() - { -#ifdef __CUDACC__ - return "CUDA"; -#elif defined( __HIPCC__ ) - return "HIP"; -#else - return "CPP"; -#endif - } - - const char* fp_label() - { -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - return "MIXED"; -#elif defined MGONGPU_FPTYPE_DOUBLE - return "DOUBLE"; -#elif defined MGONGPU_FPTYPE_FLOAT - return "FLOAT"; -#else - return "UNKNOWN"; -#endif - } - - void print_run_header( std::ostream& os ) - { - os << "Process = " << XSTRINGIFY( MG_EPOCH_PROCESS_ID ) << "_" << backend_label() -#ifdef MGONGPU_HARDCODE_PARAM - << " [hardcodePARAM=1]" << std::endl -#else - << " [hardcodePARAM=0]" << std::endl -#endif - << "FP precision = " << fp_label() << std::endl - << "Random number generation = COMMON RANDOM HOST" << std::endl; - } - - void print_momenta_table( std::ostream& os, const fptype* aosoa, unsigned int ievt ) - { - auto constexpr prec = std::numeric_limits::digits10; - constexpr int npar = CPPProcess::npar; - os << std::string( SEP79, '-' ) << std::endl - << " n E px py pz" << std::endl; - for( int ipar = 0; ipar < npar; ++ipar ) - { - double E = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 0, ipar ); - double px = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 1, ipar ); - double py = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 2, ipar ); - double pz = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 3, ipar ); - os << std::scientific << std::setprecision( prec ) - << std::setw( 4 ) << ipar + 1 - << std::setw( prec + 10 ) << E - << std::setw( prec + 10 ) << px - << std::setw( prec + 10 ) << py - << std::setw( prec + 10 ) << pz - << std::endl - << std::defaultfloat; - } - os << std::string( SEP79, '-' ) << std::endl; - } - - // Run sigmaKin via UMAMI for `nevt` events and copy back the MEs. - // Both the momenta (UMAMI SoA layout) and the per-event flavor buffer must be set - // by the caller. On GPU the buffers are device pointers and `hstMEs` receives the - // host-side copy; on CPU `umamiMEs` is the output buffer. - bool run_umami( - UmamiHandle handle, - unsigned int nevt, - mgOnGpu::TimerMap& timermap, - double& wavetime, - const std::vector& umamiMomenta, - const std::vector& flvVec, - std::vector& umamiMEs - ) - { - constexpr unsigned int UmamiInKeyNum = 2; - timermap.start( "3a SigmaKin" ); - UmamiInputKey in_keys[UmamiInKeyNum] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX }; - UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; - const void* inputs[UmamiInKeyNum] = { umamiMomenta.data(), flvVec.data() }; - void* outputs[1] = { umamiMEs.data() }; - UmamiStatus st = umami_matrix_element( - handle, nevt, nevt, 0, UmamiInKeyNum, in_keys, inputs, 1, out_keys, outputs ); - wavetime += timermap.stop(); - if( st != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; - return false; - } - - return true; - } - - // -------------------------------------------------------------------------- - // Classic MadGraph standalone RAMBO (RANMAR generator seeded with the fixed - // values 1802/9373), reproducing the exact phase-space point used by the - // Fortran and C++ standalone 'check' drivers at the same energy, so that - // matrix-mode output can be compared line by line across backends. - // Host-side only; ported from madgraph/iolibs/template_files/rambo.cc. - // -------------------------------------------------------------------------- - namespace classic_rambo - { - struct Random - { - double ranu[98]; - double ranc, rancd, rancm; - int iranmr, jranmr; - - // universal random number generator proposed by Marsaglia and Zaman - double ranmar() - { - double uni = ranu[iranmr] - ranu[jranmr]; - if( uni < 0 ) uni = uni + 1; - ranu[iranmr] = uni; - iranmr = iranmr - 1; - jranmr = jranmr - 1; - if( iranmr == 0 ) iranmr = 97; - if( jranmr == 0 ) jranmr = 97; - ranc = ranc - rancd; - if( ranc < 0 ) ranc = ranc + rancm; - uni = uni - ranc; - if( uni < 0 ) uni = uni + 1; - return uni; - } - - void rmarin( int ij, int kl ) - { - int i = ij / 177 % 177 + 2; - int j = ij % 177 + 2; - int k = ( kl / 169 ) % 178 + 1; - int l = kl % 169; - for( int ii = 1; ii < 98; ii++ ) - { - double s = 0; - double t = .5; - for( int jj = 1; jj < 25; jj++ ) - { - int m = ( ( i * j % 179 ) * k ) % 179; - i = j; - j = k; - k = m; - l = ( 53 * l + 1 ) % 169; - if( ( l * m ) % 64 >= 32 ) s = s + t; - t = .5 * t; - } - ranu[ii] = s; - } - ranc = 362436. / 16777216.; - rancd = 7654321. / 16777216.; - rancm = 16777213. / 16777216.; - iranmr = 97; - jranmr = 33; - } - }; - - inline double rn() - { - static Random rand; - static bool init = true; - if( init ) - { - init = false; - rand.rmarin( 1802, 9373 ); - } - double ran; - while( true ) - { - ran = rand.ranmar(); - if( ran > 1e-16 ) break; - } - return ran; - } - - // RAMBO: democratic multi-particle phase space generator (S.D. Ellis, - // R. Kleiss, W.J. Stirling); weights are logarithmic. - inline std::vector> - rambo( double et, const std::vector& xm, double& wt ) - { - const int n = (int)xm.size(); - std::vector> q( n, std::vector( 4 ) ); - std::vector> p( n, std::vector( 4 ) ); - std::vector z( n ), r( 4 ), b( 3 ), p2( n ), xm2( n ), e( n ), v( n ); - const double acc = 1e-14; - const int itmax = 6; - const double twopi = 8. * atan( 1. ); - const double po2log = log( twopi / 4. ); - - // factorials for the phase-space weight - z[1] = po2log; - for( int k = 2; k < n; k++ ) z[k] = z[k - 1] + po2log - 2. * log( double( k - 1 ) ); - for( int k = 2; k < n; k++ ) z[k] = z[k] - log( double( k ) ); - - if( n < 1 || n > 101 ) - { - std::cout << "Too few or many particles: " << n << std::endl; - exit( -1 ); - } - double xmt = 0.; - int nm = 0; - for( int i = 0; i < n; i++ ) - { - if( xm[i] != 0. ) nm = nm + 1; - xmt = xmt + std::abs( xm[i] ); - } - if( xmt > et ) - { - std::cout << "Too low energy: " << et << " needed " << xmt << std::endl; - exit( -1 ); - } - - // generate n massless momenta in infinite phase space - for( int i = 0; i < n; i++ ) - { - double r1 = rn(); - double c = 2. * r1 - 1.; - double s = sqrt( 1. - c * c ); - double f = twopi * rn(); - r1 = rn(); - double r2 = rn(); - q[i][0] = -log( r1 * r2 ); - q[i][3] = q[i][0] * c; - q[i][2] = q[i][0] * s * cos( f ); - q[i][1] = q[i][0] * s * sin( f ); - } - - // parameters of the conformal transformation - for( int k = 0; k < 4; k++ ) r[k] = 0.; - for( int i = 0; i < n; i++ ) - for( int k = 0; k < 4; k++ ) r[k] = r[k] + q[i][k]; - double rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); - for( int k = 1; k < 4; k++ ) b[k - 1] = -r[k] / rmas; - double g = r[0] / rmas; - double a = 1. / ( 1. + g ); - double x = et / rmas; - - // transform the q's conformally into the p's - for( int i = 0; i < n; i++ ) - { - double bq = b[0] * q[i][1] + b[1] * q[i][2] + b[2] * q[i][3]; - for( int k = 1; k < 4; k++ ) p[i][k] = x * ( q[i][k] + b[k - 1] * ( q[i][0] + a * bq ) ); - p[i][0] = x * ( g * q[i][0] + bq ); - } - - wt = po2log; - if( n != 2 ) wt = ( 2. * n - 4. ) * log( et ) + z[n - 1]; - - // massless case is done - if( nm == 0 ) return p; - - // massive particles: rescale the momenta by a factor x - double xmax = sqrt( 1. - pow( xmt / et, 2 ) ); - for( int i = 0; i < n; i++ ) - { - xm2[i] = pow( xm[i], 2 ); - p2[i] = pow( p[i][0], 2 ); - } - int iter = 0; - x = xmax; - double accu = et * acc; - while( true ) - { - double f0 = -et; - double g0 = 0.; - double x2 = x * x; - for( int i = 0; i < n; i++ ) - { - e[i] = sqrt( xm2[i] + x2 * p2[i] ); - f0 = f0 + e[i]; - g0 = g0 + p2[i] / e[i]; - } - if( std::abs( f0 ) <= accu ) break; - iter = iter + 1; - if( iter > itmax ) - { - std::cout << "Too many iterations without desired accuracy: " << itmax << std::endl; - break; - } - x = x - f0 / ( x * g0 ); - } - for( int i = 0; i < n; i++ ) - { - v[i] = x * p[i][0]; - for( int k = 1; k < 4; k++ ) p[i][k] = x * p[i][k]; - p[i][0] = e[i]; - } - - double wt2 = 1.; - double wt3 = 0.; - for( int i = 0; i < n; i++ ) - { - wt2 = wt2 * v[i] / e[i]; - wt3 = wt3 + pow( v[i], 2 ) / e[i]; - } - double wtm = ( 2. * n - 3. ) * log( x ) + log( wt2 / wt3 * et ); - wt = wt + wtm; - return p; - } - - // Auxiliary function changing convention between MadGraph5_aMC@NLO and - // RAMBO four-momenta (same as get_momenta in the standalone_cpp driver). - inline std::vector> - get_momenta( int ninitial, double energy, const std::vector& masses, double& wgt ) - { - const int nexternal = (int)masses.size(); - const int nfinal = nexternal - ninitial; - const double e2 = pow( energy, 2 ); - const double m1 = masses[0]; - - if( ninitial == 1 ) - { - std::vector> p( 1, std::vector( 4, 0. ) ); - p[0][0] = m1; - std::vector finalmasses( masses.begin() + 1, masses.end() ); - std::vector> p_rambo = rambo( m1, finalmasses, wgt ); - p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); - return p; - } - - if( ninitial != 2 ) - { - std::cout << "Rambo needs 1 or 2 incoming particles" << std::endl; - exit( -1 ); - } - - double etot = energy; - if( nfinal == 1 ) etot = m1; - const double m2 = masses[1]; - const double mom = sqrt( ( pow( e2, 2 ) - 2 * e2 * pow( m1, 2 ) + pow( m1, 4 ) - 2 * e2 * pow( m2, 2 ) - 2 * pow( m1, 2 ) * pow( m2, 2 ) + pow( m2, 4 ) ) / ( 4 * e2 ) ); - const double energy1 = sqrt( pow( mom, 2 ) + pow( m1, 2 ) ); - const double energy2 = sqrt( pow( mom, 2 ) + pow( m2, 2 ) ); - std::vector> p( 2, std::vector( 4, 0. ) ); - p[0][0] = energy1; - p[0][3] = mom; - p[1][0] = energy2; - p[1][3] = -mom; - - if( nfinal == 1 ) - { - p.push_back( std::vector( 4, 0. ) ); - p[2][0] = etot; - wgt = 1; - return p; - } - std::vector finalmasses( masses.begin() + 2, masses.end() ); - std::vector> p_rambo = rambo( etot, finalmasses, wgt ); - p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); - return p; - } - } - - // -------------------------------------------------------------------------- - // matrix mode: same PS point fed to every flavor combination, print event 0. - // The point is generated with the classic standalone RAMBO so it is - // identical to the one of the Fortran/C++ standalone 'check' drivers. - // -------------------------------------------------------------------------- - int run_matrix_mode( bool verbose ) - { - constexpr unsigned int nevt = kMatrixBlocks * kMatrixThreads; - const unsigned int nFlavors = CPPProcess::nmaxflavor; - - mgOnGpu::TimerMap timermap; - - HostBufferRndNumMomenta hstRndmom( nevt ); - HostBufferMomenta hstMomenta( nevt ); - HostBufferWeights hstWeights( nevt ); - std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - std::vector umamiMEs( nevt ); - std::vector flvVec( nevt ); - - UmamiHandle umami_handle = nullptr; - if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_initialize failed" << std::endl; - return 2; - } - - // Generate one shared phase-space point used by every flavor, with the - // classic standalone RAMBO so it matches the Fortran/C++ 'check' drivers. - CPPProcess process; - process.initProc( "../../Cards/param_card.dat" ); - double rambowgt = 0.; - - // Retrieve masses - int npar_meta = 0; - if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - std::vector massesD( npar_meta ); - if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - const std::vector masses( massesD.begin(), massesD.end() ); - - std::vector> point = - classic_rambo::get_momenta( CPPProcess::npari, (double)kEnergy, masses, rambowgt ); - - // alpha_s from the param card so the couplings match the Fortran/C++ - // 'check' drivers (UMAMI otherwise falls back to a hardcoded g_s). - SLHAReader slha( "../../Cards/param_card.dat", false ); - const double alphaS = slha.get_block_entry( "sminputs", 3, 1.180000e-01 ); - std::vector alphasVec( nevt, alphaS ); - - // Always massive RAMBO - std::unique_ptr prsk( - new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); - prsk->getMomentaInitial(); - prsk->getMomentaFinal(); - - // Fill the UMAMI SoA buffer with nevt copies of the same event: - // soa[ip4 * npar*nevt + ipar*nevt + ievt] - for( int ip4 = 0; ip4 < 4; ++ip4 ) - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - for( unsigned int ievt = 0; ievt < nevt; ++ievt ) - umamiMomenta[(std::size_t)ip4 * CPPProcess::npar * nevt + (std::size_t)ipar * nevt + ievt] = point[ipar][ip4]; - for( std::size_t ievt = 0; ievt < nevt; ++ievt ) - aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); - - if( verbose ) - { - std::cout << std::string( SEP79, '*' ) << std::endl; - print_run_header( std::cout ); - std::cout << std::string( SEP79, '*' ) << std::endl; - } - - std::cout << "Phase space point:" << std::endl - << std::string( SEP79, '-' ) << std::endl - << " n E px py pz" << std::endl; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - { - std::cout << std::scientific << std::setprecision( 7 ) - << std::setw( 2 ) << ipar + 1 - << std::setw( 16 ) << point[ipar][0] - << std::setw( 16 ) << point[ipar][1] - << std::setw( 16 ) << point[ipar][2] - << std::setw( 16 ) << point[ipar][3] - << std::endl - << std::defaultfloat; - } - std::cout << std::string( SEP79, '-' ) << std::endl; - - for( unsigned int iflav = 0; iflav < nFlavors; ++iflav ) - { - std::fill( flvVec.begin(), flvVec.end(), iflav ); - timermap.start( "3a SigmaKin" ); - UmamiInputKey in_keys[3] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S }; - UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; - const void* inputs[3] = { umamiMomenta.data(), flvVec.data(), alphasVec.data() }; - void* outputs[1] = { umamiMEs.data() }; - UmamiStatus st = umami_matrix_element( - umami_handle, nevt, nevt, 0, 3, in_keys, inputs, 1, out_keys, outputs ); - timermap.stop(); - if( st != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; - umami_free( umami_handle ); - return 3; - } - const double* mes = umamiMEs.data(); - - std::cout << " PDG"; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - std::cout << std::setw( 12 ) << CPPProcess::flavorPDG( iflav, ipar ); - std::cout << std::endl - << " Matrix element = " << std::scientific << std::setprecision( 16 ) - << mes[0] << " GeV^" << kMEGeVExponent << std::endl - << std::defaultfloat - << std::string( SEP79, '-' ) << std::endl; - } - - umami_free( umami_handle ); - return 0; - } - - // -------------------------------------------------------------------------- - // perf mode: nblocks*nthreads events per iteration on a single flavor. - // -------------------------------------------------------------------------- - int run_perf_mode( bool verbose, - unsigned int gpublocks, - unsigned int gputhreads, - unsigned int niter, - unsigned int flavorID, - RamboType ramboType, - const std::string& lheFile = "" ) - { - const unsigned int nevt = gpublocks * gputhreads; - - // LHE instead of generating. Processed in batches of nevt and - // niter is derived from the number of events read. - std::vector lheEvents; - if( !lheFile.empty() ) - { - if( !read_lhe_events( lheFile, lheEvents ) ) return 2; - niter = (unsigned int)( ( lheEvents.size() + nevt - 1 ) / nevt ); - std::cout << "Reading events from LHE file = " << lheFile - << " (" << lheEvents.size() << " events, " << niter - << " batches of " << nevt << ")" << std::endl; - } - - mgOnGpu::TimerMap timermap; - - HostBufferRndNumMomenta hstRndmom( nevt ); - HostBufferMomenta hstMomenta( nevt ); - HostBufferWeights hstWeights( nevt ); - std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - std::vector umamiMEs( nevt ); - std::vector flvVec( nevt, flavorID ); - - std::unique_ptr prnk( - new CommonRandomNumberKernel( hstRndmom ) ); - - UmamiHandle umami_handle = nullptr; - if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_initialize failed" << std::endl; - return 2; - } - - // Retrieve masses - std::vector masses; - if( ramboType != RAMBO_MASSLESS) - { - int npar_meta = 0; - if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - std::vector massesD( npar_meta ); - if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - masses.assign( massesD.begin(), massesD.end() ); - } - - std::unique_ptr prsk; - if( ramboType != RAMBO_MASSLESS ) - { - // Massive host only (copy) - prsk.reset( new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); - } - else - { - prsk.reset( new MasslessRamboSamplingKernelHost( kEnergy, hstRndmom, hstMomenta, hstWeights, nevt ) ); - } - - std::unique_ptr genrtimes( new double[niter] ); - std::unique_ptr rambtimes( new double[niter] ); - std::unique_ptr wavetimes( new double[niter] ); - - unsigned int nevtABN = 0; - unsigned int nevtZERO = 0; - double sumME = 0.; - double sumMEsq = 0.; - double minME = std::numeric_limits::infinity(); - double maxME = -std::numeric_limits::infinity(); - unsigned int nevtALL = 0; - - for( unsigned int iiter = 0; iiter < niter; ++iiter ) - { - double genrtime = 0; - double rambtime = 0; - unsigned int nreal = nevt; // number of real (non-padding) events in this batch - if( lheFile.empty() ) - { - timermap.start( "1a GenSeed " ); - prnk->seedGenerator( kSeed + iiter ); - genrtime += timermap.stop(); - timermap.start( "1b GenRnGen" ); - prnk->generateRnarray(); - genrtime += timermap.stop(); - - timermap.start( "2a RamboIni" ); - prsk->getMomentaInitial(); - rambtime += timermap.stop(); - timermap.start( "2b RamboFin" ); - prsk->getMomentaFinal(); - rambtime += timermap.stop(); - } - else - { - // Fill this batch from the LHE events (AOSOA layout, (E,px,py,pz) per leg). - // padded by repeating its last real event so the SIMD page is valid - // only the nreal real events are counted below. - timermap.start( "2e ReadLHE " ); - const std::size_t base = (std::size_t)iiter * nevt; - nreal = (unsigned int)std::min( nevt, lheEvents.size() - base ); - for( unsigned int ievt = 0; ievt < nevt; ++ievt ) - { - const std::size_t src = base + std::min( ievt, (std::size_t)nreal - 1 ); - const LheEvent& ev = lheEvents[src]; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - for( int ip4 = 0; ip4 < 4; ++ip4 ) - MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar ) = (fptype)ev[ipar][ip4]; - } - rambtime += timermap.stop(); - } - - timermap.start( "2d Aosoa2U " ); - for( std::size_t ievt = 0; ievt < nevt; ++ievt ) - aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); - rambtime += timermap.stop(); - - double wavetime = 0; - if( !run_umami( umami_handle, nevt, timermap, wavetime, - umamiMomenta, flvVec, umamiMEs - ) ) - { - umami_free( umami_handle ); - return 3; - } - - const double* mes = umamiMEs.data(); - - timermap.start( "4@ UpdtStat" ); - for( unsigned int ievt = 0; ievt < nreal; ++ievt ) - { - double me = mes[ievt]; - ++nevtALL; - if( !std::isfinite( me ) ) - ++nevtABN; - else if( me == 0. ) - ++nevtZERO; - sumME += me; - sumMEsq += me * me; - if( me < minME ) minME = me; - if( me > maxME ) maxME = me; - } - - genrtimes[iiter] = genrtime; - rambtimes[iiter] = rambtime; - wavetimes[iiter] = wavetime; - - if( verbose ) - { - std::cout << std::string( SEP79, '*' ) << std::endl - << "Iteration #" << iiter + 1 << " of " << niter << std::endl; - for( unsigned int ievt = 0; ievt < nreal; ++ievt ) - { - std::cout << "Event #" << ievt + 1 << std::endl; - print_momenta_table( std::cout, hstMomenta.data(), ievt ); - std::cout << " Matrix element = " << std::scientific << std::setprecision( 16 ) - << mes[ievt] << " GeV^" << kMEGeVExponent << std::endl - << std::defaultfloat - << std::string( SEP79, '-' ) << std::endl; - } - } - } - - double sumgtim = 0, sumrtim = 0, sumwtim = 0; - double minwtim = wavetimes[0], maxwtim = wavetimes[0]; - for( unsigned int i = 0; i < niter; ++i ) - { - sumgtim += genrtimes[i]; - sumrtim += rambtimes[i]; - sumwtim += wavetimes[i]; - minwtim = std::min( minwtim, wavetimes[i] ); - maxwtim = std::max( maxwtim, wavetimes[i] ); - } - double meanwtim = sumwtim / niter; - - unsigned int nevtGood = nevtALL - nevtABN; - double meanME = ( nevtGood > 0 ) ? sumME / nevtGood : 0.; - double varME = ( nevtGood > 0 ) ? sumMEsq / nevtGood - meanME * meanME : 0.; - double stdME = ( varME > 0 ) ? std::sqrt( varME ) : 0.; - - std::cout << std::string( SEP79, '*' ) << std::endl; - print_run_header( std::cout ); - std::cout << "NumBlocksPerGrid = " << gpublocks << std::endl - << "NumThreadsPerBlock = " << gputhreads << std::endl - << "NumIterations = " << niter << std::endl - << "FlavorIndex = " << flavorID << " / " << CPPProcess::nmaxflavor << std::endl - << std::string( SEP79, '-' ) << std::endl - << "NaN/abnormal MEs = " << nevtABN << std::endl - << "Zero MEs = " << nevtZERO << std::endl - << std::string( SEP79, '-' ) << std::endl - << "NumberOfEntries = " << niter << std::endl - << std::scientific - << "TotalTime[Rnd+Rmb+ME] (123) = ( " << sumgtim + sumrtim + sumwtim << " ) sec" << std::endl - << "TotalTime[Rambo+ME] (23) = ( " << sumrtim + sumwtim << " ) sec" << std::endl - << "TotalTime[RndNumGen] (1) = ( " << sumgtim << " ) sec" << std::endl - << "TotalTime[Rambo] (2) = ( " << sumrtim << " ) sec" << std::endl - << "TotalTime[MatrixElems] (3) = ( " << sumwtim << " ) sec" << std::endl - << "MeanTimeInMatrixElems = ( " << meanwtim << " ) sec" << std::endl - << "[Min,Max]TimeInMatrixElems = [ " << minwtim << " , " << maxwtim << " ] sec" << std::endl - << std::string( SEP79, '-' ) << std::endl - << "TotalEventsComputed = " << nevtALL << std::endl - << "EvtsPerSec[Rnd+Rmb+ME](123) = ( " << nevtALL / ( sumgtim + sumrtim + sumwtim ) << " ) sec^-1" << std::endl - << "EvtsPerSec[Rmb+ME] (23) = ( " << nevtALL / ( sumrtim + sumwtim ) << " ) sec^-1" << std::endl - << "EvtsPerSec[MatrixElems] (3) = ( " << nevtALL / sumwtim << " ) sec^-1" << std::endl - << std::defaultfloat - << std::string( SEP79, '*' ) << std::endl - << "MeanMatrixElemValue = ( " << meanME << " +- " << stdME / std::sqrt( (double)std::max( 1u, nevtGood ) ) - << " ) GeV^" << kMEGeVExponent << std::endl - << "[Min,Max]MatrixElemValue = [ " << minME << " , " << maxME << " ] GeV^" << kMEGeVExponent << std::endl - << std::string( SEP79, '*' ) << std::endl; - timermap.dump(); - std::cout << std::string( SEP79, '*' ) << std::endl; - - umami_free( umami_handle ); - return 0; - } -} - -int main( int argc, char** argv ) -{ - - Mode mode = MODE_MATRIX; - RamboType ramboType = RAMBO_MASSIVE; // default - bool ramboTypeSet = false; - bool verbose = false; - unsigned int flavorID = 0; - unsigned int gpublocks = 64; - unsigned int gputhreads = 256; - unsigned int niter = 1; - unsigned int numvec[3] = { 0, 0, 0 }; - int nnum = 0; - std::string lheFile; // -e/--events: read momenta from this LHE file (perf mode only) - - // Optional leading subcommand (no leading dash). - int firstArg = 1; - if( firstArg < argc ) - { - std::string a = argv[firstArg]; - if( a == "matrix" ) { mode = MODE_MATRIX; ++firstArg; } - else if( a == "perf" ) { mode = MODE_PERF; ++firstArg; } - } - - double energyArg = -1.; - - for( int argn = firstArg; argn < argc; ++argn ) - { - std::string arg = argv[argn]; - if( arg == "--verbose" || arg == "-v" ) - verbose = true; - else if( arg == "--performance" || arg == "-p" ) - mode = MODE_PERF; // legacy alias - else if( ( arg == "--flavor" || arg == "-f" ) && argn + 1 < argc && is_number( argv[argn + 1] ) ) - flavorID = strtoul( argv[++argn], nullptr, 0 ); - else if( arg == "--rambo-massless" ) - { - std::string r = argv[++argn]; - ramboType = RAMBO_MASSLESS; - ramboTypeSet = true; - } - else if( ( arg == "--events" || arg == "-e" ) && argn + 1 < argc ) - { - lheFile = argv[++argn]; - mode = MODE_PERF; // reading events from file only makes sense in perf mode - } - else if( is_number( argv[argn] ) && nnum < 3 ) - { - numvec[nnum++] = strtoul( argv[argn], nullptr, 0 ); - if( energyArg < 0 ) energyArg = atof( argv[argn] ); - } - else if( is_float( argv[argn] ) && energyArg < 0 ) - { - // decimal number: only meaningful as the matrix-mode energy - energyArg = atof( argv[argn] ); - } - else - return usage( argv[0] ); - } -//ENERGY CHANGE FOR THE MATRIX MODE -// (default 1000 GeV as for the Fortran/C++ standalone 'check' drivers; -// can be overridden with a single positional argument) - if( mode == MODE_MATRIX ) kEnergy = ( energyArg > 0 ) ? energyArg : 1000.; - - if( mode == MODE_MATRIX ) - { - if( ramboType == RAMBO_MASSLESS ) - { - std::cerr << "ERROR: matrix mode only supports the classic RAMBO (-r c)." << std::endl; - return usage( argv[0] ); - } - if( nnum > 1 ) - { - std::cerr << "WARNING: extra positional args are ignored in matrix mode " - << "(dimensions are fixed at " << kMatrixBlocks << " " << kMatrixThreads << " 1)." - << std::endl; - } - return run_matrix_mode( verbose ); - } - - // perf mode - if( nnum == 3 ) - { - gpublocks = numvec[0]; - gputhreads = numvec[1]; - niter = numvec[2]; - } - else if( nnum == 1 ) - { - niter = numvec[0]; - } - else if( nnum != 0 ) - { - return usage( argv[0] ); - } - if( niter == 0 && lheFile.empty() ) return usage( argv[0] ); // niter is derived from the file in LHE mode - - if( flavorID >= CPPProcess::nmaxflavor ) - { - std::cerr << "ERROR: flavor index " << flavorID - << " is out of range [0, " << CPPProcess::nmaxflavor << ")." << std::endl; - return 1; - } - - return run_perf_mode( verbose, gpublocks, gputhreads, niter, flavorID, ramboType, lheFile ); -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc deleted file mode 100644 index d0b13c937f..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.cc +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Oct 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "Bridge.h" -#include "CPPProcess.h" -#include "GpuRuntime.h" - -extern "C" -{ - /** - * The namespace where the Bridge class is taken from. - * - * In the current implementation, two separate shared libraries are created for the GPU/CUDA and CPU/C++ implementations. - * Actually, two shared libraries for GPU and CPU are created for each of the five SIMD implementations on CPUs (none, sse4, avx2, 512y, 512z). - * A single fcreatebridge_ symbol is created in each library with the same name, connected to the appropriate Bridge on CPU or GPU. - * The Fortran MadEvent code is always the same: the choice whether to use a CPU or GPU implementation is done by linking the appropriate library. - * As the names of the two CPU/GPU libraries are the same in the five SIMD implementations, the choice of SIMD is done by setting LD_LIBRARY_PATH. - * - * In a future implementation, a single heterogeneous shared library may be created, with the same interface. - * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. - * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. - */ - using namespace mg5amcCpu; - - /** - * The floating point precision used in Fortran arrays. - * This is presently hardcoded to double precision (REAL*8). - */ - using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays - //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays - - /** - * Create a Bridge and return its pointer. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param nevtF the pointer to the number of events in the Fortran arrays - * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - */ - void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) - { - // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) - // FIXME: disable OMP in Bridge when called from Fortran - *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); - } - - /** - * Delete a Bridge. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - */ - void fbridgedelete_( CppObjectInFortran** ppbridge ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); - delete pbridge; - } - - /** - * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant alphas) - * @param rndhel the pointer to the input random numbers for helicity selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the pointer to the input channels i.e. Feynman diagrams to enhance (1 to n: 0 is an invalid value!) - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void fbridgesequence_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); - // Use the host/CPU implementation in the C++ library - // (there is no device implementation in this library) - pbridge->cpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); - } - - /** - * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++, without multi-channel mode. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant alphas) - * @param rndhel the pointer to the input random numbers for helicity selection - * @param rndcol the pointer to the input random numbers for color selection - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ) - { - //printf("fbridgesequence_nomultichannel_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); - fbridgesequence_( ppbridge, momenta, gs, iflavorVec, rndhel, rndcol, nullptr, mes, selhel, selcol, pgoodHelOnly ); - } - - /** - * Retrieve the number of good helicities for helicity filtering in the Bridge. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param pngoodhel the pointer to the output number of good helicities - * @param pntothel the pointer to the output total number of helicities - */ - void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, - unsigned int* pngoodhel, - unsigned int* pntothel ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgegetngoodhel_: invalid Bridge address" ); - *pngoodhel = pbridge->nGoodHel(); - *pntothel = pbridge->nTotHel(); - } -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h deleted file mode 100644 index 4815fdb027..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fbridge.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: Z. Wettersten (Oct 2024) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, A. Thete, A. Valassi (2025). - -#include "Bridge.h" -#include "CPPProcess.h" -#include "GpuRuntime.h" - -#ifndef _FBRIDGE_H_ -#define _FBRIDGE_H_ - -extern "C" -{ - using namespace mg5amcCpu; - - using FORTRANFPTYPE = double; - - void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ); - - void fbridgedelete_( CppObjectInFortran** ppbridge ); - - void fbridgesequence_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ); - - void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ); - - void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, unsigned int* pngoodhel, unsigned int* pntothel ); -} -#endif // _FBRIDGE_H_ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc deleted file mode 100644 index 4048cce912..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/fsampler.cc +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Feb 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "mgOnGpuConfig.h" - -#include "Bridge.h" -#include "CPPProcess.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" - -//-------------------------------------------------------------------------- - -namespace mg5amcCpu -{ - template - class Sampler final : public CppObjectInFortran - { - public: - // Constructor - // @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran arrays - // @param nparF (NEXTERNAL, nexternal.inc) number of external particles in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) - // @param np4F number of momenta components, usually 4, in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) - Sampler( int nevtF, int nparF, int np4F ); - // Destructor - virtual ~Sampler() {} - // Delete copy/move constructors and assignment operators - Sampler( const Sampler& ) = delete; - Sampler( Sampler&& ) = delete; - Sampler& operator=( const Sampler& ) = delete; - Sampler& operator=( Sampler&& ) = delete; - // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta - void samplerHostSequence( FORTRANFPTYPE* fortranMomenta ); - private: - const int m_nevt; // The number of events in each iteration - int m_iiter; // The iteration counter (for random number seeding) - HostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers - HostBufferMomenta m_hstMomenta; // Memory buffers for momenta - HostBufferWeights m_hstWeights; // Memory buffers for sampling weights - std::unique_ptr m_prnk; // The appropriate RandomNumberKernel - std::unique_ptr m_prsk; // The appropriate SamplingKernel - // HARDCODED DEFAULTS - static constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) - }; - - template - Sampler::Sampler( int nevtF, int nparF, int np4F ) - : m_nevt( nevtF ) - , m_iiter( 0 ) - , m_hstRndmom( nevtF ) - , m_hstMomenta( nevtF ) - , m_hstWeights( nevtF ) - , m_prnk( new CommonRandomNumberKernel( m_hstRndmom ) ) - , m_prsk( new RamboSamplingKernelHost( energy, m_hstRndmom, m_hstMomenta, m_hstWeights, nevtF ) ) - { - if( nparF != CPPProcess::npar ) throw std::runtime_error( "Sampler constructor: npar mismatch" ); - if( np4F != CPPProcess::np4 ) throw std::runtime_error( "Sampler constructor: np4 mismatch" ); - std::cout << "WARNING! Instantiate host Sampler (nevt=" << m_nevt << ")" << std::endl; - } - - // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta - template - void Sampler::samplerHostSequence( FORTRANFPTYPE* fortranMomenta ) - { - std::cout << "Iteration #" << m_iiter + 1 << std::endl; - // === STEP 1 OF 3 - // --- 1a. Seed rnd generator (to get same results on host and device in curand) - // [NB This should not be necessary using the host API: "Generation functions - // can be called multiple times on the same generator to generate successive - // blocks of results. For pseudorandom generators, multiple calls to generation - // functions will yield the same result as a single call with a large size."] - // *** NB! REMEMBER THAT THE FORTRAN SAMPLER ALWAYS USES COMMON RANDOM NUMBERS! *** - constexpr unsigned long long seed = 20200805; - m_prnk->seedGenerator( seed + m_iiter ); - m_iiter++; - // --- 1b. Generate all relevant numbers to build nevt events (i.e. nevt phase space points) on the host - m_prnk->generateRnarray(); - //std::cout << "Got random numbers" << std::endl; - // === STEP 2 OF 3 - // --- 2a. Fill in momenta of initial state particles on the device - m_prsk->getMomentaInitial(); - //std::cout << "Got initial momenta" << std::endl; - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - m_prsk->getMomentaFinal(); - //std::cout << "Got final momenta" << std::endl; - // --- 2c. TransposeC2F - hst_transposeMomentaC2F( m_hstMomenta.data(), fortranMomenta, m_nevt ); - } -} - -//-------------------------------------------------------------------------- - -extern "C" -{ - using namespace mg5amcCpu; - - /** - * The floating point precision used in Fortran arrays. - * This is presently hardcoded to double precision (REAL*8). - */ - using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays - //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays - - /** - * Create a Sampler and return its pointer. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - * @param nevtF the pointer to the number of events in the Fortran arrays - * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - */ - void fsamplercreate_( CppObjectInFortran** ppsampler, const int* pnevtF, const int* pnparF, const int* pnp4F ) - { - *ppsampler = new Sampler( *pnevtF, *pnparF, *pnp4F ); - } - - /** - * Delete a Sampler. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - */ - void fsamplerdelete_( CppObjectInFortran** ppsampler ) - { - Sampler* psampler = dynamic_cast*>( *ppsampler ); - if( psampler == 0 ) throw std::runtime_error( "fsamplerdelete_: invalid Sampler address" ); - delete psampler; - } - - /** - * Execute the matrix-element calculation "sequence" via a Sampler on GPU/CUDA or CUDA/C++. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param mes the pointer to the output matrix elements - */ - void fsamplersequence_( CppObjectInFortran** ppsampler, FORTRANFPTYPE* momenta ) - { - Sampler* psampler = dynamic_cast*>( *ppsampler ); - if( psampler == 0 ) throw std::runtime_error( "fsamplersequence_: invalid Sampler address" ); - // Use the host/CPU implementation (there is no device implementation) - psampler->samplerHostSequence( momenta ); - } -} - -//-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h deleted file mode 100644 index d466affa16..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/massless_rambo.h +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: S. Roiser (Feb 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== - -#ifndef MASSLESS_RAMBO_H -#define MASSLESS_RAMBO_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuFptypes.h" - -#include "ProcessData.h" - -#include -#include -#include - -// Simplified rambo version for 2 to N (with N>=2) processes with massless particles -namespace mg5amcCpu -{ -namespace massless_rambo { - - constexpr int np4 = ProcessData::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npari = ProcessData::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- - constexpr int nparf = ProcessData::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- - constexpr int npar = ProcessData::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - - //-------------------------------------------------------------------------- - - // Fill in the momenta of the initial particles - // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] - template - __host__ __device__ void - ramboGetMomentaInitial( const fptype energy, // input: energy - fptype* momenta ) // output: momenta for one event or for a set of events - { - const fptype energy1 = energy / 2; - const fptype energy2 = energy / 2; - const fptype mom = energy / 2; - M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 0 ) = energy1; - M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 0 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 0 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 0 ) = mom; - M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 1 ) = energy2; - M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 1 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 1 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 1 ) = -mom; - } - - //-------------------------------------------------------------------------- - - // Fill in the momenta of the final particles using the RAMBO algorithm - // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] - template - __host__ __device__ void - ramboGetMomentaFinal( const fptype energy, // input: energy - const fptype* rndmom, // input: random numbers in [0,1] for one event or for a set of events - fptype* momenta, // output: momenta for one event or for a set of events - fptype* wgts ) // output: weights for one event or for a set of events - { - /**************************************************************************** - * rambo * - * ra(ndom) m(omenta) b(eautifully) o(rganized) * - * * - * a democratic multi-particle phase space generator * - * authors: s.d. ellis, r. kleiss, w.j. stirling * - * this is version 1.0 - written by r. kleiss * - * -- adjusted by hans kuijf, weights are logarithmic (1990-08-20) * - * -- adjusted by madgraph@sheffield_gpu_hackathon team (2020-07-29) * - * * - ****************************************************************************/ - - // output weight - fptype& wt = W_ACCESS::kernelAccess( wgts ); - - // AV special case nparf==1 (issue #358) - if constexpr( nparf == 1 ) - { - static bool first = true; - if( first ) - { - { - printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); - } - first = false; - } - const int iparf = 0; - for( int i4 = 0; i4 < np4; i4++ ) - { - M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = 0; - for( int ipari = 0; ipari < npari; ipari++ ) - { - M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) += M_ACCESS::kernelAccessIp4Ipar( momenta, i4, ipari ); - } - } - wt = 1; - return; - } - - // initialization step: factorials for the phase space weight - const fptype twopi = 8. * atan( 1. ); - const fptype po2log = log( twopi / 4. ); - fptype z[nparf]; - if constexpr( nparf > 1 ) // avoid build warning on clang (related to #358) - z[1] = po2log; - for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = z[kpar - 1] + po2log - 2. * log( fptype( kpar - 1 ) ); - for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = ( z[kpar] - log( fptype( kpar ) ) ); - - // generate n massless momenta in infinite phase space - fptype q[nparf][np4]; - for( int iparf = 0; iparf < nparf; iparf++ ) - { - const fptype r1 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 0, iparf ); - const fptype r2 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 1, iparf ); - const fptype r3 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 2, iparf ); - const fptype r4 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 3, iparf ); - const fptype c = 2. * r1 - 1.; - const fptype s = sqrt( 1. - c * c ); - const fptype f = twopi * r2; - q[iparf][0] = -log( r3 * r4 ); - q[iparf][3] = q[iparf][0] * c; - q[iparf][2] = q[iparf][0] * s * cos( f ); - q[iparf][1] = q[iparf][0] * s * sin( f ); - } - - // calculate the parameters of the conformal transformation - fptype r[np4]; - fptype b[np4 - 1]; - for( int i4 = 0; i4 < np4; i4++ ) r[i4] = 0.; - for( int iparf = 0; iparf < nparf; iparf++ ) - { - for( int i4 = 0; i4 < np4; i4++ ) r[i4] = r[i4] + q[iparf][i4]; - } - const fptype rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); - for( int i4 = 1; i4 < np4; i4++ ) b[i4 - 1] = -r[i4] / rmas; - const fptype g = r[0] / rmas; - const fptype a = 1. / ( 1. + g ); - const fptype x0 = energy / rmas; - - // transform the q's conformally into the p's (i.e. the 'momenta') - for( int iparf = 0; iparf < nparf; iparf++ ) - { - fptype bq = b[0] * q[iparf][1] + b[1] * q[iparf][2] + b[2] * q[iparf][3]; - for( int i4 = 1; i4 < np4; i4++ ) - { - M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = x0 * ( q[iparf][i4] + b[i4 - 1] * ( q[iparf][0] + a * bq ) ); - } - M_ACCESS::kernelAccessIp4Ipar( momenta, 0, iparf + npari ) = x0 * ( g * q[iparf][0] + bq ); - } - - // calculate weight (NB return log of weight) - wt = po2log; - if( nparf != 2 ) wt = ( 2. * nparf - 4. ) * log( energy ) + z[nparf - 1]; - - // issue warnings if weight is too small or too large - static int iwarn[5] = { 0, 0, 0, 0, 0 }; - if( wt < -180. ) - { - if( iwarn[0] <= 5 ) std::cout << "Too small wt, risk for underflow: " << wt << std::endl; - iwarn[0] = iwarn[0] + 1; - } - if( wt > 174. ) - { - if( iwarn[1] <= 5 ) std::cout << "Too large wt, risk for overflow: " << wt << std::endl; - iwarn[1] = iwarn[1] + 1; - } - - // return for weighted massless momenta - // nothing else to do in this event if all particles are massless (nm==0) - - return; - } - - //-------------------------------------------------------------------------- -} -} - -#endif // MASSLESS_RAMBO_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/runTest.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/runTest.cc deleted file mode 100644 index 7bd143de4a..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/runTest.cc +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Hageboeck (Nov 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. -// ---------------------------------------------------------------------------- -// Use ./runTest.exe --gtest_filter=*xxx to run only testxxx.cc tests -// ---------------------------------------------------------------------------- - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "MadgraphTest.h" -#include "MatrixElementKernels.h" -#include "MemoryAccessChannelIds.h" -#include "MemoryAccessMatrixElements.h" -#include "MemoryAccessMomenta.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" -#include "coloramps.h" -#include "epoch_process_id.h" - -#include - -using namespace mg5amcCpu; - -struct CUDA_CPU_TestBase : public TestDriverBase -{ - static constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static constexpr int np4 = CPPProcess::np4; - static constexpr int npar = CPPProcess::npar; - static_assert( gputhreads % neppM == 0, "ERROR! #threads/block should be a multiple of neppM" ); - static_assert( gputhreads <= mgOnGpu::ntpbMAX, "ERROR! #threads/block should be <= ntpbMAX" ); - CUDA_CPU_TestBase( const std::string& refFileName ) - : TestDriverBase( npar, refFileName ) {} - // Does this test use channelIds? - virtual bool useChannelIds() const = 0; - // Set channelId array (in the same way for CUDA and CPU tests) - static constexpr unsigned int warpSize = 32; // FIXME: add a sanity check in madevent that this is the minimum? (would need to expose this from cudacpp to madevent) - static void setChannelIds( BufferChannelIds& hstChannelIds, std::size_t iiter ) - { - static const char* debugC = getenv( "CUDACPP_RUNTEST_DEBUG" ); - static const bool debug = ( debugC != 0 ) && ( std::string( debugC ) != "" ); - // Fill channelIds for multi-channel tests #896 - // (NB: these are only used if useChannelIds == true) - // TEMPORARY(0): debug multichannel tests with channelId=1 for all events - //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1; - // TEMPORARY(1): debug multichannel tests with channelId=1,2,..,ndiag,1,2,..ndiag,... (every event gets a different channel, no warps) - //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1 + i % CPPProcess::ndiagrams; - // ALMOST FINAL test implementation: 1111222233331111... (every 32-event warp gets a different channel) - // FINAL(?) test implementation: 2222333344442222... (every 32-event warp gets a different channel, skip those without associated iconfig #917) - static_assert( nevt % warpSize == 0, "ERROR! nevt should be a multiple of warpSize" ); - constexpr unsigned int nWarp = nevt / warpSize; - for( unsigned int iWarp = 0; iWarp < nWarp; ++iWarp ) - { - //const unsigned int channelId = 1 + ( iWarp + iiter * nWarp ) % CPPProcess::ndiagrams; // bug #917 - const int iconfig = 1 + ( iWarp + iiter * nWarp ) % mgOnGpu::nconfigSDE; - unsigned int channelId = 0; - //for( unsigned int idiagram = 1; idiagram < CPPProcess::ndiagrams; idiagram++ ) // two bugs #920 and #919 - for( unsigned int idiagram = 0; idiagram < mgOnGpu::nchannels; idiagram++ ) // fix #920 and work around #919 - { - if( mgOnGpu::hostChannel2iconfig[idiagram] == iconfig ) - { - channelId = idiagram + 1; // fix #917 (NB add +1 because channelId uses F indexing) - break; - } - } - assert( channelId > 0 ); // sanity check that the channelId for the given iconfig was found - if( debug ) std::cout << "CUDA_CPU_TestBase::setChannelIds: iWarp=" << iWarp << ", iconfig=" << iconfig << ", channelId=" << channelId << std::endl; - for( unsigned int i = 0; i < warpSize; ++i ) - hstChannelIds[iWarp * warpSize + i] = channelId; - } - } -}; - -struct CPUTest : public CUDA_CPU_TestBase -{ - // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) - // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] - CPPProcess process; - HostBufferRndNumMomenta hstRndMom; - HostBufferChannelIds hstChannelIds; - HostBufferMomenta hstMomenta; - HostBufferGs hstGs; - HostBufferRndNumHelicity hstRndHel; - HostBufferRndNumColor hstRndCol; - HostBufferWeights hstWeights; - HostBufferMatrixElements hstMatrixElements; - HostBufferSelectedHelicity hstSelHel; - HostBufferSelectedColor hstSelCol; - HostBufferHelicityMask hstIsGoodHel; - std::unique_ptr pmek; - - // Create a process object - // Read param_card and set parameters - // ** WARNING EVIL EVIL ** - // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. - // Don't remove! - CPUTest( const std::string& refFileName ) - : CUDA_CPU_TestBase( refFileName ) - , process( /*verbose=*/false ) - , hstRndMom( nevt ) - , hstChannelIds( nevt ) - , hstMomenta( nevt ) - , hstGs( nevt ) - , hstRndHel( nevt ) - , hstRndCol( nevt ) - , hstWeights( nevt ) - , hstMatrixElements( nevt ) - , hstSelHel( nevt ) - , hstSelCol( nevt ) - , hstIsGoodHel( CPPProcess::ncomb ) - , pmek( new MatrixElementKernelHost( hstMomenta, hstGs, hstRndHel, hstRndCol, hstChannelIds, hstMatrixElements, hstSelHel, hstSelCol, nevt ) ) - { - // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? - process.initProc( "../../Cards/param_card.dat" ); - } - - virtual ~CPUTest() {} - - void prepareRandomNumbers( unsigned int iiter ) override - { - // Random numbers for momenta - CommonRandomNumberKernel rnk( hstRndMom ); - rnk.seedGenerator( 1337 + iiter ); - rnk.generateRnarray(); - // Random numbers for helicity and color selection (fix #931) - CommonRandomNumberKernel rnk2( hstRndHel ); - rnk2.seedGenerator( 1338 + iiter ); - rnk2.generateRnarray(); - CommonRandomNumberKernel rnk3( hstRndCol ); - rnk3.seedGenerator( 1339 + iiter ); - rnk3.generateRnarray(); - } - - void prepareMomenta( fptype energy ) override - { - RamboSamplingKernelHost rsk( energy, hstRndMom, hstMomenta, hstWeights, nevt ); - // --- 2a. Fill in momenta of initial state particles on the device - rsk.getMomentaInitial(); - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - rsk.getMomentaFinal(); - } - - void runSigmaKin( std::size_t iiter ) override - { - constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) - for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; - setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 - if( iiter == 0 ) pmek->computeGoodHelicities(); - pmek->computeMatrixElements( useChannelIds() ); - } - - fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override - { - assert( ipar < npar ); - assert( ip4 < np4 ); - return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); - } - - fptype getMatrixElement( std::size_t ievt ) const override - { - return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); - } - - int getChannelId( std::size_t ievt ) const override - { - return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); - } - - int getSelectedHelicity( std::size_t ievt ) const override - { - //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... - return hstSelHel.data()[ievt]; - } - - int getSelectedColor( std::size_t ievt ) const override - { - //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... - return hstSelCol.data()[ievt]; - } -}; - -// Old test with multi-channel disabled #466 -struct CPUTestNoMultiChannel : public CPUTest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return false; } - - // Constructor - CPUTestNoMultiChannel( const std::string& refFileName ) - : CPUTest( refFileName ) {} // suffix .txt - - // Destructor - virtual ~CPUTestNoMultiChannel() {} -}; - -// New test with multi-channel enabled #896 -struct CPUTestMultiChannel : public CPUTest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return true; } - - // Constructor - CPUTestMultiChannel( const std::string& refFileName ) - : CPUTest( refFileName + "2" ) {} // suffix .txt2 - - // Destructor - virtual ~CPUTestMultiChannel() {} -}; - - -// AV July 2024 much simpler class structure without the presently-unnecessary googletest templates -// This is meant as a workaround to prevent not-understood segfault #907 when adding a second test -// Note: instantiate test2 first and test1 second to ensure that the channelid printout from the dtors comes from test1 first and test2 second -// CPU test drivers -CPUTestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID2( s ) s##_CPU_MULTICHANNEL -CPUTestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID1( s ) s##_CPU_NOMULTICHANNEL -// Madgraph tests -MadgraphTest mgTest2( driver2 ); -MadgraphTest mgTest1( driver1 ); -// Instantiate Google test 1 -#define XTESTID1( s ) TESTID1( s ) -TEST( XTESTID1( MG_EPOCH_PROCESS_ID ), compareMomAndME ) -{ -#ifdef MGONGPU_CHANNELID_DEBUG - driver1.pmek->setTagForNevtProcessedByChannel( "(no multichannel)" ); -#endif - mgTest1.CompareMomentaAndME( *this ); -} -// Instantiate Google test 2 -#define XTESTID2( s ) TESTID2( s ) -TEST( XTESTID2( MG_EPOCH_PROCESS_ID ), compareMomAndME ) -{ -#ifdef MGONGPU_CHANNELID_DEBUG - driver2.pmek->setTagForNevtProcessedByChannel( "(channelid array)" ); -#endif - mgTest2.CompareMomentaAndME( *this ); -} -/* clang-format on */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc deleted file mode 100644 index 24ccfb208d..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/testmisc.cc +++ /dev/null @@ -1,437 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. -// ---------------------------------------------------------------------------- -// Use ./runTest.exe --gtest_filter=*misc to run only testmisc.cc tests -// ---------------------------------------------------------------------------- - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuVectors.h" - -#include "constexpr_math.h" -#include "epoch_process_id.h" -#include "valgrind.h" - -#include - -//#include -//#include // needs C++20... https://stackoverflow.com/a/65347016 -#include -#include -#include - -#define TESTID( s ) s##_CPU_MISC - -#define XTESTID( s ) TESTID( s ) - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu -{ -#define EXPECT_TRUE_sv( cond ) { EXPECT_TRUE( cond ); } - - inline const std::string - boolTF( const bool& b ) - { - return ( b ? "T" : "F" ); - } - -} - -TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) -{ - using namespace mg5amcCpu; - - //-------------------------------------------------------------------------- - - EXPECT_TRUE( true ); - - //-------------------------------------------------------------------------- - - // Vector initialization for fptype_sv - { - fptype_sv f{ 0 }; - EXPECT_TRUE_sv( f == 0 ); - } - { - fptype_sv f = fptype_sv{ 0 }; - EXPECT_TRUE_sv( f == 0 ); - } - - // Vector initialization for fptype_sv - demonstrate bug #339 in older cxmake implementation - { - fptype_sv f{ 1 }; - //std::cout << f << std::endl << boolTF( f == 1 ) << std::endl; - //EXPECT_TRUE_sv( f == 1 ); // this fails for vectors! TFFF - EXPECT_TRUE_sv( f == 1 ); // this succeds: T - } - - - // Vector initialization for cxtype_sv - { - cxtype_sv c = cxzero_sv(); - EXPECT_TRUE_sv( c.real() == 0 ); - EXPECT_TRUE_sv( c.imag() == 0 ); - } - { - cxtype_sv c = cxmake( 1, fptype_sv{ 0 } ); // here was a bug #339 - EXPECT_TRUE_sv( c.real() == 1 ); - EXPECT_TRUE_sv( c.imag() == 0 ); - } - { - cxtype_sv c = cxmake( fptype_sv{ 0 }, 1 ); // here was a bug #339 - EXPECT_TRUE_sv( c.real() == 0 ); - EXPECT_TRUE_sv( c.imag() == 1 ); - } - - // Array initialization for cxtype_sv array (example: jamp_sv in CPPProcess.cc) - { - cxtype_sv array[2] = {}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "= {}" is missing!) - //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; - EXPECT_TRUE_sv( array[0].real() == 0 ); - EXPECT_TRUE_sv( array[0].imag() == 0 ); - EXPECT_TRUE_sv( array[1].real() == 0 ); - EXPECT_TRUE_sv( array[1].imag() == 0 ); - } - - // Alternative array initialization for cxtype_sv array (example: was used for outwf in testxxx.cc) - { - cxtype_sv array[2]{}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "{}" is missing!) - //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; - EXPECT_TRUE_sv( array[0].real() == 0 ); - EXPECT_TRUE_sv( array[0].imag() == 0 ); - EXPECT_TRUE_sv( array[1].real() == 0 ); - EXPECT_TRUE_sv( array[1].imag() == 0 ); - } - - //-------------------------------------------------------------------------- - - // Scalar complex references - { - using namespace mgOnGpu; - // Refs to f1, f2 - fptype f1 = 1; - fptype f2 = 2; - cxtype_ref r12( f1, f2 ); // copy refs - //cxtype_ref r12a( r12 ); //deleted - cxtype_ref r12a( cxtype_ref( f1, f2 ) ); // copy refs - //cxtype_ref r12b = r12; // deleted - cxtype_ref r12b = cxtype_ref( f1, f2 ); // copy refs - EXPECT_TRUE( cxtype( r12 ).real() == 1 ); - EXPECT_TRUE( cxtype( r12 ).imag() == 2 ); - EXPECT_TRUE( cxtype( r12a ).real() == 1 ); - EXPECT_TRUE( cxtype( r12a ).imag() == 2 ); - EXPECT_TRUE( cxtype( r12b ).real() == 1 ); - EXPECT_TRUE( cxtype( r12b ).imag() == 2 ); - // Refs to f1c, f2c - fptype f1c = 0; - fptype f2c = 0; - cxtype_ref r12c( f1c, f2c ); - EXPECT_TRUE( cxtype( r12c ).real() == 0 ); - EXPECT_TRUE( cxtype( r12c ).imag() == 0 ); - //r12c = r12; // deleted - r12c = cxtype( r12 ); // copy values - EXPECT_TRUE( cxtype( r12c ).real() == 1 ); - EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); - // Update f1, f2 - f1 = 10; - f2 = 20; - EXPECT_TRUE( cxtype( r12 ).real() == 10 ); - EXPECT_TRUE( cxtype( r12 ).imag() == 20 ); - EXPECT_TRUE( cxtype( r12a ).real() == 10 ); - EXPECT_TRUE( cxtype( r12a ).imag() == 20 ); - EXPECT_TRUE( cxtype( r12b ).real() == 10 ); - EXPECT_TRUE( cxtype( r12b ).imag() == 20 ); - EXPECT_TRUE( cxtype( r12c ).real() == 1 ); // points to f1c, not to f1 - EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); // points to f2c, not to f2 - } - - // Vector complex references - { - using namespace mgOnGpu; - // Refs to f1, f2 - fptype_sv f1 = fptype_sv{ 0 } + 1; - fptype_sv f2 = fptype_sv{ 0 } + 2; - cxtype_sv_ref r12( f1, f2 ); // copy refs - //cxtype_sv_ref r12a( r12 ); //deleted - cxtype_sv_ref r12a( cxtype_sv_ref( f1, f2 ) ); // copy refs - //cxtype_sv_ref r12b = r12; // deleted - cxtype_sv_ref r12b = cxtype_sv_ref( f1, f2 ); // copy refs - EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 2 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 2 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 2 ); - // Refs to f1c, f2c - fptype_sv f1c = fptype_sv{ 0 }; - fptype_sv f2c = fptype_sv{ 0 }; - cxtype_sv_ref r12c( f1c, f2c ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 0 ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 0 ); - //r12c = r12; // deleted - r12c = cxtype_sv( r12 ); // copy values - EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); - // Update f1, f2 - f1 = fptype_sv{ 0 } + 10; - f2 = fptype_sv{ 0 } + 20; - EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 10 ); - EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 20 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 10 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 20 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 10 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 20 ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); // points to f1c, not to f1 - EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); // points to f2c, not to f2 - } - - //-------------------------------------------------------------------------- - - // Boolean vector (mask) times FP vector - /* - // From https://github.com/madgraph5/madgraph4gpu/issues/765#issuecomment-1853672838 - channelids_sv = CHANNEL_ACCESS::kernelAccess( pchannelIds ); // the 4 channels in the SIMD vector - bool_sv mask_sv = ( channelids_sv == 1 ); - numerators_sv += mask_sv * cxabs2( amp_sv[0] ); - if( pchannelIds != nullptr ) denominators_sv += cxabs2( amp_sv[0] ); - */ - { - typedef bool_sv test_int_sv; // defined as scalar_or_vector of long int (FPTYPE=double) or int (FPTYPE=float) - test_int_sv channelids0_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) - test_int_sv channelids1_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) - fptype_sv absamp0_sv{}; // mimic cxabs2( amp_sv[0] ) - fptype_sv absamp1_sv{}; // mimic cxabs2( amp_sv[0] ) - channelids0_sv = 0; - channelids1_sv = 1; - absamp0_sv = 10.; - absamp1_sv = 11.; - bool_sv mask0_sv = ( channelids0_sv % 2 == 0 ); // even channels 0123 -> TFTF (1010) - bool_sv mask1_sv = ( channelids1_sv % 2 == 0 ); // even channels 1234 -> FTFT (0101) - constexpr fptype_sv fpZERO_sv{}; // 0000 - //fptype_sv numerators0_sv = mask0_sv * absamp0_sv; // invalid operands to binary * ('__vector(4) long int' and '__vector(4) double') - fptype_sv numerators0_sv = fpternary( mask0_sv, absamp0_sv, fpZERO_sv ); // equivalent to "mask0_sv * absamp0_sv" - fptype_sv numerators1_sv = fpternary( mask1_sv, absamp1_sv, fpZERO_sv ); // equivalent to "mask1_sv * absamp1_sv" - // Values of numerators0_sv: 10.*1 - EXPECT_TRUE( numerators0_sv == 10. ); - // Values of numerators1_sv: 11.*0 - EXPECT_TRUE( numerators1_sv == 0. ); - } - - //-------------------------------------------------------------------------- - - // Test constexpr floor - EXPECT_TRUE( constexpr_floor( 1.5 ) == 1 ); - EXPECT_TRUE( constexpr_floor( 0.5 ) == 0 ); - EXPECT_TRUE( constexpr_floor( -0.5 ) == -1 ); - EXPECT_TRUE( constexpr_floor( -1.5 ) == -2 ); - - // Test constexpr pow - EXPECT_TRUE( constexpr_pow( 10, 0 ) == 1 ); - EXPECT_TRUE( constexpr_pow( 10, 1 ) == 10 ); - EXPECT_TRUE( constexpr_pow( 10, 2 ) == 100 ); - EXPECT_NEAR( constexpr_pow( 10, -1 ), 0.1, 0.1 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10, -1 ) = " << constexpr_pow( 10, -1 ); - EXPECT_NEAR( constexpr_pow( 10, -2 ), 0.01, 0.01 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10, -2 ) = " << constexpr_pow( 10, -2 ); - EXPECT_NEAR( constexpr_pow( 100, 0.5 ), 10, 10 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 100, 0.5 ) = " << constexpr_pow( 100, 0.5 ); - EXPECT_NEAR( constexpr_pow( 100, -0.5 ), 0.1, 0.1 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 100, -0.5 ) = " << constexpr_pow( 100, -0.5 ); - EXPECT_NEAR( constexpr_pow( 10000, 0.25 ), 10, 10 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10000, 0.25 ) = " << constexpr_pow( 10000, 0.25 ); - EXPECT_NEAR( constexpr_pow( 10000, -0.25 ), 0.1, 0.1 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10000, -0.25 ) = " << constexpr_pow( 10000, -0.25 ); - -#ifndef __aarch64__ // TO BE UNDERSTOOD? DISABLE CONSTEXPR_SQRT TESTS ON AARCH64 (#1064) - // Distance from the horizontal or vertical axis (i.e. from 0, pi/2, pi, or 3pi/2) - auto distance4 = []( const long double xx ) - { - const long double xx2 = mapIn0to2Pi( xx ); // in [0,2*pi) - const long double xx3 = xx2 - constexpr_floor( xx2 / constexpr_pi_by_2 ) * constexpr_pi_by_2; // in [0,pi/2) - const long double d0 = xx3; // distance from 0 - const long double d1 = constexpr_pi_by_2 - xx3; // distance from pi/2 - return ( d0 < d1 ? d0 : d1 ); - }; - - // Test constexpr sin, cos, tan - specific, problematic, points - auto testSinCosTanX = []( const long double xx, const double tolerance0, const bool debug = false, const long long istep = -999999999 ) - { - const double x = (double)xx; - const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 - if( debug ) - { - //std::cout << std::setprecision(40) << "testSinCosTanX: xx= " << xx << std::endl; - //std::cout << std::setprecision(40) << " x= " << x << std::endl; - } - //std::cout << std::setprecision(40) << "xx - 3pi/2 " << xx - 3 * constexpr_pi_by_2 << std::endl; - //int width = 46; - //char buf[128]; - //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)xx ); - //std::cout << std::setprecision(40) << "testSinCosTanX: xx=" << buf << std::endl; - //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)x ); - //std::cout << std::setprecision(40) << " x= " << buf << std::endl; - EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::abs( std::sin( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::abs( std::cos( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; -#ifndef __aarch64__ - if( !RUNNING_ON_VALGRIND ) - { - EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::abs( std::tan( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - } - else -#endif - { - // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) - const long double ctanx = constexpr_tan( x ); - const long double taninf = 4E14; // declare tan(x) as "infinity" if above this threshold - if( ctanx > -taninf && ctanx < taninf ) - EXPECT_NEAR( std::tan( x ), ctanx, std::abs( std::tan( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - else - { - // Allow tan(x)=-inf if ctan(x)=+inf and viceversa - EXPECT_GT( std::abs( std::tan( x ) ), taninf ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - /* - // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) - if( ctanx > 0 ) - EXPECT_GT( std::tan( x ), taninf ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - else - EXPECT_LT( std::tan( x ), -taninf ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - */ - } - } - std::cout << std::setprecision( 6 ); // default - }; - testSinCosTanX( constexpr_pi, 1E-3, true ); // from math.h - testSinCosTanX( (long double)3.141592653589793238462643383279502884L, 1E-3, true ); // from math.h - testSinCosTanX( 4.712388980384687897640105802565813064575L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) - testSinCosTanX( 3 * constexpr_pi_by_2 - 1.96e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x!=xx) - testSinCosTanX( 3 * constexpr_pi_by_2 - 1.9601e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) - - // Test constexpr sin, cos, tan - 8 points on (or close to) the boundaries of the 8 sectors of [0,2*pi] - auto testSinCosTan8 = [testSinCosTanX]( const double deltax, const double tolerance ) - { - for( int ioff = -1; ioff < 2; ioff++, ioff++ ) // -1, 1 - { - const bool debug = false; - const int nstep = 8; - for( int istep = 0; istep < nstep + 1; istep++ ) - { - long double x0 = deltax * ioff; - long double x1 = deltax * ioff + 2 * constexpr_pi; - double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) - testSinCosTanX( x, tolerance, debug, istep ); - } - } - }; - - // Use much lower tolerance when testing on the boundaries of the 8 sectors of [0,2*pi] - // Use progressively stricter tolerances as you move away from the boundaries of the 8 sectors of [0,2*pi] - testSinCosTan8( 0, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... - testSinCosTan8( 1E-15, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... - testSinCosTan8( 1E-14, 1E-04 ); // fails with 1E-05 - testSinCosTan8( 1E-12, 1E-06 ); // fails with 1E-07 - testSinCosTan8( 1E-09, 1E-09 ); // fails with 1E-10 - testSinCosTan8( 1E-06, 1E-12 ); // fails with 1E-13 - testSinCosTan8( 1E-03, 1E-14 ); // fails with 1E-16: could use 1E-14 but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) - testSinCosTan8( 1E-02, 1E-14 ); // never fails? could use 1E-99(?) but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) - - // Test constexpr sin, cos, tan - N points almost randomly with a varying tolerance - auto testSinCosTanN = [distance4]( const int nstep, const double x0, const double x1 ) - { - auto toleranceForX = [distance4]( const double x ) - { - const double d4 = distance4( x ); - if( d4 < 1E-14 ) - return 1E-03; // NB: absolute distance limited to 1E-14 anyway even if relative tolerance is 1E-3... - else if( d4 < 1E-13 ) - return 1E-04; - else if( d4 < 1E-12 ) - return 1E-05; - else if( d4 < 1E-11 ) - return 1E-06; - else if( d4 < 1E-10 ) - return 1E-07; - else if( d4 < 1E-09 ) - return 1E-08; - else if( d4 < 1E-08 ) - return 1E-09; - else if( d4 < 1E-07 ) - return 1E-10; - else if( d4 < 1E-06 ) - return 1E-11; - else if( d4 < 1E-05 ) - return 1E-12; - else if( d4 < 1E-04 ) - return 1E-13; - else - return 1E-14; // play it safe even if the agreement might even be better? - }; - for( int istep = 0; istep < nstep + 1; istep++ ) - { - double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) - const double tolerance0 = toleranceForX( x ); - const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 - EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::max( std::abs( std::sin( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::max( std::abs( std::cos( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); -#ifndef __aarch64__ - if( !RUNNING_ON_VALGRIND ) - { - EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - } - else -#endif - { - // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) - const long double ctanx = constexpr_tan( x ); - const long double taninf = 4E14; // declare tan(x) as "infinity if above this threshold - if( ctanx > -taninf && ctanx < taninf ) - EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - else - { - // Allow tan(x)=-inf if ctan(x)=+inf and viceversa - EXPECT_GT( std::abs( std::tan( x ) ), taninf ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - /* - // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) - if( ctanx > 0 ) - EXPECT_GT( std::tan( x ), taninf ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - else - EXPECT_LT( std::tan( x ), -taninf ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - */ - } - } - } - }; - testSinCosTanN( 100, -4 * constexpr_pi, 6 * constexpr_pi ); // this was failing at 3*pi/2 (now fixed by absolute tolerance 3E-15) - testSinCosTanN( 10000, -constexpr_pi_by_2, 5 * constexpr_pi_by_2 ); - - // Test constexpr atan - { - const double tolerance = 1E-12; - const int nstep = 1000; - for( int istep = 0; istep < nstep + 1; istep++ ) - { - long double x0 = -5, x1 = +5; - double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) - EXPECT_NEAR( std::atan( x ), constexpr_atan( x ), std::abs( std::atan( x ) * tolerance ) ) - << "x=" << x << ", istep=" << istep; - } - } -#endif - //-------------------------------------------------------------------------- -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc deleted file mode 100644 index 4c72479501..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CommonRandomNumberKernel.cc +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "CommonRandomNumbers.h" -#include "GpuAbstraction.h" -#include "MemoryBuffers.h" -#include "RandomNumberKernels.h" - -#include - -namespace mg5amcGpu -{ - //-------------------------------------------------------------------------- - - CommonRandomNumberKernel::CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ) - : RandomNumberKernelBase( rnarray ) - , m_seed( 20211220 ) - { - if( m_rnarray.isOnDevice() ) - throw std::runtime_error( "CommonRandomNumberKernel on host with a device random number array" ); - } - - //-------------------------------------------------------------------------- - - void CommonRandomNumberKernel::generateRnarray() - { - std::vector rnd = CommonRandomNumbers::generate( m_rnarray.size(), m_seed ); // NB: generate as double (HARDCODED) - std::copy( rnd.begin(), rnd.end(), m_rnarray.data() ); // NB: copy may imply a double-to-float conversion - } - - //-------------------------------------------------------------------------- -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc deleted file mode 100644 index 442c483981..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CurandRandomNumberKernel.cc +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "GpuRuntime.h" -#include "MemoryBuffers.h" -#include "RandomNumberKernels.h" - -#include - -#ifndef MGONGPU_HAS_NO_CURAND /* clang-format off */ -// NB This must come AFTER mgOnGpuConfig.h which contains our definition of __global__ when MGONGPUCPP_GPUIMPL is not defined -#include "curand.h" -#define checkCurand( code ){ assertCurand( code, __FILE__, __LINE__ ); } -inline void assertCurand( curandStatus_t code, const char *file, int line, bool abort = true ) -{ - if ( code != CURAND_STATUS_SUCCESS ) - { - printf( "CurandAssert: %s:%d code=%d\n", file, line, code ); - if ( abort ) assert( code == CURAND_STATUS_SUCCESS ); - } -} -#endif /* clang-format on */ - -namespace mg5amcGpu -{ - //-------------------------------------------------------------------------- -#ifndef MGONGPU_HAS_NO_CURAND - CurandRandomNumberKernel::CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) - : RandomNumberKernelBase( rnarray ) - , m_isOnDevice( onDevice ) - { - if( m_isOnDevice ) - { - if( !m_rnarray.isOnDevice() ) - throw std::runtime_error( "CurandRandomNumberKernel on device with a host random number array" ); - } - else - { - if( m_rnarray.isOnDevice() ) - throw std::runtime_error( "CurandRandomNumberKernel on host with a device random number array" ); - } - createGenerator(); - } - - //-------------------------------------------------------------------------- - - CurandRandomNumberKernel::~CurandRandomNumberKernel() - { - destroyGenerator(); - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::seedGenerator( const unsigned int seed ) - { - if( m_isOnDevice ) - { - destroyGenerator(); // workaround for #429 - createGenerator(); // workaround for #429 - } - //printf( "seedGenerator: seed %d\n", seed ); - checkCurand( curandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::createGenerator() - { - // [NB Timings are for GenRnGen host|device (cpp|cuda) generation of 256*32*1 events with nproc=1: rn(0) is host=0.0012s] - const curandRngType_t type = CURAND_RNG_PSEUDO_MTGP32; // 0.00082s | 0.00064s (FOR FAST TESTS) - //const curandRngType_t type = CURAND_RNG_PSEUDO_XORWOW; // 0.049s | 0.0016s - //const curandRngType_t type = CURAND_RNG_PSEUDO_MRG32K3A; // 0.71s | 0.0012s (better but slower, especially in c++) - //const curandRngType_t type = CURAND_RNG_PSEUDO_MT19937; // 21s | 0.021s - //const curandRngType_t type = CURAND_RNG_PSEUDO_PHILOX4_32_10; // 0.024s | 0.00026s (used to segfault?) - if( m_isOnDevice ) - { - checkCurand( curandCreateGenerator( &m_rnGen, type ) ); - } - else - { - checkCurand( curandCreateGeneratorHost( &m_rnGen, type ) ); - } - //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_LEGACY ) ); // fails with code=104 (see #429) - checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_BEST ) ); - //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_DYNAMIC ) ); // fails with code=104 (see #429) - //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_SEEDED ) ); // fails with code=104 (see #429) - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::destroyGenerator() - { - checkCurand( curandDestroyGenerator( m_rnGen ) ); - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::generateRnarray() - { -#if defined MGONGPU_FPTYPE_DOUBLE - checkCurand( curandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#elif defined MGONGPU_FPTYPE_FLOAT - checkCurand( curandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#endif - /* - printf( "\nCurandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); - fptype* data = m_rnarray.data(); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) - { - data = new fptype[m_rnarray.size()](); - checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); - } -#endif - for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) - printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) delete[] data; -#endif - */ - } - - //-------------------------------------------------------------------------- -#endif -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc deleted file mode 100644 index 54287cb57d..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/HiprandRandomNumberKernel.cc +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2024) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "mgOnGpuConfig.h" - -#include "GpuRuntime.h" -#include "MemoryBuffers.h" -#include "RandomNumberKernels.h" - -#include - -#ifndef MGONGPU_HAS_NO_HIPRAND /* clang-format off */ -#ifndef __HIP_PLATFORM_AMD__ -#define __HIP_PLATFORM_AMD__ 1 // enable hiprand for AMD (rocrand) -#endif -#include -#define checkHiprand( code ){ assertHiprand( code, __FILE__, __LINE__ ); } -inline void assertHiprand( hiprandStatus_t code, const char *file, int line, bool abort = true ) -{ - if ( code != HIPRAND_STATUS_SUCCESS ) - { - printf( "HiprandAssert: %s:%d code=%d\n", file, line, code ); - if ( abort ) assert( code == HIPRAND_STATUS_SUCCESS ); - } -} -#endif /* clang-format on */ - -namespace mg5amcGpu -{ - //-------------------------------------------------------------------------- -#ifndef MGONGPU_HAS_NO_HIPRAND - HiprandRandomNumberKernel::HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) - : RandomNumberKernelBase( rnarray ) - , m_isOnDevice( onDevice ) - { - if( m_isOnDevice ) - { - if( !m_rnarray.isOnDevice() ) - throw std::runtime_error( "HiprandRandomNumberKernel on device with a host random number array" ); - } - else - { - if( m_rnarray.isOnDevice() ) - throw std::runtime_error( "HiprandRandomNumberKernel on host with a device random number array" ); - } - createGenerator(); - } - - //-------------------------------------------------------------------------- - - HiprandRandomNumberKernel::~HiprandRandomNumberKernel() - { - destroyGenerator(); - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::seedGenerator( const unsigned int seed ) - { - if( m_isOnDevice ) - { - destroyGenerator(); // workaround for #429 - createGenerator(); // workaround for #429 - } - //printf( "seedGenerator: seed %d\n", seed ); - checkHiprand( hiprandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::createGenerator() - { - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_DEFAULT; - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_XORWOW; - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MRG32K3A; - const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MTGP32; // same as curand; not implemented yet (code=1000) in host code - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MT19937; - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_PHILOX4_32_10; - if( m_isOnDevice ) - { - checkHiprand( hiprandCreateGenerator( &m_rnGen, type ) ); - } - else - { - // See https://github.com/ROCm/hipRAND/issues/76 - throw std::runtime_error( "HiprandRandomNumberKernel on host is not supported yet (hiprandCreateGeneratorHost is not implemented yet)" ); - //checkHiprand( hiprandCreateGeneratorHost( &m_rnGen, type ) ); // ALWAYS FAILS WITH CODE=1000 - } - // FIXME: hiprand ordering is not implemented yet - // See https://github.com/ROCm/hipRAND/issues/75 - /* - //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_LEGACY ) ); - checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_BEST ) ); - //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_DYNAMIC ) ); - //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_SEEDED ) ); - */ - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::destroyGenerator() - { - checkHiprand( hiprandDestroyGenerator( m_rnGen ) ); - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::generateRnarray() - { -#if defined MGONGPU_FPTYPE_DOUBLE - checkHiprand( hiprandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#elif defined MGONGPU_FPTYPE_FLOAT - checkHiprand( hiprandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#endif - /* - printf( "\nHiprandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); - fptype* data = m_rnarray.data(); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) - { - data = new fptype[m_rnarray.size()](); - checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); - } -#endif - for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) - printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) delete[] data; -#endif - */ - } - - //-------------------------------------------------------------------------- -#endif -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h deleted file mode 100644 index f50ee1a0dc..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MadgraphTest.h +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Hageboeck (Dec 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, J. Teig, A. Valassi (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MADGRAPHTEST_H_ -#define MADGRAPHTEST_H_ 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" - -#include - -#include -#include -//#ifdef __HIPCC__ -//#include // see https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 -//#else -//#include // bypass this completely to ease portability on LUMI #803 -//#endif -#include -#include -#include -#include -#include -#include - -using mg5amcGpu::CPPProcess; - -namespace -{ - struct ReferenceData - { - std::vector>> momenta; - std::vector MEs; - std::vector ChanIds; - std::vector SelHels; - std::vector SelCols; - }; - - /// Read batches of reference data from a file and store them in a map. - std::map readReferenceData( const std::string& refFileName ) - { - std::cout << "INFO: Opening reference file " << refFileName << std::endl; - std::ifstream referenceFile( refFileName.c_str() ); - EXPECT_TRUE( referenceFile.is_open() ) << refFileName; - std::map referenceData; - unsigned int evtNo; - unsigned int batchNo; - for( std::string line; std::getline( referenceFile, line ); ) - { - std::stringstream lineStr( line ); - if( line.empty() || line[0] == '#' ) - { - continue; - } - else if( line.find( "Event" ) != std::string::npos ) - { - std::string dummy; - lineStr >> dummy >> evtNo >> dummy >> batchNo; - } - else if( line.find( "ME" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].MEs.size() ) - referenceData[batchNo].MEs.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].MEs[evtNo]; - } - else if( line.find( "ChanId" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].ChanIds.size() ) - referenceData[batchNo].ChanIds.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].ChanIds[evtNo]; - referenceData[batchNo].ChanIds[evtNo] = 0; // disable ChanId comparison if multichannel is not supported #976 - } - else if( line.find( "SelHel" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].SelHels.size() ) - referenceData[batchNo].SelHels.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].SelHels[evtNo]; - } - else if( line.find( "SelCol" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].SelCols.size() ) - referenceData[batchNo].SelCols.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].SelCols[evtNo]; - } - else - { - unsigned int particleIndex; - lineStr >> particleIndex; - if( evtNo <= referenceData[batchNo].momenta.size() ) - referenceData[batchNo].momenta.resize( evtNo + 1 ); - if( particleIndex <= referenceData[batchNo].momenta[evtNo].size() ) - referenceData[batchNo].momenta[evtNo].resize( particleIndex + 1 ); - auto& fourVec = referenceData[batchNo].momenta[evtNo][particleIndex]; - for( unsigned int i = 0; i < fourVec.size(); ++i ) - { - EXPECT_TRUE( lineStr.good() ); - lineStr >> fourVec[i]; - } - EXPECT_TRUE( lineStr.eof() ); - } - } - return referenceData; - } - -} - -/** - * Test driver providing a common interface for testing different implementations. - * Users need to implement: - * - Functions to retrieve matrix element and 4-momenta. These are used in the tests. - * - Driver functions that run the madgraph workflow. - */ -class TestDriverBase -{ - std::string m_refFileName; -public: - const unsigned int nparticle; - static constexpr unsigned int niter = 2; - static constexpr unsigned int gpublocks = 2; - static constexpr unsigned int gputhreads = 128; - static constexpr unsigned int nevt = gpublocks * gputhreads; - - TestDriverBase( unsigned int npart, const std::string& refFileName ) - : m_refFileName( refFileName ) - , nparticle( npart ) - { - } - TestDriverBase() = delete; - virtual ~TestDriverBase() {} - const std::string& getRefFileName() { return m_refFileName; } - - // ------------------------------------------------ - // Interface for retrieving info from madgraph - // ------------------------------------------------ - virtual fptype getMomentum( std::size_t evtNo, unsigned int particleNo, unsigned int component ) const = 0; - virtual fptype getMatrixElement( std::size_t evtNo ) const = 0; - virtual int getChannelId( std::size_t ievt ) const = 0; - virtual int getSelectedHelicity( std::size_t ievt ) const = 0; - virtual int getSelectedColor( std::size_t ievt ) const = 0; - - // ------------------------------------------------ - // Interface for steering madgraph run - // ------------------------------------------------ - virtual void prepareRandomNumbers( unsigned int iiter ) = 0; - virtual void prepareMomenta( fptype energy ) = 0; - virtual void runSigmaKin( std::size_t iiter ) = 0; - - /// Print the requested event into the stream. If the reference data has enough events, it will be printed as well. - void dumpParticles( std::ostream& stream, std::size_t ievt, unsigned int numParticles, unsigned int nDigit, const ReferenceData& referenceData ) const - { - const auto width = nDigit + 8; - for( unsigned int ipar = 0; ipar < numParticles; ipar++ ) - { - // NB: 'setw' affects only the next field (of any type) - stream << std::scientific // fixed format: affects all floats (default nDigit: 6) - << std::setprecision( nDigit ) - << std::setw( 4 ) << ipar - << std::setw( width ) << getMomentum( ievt, ipar, 0 ) - << std::setw( width ) << getMomentum( ievt, ipar, 1 ) - << std::setw( width ) << getMomentum( ievt, ipar, 2 ) - << std::setw( width ) << getMomentum( ievt, ipar, 3 ) - << "\n"; - if( ievt < referenceData.momenta.size() ) - { - stream << "ref" << ipar; - stream << std::setw( width ) << referenceData.momenta[ievt][ipar][0] - << std::setw( width ) << referenceData.momenta[ievt][ipar][1] - << std::setw( width ) << referenceData.momenta[ievt][ipar][2] - << std::setw( width ) << referenceData.momenta[ievt][ipar][3] - << "\n\n"; - } - stream << std::flush << std::defaultfloat; // default format: affects all floats - } - } -}; - -/** - * Test class that's defining all tests to run with a Madgraph workflow. - */ -class MadgraphTest -{ -public: - MadgraphTest( TestDriverBase& testDriverRef ) - : testDriver( &testDriverRef ) {} - ~MadgraphTest() {} - void CompareMomentaAndME( testing::Test& googleTest ) const; // NB: googleTest is ONLY needed for the HasFailure method... -private: - TestDriverBase* testDriver; // non-owning pointer -}; - -void -MadgraphTest::CompareMomentaAndME( testing::Test& googleTest ) const -{ - const fptype toleranceMomenta = std::is_same::value ? 1.E-10 : 4.E-2; // see #735 -#ifdef __APPLE__ - const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-2; // see #583 -#else - //const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 2.E-3; // fails smeft/hip #843 - const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-3; -#endif - constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) - // Dump events to a new reference file? - const char* dumpEventsC = getenv( "CUDACPP_RUNTEST_DUMPEVENTS" ); - const bool dumpEvents = ( dumpEventsC != 0 ) && ( std::string( dumpEventsC ) != "" ); - const std::string refFileName = testDriver->getRefFileName(); - /* -#ifdef __HIPCC__ - const std::string dumpFileName = std::experimental::filesystem::path( refFileName ).filename(); -#else - const std::string dumpFileName = std::filesystem::path( refFileName ).filename(); -#endif - */ - const std::string dumpFileName = refFileName; // bypass std::filesystem #803 - std::ofstream dumpFile; - if( dumpEvents ) - { - dumpFile.open( dumpFileName, std::ios::trunc ); - } - // Read reference data - std::map referenceData; - if( !dumpEvents ) - { - referenceData = readReferenceData( refFileName ); - } - ASSERT_FALSE( googleTest.HasFailure() ); // It doesn't make any sense to continue if we couldn't read the reference file. - // ************************************** - // *** START MAIN LOOP ON #ITERATIONS *** - // ************************************** - for( unsigned int iiter = 0; iiter < testDriver->niter; ++iiter ) - { - testDriver->prepareRandomNumbers( iiter ); - testDriver->prepareMomenta( energy ); - testDriver->runSigmaKin( iiter ); - // --- Run checks on all events produced in this iteration - for( std::size_t ievt = 0; ievt < testDriver->nevt && !googleTest.HasFailure(); ++ievt ) - { - if( dumpEvents ) - { - ASSERT_TRUE( dumpFile.is_open() ) << dumpFileName; - dumpFile << "Event " << std::setw( 8 ) << ievt << " " - << "Batch " << std::setw( 4 ) << iiter << "\n"; - testDriver->dumpParticles( dumpFile, ievt, testDriver->nparticle, 15, ReferenceData() ); - // Dump matrix element - dumpFile << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) - << testDriver->getMatrixElement( ievt ) << "\n" - << std::defaultfloat; - // Dump channelId - dumpFile << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n"; - // Dump selected helicity and color - dumpFile << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n"; - dumpFile << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" - << std::endl; // leave one line between events - continue; - } - // Check that we have the required reference data - ASSERT_GT( referenceData.size(), iiter ) - << "Don't have enough reference data for iteration " << iiter << ". Ref file:" << refFileName; - ASSERT_GT( referenceData[iiter].MEs.size(), ievt ) - << "Don't have enough reference MEs for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].ChanIds.size(), ievt ) - << "Don't have enough reference ChanIds for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].SelHels.size(), ievt ) - << "Don't have enough reference SelHels for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].SelCols.size(), ievt ) - << "Don't have enough reference SelCols for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].momenta.size(), ievt ) - << "Don't have enough reference momenta for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GE( referenceData[iiter].momenta[ievt].size(), testDriver->nparticle ) - << "Don't have enough reference particles for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - // This trace will help to understand the event that is being checked. - // It will only be printed in case of failures: - std::stringstream eventTrace; - eventTrace << "In comparing event " << ievt << " from iteration " << iiter << "\n"; - testDriver->dumpParticles( eventTrace, ievt, testDriver->nparticle, 15, referenceData[iiter] ); - eventTrace << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) - << testDriver->getMatrixElement( ievt ) << "\n" - << std::setw( 4 ) << "r.ME" << std::scientific << std::setw( 15 + 8 ) - << referenceData[iiter].MEs[ievt] << std::endl - << std::defaultfloat; - eventTrace << std::setw( 8 ) << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n" - << std::setw( 8 ) << "r.ChanId" << std::setw( 8 ) << referenceData[iiter].ChanIds[ievt] << std::endl; - eventTrace << std::setw( 8 ) << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n" - << std::setw( 8 ) << "r.SelHel" << std::setw( 8 ) << referenceData[iiter].SelHels[ievt] << std::endl; - eventTrace << std::setw( 8 ) << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" - << std::setw( 8 ) << "r.SelCol" << std::setw( 8 ) << referenceData[iiter].SelCols[ievt] << std::endl; - SCOPED_TRACE( eventTrace.str() ); - // Compare Momenta - for( unsigned int ipar = 0; ipar < testDriver->nparticle; ++ipar ) - { - std::stringstream momentumErrors; - for( unsigned int icomp = 0; icomp < CPPProcess::np4; ++icomp ) - { - const fptype pMadg = testDriver->getMomentum( ievt, ipar, icomp ); - const fptype pOrig = referenceData[iiter].momenta[ievt][ipar][icomp]; - //const fptype relDelta = fabs( ( pMadg - pOrig ) / pOrig ); // computing relDelta may lead to FPEs - const fptype delta = fabs( pMadg - pOrig ); - if( delta > toleranceMomenta * fabs( pOrig ) ) // better than "relDelta > toleranceMomenta" - { - momentumErrors << std::setprecision( 15 ) << std::scientific << "\nparticle " << ipar << "\tcomponent " << icomp - << "\n\t madGraph: " << std::setw( 22 ) << pMadg - << "\n\t reference: " << std::setw( 22 ) << pOrig - << "\n\t relative delta exceeds tolerance of " << toleranceMomenta; - } - } - ASSERT_TRUE( momentumErrors.str().empty() ) << momentumErrors.str(); - } - // Compare ME: - EXPECT_NEAR( testDriver->getMatrixElement( ievt ), - referenceData[iiter].MEs[ievt], - toleranceMEs * referenceData[iiter].MEs[ievt] ); - // Compare channelId - EXPECT_EQ( testDriver->getChannelId( ievt ), - referenceData[iiter].ChanIds[ievt] ); - // Compare selected helicity and color - EXPECT_EQ( testDriver->getSelectedHelicity( ievt ), - referenceData[iiter].SelHels[ievt] ); - EXPECT_EQ( testDriver->getSelectedColor( ievt ), - referenceData[iiter].SelCols[ievt] ); - } - } - if( dumpEvents ) - { - std::cout << "Event dump written to " << dumpFileName << std::endl; - } -} - -#endif /* MADGRAPHTEST_H_ */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h index cfff3d5d60..fa4b2a72a1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h @@ -14,7 +14,6 @@ #include "ProcessData.h" #include "GpuRuntime.h" #include "Parameters.h" -#include "processConfig.h" #include @@ -225,7 +224,7 @@ namespace mg5amcGpu // The size (number of elements) per event in a memory buffer for numerators // (should be equal to the number of diagrams in the process) - constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; + constexpr size_t sizePerEventNumerators = ProcessData::ndiagrams; // A class encapsulating a CUDA pinned host buffer for numerators typedef PinnedHostBuffer PinnedHostBufferNumerators; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc deleted file mode 100644 index ae42513fd6..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.cc +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "RamboSamplingKernels.h" - -#include "GpuRuntime.h" -#include "MemoryAccessMomenta.h" -#include "MemoryAccessRandomNumbers.h" -#include "MemoryAccessWeights.h" -#include "MemoryBuffers.h" -#include "rambo.h" // inline classic (massive) RAMBO, ported from standalone_cpp -#include "massless_rambo.h" // inline implementation of massless RAMBO algorithms and kernels - -#include - -namespace mg5amcGpu -{ - //-------------------------------------------------------------------------- - - MasslessRamboSamplingKernelHost::MasslessRamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t nevt ) - : SamplingKernelBase( energy, rndmom, momenta, weights ) - , NumberOfEvents( nevt ) - { - if( m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: rndmom must be a host array" ); - if( m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: momenta must be a host array" ); - if( m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: weights must be a host array" ); - if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with rndmom" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with momenta" ); - if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with weights" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( nevt % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Sanity checks for memory access (random number buffer) - constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout - static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); - if( nevt % neppR != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppR=" << neppR; - throw std::runtime_error( sstr.str() ); - } - } - - //-------------------------------------------------------------------------- - - void - MasslessRamboSamplingKernelHost::getMomentaInitial() - { - constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; - // ** START LOOP ON IEVT ** - for( size_t ievt = 0; ievt < nevt(); ++ievt ) - { - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); - getMomentaInitial( m_energy, ievtMomenta ); - } - // ** END LOOP ON IEVT ** - } - - //-------------------------------------------------------------------------- - - void - MasslessRamboSamplingKernelHost::getMomentaFinal() - { - constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; - // ** START LOOP ON IEVT ** - for( size_t ievt = 0; ievt < nevt(); ++ievt ) - { - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - const fptype* ievtRndmom = MemoryAccessRandomNumbers::ieventAccessRecordConst( m_rndmom.data(), ievt ); - fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); - fptype* ievtWeights = MemoryAccessWeights::ieventAccessRecord( m_weights.data(), ievt ); - getMomentaFinal( m_energy, ievtRndmom, ievtMomenta, ievtWeights ); - } - // ** END LOOP ON IEVT ** - } - - //-------------------------------------------------------------------------- - - RamboSamplingKernelHost::RamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED - const std::vector& masses, // input: external-leg masses - const int ninitial, // input: #initial-state particles - const size_t nevt, // input: #events - BufferMomenta& momenta, // output: momenta - BufferWeights& weights ) // output: weights - : SamplingKernelBase( energy, rndmom, momenta, weights ) - , NumberOfEvents( nevt ) - , m_masses( masses.begin(), masses.end() ) - , m_ninitial( ninitial ) - { - if( m_momenta.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: momenta must be a host array" ); - if( m_weights.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: weights must be a host array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with momenta" ); - if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with weights" ); - - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( nevt % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "RamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - } - - //-------------------------------------------------------------------------- - - void - RamboSamplingKernelHost::getMomentaInitial() - { - // NOOP - } - - //-------------------------------------------------------------------------- - - void - RamboSamplingKernelHost::getMomentaFinal() - { - const int npar = (int)m_masses.size(); - // ** START LOOP ON IEVT ** - for( size_t ievt = 0; ievt < nevt(); ++ievt ) - { - // Clas. RAMBO returns [E,px,py,pz] vector per ex. particle - // own RNG, intial final once - // For reproducibility betwn fptype = FP32/FP64 generation in FP64 - double wgt = 0.; - const std::vector> point = - rambo::get_momenta( m_ninitial, (double)m_energy, m_masses, wgt ); - for( int ipar = 0; ipar < npar; ++ipar ) - for( int ip4 = 0; ip4 < 4; ++ip4 ) - MemoryAccessMomenta::ieventAccessIp4Ipar( m_momenta.data(), ievt, ip4, ipar ) = (fptype)point[ipar][ip4]; - MemoryAccessWeights::ieventAccess( m_weights.data(), ievt ) = (fptype)wgt; - } - // ** END LOOP ON IEVT ** - } - - //-------------------------------------------------------------------------- - - MasslessRamboSamplingKernelDevice::MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t gpublocks, - const size_t gputhreads ) - : SamplingKernelBase( energy, rndmom, momenta, weights ) - , NumberOfEvents( gpublocks * gputhreads ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if( !m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: rndmom must be a device array" ); - if( !m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: momenta must be a device array" ); - if( !m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: weights must be a device array" ); - if( m_gpublocks == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: gputhreads must be > 0" ); - if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with rndmom" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with momenta" ); - if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelDevice: nevt mismatch with weights" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( m_gputhreads % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelHost: gputhreads should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Sanity checks for memory access (random number buffer) - constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout - static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); - if( m_gputhreads % neppR != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelDevice: gputhreads should be a multiple of neppR=" << neppR; - throw std::runtime_error( sstr.str() ); - } - } - - //-------------------------------------------------------------------------- - - __global__ void - getMomentaInitialDevice( const fptype energy, - fptype* momenta ) - { - constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; - return getMomentaInitial( energy, momenta ); - } - - //-------------------------------------------------------------------------- - - void - MasslessRamboSamplingKernelDevice::getMomentaInitial() - { - gpuLaunchKernel( getMomentaInitialDevice, m_gpublocks, m_gputhreads, m_energy, m_momenta.data() ); - } - - //-------------------------------------------------------------------------- - - __global__ void - getMomentaFinalDevice( const fptype energy, - const fptype* rndmom, - fptype* momenta, - fptype* wgts ) - { - constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; - return getMomentaFinal( energy, rndmom, momenta, wgts ); - } - - //-------------------------------------------------------------------------- - - void - MasslessRamboSamplingKernelDevice::getMomentaFinal() - { - gpuLaunchKernel( getMomentaFinalDevice, m_gpublocks, m_gputhreads, m_energy, m_rndmom.data(), m_momenta.data(), m_weights.data() ); - } - - //-------------------------------------------------------------------------- -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h deleted file mode 100644 index c7962754c0..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RamboSamplingKernels.h +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef RAMBOSAMPLINGKERNELS_H -#define RAMBOSAMPLINGKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryBuffers.h" - -#include - -namespace mg5amcGpu -{ - //-------------------------------------------------------------------------- - - // A base class encapsulating phase space sampling on a CPU host or on a GPU device - class SamplingKernelBase //: virtual public ISamplingKernel - { - protected: - - // Constructor from existing input and output buffers - SamplingKernelBase( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights ) // output: weights - : m_energy( energy ) - , m_rndmom( rndmom ) - , m_momenta( momenta ) - , m_weights( weights ) - { - } - - public: - - // Destructor - virtual ~SamplingKernelBase() {} - - // Get momenta of initial state particles - virtual void getMomentaInitial() = 0; - - // Get momenta of final state particles and weights - virtual void getMomentaFinal() = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - protected: - - // The energy - const fptype m_energy; - - // The buffer for the input random numbers - const BufferRndNumMomenta& m_rndmom; - - // The buffer for the output momenta - BufferMomenta& m_momenta; - - // The buffer for the output weights - BufferWeights& m_weights; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating RAMBO phase space sampling on a CPU host - class MasslessRamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MasslessRamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t nevt ); - - // Destructor - virtual ~MasslessRamboSamplingKernelHost() {} - - // Get momenta of initial state particles - void getMomentaInitial() override final; - - // Get momenta of final state particles and weights - void getMomentaFinal() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - }; - - //-------------------------------------------------------------------------- - - // Compability port mirroring the massless momenta implementation - // For now own RNG internally (or keep to match Fortran) TODO - // rndmom just for interface (maybe delete later) TODO - class RamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents - { - public: - - RamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED - const std::vector& masses, // input: external-leg masses - const int ninitial, // input: #n initial-state particles - const size_t nevt, // input: #n events - BufferMomenta& momenta, // output: momenta - BufferWeights& weights); // output: weights - - virtual ~RamboSamplingKernelHost() {} - - // No-op, kept to match the massless - void getMomentaInitial() override final; - - // All the magic here - void getMomentaFinal() override final; - - bool isOnDevice() const override final { return false; } - - private: - - // The EXTERNAL masses - std::vector m_masses; - - // The number of inital particles - const int m_ninitial; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating RAMBO phase space sampling on a GPU device - class MasslessRamboSamplingKernelDevice final : public SamplingKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MasslessRamboSamplingKernelDevice( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~MasslessRamboSamplingKernelDevice() {} - - // Get momenta of initial state particles - void getMomentaInitial() override final; - - // Get momenta of final state particles and weights - void getMomentaFinal() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; - - //-------------------------------------------------------------------------- -} -#endif // RAMBOSAMPLINGKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h deleted file mode 100644 index 7fbde1edd0..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/RandomNumberKernels.h +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef RANDOMNUMBERKERNELS_H -#define RANDOMNUMBERKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryBuffers.h" - -// Forward definition from curand.h (the full header is only needed in CurandRandomKernel.cc) -struct curandGenerator_st; - -// Forward definition from hiprand.h (the full header is only needed in HiprandRandomKernel.cc) -struct rocrand_generator_base_type; -typedef rocrand_generator_base_type hiprandGenerator_st; - -namespace mg5amcGpu -{ - //-------------------------------------------------------------------------- - - /* - // An interface encapsulating random number generation on a CPU host or on a GPU device - class IRandomNumberKernel - { - public: - - // Destructor - virtual ~IRandomNumberKernel(){} - - // Seed the random number generator - virtual void seedGenerator( const unsigned int seed ) = 0; - - // Generate the random number array - virtual void generateRnarray() = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - }; - */ - - //-------------------------------------------------------------------------- - - // A base class encapsulating random number generation on a CPU host or on a GPU device - class RandomNumberKernelBase //: virtual public IRandomNumberKernel - { - - protected: - - // Constructor from an existing output buffer - RandomNumberKernelBase( BufferRndNumMomenta& rnarray ) - : m_rnarray( rnarray ) {} - - public: - - // Destructor - virtual ~RandomNumberKernelBase() {} - - // Seed the random number generator - virtual void seedGenerator( const unsigned int seed ) = 0; - - // Generate the random number array - virtual void generateRnarray() = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - protected: - - // The buffer for the output random numbers - BufferRndNumMomenta& m_rnarray; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating common random number generation on a CPU host - class CommonRandomNumberKernel final : public RandomNumberKernelBase - { - public: - - // Constructor from an existing output buffer - CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ); - - // Destructor - ~CommonRandomNumberKernel() {} - - // Seed the random number generator - void seedGenerator( const unsigned int seed ) override final { m_seed = seed; }; - - // Generate the random number array - void generateRnarray() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // The generator seed - unsigned int m_seed; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating CURAND random number generation on a CPU host or on a GPU device - class CurandRandomNumberKernel final : public RandomNumberKernelBase - { - public: - - // Constructor from an existing output buffer - CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); - - // Destructor - ~CurandRandomNumberKernel(); - - // Seed the random number generator - void seedGenerator( const unsigned int seed ) override final; - - // Generate the random number array - void generateRnarray() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return m_isOnDevice; } - - private: - - // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void createGenerator(); - - // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void destroyGenerator(); - - private: - - // Is this a host or device kernel? - const bool m_isOnDevice; - - // The curand generator - // (NB: curand.h defines typedef curandGenerator_t as a pointer to forward-defined 'struct curandGenerator_st') - curandGenerator_st* m_rnGen; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating HIPRAND random number generation on a CPU host or on a GPU device - class HiprandRandomNumberKernel final : public RandomNumberKernelBase - { - public: - - // Constructor from an existing output buffer - HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); - - // Destructor - ~HiprandRandomNumberKernel(); - - // Seed the random number generator - void seedGenerator( const unsigned int seed ) override final; - - // Generate the random number array - void generateRnarray() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return m_isOnDevice; } - - private: - - // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void createGenerator(); - - // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void destroyGenerator(); - - private: - - // Is this a host or device kernel? - const bool m_isOnDevice; - - // The hiprand generator - // (NB: hiprand.h defines typedef hiprandGenerator_t as a pointer to forward-defined 'struct hiprandGenerator_st') - hiprandGenerator_st* m_rnGen; - }; - - //-------------------------------------------------------------------------- -} -#endif // RANDOMNUMBERKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc index ad271a29ad..5ca72c78e4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc @@ -25,7 +25,6 @@ #include "MemoryAccessWavefunctions.h" #include "color_sum.h" #include "coloramps.h" -#include "processConfig.h" namespace mg5amcGpu { @@ -163,7 +162,7 @@ namespace mg5amcGpu int ighel = blockIdx.y; ihel = dcGoodHel[ighel]; allJamps = allJamps + ighel * nevt; - allNumerators = allNumerators + ighel * nevt * processConfig::ndiagrams; + allNumerators = allNumerators + ighel * nevt * ndiagrams; allDenominators = allDenominators + ighel * nevt; } @@ -197,7 +196,7 @@ namespace mg5amcGpu const fptype* COUPs[nxcoup]; for( size_t ixcoup = 0; ixcoup < nxcoup; ixcoup++ ) COUPs[ixcoup] = allCOUPs[ixcoup]; const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid - fptype* numerators = &allNumerators[ievt * processConfig::ndiagrams]; + fptype* numerators = &allNumerators[ievt * ndiagrams]; fptype* denominators = allDenominators; // Create an array of views over the Flavor Couplings FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; @@ -404,9 +403,9 @@ namespace mg5amcGpu { fptype* hAllDenominators = ghelAllDenominators + ighel * nevt; totAllDenominators[ievt] += hAllDenominators[ievt]; - fptype* hAllNumerators = ghelAllNumerators + ( ievt + ighel * nevt ) * processConfig::ndiagrams; - fptype* firstNumerator = ghelAllNumerators + ievt * processConfig::ndiagrams; - for( int idiag = 0; idiag < processConfig::ndiagrams; ++idiag ) + fptype* hAllNumerators = ghelAllNumerators + ( ievt + ighel * nevt ) * ndiagrams; + fptype* firstNumerator = ghelAllNumerators + ievt * ndiagrams; + for( int idiag = 0; idiag < ndiagrams; ++idiag ) { firstNumerator[idiag] += hAllNumerators[idiag]; } @@ -414,7 +413,7 @@ namespace mg5amcGpu if( mulChannelWeight ) { unsigned int channelId = allChannelIds[ievt]; - allMEs[ievt] *= totAllNumerators[channelId - 1 + ievt * processConfig::ndiagrams] / totAllDenominators[ievt]; + allMEs[ievt] *= totAllNumerators[channelId - 1 + ievt * ndiagrams] / totAllDenominators[ievt]; } } return; @@ -472,13 +471,13 @@ namespace mg5amcGpu for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) { if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - normalization += allNumerators[ievt * processConfig::ndiagrams + ichan]; + normalization += allNumerators[ievt * ndiagrams + ichan]; } channelId = mgOnGpu::nchannels; for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) { if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - numerator_sum += allNumerators[ievt * processConfig::ndiagrams + ichan]; + numerator_sum += allNumerators[ievt * ndiagrams + ichan]; if( allrnddiagram[ievt] < numerator_sum / normalization ) { channelId = ichan + 1; @@ -579,7 +578,7 @@ namespace mg5amcGpu gpuMemset( allMEs, 0, nevt * sizeof( fptype ) ); gpuMemset( ghelAllJamps, 0, cNGoodHel * ncolor * mgOnGpu::nx2 * nevt * sizeof( fptype ) ); gpuMemset( colAllJamp2s, 0, ncolor * nevt * sizeof( fptype ) ); - gpuMemset( ghelAllNumerators, 0, cNGoodHel * processConfig::ndiagrams * nevt * sizeof( fptype ) ); + gpuMemset( ghelAllNumerators, 0, cNGoodHel * ndiagrams * nevt * sizeof( fptype ) ); gpuMemset( ghelAllDenominators, 0, cNGoodHel * nevt * sizeof( fptype ) ); gpuMemset( ghelAllMEs, 0, cNGoodHel * nevt * sizeof( fptype ) ); @@ -599,7 +598,7 @@ namespace mg5amcGpu { const int ihel = cGoodHel[ighel]; fptype* hAllJamps = ghelAllJamps + ighel * nevt; // HACK: bypass DeviceAccessJamp (consistent with layout defined there) - fptype* hAllNumerators = ghelAllNumerators + ighel * nevt * processConfig::ndiagrams; + fptype* hAllNumerators = ghelAllNumerators + ighel * nevt * ndiagrams; fptype* hAllDenominators = ghelAllDenominators + ighel * nevt; gpuLaunchKernelStream( calculate_jamps, gpublocks, gputhreads, ghelStreams[ighel], ihel, allmomenta, allcouplings, iflavorVec, hAllJamps, storeChannelWeights, hAllNumerators, hAllDenominators, colAllJamp2s, nevt, false ); } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc deleted file mode 100644 index f0517bcb07..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/check_sa.cc +++ /dev/null @@ -1,1081 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: O. Mattelaer (Nov 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, D. Massaro, O. Mattelaer, S. Roiser, J. Teig, A. Thete, A. Valassi (2020-2026). -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== -// -// Standalone script for MadGraph7 standalone mode. -// Generates phase-space points with RAMBO and evaluates the matrix element -// through the UMAMI interface (umami.h). -// -// Two run modes: -// * matrix (default): evaluates one phase-space point (generated with the -// classic standalone RAMBO, so identical to the one of -// the Fortran/C++ standalone 'check' drivers at the -// same energy) and prints it together with the matrix -// element of every flavor combination. -// * perf : runs nblocks*nthreads*niter events on a single flavor -// and prints performance counters. -// -//========================================================================== - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "GpuAbstraction.h" -#include "GpuRuntime.h" -#include "MemoryAccessMomenta.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" -#include "epoch_process_id.h" -#include "read_slha.h" -#include "timermap.h" -#include "umami.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define STRINGIFY( s ) #s -#define XSTRINGIFY( s ) STRINGIFY( s ) -#define SEP79 79 - -namespace -{ - using namespace mg5amcGpu; - - // Fixed physics inputs - fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV - constexpr unsigned long long kSeed = 20200805ULL; // reproducible RAMBO seed - - // Matrix-mode always runs 8 events on a single flavor index. - constexpr unsigned int kMatrixBlocks = 1; - constexpr unsigned int kMatrixThreads = 8; - - // Power of GeV of the matrix-element output; depends only on the number of external legs. - constexpr int kMEGeVExponent = -( 2 * CPPProcess::npar - 8 ); - - bool is_number( const char* s ) - { - const char* t = s; - while( *t != '\0' && isdigit( *t ) ) ++t; - return (int)strlen( s ) == t - s; - } - - // Accepts plain decimal numbers such as "1000" or "1000.0" (used for the - // optional energy argument of matrix mode). - bool is_float( const char* s ) - { - int ndots = 0; - const char* t = s; - while( *t != '\0' && ( isdigit( *t ) || ( *t == '.' && ndots++ == 0 ) ) ) ++t; - return t != s && (int)strlen( s ) == t - s; - } - - enum Mode { MODE_MATRIX, MODE_PERF }; - - enum RamboType { RAMBO_MASSIVE, RAMBO_MASSLESS }; - - // One external-particle list per LHE event, each particle stored as (E, px, py, pz). - using LheEvent = std::array, CPPProcess::npar>; - - bool read_lhe_events( const std::string& path, std::vector& events ) - { - constexpr int npar = CPPProcess::npar; - std::ifstream in( path ); - if( !in ) - { - std::cerr << "ERROR! cannot open LHE file '" << path << "'" << std::endl; - return false; - } - std::string line; - while( std::getline( in, line ) ) - { - if( line.find( "" ) == std::string::npos ) continue; - if( !std::getline( in, line ) ) break; - std::istringstream hdr( line ); - int nptcl = 0; - hdr >> nptcl; - if( nptcl != npar ) - { - std::cerr << "ERROR! LHE event has " << nptcl << " particles, expected " << npar << std::endl; - return false; - } - // particle lines: pdg status mother1 mother2 color1 color2 px py pz E m lifetime spin - LheEvent ev; - int ipar = 0; - while( ipar < npar && std::getline( in, line ) ) - { - if( line.empty() ) continue; - std::istringstream ls( line ); - long pdg; - int status, m1, m2, c1, c2; - double px, py, pz, E; - if( !( ls >> pdg >> status >> m1 >> m2 >> c1 >> c2 >> px >> py >> pz >> E ) ) - { - std::cerr << "ERROR! malformed LHE particle line: " << line << std::endl; - return false; - } - ev[ipar] = { E, px, py, pz }; - ++ipar; - } - if( ipar != npar ) - { - std::cerr << "ERROR! truncated LHE event (got " << ipar << " of " << npar << " particles)" << std::endl; - return false; - } - events.push_back( ev ); - } - if( events.empty() ) - { - std::cerr << "ERROR! no events found in '" << path << "'" << std::endl; - return false; - } - return true; - } - - int usage( const char* argv0, int ret = 1 ) - { - std::cout - << "Usage:\n" - << " " << argv0 << " [matrix] [-v|--verbose] []\n" - << " " << argv0 << " perf [-v|--verbose] [-f|--flavor ] [--rambo-massless]" - << " [-e|--events ] [<#blocksPerGrid> <#threadsPerBlock>] <#iterations>\n" - << " " << argv0 << " -p [opts] (legacy alias for `perf`)\n" - << "\n" - << "Subcommands:\n" - << " matrix (default) Evaluate one phase-space point (classic standalone\n" - << " RAMBO, identical to the Fortran/C++ 'check' drivers,\n" - << " Ecms = , default 1000 GeV) and print it with\n" - << " the matrix element for each flavor combination.\n" - << " With -v also prints backend/fptype/hardcodePARAM header.\n" - << " perf Run #blocks*#threads events over #iterations iterations\n" - << " on a single flavor index, then print performance counters.\n" - << " Always prints inputs + backend/fptype header.\n" - << " With -v also dumps every event's phase-space point and ME.\n" - << "\n" - << "Options:\n" - << " -e|--events (perf only) Read the external momenta from an LHE\n" - << " file instead of generating them with RAMBO. The events are\n" - << " processed in batches of #blocks*#threads; #iterations is\n" - << " ignored (derived from the number of events in the file).\n" - << "\n" - << "perf-mode defaults if positional args are omitted:\n" - << " #blocksPerGrid = 64, #threadsPerBlock = 256, #iterations = 1.\n"; - return ret; - } - - // AOSOA -> UMAMI SoA single-event helper. Layout reminder: - // AOSOA: aosoa[i_page * npar*4*neppM + ipar*4*neppM + ip4*neppM + i_vector] - // UMAMI: soa[ip4 * npar*nevt + ipar*nevt + ievt] - __host__ __device__ inline void - aosoa_to_umami_one( const fptype* aosoa, - double* soa, - std::size_t ievt, - std::size_t nevt ) - { - constexpr int npar = CPPProcess::npar; - for( int ipar = 0; ipar < npar; ++ipar ) - { - for( int ip4 = 0; ip4 < 4; ++ip4 ) - { - soa[(std::size_t)ip4 * npar * nevt + (std::size_t)ipar * nevt + ievt] = - (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, ip4, ipar ); - } - } - } - - __global__ void - aosoa_to_umami_kernel( const fptype* aosoa, - double* soa, - std::size_t nevt ) - { - std::size_t ievt = blockDim.x * blockIdx.x + threadIdx.x; - if( ievt >= nevt ) return; - aosoa_to_umami_one( aosoa, soa, ievt, nevt ); - } - - const char* backend_label() - { -#ifdef __CUDACC__ - return "CUDA"; -#elif defined( __HIPCC__ ) - return "HIP"; -#else - return "CPP"; -#endif - } - - const char* fp_label() - { -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - return "MIXED"; -#elif defined MGONGPU_FPTYPE_DOUBLE - return "DOUBLE"; -#elif defined MGONGPU_FPTYPE_FLOAT - return "FLOAT"; -#else - return "UNKNOWN"; -#endif - } - - void print_run_header( std::ostream& os ) - { - os << "Process = " << XSTRINGIFY( MG_EPOCH_PROCESS_ID ) << "_" << backend_label() -#ifdef MGONGPU_HARDCODE_PARAM - << " [hardcodePARAM=1]" << std::endl -#else - << " [hardcodePARAM=0]" << std::endl -#endif - << "FP precision = " << fp_label() << std::endl - << "Random number generation = COMMON RANDOM HOST" << std::endl; - } - - void print_momenta_table( std::ostream& os, const fptype* aosoa, unsigned int ievt ) - { - auto constexpr prec = std::numeric_limits::digits10; - constexpr int npar = CPPProcess::npar; - os << std::string( SEP79, '-' ) << std::endl - << " n E px py pz" << std::endl; - for( int ipar = 0; ipar < npar; ++ipar ) - { - double E = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 0, ipar ); - double px = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 1, ipar ); - double py = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 2, ipar ); - double pz = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 3, ipar ); - os << std::scientific << std::setprecision( prec ) - << std::setw( 4 ) << ipar + 1 - << std::setw( prec + 10 ) << E - << std::setw( prec + 10 ) << px - << std::setw( prec + 10 ) << py - << std::setw( prec + 10 ) << pz - << std::endl - << std::defaultfloat; - } - os << std::string( SEP79, '-' ) << std::endl; - } - - // Run sigmaKin via UMAMI for `nevt` events and copy back the MEs. - // Both the momenta (UMAMI SoA layout) and the per-event flavor buffer must be set - // by the caller. On GPU the buffers are device pointers and `hstMEs` receives the - // host-side copy; on CPU `umamiMEs` is the output buffer. - bool run_umami( - UmamiHandle handle, - unsigned int nevt, - mgOnGpu::TimerMap& timermap, - double& wavetime, - const DeviceBufferBase& devUmamiMomenta, - const DeviceBufferBase& devFlv, - DeviceBufferBase& devUmamiMEs, - std::vector& hstMEs - ) - { - constexpr unsigned int UmamiInKeyNum = 2; - timermap.start( "3a SigmaKin" ); - UmamiInputKey in_keys[UmamiInKeyNum] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX }; - UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; - const void* inputs[UmamiInKeyNum] = { devUmamiMomenta.data(), devFlv.data() }; - void* outputs[1] = { devUmamiMEs.data() }; - UmamiStatus st = umami_matrix_element( - handle, nevt, nevt, 0, UmamiInKeyNum, in_keys, inputs, 1, out_keys, outputs ); - wavetime += timermap.stop(); - if( st != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; - return false; - } - - timermap.start( "3b CpDTHmes" ); - gpuMemcpy( hstMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); - wavetime += timermap.stop(); - return true; - } - - // -------------------------------------------------------------------------- - // Classic MadGraph standalone RAMBO (RANMAR generator seeded with the fixed - // values 1802/9373), reproducing the exact phase-space point used by the - // Fortran and C++ standalone 'check' drivers at the same energy, so that - // matrix-mode output can be compared line by line across backends. - // Host-side only; ported from madgraph/iolibs/template_files/rambo.cc. - // -------------------------------------------------------------------------- - namespace classic_rambo - { - struct Random - { - double ranu[98]; - double ranc, rancd, rancm; - int iranmr, jranmr; - - // universal random number generator proposed by Marsaglia and Zaman - double ranmar() - { - double uni = ranu[iranmr] - ranu[jranmr]; - if( uni < 0 ) uni = uni + 1; - ranu[iranmr] = uni; - iranmr = iranmr - 1; - jranmr = jranmr - 1; - if( iranmr == 0 ) iranmr = 97; - if( jranmr == 0 ) jranmr = 97; - ranc = ranc - rancd; - if( ranc < 0 ) ranc = ranc + rancm; - uni = uni - ranc; - if( uni < 0 ) uni = uni + 1; - return uni; - } - - void rmarin( int ij, int kl ) - { - int i = ij / 177 % 177 + 2; - int j = ij % 177 + 2; - int k = ( kl / 169 ) % 178 + 1; - int l = kl % 169; - for( int ii = 1; ii < 98; ii++ ) - { - double s = 0; - double t = .5; - for( int jj = 1; jj < 25; jj++ ) - { - int m = ( ( i * j % 179 ) * k ) % 179; - i = j; - j = k; - k = m; - l = ( 53 * l + 1 ) % 169; - if( ( l * m ) % 64 >= 32 ) s = s + t; - t = .5 * t; - } - ranu[ii] = s; - } - ranc = 362436. / 16777216.; - rancd = 7654321. / 16777216.; - rancm = 16777213. / 16777216.; - iranmr = 97; - jranmr = 33; - } - }; - - inline double rn() - { - static Random rand; - static bool init = true; - if( init ) - { - init = false; - rand.rmarin( 1802, 9373 ); - } - double ran; - while( true ) - { - ran = rand.ranmar(); - if( ran > 1e-16 ) break; - } - return ran; - } - - // RAMBO: democratic multi-particle phase space generator (S.D. Ellis, - // R. Kleiss, W.J. Stirling); weights are logarithmic. - inline std::vector> - rambo( double et, const std::vector& xm, double& wt ) - { - const int n = (int)xm.size(); - std::vector> q( n, std::vector( 4 ) ); - std::vector> p( n, std::vector( 4 ) ); - std::vector z( n ), r( 4 ), b( 3 ), p2( n ), xm2( n ), e( n ), v( n ); - const double acc = 1e-14; - const int itmax = 6; - const double twopi = 8. * atan( 1. ); - const double po2log = log( twopi / 4. ); - - // factorials for the phase-space weight - z[1] = po2log; - for( int k = 2; k < n; k++ ) z[k] = z[k - 1] + po2log - 2. * log( double( k - 1 ) ); - for( int k = 2; k < n; k++ ) z[k] = z[k] - log( double( k ) ); - - if( n < 1 || n > 101 ) - { - std::cout << "Too few or many particles: " << n << std::endl; - exit( -1 ); - } - double xmt = 0.; - int nm = 0; - for( int i = 0; i < n; i++ ) - { - if( xm[i] != 0. ) nm = nm + 1; - xmt = xmt + std::abs( xm[i] ); - } - if( xmt > et ) - { - std::cout << "Too low energy: " << et << " needed " << xmt << std::endl; - exit( -1 ); - } - - // generate n massless momenta in infinite phase space - for( int i = 0; i < n; i++ ) - { - double r1 = rn(); - double c = 2. * r1 - 1.; - double s = sqrt( 1. - c * c ); - double f = twopi * rn(); - r1 = rn(); - double r2 = rn(); - q[i][0] = -log( r1 * r2 ); - q[i][3] = q[i][0] * c; - q[i][2] = q[i][0] * s * cos( f ); - q[i][1] = q[i][0] * s * sin( f ); - } - - // parameters of the conformal transformation - for( int k = 0; k < 4; k++ ) r[k] = 0.; - for( int i = 0; i < n; i++ ) - for( int k = 0; k < 4; k++ ) r[k] = r[k] + q[i][k]; - double rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); - for( int k = 1; k < 4; k++ ) b[k - 1] = -r[k] / rmas; - double g = r[0] / rmas; - double a = 1. / ( 1. + g ); - double x = et / rmas; - - // transform the q's conformally into the p's - for( int i = 0; i < n; i++ ) - { - double bq = b[0] * q[i][1] + b[1] * q[i][2] + b[2] * q[i][3]; - for( int k = 1; k < 4; k++ ) p[i][k] = x * ( q[i][k] + b[k - 1] * ( q[i][0] + a * bq ) ); - p[i][0] = x * ( g * q[i][0] + bq ); - } - - wt = po2log; - if( n != 2 ) wt = ( 2. * n - 4. ) * log( et ) + z[n - 1]; - - // massless case is done - if( nm == 0 ) return p; - - // massive particles: rescale the momenta by a factor x - double xmax = sqrt( 1. - pow( xmt / et, 2 ) ); - for( int i = 0; i < n; i++ ) - { - xm2[i] = pow( xm[i], 2 ); - p2[i] = pow( p[i][0], 2 ); - } - int iter = 0; - x = xmax; - double accu = et * acc; - while( true ) - { - double f0 = -et; - double g0 = 0.; - double x2 = x * x; - for( int i = 0; i < n; i++ ) - { - e[i] = sqrt( xm2[i] + x2 * p2[i] ); - f0 = f0 + e[i]; - g0 = g0 + p2[i] / e[i]; - } - if( std::abs( f0 ) <= accu ) break; - iter = iter + 1; - if( iter > itmax ) - { - std::cout << "Too many iterations without desired accuracy: " << itmax << std::endl; - break; - } - x = x - f0 / ( x * g0 ); - } - for( int i = 0; i < n; i++ ) - { - v[i] = x * p[i][0]; - for( int k = 1; k < 4; k++ ) p[i][k] = x * p[i][k]; - p[i][0] = e[i]; - } - - double wt2 = 1.; - double wt3 = 0.; - for( int i = 0; i < n; i++ ) - { - wt2 = wt2 * v[i] / e[i]; - wt3 = wt3 + pow( v[i], 2 ) / e[i]; - } - double wtm = ( 2. * n - 3. ) * log( x ) + log( wt2 / wt3 * et ); - wt = wt + wtm; - return p; - } - - // Auxiliary function changing convention between MadGraph5_aMC@NLO and - // RAMBO four-momenta (same as get_momenta in the standalone_cpp driver). - inline std::vector> - get_momenta( int ninitial, double energy, const std::vector& masses, double& wgt ) - { - const int nexternal = (int)masses.size(); - const int nfinal = nexternal - ninitial; - const double e2 = pow( energy, 2 ); - const double m1 = masses[0]; - - if( ninitial == 1 ) - { - std::vector> p( 1, std::vector( 4, 0. ) ); - p[0][0] = m1; - std::vector finalmasses( masses.begin() + 1, masses.end() ); - std::vector> p_rambo = rambo( m1, finalmasses, wgt ); - p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); - return p; - } - - if( ninitial != 2 ) - { - std::cout << "Rambo needs 1 or 2 incoming particles" << std::endl; - exit( -1 ); - } - - double etot = energy; - if( nfinal == 1 ) etot = m1; - const double m2 = masses[1]; - const double mom = sqrt( ( pow( e2, 2 ) - 2 * e2 * pow( m1, 2 ) + pow( m1, 4 ) - 2 * e2 * pow( m2, 2 ) - 2 * pow( m1, 2 ) * pow( m2, 2 ) + pow( m2, 4 ) ) / ( 4 * e2 ) ); - const double energy1 = sqrt( pow( mom, 2 ) + pow( m1, 2 ) ); - const double energy2 = sqrt( pow( mom, 2 ) + pow( m2, 2 ) ); - std::vector> p( 2, std::vector( 4, 0. ) ); - p[0][0] = energy1; - p[0][3] = mom; - p[1][0] = energy2; - p[1][3] = -mom; - - if( nfinal == 1 ) - { - p.push_back( std::vector( 4, 0. ) ); - p[2][0] = etot; - wgt = 1; - return p; - } - std::vector finalmasses( masses.begin() + 2, masses.end() ); - std::vector> p_rambo = rambo( etot, finalmasses, wgt ); - p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); - return p; - } - } - - // -------------------------------------------------------------------------- - // matrix mode: same PS point fed to every flavor combination, print event 0. - // The point is generated with the classic standalone RAMBO so it is - // identical to the one of the Fortran/C++ standalone 'check' drivers. - // -------------------------------------------------------------------------- - int run_matrix_mode( bool verbose ) - { - constexpr unsigned int nevt = kMatrixBlocks * kMatrixThreads; - const unsigned int nFlavors = CPPProcess::nmaxflavor; - - mgOnGpu::TimerMap timermap; - - timermap.start( "00 GpuInit" ); - GpuRuntime gpuRuntime( false ); - - PinnedHostBufferRndNumMomenta hstRndmom( nevt ); - PinnedHostBufferMomenta hstMomenta( nevt ); - PinnedHostBufferWeights hstWeights( nevt ); - DeviceBufferRndNumMomenta devRndmom( nevt ); - DeviceBufferMomenta devMomenta( nevt ); - DeviceBufferWeights devWeights( nevt ); - DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - DeviceBufferBase devUmamiMEs( nevt ); - DeviceBufferBase devFlv( nevt ); - std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - std::vector flvVec( nevt ); - std::vector hstUmamiMEs( nevt ); - - UmamiHandle umami_handle = nullptr; - if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_initialize failed" << std::endl; - return 2; - } - - // Generate one shared phase-space point used by every flavor, with the - // classic standalone RAMBO so it matches the Fortran/C++ 'check' drivers. - CPPProcess process; - process.initProc( "../../Cards/param_card.dat" ); - double rambowgt = 0.; - - // Retrieve masses - int npar_meta = 0; - if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - std::vector massesD( npar_meta ); - if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - const std::vector masses( massesD.begin(), massesD.end() ); - - std::vector> point = - classic_rambo::get_momenta( CPPProcess::npari, (double)kEnergy, masses, rambowgt ); - - // alpha_s from the param card so the couplings match the Fortran/C++ - // 'check' drivers (UMAMI otherwise falls back to a hardcoded g_s). - SLHAReader slha( "../../Cards/param_card.dat", false ); - const double alphaS = slha.get_block_entry( "sminputs", 3, 1.180000e-01 ); - std::vector alphasVec( nevt, alphaS ); - DeviceBufferBase devAlphaS( nevt ); - gpuMemcpy( devAlphaS.data(), alphasVec.data(), nevt * sizeof( double ), gpuMemcpyHostToDevice ); - - // Always massive RAMBO - std::unique_ptr prsk( - new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); - prsk->getMomentaInitial(); - prsk->getMomentaFinal(); - - // Fill the UMAMI SoA buffer with nevt copies of the same event: - // soa[ip4 * npar*nevt + ipar*nevt + ievt] - for( int ip4 = 0; ip4 < 4; ++ip4 ) - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - for( unsigned int ievt = 0; ievt < nevt; ++ievt ) - umamiMomenta[(std::size_t)ip4 * CPPProcess::npar * nevt + (std::size_t)ipar * nevt + ievt] = point[ipar][ip4]; - gpuMemcpy( devUmamiMomenta.data(), umamiMomenta.data(), umamiMomenta.size() * sizeof( double ), gpuMemcpyHostToDevice ); - // Host only implementation now (copy) - copyDeviceFromHost( devMomenta, hstMomenta ); - gpuLaunchKernel( aosoa_to_umami_kernel, kMatrixBlocks, kMatrixThreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); - checkGpu( gpuPeekAtLastError() ); - - if( verbose ) - { - std::cout << std::string( SEP79, '*' ) << std::endl; - print_run_header( std::cout ); - std::cout << std::string( SEP79, '*' ) << std::endl; - } - - std::cout << "Phase space point:" << std::endl - << std::string( SEP79, '-' ) << std::endl - << " n E px py pz" << std::endl; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - { - std::cout << std::scientific << std::setprecision( 7 ) - << std::setw( 2 ) << ipar + 1 - << std::setw( 16 ) << point[ipar][0] - << std::setw( 16 ) << point[ipar][1] - << std::setw( 16 ) << point[ipar][2] - << std::setw( 16 ) << point[ipar][3] - << std::endl - << std::defaultfloat; - } - std::cout << std::string( SEP79, '-' ) << std::endl; - - for( unsigned int iflav = 0; iflav < nFlavors; ++iflav ) - { - std::fill( flvVec.begin(), flvVec.end(), iflav ); - gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); - timermap.start( "3a SigmaKin" ); - UmamiInputKey in_keys[3] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S }; - UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; - const void* inputs[3] = { devUmamiMomenta.data(), devFlv.data(), devAlphaS.data() }; - void* outputs[1] = { devUmamiMEs.data() }; - UmamiStatus st = umami_matrix_element( - umami_handle, nevt, nevt, 0, 3, in_keys, inputs, 1, out_keys, outputs ); - timermap.stop(); - if( st != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; - umami_free( umami_handle ); - return 3; - } - gpuMemcpy( hstUmamiMEs.data(), devUmamiMEs.data(), nevt * sizeof( double ), gpuMemcpyDeviceToHost ); - const double* mes = hstUmamiMEs.data(); - - std::cout << " PDG"; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - std::cout << std::setw( 12 ) << CPPProcess::flavorPDG( iflav, ipar ); - std::cout << std::endl - << " Matrix element = " << std::scientific << std::setprecision( 16 ) - << mes[0] << " GeV^" << kMEGeVExponent << std::endl - << std::defaultfloat - << std::string( SEP79, '-' ) << std::endl; - } - - umami_free( umami_handle ); - return 0; - } - - // -------------------------------------------------------------------------- - // perf mode: nblocks*nthreads events per iteration on a single flavor. - // -------------------------------------------------------------------------- - int run_perf_mode( bool verbose, - unsigned int gpublocks, - unsigned int gputhreads, - unsigned int niter, - unsigned int flavorID, - RamboType ramboType, - const std::string& lheFile = "" ) - { - const unsigned int nevt = gpublocks * gputhreads; - - // LHE instead of generating. Processed in batches of nevt and - // niter is derived from the number of events read. - std::vector lheEvents; - if( !lheFile.empty() ) - { - if( !read_lhe_events( lheFile, lheEvents ) ) return 2; - niter = (unsigned int)( ( lheEvents.size() + nevt - 1 ) / nevt ); - std::cout << "Reading events from LHE file = " << lheFile - << " (" << lheEvents.size() << " events, " << niter - << " batches of " << nevt << ")" << std::endl; - } - - mgOnGpu::TimerMap timermap; - - timermap.start( "00 GpuInit" ); - GpuRuntime gpuRuntime( false ); - - PinnedHostBufferRndNumMomenta hstRndmom( nevt ); - PinnedHostBufferMomenta hstMomenta( nevt ); - PinnedHostBufferWeights hstWeights( nevt ); - DeviceBufferRndNumMomenta devRndmom( nevt ); - DeviceBufferMomenta devMomenta( nevt ); - DeviceBufferWeights devWeights( nevt ); - DeviceBufferBase devUmamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - DeviceBufferBase devUmamiMEs( nevt ); - DeviceBufferBase devFlv( nevt ); - std::vector flvVec( nevt, flavorID ); - std::vector hstUmamiMEs( nevt ); - // perf-mode runs a single flavor, so the device-side flavor buffer is filled once. - gpuMemcpy( devFlv.data(), flvVec.data(), nevt * sizeof( unsigned int ), gpuMemcpyHostToDevice ); - - std::unique_ptr prnk( - new CommonRandomNumberKernel( hstRndmom ) ); - - UmamiHandle umami_handle = nullptr; - if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_initialize failed" << std::endl; - return 2; - } - - // Retrieve masses - std::vector masses; - if( ramboType != RAMBO_MASSLESS) - { - int npar_meta = 0; - if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - std::vector massesD( npar_meta ); - if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - masses.assign( massesD.begin(), massesD.end() ); - } - - std::unique_ptr prsk; - if( ramboType != RAMBO_MASSLESS ) - { - // Massive host only (copy) - prsk.reset( new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); - } - else - { - prsk.reset( new MasslessRamboSamplingKernelDevice( kEnergy, devRndmom, devMomenta, devWeights, gpublocks, gputhreads ) ); - } - - std::unique_ptr genrtimes( new double[niter] ); - std::unique_ptr rambtimes( new double[niter] ); - std::unique_ptr wavetimes( new double[niter] ); - - unsigned int nevtABN = 0; - unsigned int nevtZERO = 0; - double sumME = 0.; - double sumMEsq = 0.; - double minME = std::numeric_limits::infinity(); - double maxME = -std::numeric_limits::infinity(); - unsigned int nevtALL = 0; - - for( unsigned int iiter = 0; iiter < niter; ++iiter ) - { - double genrtime = 0; - double rambtime = 0; - unsigned int nreal = nevt; // number of real (non-padding) events in this batch - if( lheFile.empty() ) - { - timermap.start( "1a GenSeed " ); - prnk->seedGenerator( kSeed + iiter ); - genrtime += timermap.stop(); - timermap.start( "1b GenRnGen" ); - prnk->generateRnarray(); - genrtime += timermap.stop(); - if( ramboType == RAMBO_MASSLESS ) - { - timermap.start( "1c CpHTDrnd" ); - copyDeviceFromHost( devRndmom, hstRndmom ); - genrtime += timermap.stop(); - } - - timermap.start( "2a RamboIni" ); - prsk->getMomentaInitial(); - rambtime += timermap.stop(); - timermap.start( "2b RamboFin" ); - prsk->getMomentaFinal(); - rambtime += timermap.stop(); - // Massive host only (copy) - if( ramboType != RAMBO_MASSLESS ) - { - timermap.start( "2c CpHTDmom" ); - copyDeviceFromHost( devMomenta, hstMomenta ); - rambtime += timermap.stop(); - } - } - else - { - // Fill this batch from the LHE events (AOSOA layout, (E,px,py,pz) per leg). - // padded by repeating its last real event so the SIMD page is valid - // only the nreal real events are counted below. - timermap.start( "2e ReadLHE " ); - const std::size_t base = (std::size_t)iiter * nevt; - nreal = (unsigned int)std::min( nevt, lheEvents.size() - base ); - for( unsigned int ievt = 0; ievt < nevt; ++ievt ) - { - const std::size_t src = base + std::min( ievt, (std::size_t)nreal - 1 ); - const LheEvent& ev = lheEvents[src]; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - for( int ip4 = 0; ip4 < 4; ++ip4 ) - MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar ) = (fptype)ev[ipar][ip4]; - } - rambtime += timermap.stop(); - timermap.start( "2c CpHTDmom" ); - copyDeviceFromHost( devMomenta, hstMomenta ); - rambtime += timermap.stop(); - } - - timermap.start( "2d Aosoa2U " ); - gpuLaunchKernel( aosoa_to_umami_kernel, gpublocks, gputhreads, devMomenta.data(), devUmamiMomenta.data(), (std::size_t)nevt ); - checkGpu( gpuPeekAtLastError() ); - rambtime += timermap.stop(); - - double wavetime = 0; - if( !run_umami( umami_handle, nevt, timermap, wavetime, - devUmamiMomenta, devFlv, devUmamiMEs, hstUmamiMEs - ) ) - { - umami_free( umami_handle ); - return 3; - } - - if( verbose ) - { - timermap.start( "3c CpDTHmom" ); - copyHostFromDevice( hstMomenta, devMomenta ); - wavetime += timermap.stop(); - } - const double* mes = hstUmamiMEs.data(); - - timermap.start( "4@ UpdtStat" ); - for( unsigned int ievt = 0; ievt < nreal; ++ievt ) - { - double me = mes[ievt]; - ++nevtALL; - if( !std::isfinite( me ) ) - ++nevtABN; - else if( me == 0. ) - ++nevtZERO; - sumME += me; - sumMEsq += me * me; - if( me < minME ) minME = me; - if( me > maxME ) maxME = me; - } - - genrtimes[iiter] = genrtime; - rambtimes[iiter] = rambtime; - wavetimes[iiter] = wavetime; - - if( verbose ) - { - std::cout << std::string( SEP79, '*' ) << std::endl - << "Iteration #" << iiter + 1 << " of " << niter << std::endl; - for( unsigned int ievt = 0; ievt < nreal; ++ievt ) - { - std::cout << "Event #" << ievt + 1 << std::endl; - print_momenta_table( std::cout, hstMomenta.data(), ievt ); - std::cout << " Matrix element = " << std::scientific << std::setprecision( 16 ) - << mes[ievt] << " GeV^" << kMEGeVExponent << std::endl - << std::defaultfloat - << std::string( SEP79, '-' ) << std::endl; - } - } - } - - double sumgtim = 0, sumrtim = 0, sumwtim = 0; - double minwtim = wavetimes[0], maxwtim = wavetimes[0]; - for( unsigned int i = 0; i < niter; ++i ) - { - sumgtim += genrtimes[i]; - sumrtim += rambtimes[i]; - sumwtim += wavetimes[i]; - minwtim = std::min( minwtim, wavetimes[i] ); - maxwtim = std::max( maxwtim, wavetimes[i] ); - } - double meanwtim = sumwtim / niter; - - unsigned int nevtGood = nevtALL - nevtABN; - double meanME = ( nevtGood > 0 ) ? sumME / nevtGood : 0.; - double varME = ( nevtGood > 0 ) ? sumMEsq / nevtGood - meanME * meanME : 0.; - double stdME = ( varME > 0 ) ? std::sqrt( varME ) : 0.; - - std::cout << std::string( SEP79, '*' ) << std::endl; - print_run_header( std::cout ); - std::cout << "NumBlocksPerGrid = " << gpublocks << std::endl - << "NumThreadsPerBlock = " << gputhreads << std::endl - << "NumIterations = " << niter << std::endl - << "FlavorIndex = " << flavorID << " / " << CPPProcess::nmaxflavor << std::endl - << std::string( SEP79, '-' ) << std::endl - << "NaN/abnormal MEs = " << nevtABN << std::endl - << "Zero MEs = " << nevtZERO << std::endl - << std::string( SEP79, '-' ) << std::endl - << "NumberOfEntries = " << niter << std::endl - << std::scientific - << "TotalTime[Rnd+Rmb+ME] (123) = ( " << sumgtim + sumrtim + sumwtim << " ) sec" << std::endl - << "TotalTime[Rambo+ME] (23) = ( " << sumrtim + sumwtim << " ) sec" << std::endl - << "TotalTime[RndNumGen] (1) = ( " << sumgtim << " ) sec" << std::endl - << "TotalTime[Rambo] (2) = ( " << sumrtim << " ) sec" << std::endl - << "TotalTime[MatrixElems] (3) = ( " << sumwtim << " ) sec" << std::endl - << "MeanTimeInMatrixElems = ( " << meanwtim << " ) sec" << std::endl - << "[Min,Max]TimeInMatrixElems = [ " << minwtim << " , " << maxwtim << " ] sec" << std::endl - << std::string( SEP79, '-' ) << std::endl - << "TotalEventsComputed = " << nevtALL << std::endl - << "EvtsPerSec[Rnd+Rmb+ME](123) = ( " << nevtALL / ( sumgtim + sumrtim + sumwtim ) << " ) sec^-1" << std::endl - << "EvtsPerSec[Rmb+ME] (23) = ( " << nevtALL / ( sumrtim + sumwtim ) << " ) sec^-1" << std::endl - << "EvtsPerSec[MatrixElems] (3) = ( " << nevtALL / sumwtim << " ) sec^-1" << std::endl - << std::defaultfloat - << std::string( SEP79, '*' ) << std::endl - << "MeanMatrixElemValue = ( " << meanME << " +- " << stdME / std::sqrt( (double)std::max( 1u, nevtGood ) ) - << " ) GeV^" << kMEGeVExponent << std::endl - << "[Min,Max]MatrixElemValue = [ " << minME << " , " << maxME << " ] GeV^" << kMEGeVExponent << std::endl - << std::string( SEP79, '*' ) << std::endl; - timermap.dump(); - std::cout << std::string( SEP79, '*' ) << std::endl; - - umami_free( umami_handle ); - return 0; - } -} - -int main( int argc, char** argv ) -{ - - Mode mode = MODE_MATRIX; - RamboType ramboType = RAMBO_MASSIVE; // default - bool ramboTypeSet = false; - bool verbose = false; - unsigned int flavorID = 0; - unsigned int gpublocks = 64; - unsigned int gputhreads = 256; - unsigned int niter = 1; - unsigned int numvec[3] = { 0, 0, 0 }; - int nnum = 0; - std::string lheFile; // -e/--events: read momenta from this LHE file (perf mode only) - - // Optional leading subcommand (no leading dash). - int firstArg = 1; - if( firstArg < argc ) - { - std::string a = argv[firstArg]; - if( a == "matrix" ) { mode = MODE_MATRIX; ++firstArg; } - else if( a == "perf" ) { mode = MODE_PERF; ++firstArg; } - } - - double energyArg = -1.; - - for( int argn = firstArg; argn < argc; ++argn ) - { - std::string arg = argv[argn]; - if( arg == "--verbose" || arg == "-v" ) - verbose = true; - else if( arg == "--performance" || arg == "-p" ) - mode = MODE_PERF; // legacy alias - else if( ( arg == "--flavor" || arg == "-f" ) && argn + 1 < argc && is_number( argv[argn + 1] ) ) - flavorID = strtoul( argv[++argn], nullptr, 0 ); - else if( arg == "--rambo-massless" ) - { - std::string r = argv[++argn]; - ramboType = RAMBO_MASSLESS; - ramboTypeSet = true; - } - else if( ( arg == "--events" || arg == "-e" ) && argn + 1 < argc ) - { - lheFile = argv[++argn]; - mode = MODE_PERF; // reading events from file only makes sense in perf mode - } - else if( is_number( argv[argn] ) && nnum < 3 ) - { - numvec[nnum++] = strtoul( argv[argn], nullptr, 0 ); - if( energyArg < 0 ) energyArg = atof( argv[argn] ); - } - else if( is_float( argv[argn] ) && energyArg < 0 ) - { - // decimal number: only meaningful as the matrix-mode energy - energyArg = atof( argv[argn] ); - } - else - return usage( argv[0] ); - } -//ENERGY CHANGE FOR THE MATRIX MODE -// (default 1000 GeV as for the Fortran/C++ standalone 'check' drivers; -// can be overridden with a single positional argument) - if( mode == MODE_MATRIX ) kEnergy = ( energyArg > 0 ) ? energyArg : 1000.; - - if( mode == MODE_MATRIX ) - { - if( ramboType == RAMBO_MASSLESS ) - { - std::cerr << "ERROR: matrix mode only supports the classic RAMBO (-r c)." << std::endl; - return usage( argv[0] ); - } - if( nnum > 1 ) - { - std::cerr << "WARNING: extra positional args are ignored in matrix mode " - << "(dimensions are fixed at " << kMatrixBlocks << " " << kMatrixThreads << " 1)." - << std::endl; - } - return run_matrix_mode( verbose ); - } - - // perf mode - if( nnum == 3 ) - { - gpublocks = numvec[0]; - gputhreads = numvec[1]; - niter = numvec[2]; - } - else if( nnum == 1 ) - { - niter = numvec[0]; - } - else if( nnum != 0 ) - { - return usage( argv[0] ); - } - if( niter == 0 && lheFile.empty() ) return usage( argv[0] ); // niter is derived from the file in LHE mode - - if( flavorID >= CPPProcess::nmaxflavor ) - { - std::cerr << "ERROR: flavor index " << flavorID - << " is out of range [0, " << CPPProcess::nmaxflavor << ")." << std::endl; - return 1; - } - - return run_perf_mode( verbose, gpublocks, gputhreads, niter, flavorID, ramboType, lheFile ); -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc deleted file mode 100644 index f7710943b4..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Oct 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "Bridge.h" -#include "CPPProcess.h" -#include "GpuRuntime.h" - -extern "C" -{ - /** - * The namespace where the Bridge class is taken from. - * - * In the current implementation, two separate shared libraries are created for the GPU/CUDA and CPU/C++ implementations. - * Actually, two shared libraries for GPU and CPU are created for each of the five SIMD implementations on CPUs (none, sse4, avx2, 512y, 512z). - * A single fcreatebridge_ symbol is created in each library with the same name, connected to the appropriate Bridge on CPU or GPU. - * The Fortran MadEvent code is always the same: the choice whether to use a CPU or GPU implementation is done by linking the appropriate library. - * As the names of the two CPU/GPU libraries are the same in the five SIMD implementations, the choice of SIMD is done by setting LD_LIBRARY_PATH. - * - * In a future implementation, a single heterogeneous shared library may be created, with the same interface. - * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. - * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. - */ - using namespace mg5amcGpu; - - /** - * The floating point precision used in Fortran arrays. - * This is presently hardcoded to double precision (REAL*8). - */ - using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays - //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays - - /** - * Create a Bridge and return its pointer. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param nevtF the pointer to the number of events in the Fortran arrays - * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - */ - void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) - { - GpuRuntime::setUp(); - // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) - // FIXME: disable OMP in Bridge when called from Fortran - *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); - } - - /** - * Delete a Bridge. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - */ - void fbridgedelete_( CppObjectInFortran** ppbridge ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); - delete pbridge; - GpuRuntime::tearDown(); - } - - /** - * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant alphas) - * @param rndhel the pointer to the input random numbers for helicity selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the pointer to the input channels i.e. Feynman diagrams to enhance (1 to n: 0 is an invalid value!) - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void fbridgesequence_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); - // Use the device/GPU implementation in the CUDA library - // (there is also a host implementation in this library) - pbridge->gpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); - } - - /** - * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++, without multi-channel mode. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant alphas) - * @param rndhel the pointer to the input random numbers for helicity selection - * @param rndcol the pointer to the input random numbers for color selection - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ) - { - //printf("fbridgesequence_nomultichannel_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); - fbridgesequence_( ppbridge, momenta, gs, iflavorVec, rndhel, rndcol, nullptr, mes, selhel, selcol, pgoodHelOnly ); - } - - /** - * Retrieve the number of good helicities for helicity filtering in the Bridge. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param pngoodhel the pointer to the output number of good helicities - * @param pntothel the pointer to the output total number of helicities - */ - void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, - unsigned int* pngoodhel, - unsigned int* pntothel ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgegetngoodhel_: invalid Bridge address" ); - *pngoodhel = pbridge->nGoodHel(); - *pntothel = pbridge->nTotHel(); - } -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h deleted file mode 100644 index 5bd0b270d2..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fbridge.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: Z. Wettersten (Oct 2024) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, A. Thete, A. Valassi (2025). - -#include "Bridge.h" -#include "CPPProcess.h" -#include "GpuRuntime.h" - -#ifndef _FBRIDGE_H_ -#define _FBRIDGE_H_ - -extern "C" -{ - using namespace mg5amcGpu; - - using FORTRANFPTYPE = double; - - void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ); - - void fbridgedelete_( CppObjectInFortran** ppbridge ); - - void fbridgesequence_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ); - - void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ); - - void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, unsigned int* pngoodhel, unsigned int* pntothel ); -} -#endif // _FBRIDGE_H_ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc deleted file mode 100644 index 8745aa6cbf..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/fsampler.cc +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Feb 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "mgOnGpuConfig.h" - -#include "Bridge.h" -#include "CPPProcess.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" - -//-------------------------------------------------------------------------- - -namespace mg5amcGpu -{ - template - class Sampler final : public CppObjectInFortran - { - public: - // Constructor - // @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran arrays - // @param nparF (NEXTERNAL, nexternal.inc) number of external particles in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) - // @param np4F number of momenta components, usually 4, in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) - Sampler( int nevtF, int nparF, int np4F ); - // Destructor - virtual ~Sampler() {} - // Delete copy/move constructors and assignment operators - Sampler( const Sampler& ) = delete; - Sampler( Sampler&& ) = delete; - Sampler& operator=( const Sampler& ) = delete; - Sampler& operator=( Sampler&& ) = delete; - // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta - void samplerHostSequence( FORTRANFPTYPE* fortranMomenta ); - private: - const int m_nevt; // The number of events in each iteration - int m_iiter; // The iteration counter (for random number seeding) - PinnedHostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers - PinnedHostBufferMomenta m_hstMomenta; // Memory buffers for momenta - PinnedHostBufferWeights m_hstWeights; // Memory buffers for sampling weights - std::unique_ptr m_prnk; // The appropriate RandomNumberKernel - std::unique_ptr m_prsk; // The appropriate SamplingKernel - // HARDCODED DEFAULTS - static constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) - }; - - template - Sampler::Sampler( int nevtF, int nparF, int np4F ) - : m_nevt( nevtF ) - , m_iiter( 0 ) - , m_hstRndmom( nevtF ) - , m_hstMomenta( nevtF ) - , m_hstWeights( nevtF ) - , m_prnk( new CommonRandomNumberKernel( m_hstRndmom ) ) - , m_prsk( new RamboSamplingKernelHost( energy, m_hstRndmom, m_hstMomenta, m_hstWeights, nevtF ) ) - { - if( nparF != CPPProcess::npar ) throw std::runtime_error( "Sampler constructor: npar mismatch" ); - if( np4F != CPPProcess::np4 ) throw std::runtime_error( "Sampler constructor: np4 mismatch" ); - std::cout << "WARNING! Instantiate host Sampler (nevt=" << m_nevt << ")" << std::endl; - } - - // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta - template - void Sampler::samplerHostSequence( FORTRANFPTYPE* fortranMomenta ) - { - std::cout << "Iteration #" << m_iiter + 1 << std::endl; - // === STEP 1 OF 3 - // --- 1a. Seed rnd generator (to get same results on host and device in curand) - // [NB This should not be necessary using the host API: "Generation functions - // can be called multiple times on the same generator to generate successive - // blocks of results. For pseudorandom generators, multiple calls to generation - // functions will yield the same result as a single call with a large size."] - // *** NB! REMEMBER THAT THE FORTRAN SAMPLER ALWAYS USES COMMON RANDOM NUMBERS! *** - constexpr unsigned long long seed = 20200805; - m_prnk->seedGenerator( seed + m_iiter ); - m_iiter++; - // --- 1b. Generate all relevant numbers to build nevt events (i.e. nevt phase space points) on the host - m_prnk->generateRnarray(); - //std::cout << "Got random numbers" << std::endl; - // === STEP 2 OF 3 - // --- 2a. Fill in momenta of initial state particles on the device - m_prsk->getMomentaInitial(); - //std::cout << "Got initial momenta" << std::endl; - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - m_prsk->getMomentaFinal(); - //std::cout << "Got final momenta" << std::endl; - // --- 2c. TransposeC2F - hst_transposeMomentaC2F( m_hstMomenta.data(), fortranMomenta, m_nevt ); - } -} - -//-------------------------------------------------------------------------- - -extern "C" -{ - using namespace mg5amcGpu; - - /** - * The floating point precision used in Fortran arrays. - * This is presently hardcoded to double precision (REAL*8). - */ - using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays - //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays - - /** - * Create a Sampler and return its pointer. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - * @param nevtF the pointer to the number of events in the Fortran arrays - * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - */ - void fsamplercreate_( CppObjectInFortran** ppsampler, const int* pnevtF, const int* pnparF, const int* pnp4F ) - { - *ppsampler = new Sampler( *pnevtF, *pnparF, *pnp4F ); - } - - /** - * Delete a Sampler. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - */ - void fsamplerdelete_( CppObjectInFortran** ppsampler ) - { - Sampler* psampler = dynamic_cast*>( *ppsampler ); - if( psampler == 0 ) throw std::runtime_error( "fsamplerdelete_: invalid Sampler address" ); - delete psampler; - } - - /** - * Execute the matrix-element calculation "sequence" via a Sampler on GPU/CUDA or CUDA/C++. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param mes the pointer to the output matrix elements - */ - void fsamplersequence_( CppObjectInFortran** ppsampler, FORTRANFPTYPE* momenta ) - { - Sampler* psampler = dynamic_cast*>( *ppsampler ); - if( psampler == 0 ) throw std::runtime_error( "fsamplersequence_: invalid Sampler address" ); - // Use the host/CPU implementation (there is no device implementation) - psampler->samplerHostSequence( momenta ); - } -} - -//-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h deleted file mode 100644 index 6a1180f7be..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/massless_rambo.h +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: S. Roiser (Feb 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== - -#ifndef MASSLESS_RAMBO_H -#define MASSLESS_RAMBO_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuFptypes.h" - -#include "ProcessData.h" - -#include -#include -#include - -// Simplified rambo version for 2 to N (with N>=2) processes with massless particles -namespace mg5amcGpu -{ -namespace massless_rambo { - - constexpr int np4 = ProcessData::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npari = ProcessData::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- - constexpr int nparf = ProcessData::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- - constexpr int npar = ProcessData::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - - //-------------------------------------------------------------------------- - - // Fill in the momenta of the initial particles - // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] - template - __host__ __device__ void - ramboGetMomentaInitial( const fptype energy, // input: energy - fptype* momenta ) // output: momenta for one event or for a set of events - { - const fptype energy1 = energy / 2; - const fptype energy2 = energy / 2; - const fptype mom = energy / 2; - M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 0 ) = energy1; - M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 0 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 0 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 0 ) = mom; - M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 1 ) = energy2; - M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 1 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 1 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 1 ) = -mom; - } - - //-------------------------------------------------------------------------- - - // Fill in the momenta of the final particles using the RAMBO algorithm - // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] - template - __host__ __device__ void - ramboGetMomentaFinal( const fptype energy, // input: energy - const fptype* rndmom, // input: random numbers in [0,1] for one event or for a set of events - fptype* momenta, // output: momenta for one event or for a set of events - fptype* wgts ) // output: weights for one event or for a set of events - { - /**************************************************************************** - * rambo * - * ra(ndom) m(omenta) b(eautifully) o(rganized) * - * * - * a democratic multi-particle phase space generator * - * authors: s.d. ellis, r. kleiss, w.j. stirling * - * this is version 1.0 - written by r. kleiss * - * -- adjusted by hans kuijf, weights are logarithmic (1990-08-20) * - * -- adjusted by madgraph@sheffield_gpu_hackathon team (2020-07-29) * - * * - ****************************************************************************/ - - // output weight - fptype& wt = W_ACCESS::kernelAccess( wgts ); - - // AV special case nparf==1 (issue #358) - if constexpr( nparf == 1 ) - { - static bool first = true; - if( first ) - { - if constexpr( M_ACCESS::isOnDevice() ) // avoid - { - const int ievt0 = 0; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid - if( ievt == ievt0 ) - printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); - } - else - { - printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); - } - first = false; - } - const int iparf = 0; - for( int i4 = 0; i4 < np4; i4++ ) - { - M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = 0; - for( int ipari = 0; ipari < npari; ipari++ ) - { - M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) += M_ACCESS::kernelAccessIp4Ipar( momenta, i4, ipari ); - } - } - wt = 1; - return; - } - - // initialization step: factorials for the phase space weight - const fptype twopi = 8. * atan( 1. ); - const fptype po2log = log( twopi / 4. ); - fptype z[nparf]; - if constexpr( nparf > 1 ) // avoid build warning on clang (related to #358) - z[1] = po2log; - for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = z[kpar - 1] + po2log - 2. * log( fptype( kpar - 1 ) ); - for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = ( z[kpar] - log( fptype( kpar ) ) ); - - // generate n massless momenta in infinite phase space - fptype q[nparf][np4]; - for( int iparf = 0; iparf < nparf; iparf++ ) - { - const fptype r1 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 0, iparf ); - const fptype r2 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 1, iparf ); - const fptype r3 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 2, iparf ); - const fptype r4 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 3, iparf ); - const fptype c = 2. * r1 - 1.; - const fptype s = sqrt( 1. - c * c ); - const fptype f = twopi * r2; - q[iparf][0] = -log( r3 * r4 ); - q[iparf][3] = q[iparf][0] * c; - q[iparf][2] = q[iparf][0] * s * cos( f ); - q[iparf][1] = q[iparf][0] * s * sin( f ); - } - - // calculate the parameters of the conformal transformation - fptype r[np4]; - fptype b[np4 - 1]; - for( int i4 = 0; i4 < np4; i4++ ) r[i4] = 0.; - for( int iparf = 0; iparf < nparf; iparf++ ) - { - for( int i4 = 0; i4 < np4; i4++ ) r[i4] = r[i4] + q[iparf][i4]; - } - const fptype rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); - for( int i4 = 1; i4 < np4; i4++ ) b[i4 - 1] = -r[i4] / rmas; - const fptype g = r[0] / rmas; - const fptype a = 1. / ( 1. + g ); - const fptype x0 = energy / rmas; - - // transform the q's conformally into the p's (i.e. the 'momenta') - for( int iparf = 0; iparf < nparf; iparf++ ) - { - fptype bq = b[0] * q[iparf][1] + b[1] * q[iparf][2] + b[2] * q[iparf][3]; - for( int i4 = 1; i4 < np4; i4++ ) - { - M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = x0 * ( q[iparf][i4] + b[i4 - 1] * ( q[iparf][0] + a * bq ) ); - } - M_ACCESS::kernelAccessIp4Ipar( momenta, 0, iparf + npari ) = x0 * ( g * q[iparf][0] + bq ); - } - - // calculate weight (NB return log of weight) - wt = po2log; - if( nparf != 2 ) wt = ( 2. * nparf - 4. ) * log( energy ) + z[nparf - 1]; - - - // return for weighted massless momenta - // nothing else to do in this event if all particles are massless (nm==0) - - return; - } - - //-------------------------------------------------------------------------- -} -} - -#endif // MASSLESS_RAMBO_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/runTest.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/runTest.cc deleted file mode 100644 index d5909873d8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/runTest.cc +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Hageboeck (Nov 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. -// ---------------------------------------------------------------------------- -// Use ./runTest.exe --gtest_filter=*xxx to run only testxxx.cc tests -// ---------------------------------------------------------------------------- - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "MadgraphTest.h" -#include "MatrixElementKernels.h" -#include "MemoryAccessChannelIds.h" -#include "MemoryAccessMatrixElements.h" -#include "MemoryAccessMomenta.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" -#include "coloramps.h" -#include "epoch_process_id.h" - -#include - -using namespace mg5amcGpu; - -struct CUDA_CPU_TestBase : public TestDriverBase -{ - static constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static constexpr int np4 = CPPProcess::np4; - static constexpr int npar = CPPProcess::npar; - static_assert( gputhreads % neppM == 0, "ERROR! #threads/block should be a multiple of neppM" ); - static_assert( gputhreads <= mgOnGpu::ntpbMAX, "ERROR! #threads/block should be <= ntpbMAX" ); - CUDA_CPU_TestBase( const std::string& refFileName ) - : TestDriverBase( npar, refFileName ) {} - // Does this test use channelIds? - virtual bool useChannelIds() const = 0; - // Set channelId array (in the same way for CUDA and CPU tests) - static constexpr unsigned int warpSize = 32; // FIXME: add a sanity check in madevent that this is the minimum? (would need to expose this from cudacpp to madevent) - static void setChannelIds( BufferChannelIds& hstChannelIds, std::size_t iiter ) - { - static const char* debugC = getenv( "CUDACPP_RUNTEST_DEBUG" ); - static const bool debug = ( debugC != 0 ) && ( std::string( debugC ) != "" ); - // Fill channelIds for multi-channel tests #896 - // (NB: these are only used if useChannelIds == true) - // TEMPORARY(0): debug multichannel tests with channelId=1 for all events - //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1; - // TEMPORARY(1): debug multichannel tests with channelId=1,2,..,ndiag,1,2,..ndiag,... (every event gets a different channel, no warps) - //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1 + i % CPPProcess::ndiagrams; - // ALMOST FINAL test implementation: 1111222233331111... (every 32-event warp gets a different channel) - // FINAL(?) test implementation: 2222333344442222... (every 32-event warp gets a different channel, skip those without associated iconfig #917) - static_assert( nevt % warpSize == 0, "ERROR! nevt should be a multiple of warpSize" ); - constexpr unsigned int nWarp = nevt / warpSize; - for( unsigned int iWarp = 0; iWarp < nWarp; ++iWarp ) - { - //const unsigned int channelId = 1 + ( iWarp + iiter * nWarp ) % CPPProcess::ndiagrams; // bug #917 - const int iconfig = 1 + ( iWarp + iiter * nWarp ) % mgOnGpu::nconfigSDE; - unsigned int channelId = 0; - //for( unsigned int idiagram = 1; idiagram < CPPProcess::ndiagrams; idiagram++ ) // two bugs #920 and #919 - for( unsigned int idiagram = 0; idiagram < mgOnGpu::nchannels; idiagram++ ) // fix #920 and work around #919 - { - if( mgOnGpu::hostChannel2iconfig[idiagram] == iconfig ) - { - channelId = idiagram + 1; // fix #917 (NB add +1 because channelId uses F indexing) - break; - } - } - assert( channelId > 0 ); // sanity check that the channelId for the given iconfig was found - if( debug ) std::cout << "CUDA_CPU_TestBase::setChannelIds: iWarp=" << iWarp << ", iconfig=" << iconfig << ", channelId=" << channelId << std::endl; - for( unsigned int i = 0; i < warpSize; ++i ) - hstChannelIds[iWarp * warpSize + i] = channelId; - } - } -}; - - -struct CUDATest : public CUDA_CPU_TestBase -{ - // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) - // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] - CPPProcess process; - PinnedHostBufferRndNumMomenta hstRndMom; - PinnedHostBufferMomenta hstMomenta; - PinnedHostBufferGs hstGs; - PinnedHostBufferRndNumHelicity hstRndHel; - PinnedHostBufferRndNumColor hstRndCol; - PinnedHostBufferWeights hstWeights; - PinnedHostBufferChannelIds hstChannelIds; - PinnedHostBufferMatrixElements hstMatrixElements; - PinnedHostBufferSelectedHelicity hstSelHel; - PinnedHostBufferSelectedColor hstSelCol; - PinnedHostBufferHelicityMask hstIsGoodHel; - DeviceBufferRndNumMomenta devRndMom; - DeviceBufferChannelIds devChannelIds; - DeviceBufferMomenta devMomenta; - DeviceBufferGs devGs; - DeviceBufferRndNumHelicity devRndHel; - DeviceBufferRndNumColor devRndCol; - DeviceBufferWeights devWeights; - DeviceBufferMatrixElements devMatrixElements; - DeviceBufferSelectedHelicity devSelHel; - DeviceBufferSelectedColor devSelCol; - DeviceBufferHelicityMask devIsGoodHel; - std::unique_ptr pmek; - - // Create a process object - // Read param_card and set parameters - // ** WARNING EVIL EVIL ** - // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. - // Don't remove! - CUDATest( const std::string& refFileName ) - : CUDA_CPU_TestBase( refFileName ) - , process( /*verbose=*/false ) - , hstRndMom( nevt ) - , hstChannelIds( nevt ) - , hstMomenta( nevt ) - , hstGs( nevt ) - , hstRndHel( nevt ) - , hstRndCol( nevt ) - , hstWeights( nevt ) - , hstMatrixElements( nevt ) - , hstSelHel( nevt ) - , hstSelCol( nevt ) - , hstIsGoodHel( CPPProcess::ncomb ) - , devRndMom( nevt ) - , devChannelIds( nevt ) - , devMomenta( nevt ) - , devGs( nevt ) - , devRndHel( nevt ) - , devRndCol( nevt ) - , devWeights( nevt ) - , devMatrixElements( nevt ) - , devSelHel( nevt ) - , devSelCol( nevt ) - , devIsGoodHel( CPPProcess::ncomb ) - , pmek( new MatrixElementKernelDevice( devMomenta, devGs, devRndHel, devRndCol, devChannelIds, devMatrixElements, devSelHel, devSelCol, gpublocks, gputhreads ) ) - { - // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? - process.initProc( "../../Cards/param_card.dat" ); - } - - virtual ~CUDATest() {} - - void prepareRandomNumbers( unsigned int iiter ) override - { - // Random numbers for momenta - CommonRandomNumberKernel rnk( hstRndMom ); - rnk.seedGenerator( 1337 + iiter ); - rnk.generateRnarray(); - copyDeviceFromHost( devRndMom, hstRndMom ); - // Random numbers for helicity and color selection (fix #931) - CommonRandomNumberKernel rnk2( hstRndHel ); - rnk2.seedGenerator( 1338 + iiter ); - rnk2.generateRnarray(); - copyDeviceFromHost( devRndHel, hstRndHel ); - CommonRandomNumberKernel rnk3( hstRndCol ); - rnk3.seedGenerator( 1339 + iiter ); - rnk3.generateRnarray(); - copyDeviceFromHost( devRndCol, hstRndCol ); - } - - void prepareMomenta( fptype energy ) override - { - RamboSamplingKernelDevice rsk( energy, devRndMom, devMomenta, devWeights, gpublocks, gputhreads ); - // --- 2a. Fill in momenta of initial state particles on the device - rsk.getMomentaInitial(); - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - rsk.getMomentaFinal(); - // --- 2c. CopyDToH Weights - copyHostFromDevice( hstWeights, devWeights ); - // --- 2d. CopyDToH Momenta - copyHostFromDevice( hstMomenta, devMomenta ); - } - - void runSigmaKin( std::size_t iiter ) override - { - constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) - for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; - copyDeviceFromHost( devGs, hstGs ); // BUG FIX #566 - setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 - copyDeviceFromHost( devChannelIds, hstChannelIds ); - if( iiter == 0 ) pmek->computeGoodHelicities(); - pmek->computeMatrixElements( useChannelIds() ); - copyHostFromDevice( hstMatrixElements, devMatrixElements ); - copyHostFromDevice( hstSelHel, devSelHel ); - copyHostFromDevice( hstSelCol, devSelCol ); - } - - fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override - { - assert( ipar < npar ); - assert( ip4 < np4 ); - return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); - } - - fptype getMatrixElement( std::size_t ievt ) const override - { - return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); - } - - int getChannelId( std::size_t ievt ) const override - { - return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); - } - - int getSelectedHelicity( std::size_t ievt ) const override - { - //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... - return hstSelHel.data()[ievt]; - } - - int getSelectedColor( std::size_t ievt ) const override - { - //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... - return hstSelCol.data()[ievt]; - } -}; - -// Old test with multi-channel disabled #466 -struct CUDATestNoMultiChannel : public CUDATest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return false; } - - // Constructor - CUDATestNoMultiChannel( const std::string& refFileName ) - : CUDATest( refFileName ) {} // suffix .txt - - // Destructor - virtual ~CUDATestNoMultiChannel() {} -}; - -// New test with multi-channel enabled #896 -struct CUDATestMultiChannel : public CUDATest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return true; } - - // Constructor - CUDATestMultiChannel( const std::string& refFileName ) - : CUDATest( refFileName + "2" ) {} // suffix .txt2 - - // Destructor - virtual ~CUDATestMultiChannel() {} -}; - -// AV July 2024 much simpler class structure without the presently-unnecessary googletest templates -// This is meant as a workaround to prevent not-understood segfault #907 when adding a second test -// Note: instantiate test2 first and test1 second to ensure that the channelid printout from the dtors comes from test1 first and test2 second -// CUDA test drivers -CUDATestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID2( s ) s##_GPU_MULTICHANNEL -CUDATestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID1( s ) s##_GPU_NOMULTICHANNEL -// Madgraph tests -MadgraphTest mgTest2( driver2 ); -MadgraphTest mgTest1( driver1 ); -// Instantiate Google test 1 -#define XTESTID1( s ) TESTID1( s ) -TEST( XTESTID1( MG_EPOCH_PROCESS_ID ), compareMomAndME ) -{ -#ifdef MGONGPU_CHANNELID_DEBUG - driver1.pmek->setTagForNevtProcessedByChannel( "(no multichannel)" ); -#endif - mgTest1.CompareMomentaAndME( *this ); -} -// Instantiate Google test 2 -#define XTESTID2( s ) TESTID2( s ) -TEST( XTESTID2( MG_EPOCH_PROCESS_ID ), compareMomAndME ) -{ -#ifdef MGONGPU_CHANNELID_DEBUG - driver2.pmek->setTagForNevtProcessedByChannel( "(channelid array)" ); -#endif - mgTest2.CompareMomentaAndME( *this ); -} -/* clang-format on */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc deleted file mode 100644 index 622d26b010..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/testmisc.cc +++ /dev/null @@ -1,437 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. -// ---------------------------------------------------------------------------- -// Use ./runTest.exe --gtest_filter=*misc to run only testmisc.cc tests -// ---------------------------------------------------------------------------- - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuVectors.h" - -#include "constexpr_math.h" -#include "epoch_process_id.h" -#include "valgrind.h" - -#include - -//#include -//#include // needs C++20... https://stackoverflow.com/a/65347016 -#include -#include -#include - -#define TESTID( s ) s##_GPU_MISC - -#define XTESTID( s ) TESTID( s ) - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu -{ -#define EXPECT_TRUE_sv( cond ) { EXPECT_TRUE( cond ); } - - inline const std::string - boolTF( const bool& b ) - { - return ( b ? "T" : "F" ); - } - -} - -TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) -{ - using namespace mg5amcGpu; - - //-------------------------------------------------------------------------- - - EXPECT_TRUE( true ); - - //-------------------------------------------------------------------------- - - // Vector initialization for fptype_sv - { - fptype_sv f{ 0 }; - EXPECT_TRUE_sv( f == 0 ); - } - { - fptype_sv f = fptype_sv{ 0 }; - EXPECT_TRUE_sv( f == 0 ); - } - - // Vector initialization for fptype_sv - demonstrate bug #339 in older cxmake implementation - { - fptype_sv f{ 1 }; - //std::cout << f << std::endl << boolTF( f == 1 ) << std::endl; - //EXPECT_TRUE_sv( f == 1 ); // this fails for vectors! TFFF - EXPECT_TRUE_sv( f == 1 ); // this succeds: T - } - - - // Vector initialization for cxtype_sv - { - cxtype_sv c = cxzero_sv(); - EXPECT_TRUE_sv( c.real() == 0 ); - EXPECT_TRUE_sv( c.imag() == 0 ); - } - { - cxtype_sv c = cxmake( 1, fptype_sv{ 0 } ); // here was a bug #339 - EXPECT_TRUE_sv( c.real() == 1 ); - EXPECT_TRUE_sv( c.imag() == 0 ); - } - { - cxtype_sv c = cxmake( fptype_sv{ 0 }, 1 ); // here was a bug #339 - EXPECT_TRUE_sv( c.real() == 0 ); - EXPECT_TRUE_sv( c.imag() == 1 ); - } - - // Array initialization for cxtype_sv array (example: jamp_sv in CPPProcess.cc) - { - cxtype_sv array[2] = {}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "= {}" is missing!) - //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; - EXPECT_TRUE_sv( array[0].real() == 0 ); - EXPECT_TRUE_sv( array[0].imag() == 0 ); - EXPECT_TRUE_sv( array[1].real() == 0 ); - EXPECT_TRUE_sv( array[1].imag() == 0 ); - } - - // Alternative array initialization for cxtype_sv array (example: was used for outwf in testxxx.cc) - { - cxtype_sv array[2]{}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "{}" is missing!) - //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; - EXPECT_TRUE_sv( array[0].real() == 0 ); - EXPECT_TRUE_sv( array[0].imag() == 0 ); - EXPECT_TRUE_sv( array[1].real() == 0 ); - EXPECT_TRUE_sv( array[1].imag() == 0 ); - } - - //-------------------------------------------------------------------------- - - // Scalar complex references - { - using namespace mgOnGpu; - // Refs to f1, f2 - fptype f1 = 1; - fptype f2 = 2; - cxtype_ref r12( f1, f2 ); // copy refs - //cxtype_ref r12a( r12 ); //deleted - cxtype_ref r12a( cxtype_ref( f1, f2 ) ); // copy refs - //cxtype_ref r12b = r12; // deleted - cxtype_ref r12b = cxtype_ref( f1, f2 ); // copy refs - EXPECT_TRUE( cxtype( r12 ).real() == 1 ); - EXPECT_TRUE( cxtype( r12 ).imag() == 2 ); - EXPECT_TRUE( cxtype( r12a ).real() == 1 ); - EXPECT_TRUE( cxtype( r12a ).imag() == 2 ); - EXPECT_TRUE( cxtype( r12b ).real() == 1 ); - EXPECT_TRUE( cxtype( r12b ).imag() == 2 ); - // Refs to f1c, f2c - fptype f1c = 0; - fptype f2c = 0; - cxtype_ref r12c( f1c, f2c ); - EXPECT_TRUE( cxtype( r12c ).real() == 0 ); - EXPECT_TRUE( cxtype( r12c ).imag() == 0 ); - //r12c = r12; // deleted - r12c = cxtype( r12 ); // copy values - EXPECT_TRUE( cxtype( r12c ).real() == 1 ); - EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); - // Update f1, f2 - f1 = 10; - f2 = 20; - EXPECT_TRUE( cxtype( r12 ).real() == 10 ); - EXPECT_TRUE( cxtype( r12 ).imag() == 20 ); - EXPECT_TRUE( cxtype( r12a ).real() == 10 ); - EXPECT_TRUE( cxtype( r12a ).imag() == 20 ); - EXPECT_TRUE( cxtype( r12b ).real() == 10 ); - EXPECT_TRUE( cxtype( r12b ).imag() == 20 ); - EXPECT_TRUE( cxtype( r12c ).real() == 1 ); // points to f1c, not to f1 - EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); // points to f2c, not to f2 - } - - // Vector complex references - { - using namespace mgOnGpu; - // Refs to f1, f2 - fptype_sv f1 = fptype_sv{ 0 } + 1; - fptype_sv f2 = fptype_sv{ 0 } + 2; - cxtype_sv_ref r12( f1, f2 ); // copy refs - //cxtype_sv_ref r12a( r12 ); //deleted - cxtype_sv_ref r12a( cxtype_sv_ref( f1, f2 ) ); // copy refs - //cxtype_sv_ref r12b = r12; // deleted - cxtype_sv_ref r12b = cxtype_sv_ref( f1, f2 ); // copy refs - EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 2 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 2 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 2 ); - // Refs to f1c, f2c - fptype_sv f1c = fptype_sv{ 0 }; - fptype_sv f2c = fptype_sv{ 0 }; - cxtype_sv_ref r12c( f1c, f2c ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 0 ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 0 ); - //r12c = r12; // deleted - r12c = cxtype_sv( r12 ); // copy values - EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); - // Update f1, f2 - f1 = fptype_sv{ 0 } + 10; - f2 = fptype_sv{ 0 } + 20; - EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 10 ); - EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 20 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 10 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 20 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 10 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 20 ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); // points to f1c, not to f1 - EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); // points to f2c, not to f2 - } - - //-------------------------------------------------------------------------- - - // Boolean vector (mask) times FP vector - /* - // From https://github.com/madgraph5/madgraph4gpu/issues/765#issuecomment-1853672838 - channelids_sv = CHANNEL_ACCESS::kernelAccess( pchannelIds ); // the 4 channels in the SIMD vector - bool_sv mask_sv = ( channelids_sv == 1 ); - numerators_sv += mask_sv * cxabs2( amp_sv[0] ); - if( pchannelIds != nullptr ) denominators_sv += cxabs2( amp_sv[0] ); - */ - { - typedef bool_sv test_int_sv; // defined as scalar_or_vector of long int (FPTYPE=double) or int (FPTYPE=float) - test_int_sv channelids0_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) - test_int_sv channelids1_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) - fptype_sv absamp0_sv{}; // mimic cxabs2( amp_sv[0] ) - fptype_sv absamp1_sv{}; // mimic cxabs2( amp_sv[0] ) - channelids0_sv = 0; - channelids1_sv = 1; - absamp0_sv = 10.; - absamp1_sv = 11.; - bool_sv mask0_sv = ( channelids0_sv % 2 == 0 ); // even channels 0123 -> TFTF (1010) - bool_sv mask1_sv = ( channelids1_sv % 2 == 0 ); // even channels 1234 -> FTFT (0101) - constexpr fptype_sv fpZERO_sv{}; // 0000 - //fptype_sv numerators0_sv = mask0_sv * absamp0_sv; // invalid operands to binary * ('__vector(4) long int' and '__vector(4) double') - fptype_sv numerators0_sv = fpternary( mask0_sv, absamp0_sv, fpZERO_sv ); // equivalent to "mask0_sv * absamp0_sv" - fptype_sv numerators1_sv = fpternary( mask1_sv, absamp1_sv, fpZERO_sv ); // equivalent to "mask1_sv * absamp1_sv" - // Values of numerators0_sv: 10.*1 - EXPECT_TRUE( numerators0_sv == 10. ); - // Values of numerators1_sv: 11.*0 - EXPECT_TRUE( numerators1_sv == 0. ); - } - - //-------------------------------------------------------------------------- - - // Test constexpr floor - EXPECT_TRUE( constexpr_floor( 1.5 ) == 1 ); - EXPECT_TRUE( constexpr_floor( 0.5 ) == 0 ); - EXPECT_TRUE( constexpr_floor( -0.5 ) == -1 ); - EXPECT_TRUE( constexpr_floor( -1.5 ) == -2 ); - - // Test constexpr pow - EXPECT_TRUE( constexpr_pow( 10, 0 ) == 1 ); - EXPECT_TRUE( constexpr_pow( 10, 1 ) == 10 ); - EXPECT_TRUE( constexpr_pow( 10, 2 ) == 100 ); - EXPECT_NEAR( constexpr_pow( 10, -1 ), 0.1, 0.1 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10, -1 ) = " << constexpr_pow( 10, -1 ); - EXPECT_NEAR( constexpr_pow( 10, -2 ), 0.01, 0.01 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10, -2 ) = " << constexpr_pow( 10, -2 ); - EXPECT_NEAR( constexpr_pow( 100, 0.5 ), 10, 10 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 100, 0.5 ) = " << constexpr_pow( 100, 0.5 ); - EXPECT_NEAR( constexpr_pow( 100, -0.5 ), 0.1, 0.1 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 100, -0.5 ) = " << constexpr_pow( 100, -0.5 ); - EXPECT_NEAR( constexpr_pow( 10000, 0.25 ), 10, 10 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10000, 0.25 ) = " << constexpr_pow( 10000, 0.25 ); - EXPECT_NEAR( constexpr_pow( 10000, -0.25 ), 0.1, 0.1 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10000, -0.25 ) = " << constexpr_pow( 10000, -0.25 ); - -#ifndef __aarch64__ // TO BE UNDERSTOOD? DISABLE CONSTEXPR_SQRT TESTS ON AARCH64 (#1064) - // Distance from the horizontal or vertical axis (i.e. from 0, pi/2, pi, or 3pi/2) - auto distance4 = []( const long double xx ) - { - const long double xx2 = mapIn0to2Pi( xx ); // in [0,2*pi) - const long double xx3 = xx2 - constexpr_floor( xx2 / constexpr_pi_by_2 ) * constexpr_pi_by_2; // in [0,pi/2) - const long double d0 = xx3; // distance from 0 - const long double d1 = constexpr_pi_by_2 - xx3; // distance from pi/2 - return ( d0 < d1 ? d0 : d1 ); - }; - - // Test constexpr sin, cos, tan - specific, problematic, points - auto testSinCosTanX = []( const long double xx, const double tolerance0, const bool debug = false, const long long istep = -999999999 ) - { - const double x = (double)xx; - const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 - if( debug ) - { - //std::cout << std::setprecision(40) << "testSinCosTanX: xx= " << xx << std::endl; - //std::cout << std::setprecision(40) << " x= " << x << std::endl; - } - //std::cout << std::setprecision(40) << "xx - 3pi/2 " << xx - 3 * constexpr_pi_by_2 << std::endl; - //int width = 46; - //char buf[128]; - //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)xx ); - //std::cout << std::setprecision(40) << "testSinCosTanX: xx=" << buf << std::endl; - //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)x ); - //std::cout << std::setprecision(40) << " x= " << buf << std::endl; - EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::abs( std::sin( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::abs( std::cos( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; -#ifndef __aarch64__ - if( !RUNNING_ON_VALGRIND ) - { - EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::abs( std::tan( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - } - else -#endif - { - // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) - const long double ctanx = constexpr_tan( x ); - const long double taninf = 4E14; // declare tan(x) as "infinity" if above this threshold - if( ctanx > -taninf && ctanx < taninf ) - EXPECT_NEAR( std::tan( x ), ctanx, std::abs( std::tan( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - else - { - // Allow tan(x)=-inf if ctan(x)=+inf and viceversa - EXPECT_GT( std::abs( std::tan( x ) ), taninf ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - /* - // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) - if( ctanx > 0 ) - EXPECT_GT( std::tan( x ), taninf ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - else - EXPECT_LT( std::tan( x ), -taninf ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - */ - } - } - std::cout << std::setprecision( 6 ); // default - }; - testSinCosTanX( constexpr_pi, 1E-3, true ); // from math.h - testSinCosTanX( (long double)3.141592653589793238462643383279502884L, 1E-3, true ); // from math.h - testSinCosTanX( 4.712388980384687897640105802565813064575L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) - testSinCosTanX( 3 * constexpr_pi_by_2 - 1.96e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x!=xx) - testSinCosTanX( 3 * constexpr_pi_by_2 - 1.9601e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) - - // Test constexpr sin, cos, tan - 8 points on (or close to) the boundaries of the 8 sectors of [0,2*pi] - auto testSinCosTan8 = [testSinCosTanX]( const double deltax, const double tolerance ) - { - for( int ioff = -1; ioff < 2; ioff++, ioff++ ) // -1, 1 - { - const bool debug = false; - const int nstep = 8; - for( int istep = 0; istep < nstep + 1; istep++ ) - { - long double x0 = deltax * ioff; - long double x1 = deltax * ioff + 2 * constexpr_pi; - double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) - testSinCosTanX( x, tolerance, debug, istep ); - } - } - }; - - // Use much lower tolerance when testing on the boundaries of the 8 sectors of [0,2*pi] - // Use progressively stricter tolerances as you move away from the boundaries of the 8 sectors of [0,2*pi] - testSinCosTan8( 0, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... - testSinCosTan8( 1E-15, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... - testSinCosTan8( 1E-14, 1E-04 ); // fails with 1E-05 - testSinCosTan8( 1E-12, 1E-06 ); // fails with 1E-07 - testSinCosTan8( 1E-09, 1E-09 ); // fails with 1E-10 - testSinCosTan8( 1E-06, 1E-12 ); // fails with 1E-13 - testSinCosTan8( 1E-03, 1E-14 ); // fails with 1E-16: could use 1E-14 but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) - testSinCosTan8( 1E-02, 1E-14 ); // never fails? could use 1E-99(?) but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) - - // Test constexpr sin, cos, tan - N points almost randomly with a varying tolerance - auto testSinCosTanN = [distance4]( const int nstep, const double x0, const double x1 ) - { - auto toleranceForX = [distance4]( const double x ) - { - const double d4 = distance4( x ); - if( d4 < 1E-14 ) - return 1E-03; // NB: absolute distance limited to 1E-14 anyway even if relative tolerance is 1E-3... - else if( d4 < 1E-13 ) - return 1E-04; - else if( d4 < 1E-12 ) - return 1E-05; - else if( d4 < 1E-11 ) - return 1E-06; - else if( d4 < 1E-10 ) - return 1E-07; - else if( d4 < 1E-09 ) - return 1E-08; - else if( d4 < 1E-08 ) - return 1E-09; - else if( d4 < 1E-07 ) - return 1E-10; - else if( d4 < 1E-06 ) - return 1E-11; - else if( d4 < 1E-05 ) - return 1E-12; - else if( d4 < 1E-04 ) - return 1E-13; - else - return 1E-14; // play it safe even if the agreement might even be better? - }; - for( int istep = 0; istep < nstep + 1; istep++ ) - { - double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) - const double tolerance0 = toleranceForX( x ); - const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 - EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::max( std::abs( std::sin( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::max( std::abs( std::cos( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); -#ifndef __aarch64__ - if( !RUNNING_ON_VALGRIND ) - { - EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - } - else -#endif - { - // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) - const long double ctanx = constexpr_tan( x ); - const long double taninf = 4E14; // declare tan(x) as "infinity if above this threshold - if( ctanx > -taninf && ctanx < taninf ) - EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - else - { - // Allow tan(x)=-inf if ctan(x)=+inf and viceversa - EXPECT_GT( std::abs( std::tan( x ) ), taninf ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - /* - // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) - if( ctanx > 0 ) - EXPECT_GT( std::tan( x ), taninf ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - else - EXPECT_LT( std::tan( x ), -taninf ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - */ - } - } - } - }; - testSinCosTanN( 100, -4 * constexpr_pi, 6 * constexpr_pi ); // this was failing at 3*pi/2 (now fixed by absolute tolerance 3E-15) - testSinCosTanN( 10000, -constexpr_pi_by_2, 5 * constexpr_pi_by_2 ); - - // Test constexpr atan - { - const double tolerance = 1E-12; - const int nstep = 1000; - for( int istep = 0; istep < nstep + 1; istep++ ) - { - long double x0 = -5, x1 = +5; - double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) - EXPECT_NEAR( std::atan( x ), constexpr_atan( x ), std::abs( std::atan( x ) * tolerance ) ) - << "x=" << x << ", istep=" << istep; - } - } -#endif - //-------------------------------------------------------------------------- -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc deleted file mode 100644 index 14c1ceeaf5..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CommonRandomNumberKernel.cc +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "CommonRandomNumbers.h" -#include "GpuAbstraction.h" -#include "MemoryBuffers.h" -#include "RandomNumberKernels.h" - -#include - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- - - CommonRandomNumberKernel::CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ) - : RandomNumberKernelBase( rnarray ) - , m_seed( 20211220 ) - { - if( m_rnarray.isOnDevice() ) - throw std::runtime_error( "CommonRandomNumberKernel on host with a device random number array" ); - } - - //-------------------------------------------------------------------------- - - void CommonRandomNumberKernel::generateRnarray() - { - std::vector rnd = CommonRandomNumbers::generate( m_rnarray.size(), m_seed ); // NB: generate as double (HARDCODED) - std::copy( rnd.begin(), rnd.end(), m_rnarray.data() ); // NB: copy may imply a double-to-float conversion - } - - //-------------------------------------------------------------------------- -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc deleted file mode 100644 index 24ee78772b..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CurandRandomNumberKernel.cc +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "GpuRuntime.h" -#include "MemoryBuffers.h" -#include "RandomNumberKernels.h" - -#include - -#ifndef MGONGPU_HAS_NO_CURAND /* clang-format off */ -// NB This must come AFTER mgOnGpuConfig.h which contains our definition of __global__ when MGONGPUCPP_GPUIMPL is not defined -#include "curand.h" -#define checkCurand( code ){ assertCurand( code, __FILE__, __LINE__ ); } -inline void assertCurand( curandStatus_t code, const char *file, int line, bool abort = true ) -{ - if ( code != CURAND_STATUS_SUCCESS ) - { - printf( "CurandAssert: %s:%d code=%d\n", file, line, code ); - if ( abort ) assert( code == CURAND_STATUS_SUCCESS ); - } -} -#endif /* clang-format on */ - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- -#ifndef MGONGPU_HAS_NO_CURAND - CurandRandomNumberKernel::CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) - : RandomNumberKernelBase( rnarray ) - , m_isOnDevice( onDevice ) - { - if( m_isOnDevice ) - { - throw std::runtime_error( "CurandRandomNumberKernel does not support CurandDevice on CPU host" ); - } - else - { - if( m_rnarray.isOnDevice() ) - throw std::runtime_error( "CurandRandomNumberKernel on host with a device random number array" ); - } - createGenerator(); - } - - //-------------------------------------------------------------------------- - - CurandRandomNumberKernel::~CurandRandomNumberKernel() - { - destroyGenerator(); - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::seedGenerator( const unsigned int seed ) - { - if( m_isOnDevice ) - { - destroyGenerator(); // workaround for #429 - createGenerator(); // workaround for #429 - } - //printf( "seedGenerator: seed %d\n", seed ); - checkCurand( curandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::createGenerator() - { - // [NB Timings are for GenRnGen host|device (cpp|cuda) generation of 256*32*1 events with nproc=1: rn(0) is host=0.0012s] - const curandRngType_t type = CURAND_RNG_PSEUDO_MTGP32; // 0.00082s | 0.00064s (FOR FAST TESTS) - //const curandRngType_t type = CURAND_RNG_PSEUDO_XORWOW; // 0.049s | 0.0016s - //const curandRngType_t type = CURAND_RNG_PSEUDO_MRG32K3A; // 0.71s | 0.0012s (better but slower, especially in c++) - //const curandRngType_t type = CURAND_RNG_PSEUDO_MT19937; // 21s | 0.021s - //const curandRngType_t type = CURAND_RNG_PSEUDO_PHILOX4_32_10; // 0.024s | 0.00026s (used to segfault?) - if( m_isOnDevice ) - { - checkCurand( curandCreateGenerator( &m_rnGen, type ) ); - } - else - { - checkCurand( curandCreateGeneratorHost( &m_rnGen, type ) ); - } - //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_LEGACY ) ); // fails with code=104 (see #429) - checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_BEST ) ); - //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_DYNAMIC ) ); // fails with code=104 (see #429) - //checkCurand( curandSetGeneratorOrdering( *&m_rnGen, CURAND_ORDERING_PSEUDO_SEEDED ) ); // fails with code=104 (see #429) - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::destroyGenerator() - { - checkCurand( curandDestroyGenerator( m_rnGen ) ); - } - - //-------------------------------------------------------------------------- - - void CurandRandomNumberKernel::generateRnarray() - { -#if defined MGONGPU_FPTYPE_DOUBLE - checkCurand( curandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#elif defined MGONGPU_FPTYPE_FLOAT - checkCurand( curandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#endif - /* - printf( "\nCurandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); - fptype* data = m_rnarray.data(); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) - { - data = new fptype[m_rnarray.size()](); - checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); - } -#endif - for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) - printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) delete[] data; -#endif - */ - } - - //-------------------------------------------------------------------------- -#endif -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc deleted file mode 100644 index 2fd64a2dce..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/HiprandRandomNumberKernel.cc +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2024) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "mgOnGpuConfig.h" - -#include "GpuRuntime.h" -#include "MemoryBuffers.h" -#include "RandomNumberKernels.h" - -#include - -#ifndef MGONGPU_HAS_NO_HIPRAND /* clang-format off */ -#ifndef __HIP_PLATFORM_AMD__ -#define __HIP_PLATFORM_AMD__ 1 // enable hiprand for AMD (rocrand) -#endif -#include -#define checkHiprand( code ){ assertHiprand( code, __FILE__, __LINE__ ); } -inline void assertHiprand( hiprandStatus_t code, const char *file, int line, bool abort = true ) -{ - if ( code != HIPRAND_STATUS_SUCCESS ) - { - printf( "HiprandAssert: %s:%d code=%d\n", file, line, code ); - if ( abort ) assert( code == HIPRAND_STATUS_SUCCESS ); - } -} -#endif /* clang-format on */ - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- -#ifndef MGONGPU_HAS_NO_HIPRAND - HiprandRandomNumberKernel::HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ) - : RandomNumberKernelBase( rnarray ) - , m_isOnDevice( onDevice ) - { - if( m_isOnDevice ) - { - throw std::runtime_error( "HiprandRandomNumberKernel does not support HiprandDevice on CPU host" ); - } - else - { - if( m_rnarray.isOnDevice() ) - throw std::runtime_error( "HiprandRandomNumberKernel on host with a device random number array" ); - } - createGenerator(); - } - - //-------------------------------------------------------------------------- - - HiprandRandomNumberKernel::~HiprandRandomNumberKernel() - { - destroyGenerator(); - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::seedGenerator( const unsigned int seed ) - { - if( m_isOnDevice ) - { - destroyGenerator(); // workaround for #429 - createGenerator(); // workaround for #429 - } - //printf( "seedGenerator: seed %d\n", seed ); - checkHiprand( hiprandSetPseudoRandomGeneratorSeed( m_rnGen, seed ) ); - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::createGenerator() - { - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_DEFAULT; - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_XORWOW; - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MRG32K3A; - const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MTGP32; // same as curand; not implemented yet (code=1000) in host code - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_MT19937; - //const hiprandRngType_t type = HIPRAND_RNG_PSEUDO_PHILOX4_32_10; - if( m_isOnDevice ) - { - checkHiprand( hiprandCreateGenerator( &m_rnGen, type ) ); - } - else - { - // See https://github.com/ROCm/hipRAND/issues/76 - throw std::runtime_error( "HiprandRandomNumberKernel on host is not supported yet (hiprandCreateGeneratorHost is not implemented yet)" ); - //checkHiprand( hiprandCreateGeneratorHost( &m_rnGen, type ) ); // ALWAYS FAILS WITH CODE=1000 - } - // FIXME: hiprand ordering is not implemented yet - // See https://github.com/ROCm/hipRAND/issues/75 - /* - //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_LEGACY ) ); - checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_BEST ) ); - //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_DYNAMIC ) ); - //checkHiprand( hiprandSetGeneratorOrdering( *&m_rnGen, HIPRAND_ORDERING_PSEUDO_SEEDED ) ); - */ - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::destroyGenerator() - { - checkHiprand( hiprandDestroyGenerator( m_rnGen ) ); - } - - //-------------------------------------------------------------------------- - - void HiprandRandomNumberKernel::generateRnarray() - { -#if defined MGONGPU_FPTYPE_DOUBLE - checkHiprand( hiprandGenerateUniformDouble( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#elif defined MGONGPU_FPTYPE_FLOAT - checkHiprand( hiprandGenerateUniform( m_rnGen, m_rnarray.data(), m_rnarray.size() ) ); -#endif - /* - printf( "\nHiprandRandomNumberKernel::generateRnarray size = %d\n", (int)m_rnarray.size() ); - fptype* data = m_rnarray.data(); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) - { - data = new fptype[m_rnarray.size()](); - checkCuda( cudaMemcpy( data, m_rnarray.data(), m_rnarray.bytes(), cudaMemcpyDeviceToHost ) ); - } -#endif - for( int i = 0; i < ( (int)m_rnarray.size() / 4 ); i++ ) - printf( "[%4d] %f %f %f %f\n", i * 4, data[i * 4], data[i * 4 + 2], data[i * 4 + 2], data[i * 4 + 3] ); -#ifdef MGONGPUCPP_GPUIMPL - if( m_rnarray.isOnDevice() ) delete[] data; -#endif - */ - } - - //-------------------------------------------------------------------------- -#endif -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h deleted file mode 100644 index 641917d8ff..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MadgraphTest.h +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Hageboeck (Dec 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, J. Teig, A. Valassi (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MADGRAPHTEST_H_ -#define MADGRAPHTEST_H_ 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" - -#include - -#include -#include -//#ifdef __HIPCC__ -//#include // see https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 -//#else -//#include // bypass this completely to ease portability on LUMI #803 -//#endif -#include -#include -#include -#include -#include -#include - -using mg5amcCpu::CPPProcess; - -namespace -{ - struct ReferenceData - { - std::vector>> momenta; - std::vector MEs; - std::vector ChanIds; - std::vector SelHels; - std::vector SelCols; - }; - - /// Read batches of reference data from a file and store them in a map. - std::map readReferenceData( const std::string& refFileName ) - { - std::cout << "INFO: Opening reference file " << refFileName << std::endl; - std::ifstream referenceFile( refFileName.c_str() ); - EXPECT_TRUE( referenceFile.is_open() ) << refFileName; - std::map referenceData; - unsigned int evtNo; - unsigned int batchNo; - for( std::string line; std::getline( referenceFile, line ); ) - { - std::stringstream lineStr( line ); - if( line.empty() || line[0] == '#' ) - { - continue; - } - else if( line.find( "Event" ) != std::string::npos ) - { - std::string dummy; - lineStr >> dummy >> evtNo >> dummy >> batchNo; - } - else if( line.find( "ME" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].MEs.size() ) - referenceData[batchNo].MEs.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].MEs[evtNo]; - } - else if( line.find( "ChanId" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].ChanIds.size() ) - referenceData[batchNo].ChanIds.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].ChanIds[evtNo]; - referenceData[batchNo].ChanIds[evtNo] = 0; // disable ChanId comparison if multichannel is not supported #976 - } - else if( line.find( "SelHel" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].SelHels.size() ) - referenceData[batchNo].SelHels.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].SelHels[evtNo]; - } - else if( line.find( "SelCol" ) != std::string::npos ) - { - if( evtNo <= referenceData[batchNo].SelCols.size() ) - referenceData[batchNo].SelCols.resize( evtNo + 1 ); - std::string dummy; - lineStr >> dummy >> referenceData[batchNo].SelCols[evtNo]; - } - else - { - unsigned int particleIndex; - lineStr >> particleIndex; - if( evtNo <= referenceData[batchNo].momenta.size() ) - referenceData[batchNo].momenta.resize( evtNo + 1 ); - if( particleIndex <= referenceData[batchNo].momenta[evtNo].size() ) - referenceData[batchNo].momenta[evtNo].resize( particleIndex + 1 ); - auto& fourVec = referenceData[batchNo].momenta[evtNo][particleIndex]; - for( unsigned int i = 0; i < fourVec.size(); ++i ) - { - EXPECT_TRUE( lineStr.good() ); - lineStr >> fourVec[i]; - } - EXPECT_TRUE( lineStr.eof() ); - } - } - return referenceData; - } - -} - -/** - * Test driver providing a common interface for testing different implementations. - * Users need to implement: - * - Functions to retrieve matrix element and 4-momenta. These are used in the tests. - * - Driver functions that run the madgraph workflow. - */ -class TestDriverBase -{ - std::string m_refFileName; -public: - const unsigned int nparticle; - static constexpr unsigned int niter = 2; - static constexpr unsigned int gpublocks = 2; - static constexpr unsigned int gputhreads = 128; - static constexpr unsigned int nevt = gpublocks * gputhreads; - - TestDriverBase( unsigned int npart, const std::string& refFileName ) - : m_refFileName( refFileName ) - , nparticle( npart ) - { - } - TestDriverBase() = delete; - virtual ~TestDriverBase() {} - const std::string& getRefFileName() { return m_refFileName; } - - // ------------------------------------------------ - // Interface for retrieving info from madgraph - // ------------------------------------------------ - virtual fptype getMomentum( std::size_t evtNo, unsigned int particleNo, unsigned int component ) const = 0; - virtual fptype getMatrixElement( std::size_t evtNo ) const = 0; - virtual int getChannelId( std::size_t ievt ) const = 0; - virtual int getSelectedHelicity( std::size_t ievt ) const = 0; - virtual int getSelectedColor( std::size_t ievt ) const = 0; - - // ------------------------------------------------ - // Interface for steering madgraph run - // ------------------------------------------------ - virtual void prepareRandomNumbers( unsigned int iiter ) = 0; - virtual void prepareMomenta( fptype energy ) = 0; - virtual void runSigmaKin( std::size_t iiter ) = 0; - - /// Print the requested event into the stream. If the reference data has enough events, it will be printed as well. - void dumpParticles( std::ostream& stream, std::size_t ievt, unsigned int numParticles, unsigned int nDigit, const ReferenceData& referenceData ) const - { - const auto width = nDigit + 8; - for( unsigned int ipar = 0; ipar < numParticles; ipar++ ) - { - // NB: 'setw' affects only the next field (of any type) - stream << std::scientific // fixed format: affects all floats (default nDigit: 6) - << std::setprecision( nDigit ) - << std::setw( 4 ) << ipar - << std::setw( width ) << getMomentum( ievt, ipar, 0 ) - << std::setw( width ) << getMomentum( ievt, ipar, 1 ) - << std::setw( width ) << getMomentum( ievt, ipar, 2 ) - << std::setw( width ) << getMomentum( ievt, ipar, 3 ) - << "\n"; - if( ievt < referenceData.momenta.size() ) - { - stream << "ref" << ipar; - stream << std::setw( width ) << referenceData.momenta[ievt][ipar][0] - << std::setw( width ) << referenceData.momenta[ievt][ipar][1] - << std::setw( width ) << referenceData.momenta[ievt][ipar][2] - << std::setw( width ) << referenceData.momenta[ievt][ipar][3] - << "\n\n"; - } - stream << std::flush << std::defaultfloat; // default format: affects all floats - } - } -}; - -/** - * Test class that's defining all tests to run with a Madgraph workflow. - */ -class MadgraphTest -{ -public: - MadgraphTest( TestDriverBase& testDriverRef ) - : testDriver( &testDriverRef ) {} - ~MadgraphTest() {} - void CompareMomentaAndME( testing::Test& googleTest ) const; // NB: googleTest is ONLY needed for the HasFailure method... -private: - TestDriverBase* testDriver; // non-owning pointer -}; - -void -MadgraphTest::CompareMomentaAndME( testing::Test& googleTest ) const -{ - const fptype toleranceMomenta = std::is_same::value ? 1.E-10 : 4.E-2; // see #735 -#ifdef __APPLE__ - const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-2; // see #583 -#else - //const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 2.E-3; // fails smeft/hip #843 - const fptype toleranceMEs = std::is_same::value ? 1.E-6 : 3.E-3; -#endif - constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) - // Dump events to a new reference file? - const char* dumpEventsC = getenv( "CUDACPP_RUNTEST_DUMPEVENTS" ); - const bool dumpEvents = ( dumpEventsC != 0 ) && ( std::string( dumpEventsC ) != "" ); - const std::string refFileName = testDriver->getRefFileName(); - /* -#ifdef __HIPCC__ - const std::string dumpFileName = std::experimental::filesystem::path( refFileName ).filename(); -#else - const std::string dumpFileName = std::filesystem::path( refFileName ).filename(); -#endif - */ - const std::string dumpFileName = refFileName; // bypass std::filesystem #803 - std::ofstream dumpFile; - if( dumpEvents ) - { - dumpFile.open( dumpFileName, std::ios::trunc ); - } - // Read reference data - std::map referenceData; - if( !dumpEvents ) - { - referenceData = readReferenceData( refFileName ); - } - ASSERT_FALSE( googleTest.HasFailure() ); // It doesn't make any sense to continue if we couldn't read the reference file. - // ************************************** - // *** START MAIN LOOP ON #ITERATIONS *** - // ************************************** - for( unsigned int iiter = 0; iiter < testDriver->niter; ++iiter ) - { - testDriver->prepareRandomNumbers( iiter ); - testDriver->prepareMomenta( energy ); - testDriver->runSigmaKin( iiter ); - // --- Run checks on all events produced in this iteration - for( std::size_t ievt = 0; ievt < testDriver->nevt && !googleTest.HasFailure(); ++ievt ) - { - if( dumpEvents ) - { - ASSERT_TRUE( dumpFile.is_open() ) << dumpFileName; - dumpFile << "Event " << std::setw( 8 ) << ievt << " " - << "Batch " << std::setw( 4 ) << iiter << "\n"; - testDriver->dumpParticles( dumpFile, ievt, testDriver->nparticle, 15, ReferenceData() ); - // Dump matrix element - dumpFile << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) - << testDriver->getMatrixElement( ievt ) << "\n" - << std::defaultfloat; - // Dump channelId - dumpFile << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n"; - // Dump selected helicity and color - dumpFile << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n"; - dumpFile << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" - << std::endl; // leave one line between events - continue; - } - // Check that we have the required reference data - ASSERT_GT( referenceData.size(), iiter ) - << "Don't have enough reference data for iteration " << iiter << ". Ref file:" << refFileName; - ASSERT_GT( referenceData[iiter].MEs.size(), ievt ) - << "Don't have enough reference MEs for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].ChanIds.size(), ievt ) - << "Don't have enough reference ChanIds for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].SelHels.size(), ievt ) - << "Don't have enough reference SelHels for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].SelCols.size(), ievt ) - << "Don't have enough reference SelCols for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GT( referenceData[iiter].momenta.size(), ievt ) - << "Don't have enough reference momenta for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - ASSERT_GE( referenceData[iiter].momenta[ievt].size(), testDriver->nparticle ) - << "Don't have enough reference particles for iteration " << iiter << " event " << ievt << ".\nRef file: " << refFileName; - // This trace will help to understand the event that is being checked. - // It will only be printed in case of failures: - std::stringstream eventTrace; - eventTrace << "In comparing event " << ievt << " from iteration " << iiter << "\n"; - testDriver->dumpParticles( eventTrace, ievt, testDriver->nparticle, 15, referenceData[iiter] ); - eventTrace << std::setw( 4 ) << "ME" << std::scientific << std::setw( 15 + 8 ) - << testDriver->getMatrixElement( ievt ) << "\n" - << std::setw( 4 ) << "r.ME" << std::scientific << std::setw( 15 + 8 ) - << referenceData[iiter].MEs[ievt] << std::endl - << std::defaultfloat; - eventTrace << std::setw( 8 ) << "ChanId" << std::setw( 8 ) << testDriver->getChannelId( ievt ) << "\n" - << std::setw( 8 ) << "r.ChanId" << std::setw( 8 ) << referenceData[iiter].ChanIds[ievt] << std::endl; - eventTrace << std::setw( 8 ) << "SelHel" << std::setw( 8 ) << testDriver->getSelectedHelicity( ievt ) << "\n" - << std::setw( 8 ) << "r.SelHel" << std::setw( 8 ) << referenceData[iiter].SelHels[ievt] << std::endl; - eventTrace << std::setw( 8 ) << "SelCol" << std::setw( 8 ) << testDriver->getSelectedColor( ievt ) << "\n" - << std::setw( 8 ) << "r.SelCol" << std::setw( 8 ) << referenceData[iiter].SelCols[ievt] << std::endl; - SCOPED_TRACE( eventTrace.str() ); - // Compare Momenta - for( unsigned int ipar = 0; ipar < testDriver->nparticle; ++ipar ) - { - std::stringstream momentumErrors; - for( unsigned int icomp = 0; icomp < CPPProcess::np4; ++icomp ) - { - const fptype pMadg = testDriver->getMomentum( ievt, ipar, icomp ); - const fptype pOrig = referenceData[iiter].momenta[ievt][ipar][icomp]; - //const fptype relDelta = fabs( ( pMadg - pOrig ) / pOrig ); // computing relDelta may lead to FPEs - const fptype delta = fabs( pMadg - pOrig ); - if( delta > toleranceMomenta * fabs( pOrig ) ) // better than "relDelta > toleranceMomenta" - { - momentumErrors << std::setprecision( 15 ) << std::scientific << "\nparticle " << ipar << "\tcomponent " << icomp - << "\n\t madGraph: " << std::setw( 22 ) << pMadg - << "\n\t reference: " << std::setw( 22 ) << pOrig - << "\n\t relative delta exceeds tolerance of " << toleranceMomenta; - } - } - ASSERT_TRUE( momentumErrors.str().empty() ) << momentumErrors.str(); - } - // Compare ME: - EXPECT_NEAR( testDriver->getMatrixElement( ievt ), - referenceData[iiter].MEs[ievt], - toleranceMEs * referenceData[iiter].MEs[ievt] ); - // Compare channelId - EXPECT_EQ( testDriver->getChannelId( ievt ), - referenceData[iiter].ChanIds[ievt] ); - // Compare selected helicity and color - EXPECT_EQ( testDriver->getSelectedHelicity( ievt ), - referenceData[iiter].SelHels[ievt] ); - EXPECT_EQ( testDriver->getSelectedColor( ievt ), - referenceData[iiter].SelCols[ievt] ); - } - } - if( dumpEvents ) - { - std::cout << "Event dump written to " << dumpFileName << std::endl; - } -} - -#endif /* MADGRAPHTEST_H_ */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h index 0dd7244541..4d7edc9f0d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h @@ -14,7 +14,6 @@ #include "ProcessData.h" #include "GpuRuntime.h" #include "Parameters.h" -#include "processConfig.h" #include @@ -204,7 +203,7 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for numerators // (should be equal to the number of diagrams in the process) - constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; + constexpr size_t sizePerEventNumerators = ProcessData::ndiagrams; // A class encapsulating a C++ host buffer for numerators typedef HostBuffer HostBufferNumerators; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc deleted file mode 100644 index e1e17d6247..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.cc +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "RamboSamplingKernels.h" - -#include "GpuRuntime.h" -#include "MemoryAccessMomenta.h" -#include "MemoryAccessRandomNumbers.h" -#include "MemoryAccessWeights.h" -#include "MemoryBuffers.h" -#include "rambo.h" // inline classic (massive) RAMBO, ported from standalone_cpp -#include "massless_rambo.h" // inline implementation of massless RAMBO algorithms and kernels - -#include - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- - - MasslessRamboSamplingKernelHost::MasslessRamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t nevt ) - : SamplingKernelBase( energy, rndmom, momenta, weights ) - , NumberOfEvents( nevt ) - { - if( m_rndmom.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: rndmom must be a host array" ); - if( m_momenta.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: momenta must be a host array" ); - if( m_weights.isOnDevice() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: weights must be a host array" ); - if( this->nevt() != m_rndmom.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with rndmom" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with momenta" ); - if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "MasslessRamboSamplingKernelHost: nevt mismatch with weights" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( nevt % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Sanity checks for memory access (random number buffer) - constexpr int neppR = MemoryAccessRandomNumbers::neppR; // AOSOA layout - static_assert( ispoweroftwo( neppR ), "neppR is not a power of 2" ); - if( nevt % neppR != 0 ) - { - std::ostringstream sstr; - sstr << "MasslessRamboSamplingKernelHost: nevt should be a multiple of neppR=" << neppR; - throw std::runtime_error( sstr.str() ); - } - } - - //-------------------------------------------------------------------------- - - void - MasslessRamboSamplingKernelHost::getMomentaInitial() - { - constexpr auto getMomentaInitial = massless_rambo::ramboGetMomentaInitial; - // ** START LOOP ON IEVT ** - for( size_t ievt = 0; ievt < nevt(); ++ievt ) - { - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); - getMomentaInitial( m_energy, ievtMomenta ); - } - // ** END LOOP ON IEVT ** - } - - //-------------------------------------------------------------------------- - - void - MasslessRamboSamplingKernelHost::getMomentaFinal() - { - constexpr auto getMomentaFinal = massless_rambo::ramboGetMomentaFinal; - // ** START LOOP ON IEVT ** - for( size_t ievt = 0; ievt < nevt(); ++ievt ) - { - // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" - const fptype* ievtRndmom = MemoryAccessRandomNumbers::ieventAccessRecordConst( m_rndmom.data(), ievt ); - fptype* ievtMomenta = MemoryAccessMomenta::ieventAccessRecord( m_momenta.data(), ievt ); - fptype* ievtWeights = MemoryAccessWeights::ieventAccessRecord( m_weights.data(), ievt ); - getMomentaFinal( m_energy, ievtRndmom, ievtMomenta, ievtWeights ); - } - // ** END LOOP ON IEVT ** - } - - //-------------------------------------------------------------------------- - - RamboSamplingKernelHost::RamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED - const std::vector& masses, // input: external-leg masses - const int ninitial, // input: #initial-state particles - const size_t nevt, // input: #events - BufferMomenta& momenta, // output: momenta - BufferWeights& weights ) // output: weights - : SamplingKernelBase( energy, rndmom, momenta, weights ) - , NumberOfEvents( nevt ) - , m_masses( masses.begin(), masses.end() ) - , m_ninitial( ninitial ) - { - if( m_momenta.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: momenta must be a host array" ); - if( m_weights.isOnDevice() ) throw std::runtime_error( "RamboSamplingKernelHost: weights must be a host array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with momenta" ); - if( this->nevt() != m_weights.nevt() ) throw std::runtime_error( "RamboSamplingKernelHost: nevt mismatch with weights" ); - - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( nevt % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "RamboSamplingKernelHost: nevt should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - } - - //-------------------------------------------------------------------------- - - void - RamboSamplingKernelHost::getMomentaInitial() - { - // NOOP - } - - //-------------------------------------------------------------------------- - - void - RamboSamplingKernelHost::getMomentaFinal() - { - const int npar = (int)m_masses.size(); - // ** START LOOP ON IEVT ** - for( size_t ievt = 0; ievt < nevt(); ++ievt ) - { - // Clas. RAMBO returns [E,px,py,pz] vector per ex. particle - // own RNG, intial final once - // For reproducibility betwn fptype = FP32/FP64 generation in FP64 - double wgt = 0.; - const std::vector> point = - rambo::get_momenta( m_ninitial, (double)m_energy, m_masses, wgt ); - for( int ipar = 0; ipar < npar; ++ipar ) - for( int ip4 = 0; ip4 < 4; ++ip4 ) - MemoryAccessMomenta::ieventAccessIp4Ipar( m_momenta.data(), ievt, ip4, ipar ) = (fptype)point[ipar][ip4]; - MemoryAccessWeights::ieventAccess( m_weights.data(), ievt ) = (fptype)wgt; - } - // ** END LOOP ON IEVT ** - } - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h deleted file mode 100644 index e0aa2aea7f..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/RamboSamplingKernels.h +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef RAMBOSAMPLINGKERNELS_H -#define RAMBOSAMPLINGKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryBuffers.h" - -#include - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- - - // A base class encapsulating phase space sampling on a CPU host or on a GPU device - class SamplingKernelBase //: virtual public ISamplingKernel - { - protected: - - // Constructor from existing input and output buffers - SamplingKernelBase( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights ) // output: weights - : m_energy( energy ) - , m_rndmom( rndmom ) - , m_momenta( momenta ) - , m_weights( weights ) - { - } - - public: - - // Destructor - virtual ~SamplingKernelBase() {} - - // Get momenta of initial state particles - virtual void getMomentaInitial() = 0; - - // Get momenta of final state particles and weights - virtual void getMomentaFinal() = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - protected: - - // The energy - const fptype m_energy; - - // The buffer for the input random numbers - const BufferRndNumMomenta& m_rndmom; - - // The buffer for the output momenta - BufferMomenta& m_momenta; - - // The buffer for the output weights - BufferWeights& m_weights; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating RAMBO phase space sampling on a CPU host - class MasslessRamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MasslessRamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random numbers in [0,1] - BufferMomenta& momenta, // output: momenta - BufferWeights& weights, // output: weights - const size_t nevt ); - - // Destructor - virtual ~MasslessRamboSamplingKernelHost() {} - - // Get momenta of initial state particles - void getMomentaInitial() override final; - - // Get momenta of final state particles and weights - void getMomentaFinal() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - }; - - //-------------------------------------------------------------------------- - - // Compability port mirroring the massless momenta implementation - // For now own RNG internally (or keep to match Fortran) TODO - // rndmom just for interface (maybe delete later) TODO - class RamboSamplingKernelHost final : public SamplingKernelBase, public NumberOfEvents - { - public: - - RamboSamplingKernelHost( const fptype energy, // input: energy - const BufferRndNumMomenta& rndmom, // input: random [0,1] UNUSED - const std::vector& masses, // input: external-leg masses - const int ninitial, // input: #n initial-state particles - const size_t nevt, // input: #n events - BufferMomenta& momenta, // output: momenta - BufferWeights& weights); // output: weights - - virtual ~RamboSamplingKernelHost() {} - - // No-op, kept to match the massless - void getMomentaInitial() override final; - - // All the magic here - void getMomentaFinal() override final; - - bool isOnDevice() const override final { return false; } - - private: - - // The EXTERNAL masses - std::vector m_masses; - - // The number of inital particles - const int m_ninitial; - }; - - //-------------------------------------------------------------------------- - - - //-------------------------------------------------------------------------- -} -#endif // RAMBOSAMPLINGKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h deleted file mode 100644 index 480824a7e5..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/RandomNumberKernels.h +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef RANDOMNUMBERKERNELS_H -#define RANDOMNUMBERKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "MemoryBuffers.h" - -// Forward definition from curand.h (the full header is only needed in CurandRandomKernel.cc) -struct curandGenerator_st; - -// Forward definition from hiprand.h (the full header is only needed in HiprandRandomKernel.cc) -struct rocrand_generator_base_type; -typedef rocrand_generator_base_type hiprandGenerator_st; - -namespace mg5amcCpu -{ - //-------------------------------------------------------------------------- - - /* - // An interface encapsulating random number generation on a CPU host or on a GPU device - class IRandomNumberKernel - { - public: - - // Destructor - virtual ~IRandomNumberKernel(){} - - // Seed the random number generator - virtual void seedGenerator( const unsigned int seed ) = 0; - - // Generate the random number array - virtual void generateRnarray() = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - }; - */ - - //-------------------------------------------------------------------------- - - // A base class encapsulating random number generation on a CPU host or on a GPU device - class RandomNumberKernelBase //: virtual public IRandomNumberKernel - { - - protected: - - // Constructor from an existing output buffer - RandomNumberKernelBase( BufferRndNumMomenta& rnarray ) - : m_rnarray( rnarray ) {} - - public: - - // Destructor - virtual ~RandomNumberKernelBase() {} - - // Seed the random number generator - virtual void seedGenerator( const unsigned int seed ) = 0; - - // Generate the random number array - virtual void generateRnarray() = 0; - - // Is this a host or device kernel? - virtual bool isOnDevice() const = 0; - - protected: - - // The buffer for the output random numbers - BufferRndNumMomenta& m_rnarray; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating common random number generation on a CPU host - class CommonRandomNumberKernel final : public RandomNumberKernelBase - { - public: - - // Constructor from an existing output buffer - CommonRandomNumberKernel( BufferRndNumMomenta& rnarray ); - - // Destructor - ~CommonRandomNumberKernel() {} - - // Seed the random number generator - void seedGenerator( const unsigned int seed ) override final { m_seed = seed; }; - - // Generate the random number array - void generateRnarray() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // The generator seed - unsigned int m_seed; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating CURAND random number generation on a CPU host or on a GPU device - class CurandRandomNumberKernel final : public RandomNumberKernelBase - { - public: - - // Constructor from an existing output buffer - CurandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); - - // Destructor - ~CurandRandomNumberKernel(); - - // Seed the random number generator - void seedGenerator( const unsigned int seed ) override final; - - // Generate the random number array - void generateRnarray() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return m_isOnDevice; } - - private: - - // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void createGenerator(); - - // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void destroyGenerator(); - - private: - - // Is this a host or device kernel? - const bool m_isOnDevice; - - // The curand generator - // (NB: curand.h defines typedef curandGenerator_t as a pointer to forward-defined 'struct curandGenerator_st') - curandGenerator_st* m_rnGen; - }; - - //-------------------------------------------------------------------------- - - // A class encapsulating HIPRAND random number generation on a CPU host or on a GPU device - class HiprandRandomNumberKernel final : public RandomNumberKernelBase - { - public: - - // Constructor from an existing output buffer - HiprandRandomNumberKernel( BufferRndNumMomenta& rnarray, const bool onDevice ); - - // Destructor - ~HiprandRandomNumberKernel(); - - // Seed the random number generator - void seedGenerator( const unsigned int seed ) override final; - - // Generate the random number array - void generateRnarray() override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return m_isOnDevice; } - - private: - - // Create the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void createGenerator(); - - // Destroy the generator (workaround for #429: do this in every seedGenerator call rather than only in the ctor) - void destroyGenerator(); - - private: - - // Is this a host or device kernel? - const bool m_isOnDevice; - - // The hiprand generator - // (NB: hiprand.h defines typedef hiprandGenerator_t as a pointer to forward-defined 'struct hiprandGenerator_st') - hiprandGenerator_st* m_rnGen; - }; - - //-------------------------------------------------------------------------- -} -#endif // RANDOMNUMBERKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc index 0bd5c9652d..6bb2eae817 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc @@ -25,7 +25,6 @@ #include "MemoryAccessWavefunctions.h" #include "color_sum.h" #include "coloramps.h" -#include "processConfig.h" namespace mg5amcCpu { @@ -194,7 +193,7 @@ namespace mg5amcCpu COUPs[idcoup] = CD_ACCESS::ieventAccessRecordConst( allCOUPs[idcoup], ievt0 ); // dependent couplings, vary event-by-event for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) COUPs[ndcoup + iicoup] = allCOUPs[ndcoup + iicoup]; // independent couplings, fixed for all events - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); // Create an array of views over the Flavor Couplings FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; @@ -445,11 +444,11 @@ namespace mg5amcCpu fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); MEs_sv = fptype_sv{ 0 }; - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); - for( int i = 0; i < processConfig::ndiagrams; ++i ) + for( int i = 0; i < ndiagrams; ++i ) { numerators_sv[i] = fptype_sv{ 0 }; } @@ -550,14 +549,14 @@ namespace mg5amcCpu for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) { if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - normalization += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + normalization += allNumerators[ievt / neppV * neppV * ndiagrams + ichan * neppV + ieppV % neppV]; } channelIdVec[ieppV] = mgOnGpu::nchannels; for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) { if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - numerator_sum += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + numerator_sum += allNumerators[ievt / neppV * neppV * ndiagrams + ichan * neppV + ieppV % neppV]; if( allrnddiagram[ievt] < numerator_sum / normalization ) { @@ -638,7 +637,7 @@ namespace mg5amcCpu if( mulChannelWeight && allChannelIds != nullptr ) // fix segfault #892 (not 'channelIds[0] != 0') { const unsigned int channelId = getChannelId( allChannelIds, ievt0, false ); - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc deleted file mode 100644 index ebce23a97b..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/check_sa.cc +++ /dev/null @@ -1,1020 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: O. Mattelaer (Nov 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, D. Massaro, O. Mattelaer, S. Roiser, J. Teig, A. Thete, A. Valassi (2020-2026). -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== -// -// Standalone script for MadGraph7 standalone mode. -// Generates phase-space points with RAMBO and evaluates the matrix element -// through the UMAMI interface (umami.h). -// -// Two run modes: -// * matrix (default): evaluates one phase-space point (generated with the -// classic standalone RAMBO, so identical to the one of -// the Fortran/C++ standalone 'check' drivers at the -// same energy) and prints it together with the matrix -// element of every flavor combination. -// * perf : runs nblocks*nthreads*niter events on a single flavor -// and prints performance counters. -// -//========================================================================== - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "GpuAbstraction.h" -#include "GpuRuntime.h" -#include "MemoryAccessMomenta.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" -#include "epoch_process_id.h" -#include "read_slha.h" -#include "timermap.h" -#include "umami.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define STRINGIFY( s ) #s -#define XSTRINGIFY( s ) STRINGIFY( s ) -#define SEP79 79 - -namespace -{ - using namespace mg5amcCpu; - - // Fixed physics inputs - fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV - constexpr unsigned long long kSeed = 20200805ULL; // reproducible RAMBO seed - - // Matrix-mode always runs 8 events on a single flavor index. - constexpr unsigned int kMatrixBlocks = 1; - constexpr unsigned int kMatrixThreads = 8; - - // Power of GeV of the matrix-element output; depends only on the number of external legs. - constexpr int kMEGeVExponent = -( 2 * CPPProcess::npar - 8 ); - - bool is_number( const char* s ) - { - const char* t = s; - while( *t != '\0' && isdigit( *t ) ) ++t; - return (int)strlen( s ) == t - s; - } - - // Accepts plain decimal numbers such as "1000" or "1000.0" (used for the - // optional energy argument of matrix mode). - bool is_float( const char* s ) - { - int ndots = 0; - const char* t = s; - while( *t != '\0' && ( isdigit( *t ) || ( *t == '.' && ndots++ == 0 ) ) ) ++t; - return t != s && (int)strlen( s ) == t - s; - } - - enum Mode { MODE_MATRIX, MODE_PERF }; - - enum RamboType { RAMBO_MASSIVE, RAMBO_MASSLESS }; - - // One external-particle list per LHE event, each particle stored as (E, px, py, pz). - using LheEvent = std::array, CPPProcess::npar>; - - bool read_lhe_events( const std::string& path, std::vector& events ) - { - constexpr int npar = CPPProcess::npar; - std::ifstream in( path ); - if( !in ) - { - std::cerr << "ERROR! cannot open LHE file '" << path << "'" << std::endl; - return false; - } - std::string line; - while( std::getline( in, line ) ) - { - if( line.find( "" ) == std::string::npos ) continue; - if( !std::getline( in, line ) ) break; - std::istringstream hdr( line ); - int nptcl = 0; - hdr >> nptcl; - if( nptcl != npar ) - { - std::cerr << "ERROR! LHE event has " << nptcl << " particles, expected " << npar << std::endl; - return false; - } - // particle lines: pdg status mother1 mother2 color1 color2 px py pz E m lifetime spin - LheEvent ev; - int ipar = 0; - while( ipar < npar && std::getline( in, line ) ) - { - if( line.empty() ) continue; - std::istringstream ls( line ); - long pdg; - int status, m1, m2, c1, c2; - double px, py, pz, E; - if( !( ls >> pdg >> status >> m1 >> m2 >> c1 >> c2 >> px >> py >> pz >> E ) ) - { - std::cerr << "ERROR! malformed LHE particle line: " << line << std::endl; - return false; - } - ev[ipar] = { E, px, py, pz }; - ++ipar; - } - if( ipar != npar ) - { - std::cerr << "ERROR! truncated LHE event (got " << ipar << " of " << npar << " particles)" << std::endl; - return false; - } - events.push_back( ev ); - } - if( events.empty() ) - { - std::cerr << "ERROR! no events found in '" << path << "'" << std::endl; - return false; - } - return true; - } - - int usage( const char* argv0, int ret = 1 ) - { - std::cout - << "Usage:\n" - << " " << argv0 << " [matrix] [-v|--verbose] []\n" - << " " << argv0 << " perf [-v|--verbose] [-f|--flavor ] [--rambo-massless]" - << " [-e|--events ] [<#blocksPerGrid> <#threadsPerBlock>] <#iterations>\n" - << " " << argv0 << " -p [opts] (legacy alias for `perf`)\n" - << "\n" - << "Subcommands:\n" - << " matrix (default) Evaluate one phase-space point (classic standalone\n" - << " RAMBO, identical to the Fortran/C++ 'check' drivers,\n" - << " Ecms = , default 1000 GeV) and print it with\n" - << " the matrix element for each flavor combination.\n" - << " With -v also prints backend/fptype/hardcodePARAM header.\n" - << " perf Run #blocks*#threads events over #iterations iterations\n" - << " on a single flavor index, then print performance counters.\n" - << " Always prints inputs + backend/fptype header.\n" - << " With -v also dumps every event's phase-space point and ME.\n" - << "\n" - << "Options:\n" - << " -e|--events (perf only) Read the external momenta from an LHE\n" - << " file instead of generating them with RAMBO. The events are\n" - << " processed in batches of #blocks*#threads; #iterations is\n" - << " ignored (derived from the number of events in the file).\n" - << "\n" - << "perf-mode defaults if positional args are omitted:\n" - << " #blocksPerGrid = 64, #threadsPerBlock = 256, #iterations = 1.\n"; - return ret; - } - - // AOSOA -> UMAMI SoA single-event helper. Layout reminder: - // AOSOA: aosoa[i_page * npar*4*neppM + ipar*4*neppM + ip4*neppM + i_vector] - // UMAMI: soa[ip4 * npar*nevt + ipar*nevt + ievt] - __host__ __device__ inline void - aosoa_to_umami_one( const fptype* aosoa, - double* soa, - std::size_t ievt, - std::size_t nevt ) - { - constexpr int npar = CPPProcess::npar; - for( int ipar = 0; ipar < npar; ++ipar ) - { - for( int ip4 = 0; ip4 < 4; ++ip4 ) - { - soa[(std::size_t)ip4 * npar * nevt + (std::size_t)ipar * nevt + ievt] = - (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, ip4, ipar ); - } - } - } - - - const char* backend_label() - { -#ifdef __CUDACC__ - return "CUDA"; -#elif defined( __HIPCC__ ) - return "HIP"; -#else - return "CPP"; -#endif - } - - const char* fp_label() - { -#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - return "MIXED"; -#elif defined MGONGPU_FPTYPE_DOUBLE - return "DOUBLE"; -#elif defined MGONGPU_FPTYPE_FLOAT - return "FLOAT"; -#else - return "UNKNOWN"; -#endif - } - - void print_run_header( std::ostream& os ) - { - os << "Process = " << XSTRINGIFY( MG_EPOCH_PROCESS_ID ) << "_" << backend_label() -#ifdef MGONGPU_HARDCODE_PARAM - << " [hardcodePARAM=1]" << std::endl -#else - << " [hardcodePARAM=0]" << std::endl -#endif - << "FP precision = " << fp_label() << std::endl - << "Random number generation = COMMON RANDOM HOST" << std::endl; - } - - void print_momenta_table( std::ostream& os, const fptype* aosoa, unsigned int ievt ) - { - auto constexpr prec = std::numeric_limits::digits10; - constexpr int npar = CPPProcess::npar; - os << std::string( SEP79, '-' ) << std::endl - << " n E px py pz" << std::endl; - for( int ipar = 0; ipar < npar; ++ipar ) - { - double E = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 0, ipar ); - double px = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 1, ipar ); - double py = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 2, ipar ); - double pz = (double)MemoryAccessMomenta::ieventAccessIp4IparConst( aosoa, ievt, 3, ipar ); - os << std::scientific << std::setprecision( prec ) - << std::setw( 4 ) << ipar + 1 - << std::setw( prec + 10 ) << E - << std::setw( prec + 10 ) << px - << std::setw( prec + 10 ) << py - << std::setw( prec + 10 ) << pz - << std::endl - << std::defaultfloat; - } - os << std::string( SEP79, '-' ) << std::endl; - } - - // Run sigmaKin via UMAMI for `nevt` events and copy back the MEs. - // Both the momenta (UMAMI SoA layout) and the per-event flavor buffer must be set - // by the caller. On GPU the buffers are device pointers and `hstMEs` receives the - // host-side copy; on CPU `umamiMEs` is the output buffer. - bool run_umami( - UmamiHandle handle, - unsigned int nevt, - mgOnGpu::TimerMap& timermap, - double& wavetime, - const std::vector& umamiMomenta, - const std::vector& flvVec, - std::vector& umamiMEs - ) - { - constexpr unsigned int UmamiInKeyNum = 2; - timermap.start( "3a SigmaKin" ); - UmamiInputKey in_keys[UmamiInKeyNum] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX }; - UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; - const void* inputs[UmamiInKeyNum] = { umamiMomenta.data(), flvVec.data() }; - void* outputs[1] = { umamiMEs.data() }; - UmamiStatus st = umami_matrix_element( - handle, nevt, nevt, 0, UmamiInKeyNum, in_keys, inputs, 1, out_keys, outputs ); - wavetime += timermap.stop(); - if( st != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; - return false; - } - - return true; - } - - // -------------------------------------------------------------------------- - // Classic MadGraph standalone RAMBO (RANMAR generator seeded with the fixed - // values 1802/9373), reproducing the exact phase-space point used by the - // Fortran and C++ standalone 'check' drivers at the same energy, so that - // matrix-mode output can be compared line by line across backends. - // Host-side only; ported from madgraph/iolibs/template_files/rambo.cc. - // -------------------------------------------------------------------------- - namespace classic_rambo - { - struct Random - { - double ranu[98]; - double ranc, rancd, rancm; - int iranmr, jranmr; - - // universal random number generator proposed by Marsaglia and Zaman - double ranmar() - { - double uni = ranu[iranmr] - ranu[jranmr]; - if( uni < 0 ) uni = uni + 1; - ranu[iranmr] = uni; - iranmr = iranmr - 1; - jranmr = jranmr - 1; - if( iranmr == 0 ) iranmr = 97; - if( jranmr == 0 ) jranmr = 97; - ranc = ranc - rancd; - if( ranc < 0 ) ranc = ranc + rancm; - uni = uni - ranc; - if( uni < 0 ) uni = uni + 1; - return uni; - } - - void rmarin( int ij, int kl ) - { - int i = ij / 177 % 177 + 2; - int j = ij % 177 + 2; - int k = ( kl / 169 ) % 178 + 1; - int l = kl % 169; - for( int ii = 1; ii < 98; ii++ ) - { - double s = 0; - double t = .5; - for( int jj = 1; jj < 25; jj++ ) - { - int m = ( ( i * j % 179 ) * k ) % 179; - i = j; - j = k; - k = m; - l = ( 53 * l + 1 ) % 169; - if( ( l * m ) % 64 >= 32 ) s = s + t; - t = .5 * t; - } - ranu[ii] = s; - } - ranc = 362436. / 16777216.; - rancd = 7654321. / 16777216.; - rancm = 16777213. / 16777216.; - iranmr = 97; - jranmr = 33; - } - }; - - inline double rn() - { - static Random rand; - static bool init = true; - if( init ) - { - init = false; - rand.rmarin( 1802, 9373 ); - } - double ran; - while( true ) - { - ran = rand.ranmar(); - if( ran > 1e-16 ) break; - } - return ran; - } - - // RAMBO: democratic multi-particle phase space generator (S.D. Ellis, - // R. Kleiss, W.J. Stirling); weights are logarithmic. - inline std::vector> - rambo( double et, const std::vector& xm, double& wt ) - { - const int n = (int)xm.size(); - std::vector> q( n, std::vector( 4 ) ); - std::vector> p( n, std::vector( 4 ) ); - std::vector z( n ), r( 4 ), b( 3 ), p2( n ), xm2( n ), e( n ), v( n ); - const double acc = 1e-14; - const int itmax = 6; - const double twopi = 8. * atan( 1. ); - const double po2log = log( twopi / 4. ); - - // factorials for the phase-space weight - z[1] = po2log; - for( int k = 2; k < n; k++ ) z[k] = z[k - 1] + po2log - 2. * log( double( k - 1 ) ); - for( int k = 2; k < n; k++ ) z[k] = z[k] - log( double( k ) ); - - if( n < 1 || n > 101 ) - { - std::cout << "Too few or many particles: " << n << std::endl; - exit( -1 ); - } - double xmt = 0.; - int nm = 0; - for( int i = 0; i < n; i++ ) - { - if( xm[i] != 0. ) nm = nm + 1; - xmt = xmt + std::abs( xm[i] ); - } - if( xmt > et ) - { - std::cout << "Too low energy: " << et << " needed " << xmt << std::endl; - exit( -1 ); - } - - // generate n massless momenta in infinite phase space - for( int i = 0; i < n; i++ ) - { - double r1 = rn(); - double c = 2. * r1 - 1.; - double s = sqrt( 1. - c * c ); - double f = twopi * rn(); - r1 = rn(); - double r2 = rn(); - q[i][0] = -log( r1 * r2 ); - q[i][3] = q[i][0] * c; - q[i][2] = q[i][0] * s * cos( f ); - q[i][1] = q[i][0] * s * sin( f ); - } - - // parameters of the conformal transformation - for( int k = 0; k < 4; k++ ) r[k] = 0.; - for( int i = 0; i < n; i++ ) - for( int k = 0; k < 4; k++ ) r[k] = r[k] + q[i][k]; - double rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); - for( int k = 1; k < 4; k++ ) b[k - 1] = -r[k] / rmas; - double g = r[0] / rmas; - double a = 1. / ( 1. + g ); - double x = et / rmas; - - // transform the q's conformally into the p's - for( int i = 0; i < n; i++ ) - { - double bq = b[0] * q[i][1] + b[1] * q[i][2] + b[2] * q[i][3]; - for( int k = 1; k < 4; k++ ) p[i][k] = x * ( q[i][k] + b[k - 1] * ( q[i][0] + a * bq ) ); - p[i][0] = x * ( g * q[i][0] + bq ); - } - - wt = po2log; - if( n != 2 ) wt = ( 2. * n - 4. ) * log( et ) + z[n - 1]; - - // massless case is done - if( nm == 0 ) return p; - - // massive particles: rescale the momenta by a factor x - double xmax = sqrt( 1. - pow( xmt / et, 2 ) ); - for( int i = 0; i < n; i++ ) - { - xm2[i] = pow( xm[i], 2 ); - p2[i] = pow( p[i][0], 2 ); - } - int iter = 0; - x = xmax; - double accu = et * acc; - while( true ) - { - double f0 = -et; - double g0 = 0.; - double x2 = x * x; - for( int i = 0; i < n; i++ ) - { - e[i] = sqrt( xm2[i] + x2 * p2[i] ); - f0 = f0 + e[i]; - g0 = g0 + p2[i] / e[i]; - } - if( std::abs( f0 ) <= accu ) break; - iter = iter + 1; - if( iter > itmax ) - { - std::cout << "Too many iterations without desired accuracy: " << itmax << std::endl; - break; - } - x = x - f0 / ( x * g0 ); - } - for( int i = 0; i < n; i++ ) - { - v[i] = x * p[i][0]; - for( int k = 1; k < 4; k++ ) p[i][k] = x * p[i][k]; - p[i][0] = e[i]; - } - - double wt2 = 1.; - double wt3 = 0.; - for( int i = 0; i < n; i++ ) - { - wt2 = wt2 * v[i] / e[i]; - wt3 = wt3 + pow( v[i], 2 ) / e[i]; - } - double wtm = ( 2. * n - 3. ) * log( x ) + log( wt2 / wt3 * et ); - wt = wt + wtm; - return p; - } - - // Auxiliary function changing convention between MadGraph5_aMC@NLO and - // RAMBO four-momenta (same as get_momenta in the standalone_cpp driver). - inline std::vector> - get_momenta( int ninitial, double energy, const std::vector& masses, double& wgt ) - { - const int nexternal = (int)masses.size(); - const int nfinal = nexternal - ninitial; - const double e2 = pow( energy, 2 ); - const double m1 = masses[0]; - - if( ninitial == 1 ) - { - std::vector> p( 1, std::vector( 4, 0. ) ); - p[0][0] = m1; - std::vector finalmasses( masses.begin() + 1, masses.end() ); - std::vector> p_rambo = rambo( m1, finalmasses, wgt ); - p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); - return p; - } - - if( ninitial != 2 ) - { - std::cout << "Rambo needs 1 or 2 incoming particles" << std::endl; - exit( -1 ); - } - - double etot = energy; - if( nfinal == 1 ) etot = m1; - const double m2 = masses[1]; - const double mom = sqrt( ( pow( e2, 2 ) - 2 * e2 * pow( m1, 2 ) + pow( m1, 4 ) - 2 * e2 * pow( m2, 2 ) - 2 * pow( m1, 2 ) * pow( m2, 2 ) + pow( m2, 4 ) ) / ( 4 * e2 ) ); - const double energy1 = sqrt( pow( mom, 2 ) + pow( m1, 2 ) ); - const double energy2 = sqrt( pow( mom, 2 ) + pow( m2, 2 ) ); - std::vector> p( 2, std::vector( 4, 0. ) ); - p[0][0] = energy1; - p[0][3] = mom; - p[1][0] = energy2; - p[1][3] = -mom; - - if( nfinal == 1 ) - { - p.push_back( std::vector( 4, 0. ) ); - p[2][0] = etot; - wgt = 1; - return p; - } - std::vector finalmasses( masses.begin() + 2, masses.end() ); - std::vector> p_rambo = rambo( etot, finalmasses, wgt ); - p.insert( p.end(), p_rambo.begin(), p_rambo.end() ); - return p; - } - } - - // -------------------------------------------------------------------------- - // matrix mode: same PS point fed to every flavor combination, print event 0. - // The point is generated with the classic standalone RAMBO so it is - // identical to the one of the Fortran/C++ standalone 'check' drivers. - // -------------------------------------------------------------------------- - int run_matrix_mode( bool verbose ) - { - constexpr unsigned int nevt = kMatrixBlocks * kMatrixThreads; - const unsigned int nFlavors = CPPProcess::nmaxflavor; - - mgOnGpu::TimerMap timermap; - - HostBufferRndNumMomenta hstRndmom( nevt ); - HostBufferMomenta hstMomenta( nevt ); - HostBufferWeights hstWeights( nevt ); - std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - std::vector umamiMEs( nevt ); - std::vector flvVec( nevt ); - - UmamiHandle umami_handle = nullptr; - if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_initialize failed" << std::endl; - return 2; - } - - // Generate one shared phase-space point used by every flavor, with the - // classic standalone RAMBO so it matches the Fortran/C++ 'check' drivers. - CPPProcess process; - process.initProc( "../../Cards/param_card.dat" ); - double rambowgt = 0.; - - // Retrieve masses - int npar_meta = 0; - if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - std::vector massesD( npar_meta ); - if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - const std::vector masses( massesD.begin(), massesD.end() ); - - std::vector> point = - classic_rambo::get_momenta( CPPProcess::npari, (double)kEnergy, masses, rambowgt ); - - // alpha_s from the param card so the couplings match the Fortran/C++ - // 'check' drivers (UMAMI otherwise falls back to a hardcoded g_s). - SLHAReader slha( "../../Cards/param_card.dat", false ); - const double alphaS = slha.get_block_entry( "sminputs", 3, 1.180000e-01 ); - std::vector alphasVec( nevt, alphaS ); - - // Always massive RAMBO - std::unique_ptr prsk( - new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); - prsk->getMomentaInitial(); - prsk->getMomentaFinal(); - - // Fill the UMAMI SoA buffer with nevt copies of the same event: - // soa[ip4 * npar*nevt + ipar*nevt + ievt] - for( int ip4 = 0; ip4 < 4; ++ip4 ) - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - for( unsigned int ievt = 0; ievt < nevt; ++ievt ) - umamiMomenta[(std::size_t)ip4 * CPPProcess::npar * nevt + (std::size_t)ipar * nevt + ievt] = point[ipar][ip4]; - for( std::size_t ievt = 0; ievt < nevt; ++ievt ) - aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); - - if( verbose ) - { - std::cout << std::string( SEP79, '*' ) << std::endl; - print_run_header( std::cout ); - std::cout << std::string( SEP79, '*' ) << std::endl; - } - - std::cout << "Phase space point:" << std::endl - << std::string( SEP79, '-' ) << std::endl - << " n E px py pz" << std::endl; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - { - std::cout << std::scientific << std::setprecision( 7 ) - << std::setw( 2 ) << ipar + 1 - << std::setw( 16 ) << point[ipar][0] - << std::setw( 16 ) << point[ipar][1] - << std::setw( 16 ) << point[ipar][2] - << std::setw( 16 ) << point[ipar][3] - << std::endl - << std::defaultfloat; - } - std::cout << std::string( SEP79, '-' ) << std::endl; - - for( unsigned int iflav = 0; iflav < nFlavors; ++iflav ) - { - std::fill( flvVec.begin(), flvVec.end(), iflav ); - timermap.start( "3a SigmaKin" ); - UmamiInputKey in_keys[3] = { UMAMI_IN_MOMENTA, UMAMI_IN_FLAVOR_INDEX, UMAMI_IN_ALPHA_S }; - UmamiOutputKey out_keys[1] = { UMAMI_OUT_MATRIX_ELEMENT }; - const void* inputs[3] = { umamiMomenta.data(), flvVec.data(), alphasVec.data() }; - void* outputs[1] = { umamiMEs.data() }; - UmamiStatus st = umami_matrix_element( - umami_handle, nevt, nevt, 0, 3, in_keys, inputs, 1, out_keys, outputs ); - timermap.stop(); - if( st != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_matrix_element failed (status=" << st << ")" << std::endl; - umami_free( umami_handle ); - return 3; - } - const double* mes = umamiMEs.data(); - - std::cout << " PDG"; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - std::cout << std::setw( 12 ) << CPPProcess::flavorPDG( iflav, ipar ); - std::cout << std::endl - << " Matrix element = " << std::scientific << std::setprecision( 16 ) - << mes[0] << " GeV^" << kMEGeVExponent << std::endl - << std::defaultfloat - << std::string( SEP79, '-' ) << std::endl; - } - - umami_free( umami_handle ); - return 0; - } - - // -------------------------------------------------------------------------- - // perf mode: nblocks*nthreads events per iteration on a single flavor. - // -------------------------------------------------------------------------- - int run_perf_mode( bool verbose, - unsigned int gpublocks, - unsigned int gputhreads, - unsigned int niter, - unsigned int flavorID, - RamboType ramboType, - const std::string& lheFile = "" ) - { - const unsigned int nevt = gpublocks * gputhreads; - - // LHE instead of generating. Processed in batches of nevt and - // niter is derived from the number of events read. - std::vector lheEvents; - if( !lheFile.empty() ) - { - if( !read_lhe_events( lheFile, lheEvents ) ) return 2; - niter = (unsigned int)( ( lheEvents.size() + nevt - 1 ) / nevt ); - std::cout << "Reading events from LHE file = " << lheFile - << " (" << lheEvents.size() << " events, " << niter - << " batches of " << nevt << ")" << std::endl; - } - - mgOnGpu::TimerMap timermap; - - HostBufferRndNumMomenta hstRndmom( nevt ); - HostBufferMomenta hstMomenta( nevt ); - HostBufferWeights hstWeights( nevt ); - std::vector umamiMomenta( (std::size_t)4 * CPPProcess::npar * nevt ); - std::vector umamiMEs( nevt ); - std::vector flvVec( nevt, flavorID ); - - std::unique_ptr prnk( - new CommonRandomNumberKernel( hstRndmom ) ); - - UmamiHandle umami_handle = nullptr; - if( umami_initialize( &umami_handle, "../../Cards/param_card.dat" ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_initialize failed" << std::endl; - return 2; - } - - // Retrieve masses - std::vector masses; - if( ramboType != RAMBO_MASSLESS) - { - int npar_meta = 0; - if( umami_get_meta( UMAMI_META_PARTICLE_COUNT, &npar_meta ) != UMAMI_SUCCESS || npar_meta != CPPProcess::npar ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_PARTICLE_COUNT) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - std::vector massesD( npar_meta ); - if( umami_get_meta( UMAMI_META_MASSES, massesD.data() ) != UMAMI_SUCCESS ) - { - std::cerr << "ERROR! umami_get_meta(UMAMI_META_MASSES) failed" << std::endl; - umami_free( umami_handle ); - return 2; - } - masses.assign( massesD.begin(), massesD.end() ); - } - - std::unique_ptr prsk; - if( ramboType != RAMBO_MASSLESS ) - { - // Massive host only (copy) - prsk.reset( new RamboSamplingKernelHost( kEnergy, hstRndmom, masses, CPPProcess::npari, nevt, hstMomenta, hstWeights ) ); - } - else - { - prsk.reset( new MasslessRamboSamplingKernelHost( kEnergy, hstRndmom, hstMomenta, hstWeights, nevt ) ); - } - - std::unique_ptr genrtimes( new double[niter] ); - std::unique_ptr rambtimes( new double[niter] ); - std::unique_ptr wavetimes( new double[niter] ); - - unsigned int nevtABN = 0; - unsigned int nevtZERO = 0; - double sumME = 0.; - double sumMEsq = 0.; - double minME = std::numeric_limits::infinity(); - double maxME = -std::numeric_limits::infinity(); - unsigned int nevtALL = 0; - - for( unsigned int iiter = 0; iiter < niter; ++iiter ) - { - double genrtime = 0; - double rambtime = 0; - unsigned int nreal = nevt; // number of real (non-padding) events in this batch - if( lheFile.empty() ) - { - timermap.start( "1a GenSeed " ); - prnk->seedGenerator( kSeed + iiter ); - genrtime += timermap.stop(); - timermap.start( "1b GenRnGen" ); - prnk->generateRnarray(); - genrtime += timermap.stop(); - - timermap.start( "2a RamboIni" ); - prsk->getMomentaInitial(); - rambtime += timermap.stop(); - timermap.start( "2b RamboFin" ); - prsk->getMomentaFinal(); - rambtime += timermap.stop(); - } - else - { - // Fill this batch from the LHE events (AOSOA layout, (E,px,py,pz) per leg). - // padded by repeating its last real event so the SIMD page is valid - // only the nreal real events are counted below. - timermap.start( "2e ReadLHE " ); - const std::size_t base = (std::size_t)iiter * nevt; - nreal = (unsigned int)std::min( nevt, lheEvents.size() - base ); - for( unsigned int ievt = 0; ievt < nevt; ++ievt ) - { - const std::size_t src = base + std::min( ievt, (std::size_t)nreal - 1 ); - const LheEvent& ev = lheEvents[src]; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) - for( int ip4 = 0; ip4 < 4; ++ip4 ) - MemoryAccessMomenta::ieventAccessIp4Ipar( hstMomenta.data(), ievt, ip4, ipar ) = (fptype)ev[ipar][ip4]; - } - rambtime += timermap.stop(); - } - - timermap.start( "2d Aosoa2U " ); - for( std::size_t ievt = 0; ievt < nevt; ++ievt ) - aosoa_to_umami_one( hstMomenta.data(), umamiMomenta.data(), ievt, nevt ); - rambtime += timermap.stop(); - - double wavetime = 0; - if( !run_umami( umami_handle, nevt, timermap, wavetime, - umamiMomenta, flvVec, umamiMEs - ) ) - { - umami_free( umami_handle ); - return 3; - } - - const double* mes = umamiMEs.data(); - - timermap.start( "4@ UpdtStat" ); - for( unsigned int ievt = 0; ievt < nreal; ++ievt ) - { - double me = mes[ievt]; - ++nevtALL; - if( !std::isfinite( me ) ) - ++nevtABN; - else if( me == 0. ) - ++nevtZERO; - sumME += me; - sumMEsq += me * me; - if( me < minME ) minME = me; - if( me > maxME ) maxME = me; - } - - genrtimes[iiter] = genrtime; - rambtimes[iiter] = rambtime; - wavetimes[iiter] = wavetime; - - if( verbose ) - { - std::cout << std::string( SEP79, '*' ) << std::endl - << "Iteration #" << iiter + 1 << " of " << niter << std::endl; - for( unsigned int ievt = 0; ievt < nreal; ++ievt ) - { - std::cout << "Event #" << ievt + 1 << std::endl; - print_momenta_table( std::cout, hstMomenta.data(), ievt ); - std::cout << " Matrix element = " << std::scientific << std::setprecision( 16 ) - << mes[ievt] << " GeV^" << kMEGeVExponent << std::endl - << std::defaultfloat - << std::string( SEP79, '-' ) << std::endl; - } - } - } - - double sumgtim = 0, sumrtim = 0, sumwtim = 0; - double minwtim = wavetimes[0], maxwtim = wavetimes[0]; - for( unsigned int i = 0; i < niter; ++i ) - { - sumgtim += genrtimes[i]; - sumrtim += rambtimes[i]; - sumwtim += wavetimes[i]; - minwtim = std::min( minwtim, wavetimes[i] ); - maxwtim = std::max( maxwtim, wavetimes[i] ); - } - double meanwtim = sumwtim / niter; - - unsigned int nevtGood = nevtALL - nevtABN; - double meanME = ( nevtGood > 0 ) ? sumME / nevtGood : 0.; - double varME = ( nevtGood > 0 ) ? sumMEsq / nevtGood - meanME * meanME : 0.; - double stdME = ( varME > 0 ) ? std::sqrt( varME ) : 0.; - - std::cout << std::string( SEP79, '*' ) << std::endl; - print_run_header( std::cout ); - std::cout << "NumBlocksPerGrid = " << gpublocks << std::endl - << "NumThreadsPerBlock = " << gputhreads << std::endl - << "NumIterations = " << niter << std::endl - << "FlavorIndex = " << flavorID << " / " << CPPProcess::nmaxflavor << std::endl - << std::string( SEP79, '-' ) << std::endl - << "NaN/abnormal MEs = " << nevtABN << std::endl - << "Zero MEs = " << nevtZERO << std::endl - << std::string( SEP79, '-' ) << std::endl - << "NumberOfEntries = " << niter << std::endl - << std::scientific - << "TotalTime[Rnd+Rmb+ME] (123) = ( " << sumgtim + sumrtim + sumwtim << " ) sec" << std::endl - << "TotalTime[Rambo+ME] (23) = ( " << sumrtim + sumwtim << " ) sec" << std::endl - << "TotalTime[RndNumGen] (1) = ( " << sumgtim << " ) sec" << std::endl - << "TotalTime[Rambo] (2) = ( " << sumrtim << " ) sec" << std::endl - << "TotalTime[MatrixElems] (3) = ( " << sumwtim << " ) sec" << std::endl - << "MeanTimeInMatrixElems = ( " << meanwtim << " ) sec" << std::endl - << "[Min,Max]TimeInMatrixElems = [ " << minwtim << " , " << maxwtim << " ] sec" << std::endl - << std::string( SEP79, '-' ) << std::endl - << "TotalEventsComputed = " << nevtALL << std::endl - << "EvtsPerSec[Rnd+Rmb+ME](123) = ( " << nevtALL / ( sumgtim + sumrtim + sumwtim ) << " ) sec^-1" << std::endl - << "EvtsPerSec[Rmb+ME] (23) = ( " << nevtALL / ( sumrtim + sumwtim ) << " ) sec^-1" << std::endl - << "EvtsPerSec[MatrixElems] (3) = ( " << nevtALL / sumwtim << " ) sec^-1" << std::endl - << std::defaultfloat - << std::string( SEP79, '*' ) << std::endl - << "MeanMatrixElemValue = ( " << meanME << " +- " << stdME / std::sqrt( (double)std::max( 1u, nevtGood ) ) - << " ) GeV^" << kMEGeVExponent << std::endl - << "[Min,Max]MatrixElemValue = [ " << minME << " , " << maxME << " ] GeV^" << kMEGeVExponent << std::endl - << std::string( SEP79, '*' ) << std::endl; - timermap.dump(); - std::cout << std::string( SEP79, '*' ) << std::endl; - - umami_free( umami_handle ); - return 0; - } -} - -int main( int argc, char** argv ) -{ - - Mode mode = MODE_MATRIX; - RamboType ramboType = RAMBO_MASSIVE; // default - bool ramboTypeSet = false; - bool verbose = false; - unsigned int flavorID = 0; - unsigned int gpublocks = 64; - unsigned int gputhreads = 256; - unsigned int niter = 1; - unsigned int numvec[3] = { 0, 0, 0 }; - int nnum = 0; - std::string lheFile; // -e/--events: read momenta from this LHE file (perf mode only) - - // Optional leading subcommand (no leading dash). - int firstArg = 1; - if( firstArg < argc ) - { - std::string a = argv[firstArg]; - if( a == "matrix" ) { mode = MODE_MATRIX; ++firstArg; } - else if( a == "perf" ) { mode = MODE_PERF; ++firstArg; } - } - - double energyArg = -1.; - - for( int argn = firstArg; argn < argc; ++argn ) - { - std::string arg = argv[argn]; - if( arg == "--verbose" || arg == "-v" ) - verbose = true; - else if( arg == "--performance" || arg == "-p" ) - mode = MODE_PERF; // legacy alias - else if( ( arg == "--flavor" || arg == "-f" ) && argn + 1 < argc && is_number( argv[argn + 1] ) ) - flavorID = strtoul( argv[++argn], nullptr, 0 ); - else if( arg == "--rambo-massless" ) - { - std::string r = argv[++argn]; - ramboType = RAMBO_MASSLESS; - ramboTypeSet = true; - } - else if( ( arg == "--events" || arg == "-e" ) && argn + 1 < argc ) - { - lheFile = argv[++argn]; - mode = MODE_PERF; // reading events from file only makes sense in perf mode - } - else if( is_number( argv[argn] ) && nnum < 3 ) - { - numvec[nnum++] = strtoul( argv[argn], nullptr, 0 ); - if( energyArg < 0 ) energyArg = atof( argv[argn] ); - } - else if( is_float( argv[argn] ) && energyArg < 0 ) - { - // decimal number: only meaningful as the matrix-mode energy - energyArg = atof( argv[argn] ); - } - else - return usage( argv[0] ); - } -//ENERGY CHANGE FOR THE MATRIX MODE -// (default 1000 GeV as for the Fortran/C++ standalone 'check' drivers; -// can be overridden with a single positional argument) - if( mode == MODE_MATRIX ) kEnergy = ( energyArg > 0 ) ? energyArg : 1000.; - - if( mode == MODE_MATRIX ) - { - if( ramboType == RAMBO_MASSLESS ) - { - std::cerr << "ERROR: matrix mode only supports the classic RAMBO (-r c)." << std::endl; - return usage( argv[0] ); - } - if( nnum > 1 ) - { - std::cerr << "WARNING: extra positional args are ignored in matrix mode " - << "(dimensions are fixed at " << kMatrixBlocks << " " << kMatrixThreads << " 1)." - << std::endl; - } - return run_matrix_mode( verbose ); - } - - // perf mode - if( nnum == 3 ) - { - gpublocks = numvec[0]; - gputhreads = numvec[1]; - niter = numvec[2]; - } - else if( nnum == 1 ) - { - niter = numvec[0]; - } - else if( nnum != 0 ) - { - return usage( argv[0] ); - } - if( niter == 0 && lheFile.empty() ) return usage( argv[0] ); // niter is derived from the file in LHE mode - - if( flavorID >= CPPProcess::nmaxflavor ) - { - std::cerr << "ERROR: flavor index " << flavorID - << " is out of range [0, " << CPPProcess::nmaxflavor << ")." << std::endl; - return 1; - } - - return run_perf_mode( verbose, gpublocks, gputhreads, niter, flavorID, ramboType, lheFile ); -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc deleted file mode 100644 index d0b13c937f..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.cc +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Oct 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "Bridge.h" -#include "CPPProcess.h" -#include "GpuRuntime.h" - -extern "C" -{ - /** - * The namespace where the Bridge class is taken from. - * - * In the current implementation, two separate shared libraries are created for the GPU/CUDA and CPU/C++ implementations. - * Actually, two shared libraries for GPU and CPU are created for each of the five SIMD implementations on CPUs (none, sse4, avx2, 512y, 512z). - * A single fcreatebridge_ symbol is created in each library with the same name, connected to the appropriate Bridge on CPU or GPU. - * The Fortran MadEvent code is always the same: the choice whether to use a CPU or GPU implementation is done by linking the appropriate library. - * As the names of the two CPU/GPU libraries are the same in the five SIMD implementations, the choice of SIMD is done by setting LD_LIBRARY_PATH. - * - * In a future implementation, a single heterogeneous shared library may be created, with the same interface. - * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. - * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. - */ - using namespace mg5amcCpu; - - /** - * The floating point precision used in Fortran arrays. - * This is presently hardcoded to double precision (REAL*8). - */ - using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays - //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays - - /** - * Create a Bridge and return its pointer. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param nevtF the pointer to the number of events in the Fortran arrays - * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - */ - void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) - { - // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) - // FIXME: disable OMP in Bridge when called from Fortran - *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); - } - - /** - * Delete a Bridge. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - */ - void fbridgedelete_( CppObjectInFortran** ppbridge ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); - delete pbridge; - } - - /** - * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant alphas) - * @param rndhel the pointer to the input random numbers for helicity selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the pointer to the input channels i.e. Feynman diagrams to enhance (1 to n: 0 is an invalid value!) - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void fbridgesequence_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); - // Use the host/CPU implementation in the C++ library - // (there is no device implementation in this library) - pbridge->cpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); - } - - /** - * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++, without multi-channel mode. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant alphas) - * @param rndhel the pointer to the input random numbers for helicity selection - * @param rndcol the pointer to the input random numbers for color selection - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ) - { - //printf("fbridgesequence_nomultichannel_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); - fbridgesequence_( ppbridge, momenta, gs, iflavorVec, rndhel, rndcol, nullptr, mes, selhel, selcol, pgoodHelOnly ); - } - - /** - * Retrieve the number of good helicities for helicity filtering in the Bridge. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param pngoodhel the pointer to the output number of good helicities - * @param pntothel the pointer to the output total number of helicities - */ - void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, - unsigned int* pngoodhel, - unsigned int* pntothel ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgegetngoodhel_: invalid Bridge address" ); - *pngoodhel = pbridge->nGoodHel(); - *pntothel = pbridge->nTotHel(); - } -} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h deleted file mode 100644 index 4815fdb027..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/fbridge.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: Z. Wettersten (Oct 2024) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, A. Thete, A. Valassi (2025). - -#include "Bridge.h" -#include "CPPProcess.h" -#include "GpuRuntime.h" - -#ifndef _FBRIDGE_H_ -#define _FBRIDGE_H_ - -extern "C" -{ - using namespace mg5amcCpu; - - using FORTRANFPTYPE = double; - - void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ); - - void fbridgedelete_( CppObjectInFortran** ppbridge ); - - void fbridgesequence_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ); - - void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ); - - void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, unsigned int* pngoodhel, unsigned int* pntothel ); -} -#endif // _FBRIDGE_H_ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc deleted file mode 100644 index 4048cce912..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/fsampler.cc +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Feb 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "mgOnGpuConfig.h" - -#include "Bridge.h" -#include "CPPProcess.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" - -//-------------------------------------------------------------------------- - -namespace mg5amcCpu -{ - template - class Sampler final : public CppObjectInFortran - { - public: - // Constructor - // @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran arrays - // @param nparF (NEXTERNAL, nexternal.inc) number of external particles in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) - // @param np4F number of momenta components, usually 4, in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) - Sampler( int nevtF, int nparF, int np4F ); - // Destructor - virtual ~Sampler() {} - // Delete copy/move constructors and assignment operators - Sampler( const Sampler& ) = delete; - Sampler( Sampler&& ) = delete; - Sampler& operator=( const Sampler& ) = delete; - Sampler& operator=( Sampler&& ) = delete; - // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta - void samplerHostSequence( FORTRANFPTYPE* fortranMomenta ); - private: - const int m_nevt; // The number of events in each iteration - int m_iiter; // The iteration counter (for random number seeding) - HostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers - HostBufferMomenta m_hstMomenta; // Memory buffers for momenta - HostBufferWeights m_hstWeights; // Memory buffers for sampling weights - std::unique_ptr m_prnk; // The appropriate RandomNumberKernel - std::unique_ptr m_prsk; // The appropriate SamplingKernel - // HARDCODED DEFAULTS - static constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) - }; - - template - Sampler::Sampler( int nevtF, int nparF, int np4F ) - : m_nevt( nevtF ) - , m_iiter( 0 ) - , m_hstRndmom( nevtF ) - , m_hstMomenta( nevtF ) - , m_hstWeights( nevtF ) - , m_prnk( new CommonRandomNumberKernel( m_hstRndmom ) ) - , m_prsk( new RamboSamplingKernelHost( energy, m_hstRndmom, m_hstMomenta, m_hstWeights, nevtF ) ) - { - if( nparF != CPPProcess::npar ) throw std::runtime_error( "Sampler constructor: npar mismatch" ); - if( np4F != CPPProcess::np4 ) throw std::runtime_error( "Sampler constructor: np4 mismatch" ); - std::cout << "WARNING! Instantiate host Sampler (nevt=" << m_nevt << ")" << std::endl; - } - - // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta - template - void Sampler::samplerHostSequence( FORTRANFPTYPE* fortranMomenta ) - { - std::cout << "Iteration #" << m_iiter + 1 << std::endl; - // === STEP 1 OF 3 - // --- 1a. Seed rnd generator (to get same results on host and device in curand) - // [NB This should not be necessary using the host API: "Generation functions - // can be called multiple times on the same generator to generate successive - // blocks of results. For pseudorandom generators, multiple calls to generation - // functions will yield the same result as a single call with a large size."] - // *** NB! REMEMBER THAT THE FORTRAN SAMPLER ALWAYS USES COMMON RANDOM NUMBERS! *** - constexpr unsigned long long seed = 20200805; - m_prnk->seedGenerator( seed + m_iiter ); - m_iiter++; - // --- 1b. Generate all relevant numbers to build nevt events (i.e. nevt phase space points) on the host - m_prnk->generateRnarray(); - //std::cout << "Got random numbers" << std::endl; - // === STEP 2 OF 3 - // --- 2a. Fill in momenta of initial state particles on the device - m_prsk->getMomentaInitial(); - //std::cout << "Got initial momenta" << std::endl; - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - m_prsk->getMomentaFinal(); - //std::cout << "Got final momenta" << std::endl; - // --- 2c. TransposeC2F - hst_transposeMomentaC2F( m_hstMomenta.data(), fortranMomenta, m_nevt ); - } -} - -//-------------------------------------------------------------------------- - -extern "C" -{ - using namespace mg5amcCpu; - - /** - * The floating point precision used in Fortran arrays. - * This is presently hardcoded to double precision (REAL*8). - */ - using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays - //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays - - /** - * Create a Sampler and return its pointer. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - * @param nevtF the pointer to the number of events in the Fortran arrays - * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - */ - void fsamplercreate_( CppObjectInFortran** ppsampler, const int* pnevtF, const int* pnparF, const int* pnp4F ) - { - *ppsampler = new Sampler( *pnevtF, *pnparF, *pnp4F ); - } - - /** - * Delete a Sampler. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - */ - void fsamplerdelete_( CppObjectInFortran** ppsampler ) - { - Sampler* psampler = dynamic_cast*>( *ppsampler ); - if( psampler == 0 ) throw std::runtime_error( "fsamplerdelete_: invalid Sampler address" ); - delete psampler; - } - - /** - * Execute the matrix-element calculation "sequence" via a Sampler on GPU/CUDA or CUDA/C++. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param mes the pointer to the output matrix elements - */ - void fsamplersequence_( CppObjectInFortran** ppsampler, FORTRANFPTYPE* momenta ) - { - Sampler* psampler = dynamic_cast*>( *ppsampler ); - if( psampler == 0 ) throw std::runtime_error( "fsamplersequence_: invalid Sampler address" ); - // Use the host/CPU implementation (there is no device implementation) - psampler->samplerHostSequence( momenta ); - } -} - -//-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h deleted file mode 100644 index d466affa16..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/massless_rambo.h +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright (C) 2010 The MadGraph5_aMC@NLO development team and contributors. -// Created by: J. Alwall (Oct 2010) for the MG5aMC CPP backend. -//========================================================================== -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Modified originally by: S. Roiser (Feb 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. -//========================================================================== - -#ifndef MASSLESS_RAMBO_H -#define MASSLESS_RAMBO_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuFptypes.h" - -#include "ProcessData.h" - -#include -#include -#include - -// Simplified rambo version for 2 to N (with N>=2) processes with massless particles -namespace mg5amcCpu -{ -namespace massless_rambo { - - constexpr int np4 = ProcessData::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npari = ProcessData::npari; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- - constexpr int nparf = ProcessData::nparf; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- - constexpr int npar = ProcessData::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - - //-------------------------------------------------------------------------- - - // Fill in the momenta of the initial particles - // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] - template - __host__ __device__ void - ramboGetMomentaInitial( const fptype energy, // input: energy - fptype* momenta ) // output: momenta for one event or for a set of events - { - const fptype energy1 = energy / 2; - const fptype energy2 = energy / 2; - const fptype mom = energy / 2; - M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 0 ) = energy1; - M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 0 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 0 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 0 ) = mom; - M_ACCESS::kernelAccessIp4Ipar( momenta, 0, 1 ) = energy2; - M_ACCESS::kernelAccessIp4Ipar( momenta, 1, 1 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 2, 1 ) = 0; - M_ACCESS::kernelAccessIp4Ipar( momenta, 3, 1 ) = -mom; - } - - //-------------------------------------------------------------------------- - - // Fill in the momenta of the final particles using the RAMBO algorithm - // [NB: the output buffer includes both initial and final momenta, but only initial momenta are filled in] - template - __host__ __device__ void - ramboGetMomentaFinal( const fptype energy, // input: energy - const fptype* rndmom, // input: random numbers in [0,1] for one event or for a set of events - fptype* momenta, // output: momenta for one event or for a set of events - fptype* wgts ) // output: weights for one event or for a set of events - { - /**************************************************************************** - * rambo * - * ra(ndom) m(omenta) b(eautifully) o(rganized) * - * * - * a democratic multi-particle phase space generator * - * authors: s.d. ellis, r. kleiss, w.j. stirling * - * this is version 1.0 - written by r. kleiss * - * -- adjusted by hans kuijf, weights are logarithmic (1990-08-20) * - * -- adjusted by madgraph@sheffield_gpu_hackathon team (2020-07-29) * - * * - ****************************************************************************/ - - // output weight - fptype& wt = W_ACCESS::kernelAccess( wgts ); - - // AV special case nparf==1 (issue #358) - if constexpr( nparf == 1 ) - { - static bool first = true; - if( first ) - { - { - printf( "WARNING! Rambo called with 1 final particle: random numbers will be ignored\n" ); - } - first = false; - } - const int iparf = 0; - for( int i4 = 0; i4 < np4; i4++ ) - { - M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = 0; - for( int ipari = 0; ipari < npari; ipari++ ) - { - M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) += M_ACCESS::kernelAccessIp4Ipar( momenta, i4, ipari ); - } - } - wt = 1; - return; - } - - // initialization step: factorials for the phase space weight - const fptype twopi = 8. * atan( 1. ); - const fptype po2log = log( twopi / 4. ); - fptype z[nparf]; - if constexpr( nparf > 1 ) // avoid build warning on clang (related to #358) - z[1] = po2log; - for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = z[kpar - 1] + po2log - 2. * log( fptype( kpar - 1 ) ); - for( int kpar = 2; kpar < nparf; kpar++ ) z[kpar] = ( z[kpar] - log( fptype( kpar ) ) ); - - // generate n massless momenta in infinite phase space - fptype q[nparf][np4]; - for( int iparf = 0; iparf < nparf; iparf++ ) - { - const fptype r1 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 0, iparf ); - const fptype r2 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 1, iparf ); - const fptype r3 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 2, iparf ); - const fptype r4 = R_ACCESS::kernelAccessIp4IparfConst( rndmom, 3, iparf ); - const fptype c = 2. * r1 - 1.; - const fptype s = sqrt( 1. - c * c ); - const fptype f = twopi * r2; - q[iparf][0] = -log( r3 * r4 ); - q[iparf][3] = q[iparf][0] * c; - q[iparf][2] = q[iparf][0] * s * cos( f ); - q[iparf][1] = q[iparf][0] * s * sin( f ); - } - - // calculate the parameters of the conformal transformation - fptype r[np4]; - fptype b[np4 - 1]; - for( int i4 = 0; i4 < np4; i4++ ) r[i4] = 0.; - for( int iparf = 0; iparf < nparf; iparf++ ) - { - for( int i4 = 0; i4 < np4; i4++ ) r[i4] = r[i4] + q[iparf][i4]; - } - const fptype rmas = sqrt( pow( r[0], 2 ) - pow( r[3], 2 ) - pow( r[2], 2 ) - pow( r[1], 2 ) ); - for( int i4 = 1; i4 < np4; i4++ ) b[i4 - 1] = -r[i4] / rmas; - const fptype g = r[0] / rmas; - const fptype a = 1. / ( 1. + g ); - const fptype x0 = energy / rmas; - - // transform the q's conformally into the p's (i.e. the 'momenta') - for( int iparf = 0; iparf < nparf; iparf++ ) - { - fptype bq = b[0] * q[iparf][1] + b[1] * q[iparf][2] + b[2] * q[iparf][3]; - for( int i4 = 1; i4 < np4; i4++ ) - { - M_ACCESS::kernelAccessIp4Ipar( momenta, i4, iparf + npari ) = x0 * ( q[iparf][i4] + b[i4 - 1] * ( q[iparf][0] + a * bq ) ); - } - M_ACCESS::kernelAccessIp4Ipar( momenta, 0, iparf + npari ) = x0 * ( g * q[iparf][0] + bq ); - } - - // calculate weight (NB return log of weight) - wt = po2log; - if( nparf != 2 ) wt = ( 2. * nparf - 4. ) * log( energy ) + z[nparf - 1]; - - // issue warnings if weight is too small or too large - static int iwarn[5] = { 0, 0, 0, 0, 0 }; - if( wt < -180. ) - { - if( iwarn[0] <= 5 ) std::cout << "Too small wt, risk for underflow: " << wt << std::endl; - iwarn[0] = iwarn[0] + 1; - } - if( wt > 174. ) - { - if( iwarn[1] <= 5 ) std::cout << "Too large wt, risk for overflow: " << wt << std::endl; - iwarn[1] = iwarn[1] + 1; - } - - // return for weighted massless momenta - // nothing else to do in this event if all particles are massless (nm==0) - - return; - } - - //-------------------------------------------------------------------------- -} -} - -#endif // MASSLESS_RAMBO_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/runTest.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/runTest.cc deleted file mode 100644 index 7bd143de4a..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/runTest.cc +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Hageboeck (Nov 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. -// ---------------------------------------------------------------------------- -// Use ./runTest.exe --gtest_filter=*xxx to run only testxxx.cc tests -// ---------------------------------------------------------------------------- - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" -#include "MadgraphTest.h" -#include "MatrixElementKernels.h" -#include "MemoryAccessChannelIds.h" -#include "MemoryAccessMatrixElements.h" -#include "MemoryAccessMomenta.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" -#include "coloramps.h" -#include "epoch_process_id.h" - -#include - -using namespace mg5amcCpu; - -struct CUDA_CPU_TestBase : public TestDriverBase -{ - static constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static constexpr int np4 = CPPProcess::np4; - static constexpr int npar = CPPProcess::npar; - static_assert( gputhreads % neppM == 0, "ERROR! #threads/block should be a multiple of neppM" ); - static_assert( gputhreads <= mgOnGpu::ntpbMAX, "ERROR! #threads/block should be <= ntpbMAX" ); - CUDA_CPU_TestBase( const std::string& refFileName ) - : TestDriverBase( npar, refFileName ) {} - // Does this test use channelIds? - virtual bool useChannelIds() const = 0; - // Set channelId array (in the same way for CUDA and CPU tests) - static constexpr unsigned int warpSize = 32; // FIXME: add a sanity check in madevent that this is the minimum? (would need to expose this from cudacpp to madevent) - static void setChannelIds( BufferChannelIds& hstChannelIds, std::size_t iiter ) - { - static const char* debugC = getenv( "CUDACPP_RUNTEST_DEBUG" ); - static const bool debug = ( debugC != 0 ) && ( std::string( debugC ) != "" ); - // Fill channelIds for multi-channel tests #896 - // (NB: these are only used if useChannelIds == true) - // TEMPORARY(0): debug multichannel tests with channelId=1 for all events - //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1; - // TEMPORARY(1): debug multichannel tests with channelId=1,2,..,ndiag,1,2,..ndiag,... (every event gets a different channel, no warps) - //for( unsigned int i = 0; i < nevt; ++i ) hstChannelIds[i] = 1 + i % CPPProcess::ndiagrams; - // ALMOST FINAL test implementation: 1111222233331111... (every 32-event warp gets a different channel) - // FINAL(?) test implementation: 2222333344442222... (every 32-event warp gets a different channel, skip those without associated iconfig #917) - static_assert( nevt % warpSize == 0, "ERROR! nevt should be a multiple of warpSize" ); - constexpr unsigned int nWarp = nevt / warpSize; - for( unsigned int iWarp = 0; iWarp < nWarp; ++iWarp ) - { - //const unsigned int channelId = 1 + ( iWarp + iiter * nWarp ) % CPPProcess::ndiagrams; // bug #917 - const int iconfig = 1 + ( iWarp + iiter * nWarp ) % mgOnGpu::nconfigSDE; - unsigned int channelId = 0; - //for( unsigned int idiagram = 1; idiagram < CPPProcess::ndiagrams; idiagram++ ) // two bugs #920 and #919 - for( unsigned int idiagram = 0; idiagram < mgOnGpu::nchannels; idiagram++ ) // fix #920 and work around #919 - { - if( mgOnGpu::hostChannel2iconfig[idiagram] == iconfig ) - { - channelId = idiagram + 1; // fix #917 (NB add +1 because channelId uses F indexing) - break; - } - } - assert( channelId > 0 ); // sanity check that the channelId for the given iconfig was found - if( debug ) std::cout << "CUDA_CPU_TestBase::setChannelIds: iWarp=" << iWarp << ", iconfig=" << iconfig << ", channelId=" << channelId << std::endl; - for( unsigned int i = 0; i < warpSize; ++i ) - hstChannelIds[iWarp * warpSize + i] = channelId; - } - } -}; - -struct CPUTest : public CUDA_CPU_TestBase -{ - // Struct data members (process, and memory structures for random numbers, momenta, matrix elements and weights on host and device) - // [NB the hst/dev memory arrays must be initialised in the constructor, see issue #290] - CPPProcess process; - HostBufferRndNumMomenta hstRndMom; - HostBufferChannelIds hstChannelIds; - HostBufferMomenta hstMomenta; - HostBufferGs hstGs; - HostBufferRndNumHelicity hstRndHel; - HostBufferRndNumColor hstRndCol; - HostBufferWeights hstWeights; - HostBufferMatrixElements hstMatrixElements; - HostBufferSelectedHelicity hstSelHel; - HostBufferSelectedColor hstSelCol; - HostBufferHelicityMask hstIsGoodHel; - std::unique_ptr pmek; - - // Create a process object - // Read param_card and set parameters - // ** WARNING EVIL EVIL ** - // The CPPProcess constructor has side effects on the globals Proc::cHel, which is needed in ME calculations. - // Don't remove! - CPUTest( const std::string& refFileName ) - : CUDA_CPU_TestBase( refFileName ) - , process( /*verbose=*/false ) - , hstRndMom( nevt ) - , hstChannelIds( nevt ) - , hstMomenta( nevt ) - , hstGs( nevt ) - , hstRndHel( nevt ) - , hstRndCol( nevt ) - , hstWeights( nevt ) - , hstMatrixElements( nevt ) - , hstSelHel( nevt ) - , hstSelCol( nevt ) - , hstIsGoodHel( CPPProcess::ncomb ) - , pmek( new MatrixElementKernelHost( hstMomenta, hstGs, hstRndHel, hstRndCol, hstChannelIds, hstMatrixElements, hstSelHel, hstSelCol, nevt ) ) - { - // FIXME: the process instance can happily go out of scope because it is only needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? - process.initProc( "../../Cards/param_card.dat" ); - } - - virtual ~CPUTest() {} - - void prepareRandomNumbers( unsigned int iiter ) override - { - // Random numbers for momenta - CommonRandomNumberKernel rnk( hstRndMom ); - rnk.seedGenerator( 1337 + iiter ); - rnk.generateRnarray(); - // Random numbers for helicity and color selection (fix #931) - CommonRandomNumberKernel rnk2( hstRndHel ); - rnk2.seedGenerator( 1338 + iiter ); - rnk2.generateRnarray(); - CommonRandomNumberKernel rnk3( hstRndCol ); - rnk3.seedGenerator( 1339 + iiter ); - rnk3.generateRnarray(); - } - - void prepareMomenta( fptype energy ) override - { - RamboSamplingKernelHost rsk( energy, hstRndMom, hstMomenta, hstWeights, nevt ); - // --- 2a. Fill in momenta of initial state particles on the device - rsk.getMomentaInitial(); - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - rsk.getMomentaFinal(); - } - - void runSigmaKin( std::size_t iiter ) override - { - constexpr fptype fixedG = 1.2177157847767195; // fixed G for aS=0.118 (hardcoded for now in check_sa.cc, fcheck_sa.f, runTest.cc) - for( unsigned int i = 0; i < nevt; ++i ) hstGs[i] = fixedG; - setChannelIds( hstChannelIds, iiter ); // fill channelIds for multi-channel tests #896 - if( iiter == 0 ) pmek->computeGoodHelicities(); - pmek->computeMatrixElements( useChannelIds() ); - } - - fptype getMomentum( std::size_t ievt, unsigned int ipar, unsigned int ip4 ) const override - { - assert( ipar < npar ); - assert( ip4 < np4 ); - return MemoryAccessMomenta::ieventAccessIp4IparConst( hstMomenta.data(), ievt, ip4, ipar ); - } - - fptype getMatrixElement( std::size_t ievt ) const override - { - return MemoryAccessMatrixElements::ieventAccessConst( hstMatrixElements.data(), ievt ); - } - - int getChannelId( std::size_t ievt ) const override - { - return MemoryAccessChannelIds::ieventAccessConst( hstChannelIds.data(), ievt ); - } - - int getSelectedHelicity( std::size_t ievt ) const override - { - //return MemoryAccessSelectedHelicity::ieventAccessConst( hstSelHel.data(), ievt ); // does not exist yet... - return hstSelHel.data()[ievt]; - } - - int getSelectedColor( std::size_t ievt ) const override - { - //return MemoryAccessSelectedColor::ieventAccessConst( hstSelCol.data(), ievt ); // does not exist yet... - return hstSelCol.data()[ievt]; - } -}; - -// Old test with multi-channel disabled #466 -struct CPUTestNoMultiChannel : public CPUTest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return false; } - - // Constructor - CPUTestNoMultiChannel( const std::string& refFileName ) - : CPUTest( refFileName ) {} // suffix .txt - - // Destructor - virtual ~CPUTestNoMultiChannel() {} -}; - -// New test with multi-channel enabled #896 -struct CPUTestMultiChannel : public CPUTest -{ - // Does this test use channelIds? - bool useChannelIds() const override final { return true; } - - // Constructor - CPUTestMultiChannel( const std::string& refFileName ) - : CPUTest( refFileName + "2" ) {} // suffix .txt2 - - // Destructor - virtual ~CPUTestMultiChannel() {} -}; - - -// AV July 2024 much simpler class structure without the presently-unnecessary googletest templates -// This is meant as a workaround to prevent not-understood segfault #907 when adding a second test -// Note: instantiate test2 first and test1 second to ensure that the channelid printout from the dtors comes from test1 first and test2 second -// CPU test drivers -CPUTestMultiChannel driver2( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID2( s ) s##_CPU_MULTICHANNEL -CPUTestNoMultiChannel driver1( MG_EPOCH_REFERENCE_FILE_NAME ); -#define TESTID1( s ) s##_CPU_NOMULTICHANNEL -// Madgraph tests -MadgraphTest mgTest2( driver2 ); -MadgraphTest mgTest1( driver1 ); -// Instantiate Google test 1 -#define XTESTID1( s ) TESTID1( s ) -TEST( XTESTID1( MG_EPOCH_PROCESS_ID ), compareMomAndME ) -{ -#ifdef MGONGPU_CHANNELID_DEBUG - driver1.pmek->setTagForNevtProcessedByChannel( "(no multichannel)" ); -#endif - mgTest1.CompareMomentaAndME( *this ); -} -// Instantiate Google test 2 -#define XTESTID2( s ) TESTID2( s ) -TEST( XTESTID2( MG_EPOCH_PROCESS_ID ), compareMomAndME ) -{ -#ifdef MGONGPU_CHANNELID_DEBUG - driver2.pmek->setTagForNevtProcessedByChannel( "(channelid array)" ); -#endif - mgTest2.CompareMomentaAndME( *this ); -} -/* clang-format on */ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc deleted file mode 100644 index ce876db2cf..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/testmisc.cc +++ /dev/null @@ -1,479 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. -// ---------------------------------------------------------------------------- -// Use ./runTest.exe --gtest_filter=*misc to run only testmisc.cc tests -// ---------------------------------------------------------------------------- - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuVectors.h" - -#include "constexpr_math.h" -#include "epoch_process_id.h" -#include "valgrind.h" - -#include - -//#include -//#include // needs C++20... https://stackoverflow.com/a/65347016 -#include -#include -#include - -#define TESTID( s ) s##_CPU_MISC - -#define XTESTID( s ) TESTID( s ) - -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu -{ -#define EXPECT_TRUE_sv( cond ) { bool_v mask( cond ); EXPECT_TRUE( maskand( mask ) ); } - - inline const std::string - boolTF( const bool& b ) - { - return ( b ? "T" : "F" ); - } - - inline const std::string - boolTF( const bool_v& v ) - { - std::stringstream out; - out << "{ " << ( v[0] ? "T" : "F" ); - for( int i = 1; i < neppV; i++ ) out << ", " << ( v[i] ? "T" : "F" ); - out << " }"; - return out.str(); - } -} - -TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) -{ - using namespace mg5amcCpu; - - //-------------------------------------------------------------------------- - - EXPECT_TRUE( true ); - - //-------------------------------------------------------------------------- - - // Vector initialization for fptype_sv - { - fptype_sv f{ 0 }; - EXPECT_TRUE_sv( f == 0 ); - } - { - fptype_sv f = fptype_sv{ 0 }; - EXPECT_TRUE_sv( f == 0 ); - } - - // Vector initialization for fptype_sv - demonstrate bug #339 in older cxmake implementation - { - fptype_sv f{ 1 }; - //std::cout << f << std::endl << boolTF( f == 1 ) << std::endl; - //EXPECT_TRUE_sv( f == 1 ); // this fails for vectors! TFFF - EXPECT_TRUE( ( f == 1 )[0] ); // this succeds: TFFF[0] - EXPECT_TRUE( ( f[0] == 1 ) ); - for( int i = 1; i < neppV; i++ ) - { - EXPECT_TRUE( !( ( f == 1 )[i] ) ); // this succeds: FTTT[i>=1] - EXPECT_TRUE( ( f[i] == 0 ) ); // equals 0, not 1 - } - } - - // Vector initialization for cxtype_sv - demonstrate fix for bug #339 - { - fptype_sv f1 = fptype_v{ 0 } + 1; - EXPECT_TRUE_sv( f1 == 1 ); - cxtype_v c12 = cxmake( f1, 2 ); - //std::cout << c12 << std::endl << boolTF( c12.real() == 1 ) << std::endl << boolTF( c12.imag() == 2 ) << std::endl; - EXPECT_TRUE_sv( c12.real() == 1 ); - EXPECT_TRUE_sv( c12.imag() == 2 ); - cxtype_v c21 = cxmake( 2, f1 ); - //std::cout << c21 << std::endl << boolTF( c21.real() == 2 ) << std::endl << boolTF( c21.imag() == 1 ) << std::endl; - EXPECT_TRUE_sv( c21.real() == 2 ); - EXPECT_TRUE_sv( c21.imag() == 1 ); - } - - // Vector initialization for cxtype_sv - { - cxtype_sv c = cxzero_sv(); - EXPECT_TRUE_sv( c.real() == 0 ); - EXPECT_TRUE_sv( c.imag() == 0 ); - } - { - cxtype_sv c = cxmake( 1, fptype_sv{ 0 } ); // here was a bug #339 - EXPECT_TRUE_sv( c.real() == 1 ); - EXPECT_TRUE_sv( c.imag() == 0 ); - } - { - cxtype_sv c = cxmake( fptype_sv{ 0 }, 1 ); // here was a bug #339 - EXPECT_TRUE_sv( c.real() == 0 ); - EXPECT_TRUE_sv( c.imag() == 1 ); - } - - // Array initialization for cxtype_sv array (example: jamp_sv in CPPProcess.cc) - { - cxtype_sv array[2] = {}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "= {}" is missing!) - //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; - EXPECT_TRUE_sv( array[0].real() == 0 ); - EXPECT_TRUE_sv( array[0].imag() == 0 ); - EXPECT_TRUE_sv( array[1].real() == 0 ); - EXPECT_TRUE_sv( array[1].imag() == 0 ); - } - - // Alternative array initialization for cxtype_sv array (example: was used for outwf in testxxx.cc) - { - cxtype_sv array[2]{}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxype is NOT, if "{}" is missing!) - //std::cout << array[0].real() << std::endl; std::cout << boolTF( array[0].real() == 0 ) << std::endl; - EXPECT_TRUE_sv( array[0].real() == 0 ); - EXPECT_TRUE_sv( array[0].imag() == 0 ); - EXPECT_TRUE_sv( array[1].real() == 0 ); - EXPECT_TRUE_sv( array[1].imag() == 0 ); - } - - //-------------------------------------------------------------------------- - - // Scalar complex references - { - using namespace mgOnGpu; - // Refs to f1, f2 - fptype f1 = 1; - fptype f2 = 2; - cxtype_ref r12( f1, f2 ); // copy refs - //cxtype_ref r12a( r12 ); //deleted - cxtype_ref r12a( cxtype_ref( f1, f2 ) ); // copy refs - //cxtype_ref r12b = r12; // deleted - cxtype_ref r12b = cxtype_ref( f1, f2 ); // copy refs - EXPECT_TRUE( cxtype( r12 ).real() == 1 ); - EXPECT_TRUE( cxtype( r12 ).imag() == 2 ); - EXPECT_TRUE( cxtype( r12a ).real() == 1 ); - EXPECT_TRUE( cxtype( r12a ).imag() == 2 ); - EXPECT_TRUE( cxtype( r12b ).real() == 1 ); - EXPECT_TRUE( cxtype( r12b ).imag() == 2 ); - // Refs to f1c, f2c - fptype f1c = 0; - fptype f2c = 0; - cxtype_ref r12c( f1c, f2c ); - EXPECT_TRUE( cxtype( r12c ).real() == 0 ); - EXPECT_TRUE( cxtype( r12c ).imag() == 0 ); - //r12c = r12; // deleted - r12c = cxtype( r12 ); // copy values - EXPECT_TRUE( cxtype( r12c ).real() == 1 ); - EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); - // Update f1, f2 - f1 = 10; - f2 = 20; - EXPECT_TRUE( cxtype( r12 ).real() == 10 ); - EXPECT_TRUE( cxtype( r12 ).imag() == 20 ); - EXPECT_TRUE( cxtype( r12a ).real() == 10 ); - EXPECT_TRUE( cxtype( r12a ).imag() == 20 ); - EXPECT_TRUE( cxtype( r12b ).real() == 10 ); - EXPECT_TRUE( cxtype( r12b ).imag() == 20 ); - EXPECT_TRUE( cxtype( r12c ).real() == 1 ); // points to f1c, not to f1 - EXPECT_TRUE( cxtype( r12c ).imag() == 2 ); // points to f2c, not to f2 - } - - // Vector complex references - { - using namespace mgOnGpu; - // Refs to f1, f2 - fptype_sv f1 = fptype_sv{ 0 } + 1; - fptype_sv f2 = fptype_sv{ 0 } + 2; - cxtype_sv_ref r12( f1, f2 ); // copy refs - //cxtype_sv_ref r12a( r12 ); //deleted - cxtype_sv_ref r12a( cxtype_sv_ref( f1, f2 ) ); // copy refs - //cxtype_sv_ref r12b = r12; // deleted - cxtype_sv_ref r12b = cxtype_sv_ref( f1, f2 ); // copy refs - EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 2 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 2 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 2 ); - // Refs to f1c, f2c - fptype_sv f1c = fptype_sv{ 0 }; - fptype_sv f2c = fptype_sv{ 0 }; - cxtype_sv_ref r12c( f1c, f2c ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 0 ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 0 ); - //r12c = r12; // deleted - r12c = cxtype_sv( r12 ); // copy values - EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); - // Update f1, f2 - f1 = fptype_sv{ 0 } + 10; - f2 = fptype_sv{ 0 } + 20; - EXPECT_TRUE_sv( cxtype_sv( r12 ).real() == 10 ); - EXPECT_TRUE_sv( cxtype_sv( r12 ).imag() == 20 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).real() == 10 ); - EXPECT_TRUE_sv( cxtype_sv( r12a ).imag() == 20 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).real() == 10 ); - EXPECT_TRUE_sv( cxtype_sv( r12b ).imag() == 20 ); - EXPECT_TRUE_sv( cxtype_sv( r12c ).real() == 1 ); // points to f1c, not to f1 - EXPECT_TRUE_sv( cxtype_sv( r12c ).imag() == 2 ); // points to f2c, not to f2 - } - - //-------------------------------------------------------------------------- - - // Boolean vector (mask) times FP vector - /* - // From https://github.com/madgraph5/madgraph4gpu/issues/765#issuecomment-1853672838 - channelids_sv = CHANNEL_ACCESS::kernelAccess( pchannelIds ); // the 4 channels in the SIMD vector - bool_sv mask_sv = ( channelids_sv == 1 ); - numerators_sv += mask_sv * cxabs2( amp_sv[0] ); - if( pchannelIds != nullptr ) denominators_sv += cxabs2( amp_sv[0] ); - */ - { - typedef bool_sv test_int_sv; // defined as scalar_or_vector of long int (FPTYPE=double) or int (FPTYPE=float) - test_int_sv channelids0_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) - test_int_sv channelids1_sv{}; // mimic CHANNEL_ACCESS::kernelAccess( pchannelIds ) - fptype_sv absamp0_sv{}; // mimic cxabs2( amp_sv[0] ) - fptype_sv absamp1_sv{}; // mimic cxabs2( amp_sv[0] ) - for( int i = 0; i < neppV; i++ ) - { - channelids0_sv[i] = i; // 0123 - channelids1_sv[i] = i; // 1234 - absamp0_sv[i] = 10. + i; // 10. 11. 12. 13. - absamp1_sv[i] = 11. + i; // 11. 12. 13. 14. - } - bool_sv mask0_sv = ( channelids0_sv % 2 == 0 ); // even channels 0123 -> TFTF (1010) - bool_sv mask1_sv = ( channelids1_sv % 2 == 0 ); // even channels 1234 -> FTFT (0101) - constexpr fptype_sv fpZERO_sv{}; // 0000 - //fptype_sv numerators0_sv = mask0_sv * absamp0_sv; // invalid operands to binary * ('__vector(4) long int' and '__vector(4) double') - fptype_sv numerators0_sv = fpternary( mask0_sv, absamp0_sv, fpZERO_sv ); // equivalent to "mask0_sv * absamp0_sv" - fptype_sv numerators1_sv = fpternary( mask1_sv, absamp1_sv, fpZERO_sv ); // equivalent to "mask1_sv * absamp1_sv" - //std::cout << "numerators0_sv: " << numerators0_sv << std::endl; - //std::cout << "numerators1_sv: " << numerators1_sv << std::endl; - for( int i = 0; i < neppV; i++ ) - { - // Values of numerators0_sv: 10.*1 11.*0 12.*1 13.*0 - if( channelids0_sv[i] % 2 == 0 ) // even channels - EXPECT_TRUE( numerators0_sv[i] == ( 10. + i ) ); - else // odd channels - EXPECT_TRUE( numerators0_sv[i] == 0. ); - // Values of numerators1_sv: 11.*0 12.*1 13.*0 14.*1 - if( channelids1_sv[i] % 2 == 0 ) // even channels - EXPECT_TRUE( numerators1_sv[i] == ( 11. + i ) ); - else // odd channels - EXPECT_TRUE( numerators1_sv[i] == 0. ); - } - } - - //-------------------------------------------------------------------------- - - // Test constexpr floor - EXPECT_TRUE( constexpr_floor( 1.5 ) == 1 ); - EXPECT_TRUE( constexpr_floor( 0.5 ) == 0 ); - EXPECT_TRUE( constexpr_floor( -0.5 ) == -1 ); - EXPECT_TRUE( constexpr_floor( -1.5 ) == -2 ); - - // Test constexpr pow - EXPECT_TRUE( constexpr_pow( 10, 0 ) == 1 ); - EXPECT_TRUE( constexpr_pow( 10, 1 ) == 10 ); - EXPECT_TRUE( constexpr_pow( 10, 2 ) == 100 ); - EXPECT_NEAR( constexpr_pow( 10, -1 ), 0.1, 0.1 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10, -1 ) = " << constexpr_pow( 10, -1 ); - EXPECT_NEAR( constexpr_pow( 10, -2 ), 0.01, 0.01 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10, -2 ) = " << constexpr_pow( 10, -2 ); - EXPECT_NEAR( constexpr_pow( 100, 0.5 ), 10, 10 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 100, 0.5 ) = " << constexpr_pow( 100, 0.5 ); - EXPECT_NEAR( constexpr_pow( 100, -0.5 ), 0.1, 0.1 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 100, -0.5 ) = " << constexpr_pow( 100, -0.5 ); - EXPECT_NEAR( constexpr_pow( 10000, 0.25 ), 10, 10 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10000, 0.25 ) = " << constexpr_pow( 10000, 0.25 ); - EXPECT_NEAR( constexpr_pow( 10000, -0.25 ), 0.1, 0.1 * 1E-14 ) - << std::setprecision( 40 ) << "constexpr_pow( 10000, -0.25 ) = " << constexpr_pow( 10000, -0.25 ); - -#ifndef __aarch64__ // TO BE UNDERSTOOD? DISABLE CONSTEXPR_SQRT TESTS ON AARCH64 (#1064) - // Distance from the horizontal or vertical axis (i.e. from 0, pi/2, pi, or 3pi/2) - auto distance4 = []( const long double xx ) - { - const long double xx2 = mapIn0to2Pi( xx ); // in [0,2*pi) - const long double xx3 = xx2 - constexpr_floor( xx2 / constexpr_pi_by_2 ) * constexpr_pi_by_2; // in [0,pi/2) - const long double d0 = xx3; // distance from 0 - const long double d1 = constexpr_pi_by_2 - xx3; // distance from pi/2 - return ( d0 < d1 ? d0 : d1 ); - }; - - // Test constexpr sin, cos, tan - specific, problematic, points - auto testSinCosTanX = []( const long double xx, const double tolerance0, const bool debug = false, const long long istep = -999999999 ) - { - const double x = (double)xx; - const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 - if( debug ) - { - //std::cout << std::setprecision(40) << "testSinCosTanX: xx= " << xx << std::endl; - //std::cout << std::setprecision(40) << " x= " << x << std::endl; - } - //std::cout << std::setprecision(40) << "xx - 3pi/2 " << xx - 3 * constexpr_pi_by_2 << std::endl; - //int width = 46; - //char buf[128]; - //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)xx ); - //std::cout << std::setprecision(40) << "testSinCosTanX: xx=" << buf << std::endl; - //quadmath_snprintf( buf, sizeof( buf ), "%+-#*.40Qe", width, (__float128)x ); - //std::cout << std::setprecision(40) << " x= " << buf << std::endl; - EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::abs( std::sin( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::abs( std::cos( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; -#ifndef __aarch64__ - if( !RUNNING_ON_VALGRIND ) - { - EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::abs( std::tan( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - } - else -#endif - { - // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) - const long double ctanx = constexpr_tan( x ); - const long double taninf = 4E14; // declare tan(x) as "infinity" if above this threshold - if( ctanx > -taninf && ctanx < taninf ) - EXPECT_NEAR( std::tan( x ), ctanx, std::abs( std::tan( x ) * tolerance ) ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - else - { - // Allow tan(x)=-inf if ctan(x)=+inf and viceversa - EXPECT_GT( std::abs( std::tan( x ) ), taninf ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - /* - // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) - if( ctanx > 0 ) - EXPECT_GT( std::tan( x ), taninf ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - else - EXPECT_LT( std::tan( x ), -taninf ) - << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ", istep=" << istep; - */ - } - } - std::cout << std::setprecision( 6 ); // default - }; - testSinCosTanX( constexpr_pi, 1E-3, true ); // from math.h - testSinCosTanX( (long double)3.141592653589793238462643383279502884L, 1E-3, true ); // from math.h - testSinCosTanX( 4.712388980384687897640105802565813064575L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) - testSinCosTanX( 3 * constexpr_pi_by_2 - 1.96e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x!=xx) - testSinCosTanX( 3 * constexpr_pi_by_2 - 1.9601e-15L, 1E-3, true ); // from 100 steps n [-4*pi,6*pi]... succeeds? (note x==xx) - - // Test constexpr sin, cos, tan - 8 points on (or close to) the boundaries of the 8 sectors of [0,2*pi] - auto testSinCosTan8 = [testSinCosTanX]( const double deltax, const double tolerance ) - { - for( int ioff = -1; ioff < 2; ioff++, ioff++ ) // -1, 1 - { - const bool debug = false; - const int nstep = 8; - for( int istep = 0; istep < nstep + 1; istep++ ) - { - long double x0 = deltax * ioff; - long double x1 = deltax * ioff + 2 * constexpr_pi; - double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) - testSinCosTanX( x, tolerance, debug, istep ); - } - } - }; - - // Use much lower tolerance when testing on the boundaries of the 8 sectors of [0,2*pi] - // Use progressively stricter tolerances as you move away from the boundaries of the 8 sectors of [0,2*pi] - testSinCosTan8( 0, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... - testSinCosTan8( 1E-15, 1E-03 ); // fails with 1E-04 - DANGEROUS ANYWAY... - testSinCosTan8( 1E-14, 1E-04 ); // fails with 1E-05 - testSinCosTan8( 1E-12, 1E-06 ); // fails with 1E-07 - testSinCosTan8( 1E-09, 1E-09 ); // fails with 1E-10 - testSinCosTan8( 1E-06, 1E-12 ); // fails with 1E-13 - testSinCosTan8( 1E-03, 1E-14 ); // fails with 1E-16: could use 1E-14 but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) - testSinCosTan8( 1E-02, 1E-14 ); // never fails? could use 1E-99(?) but keep it at 1E-14 (avoid 'EXPECT_NEAR equivalent to EXPECT_EQUAL' on Mac) - - // Test constexpr sin, cos, tan - N points almost randomly with a varying tolerance - auto testSinCosTanN = [distance4]( const int nstep, const double x0, const double x1 ) - { - auto toleranceForX = [distance4]( const double x ) - { - const double d4 = distance4( x ); - if( d4 < 1E-14 ) - return 1E-03; // NB: absolute distance limited to 1E-14 anyway even if relative tolerance is 1E-3... - else if( d4 < 1E-13 ) - return 1E-04; - else if( d4 < 1E-12 ) - return 1E-05; - else if( d4 < 1E-11 ) - return 1E-06; - else if( d4 < 1E-10 ) - return 1E-07; - else if( d4 < 1E-09 ) - return 1E-08; - else if( d4 < 1E-08 ) - return 1E-09; - else if( d4 < 1E-07 ) - return 1E-10; - else if( d4 < 1E-06 ) - return 1E-11; - else if( d4 < 1E-05 ) - return 1E-12; - else if( d4 < 1E-04 ) - return 1E-13; - else - return 1E-14; // play it safe even if the agreement might even be better? - }; - for( int istep = 0; istep < nstep + 1; istep++ ) - { - double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) - const double tolerance0 = toleranceForX( x ); - const double tolerance = tolerance0 * ( !RUNNING_ON_VALGRIND ? 1 : 1100 ); // higher tolerance when running through valgrind #906 - EXPECT_NEAR( std::sin( x ), constexpr_sin( x ), std::max( std::abs( std::sin( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - EXPECT_NEAR( std::cos( x ), constexpr_cos( x ), std::max( std::abs( std::cos( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); -#ifndef __aarch64__ - if( !RUNNING_ON_VALGRIND ) - { - EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - } - else -#endif - { - // Higher tolerance when running through valgrind #906 (or on aarch64 #1064) - const long double ctanx = constexpr_tan( x ); - const long double taninf = 4E14; // declare tan(x) as "infinity if above this threshold - if( ctanx > -taninf && ctanx < taninf ) - EXPECT_NEAR( std::tan( x ), constexpr_tan( x ), std::max( std::abs( std::tan( x ) * tolerance ), 3E-15 ) ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - else - { - // Allow tan(x)=-inf if ctan(x)=+inf and viceversa - EXPECT_GT( std::abs( std::tan( x ) ), taninf ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - /* - // Require tan(x)=+inf if ctan(x)=+inf and similarly for -inf (this fails around 3*pi/2) - if( ctanx > 0 ) - EXPECT_GT( std::tan( x ), taninf ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - else - EXPECT_LT( std::tan( x ), -taninf ) - << std::setprecision( 40 ) << "x=" << x << ", x(0to2Pi)=" << mapIn0to2Pi( x ) << ",\n istep=" << istep << ", distance4=" << distance4( x ); - */ - } - } - } - }; - testSinCosTanN( 100, -4 * constexpr_pi, 6 * constexpr_pi ); // this was failing at 3*pi/2 (now fixed by absolute tolerance 3E-15) - testSinCosTanN( 10000, -constexpr_pi_by_2, 5 * constexpr_pi_by_2 ); - - // Test constexpr atan - { - const double tolerance = 1E-12; - const int nstep = 1000; - for( int istep = 0; istep < nstep + 1; istep++ ) - { - long double x0 = -5, x1 = +5; - double x = x0 + istep * ( x1 - x0 ) / nstep; // test this for double (else std::cos and std::sin use long double) - EXPECT_NEAR( std::atan( x ), constexpr_atan( x ), std::abs( std::atan( x ) * tolerance ) ) - << "x=" << x << ", istep=" << istep; - } - } -#endif - //-------------------------------------------------------------------------- -} diff --git a/madgraph/iolibs/template_files/madmatrix/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/check_sa.cc index 68e93edb50..50af5a947c 100644 --- a/madgraph/iolibs/template_files/madmatrix/check_sa.cc +++ b/madgraph/iolibs/template_files/madmatrix/check_sa.cc @@ -32,7 +32,7 @@ #include "MemoryBuffers.h" #include "RamboSamplingKernels.h" #include "RandomNumberKernels.h" -#include "epoch_process_id.h" +#include "ProcessData.h" #include "read_slha.h" #include "timermap.h" #include "umami.h" diff --git a/madgraph/iolibs/template_files/madmatrix/epoch_process_id.h b/madgraph/iolibs/template_files/madmatrix/epoch_process_id.h deleted file mode 100644 index bdf579affd..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/epoch_process_id.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Oct 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef EPOCH_PROCESS_ID_H -#define EPOCH_PROCESS_ID_H 1 - -// No need to indicate EPOCHX_ any longer for auto-generated code -// However, keep the name of the file as it may be useful again for new manual developments -#define MG_EPOCH_PROCESS_ID %(processid_uppercase)s - -// For simplicity, define here the name of the process-dependent reference file for tests -#define MG_EPOCH_REFERENCE_FILE_NAME "../../test/ref/dump_CPUTest.%(processid)s.txt" - -#endif // EPOCH_PROCESS_ID_H diff --git a/madgraph/iolibs/template_files/madmatrix/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/fbridge.cc deleted file mode 100644 index 05c3b3e0c4..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/fbridge.cc +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Oct 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "Bridge.h" -#include "CPPProcess.h" -#include "GpuRuntime.h" - -extern "C" -{ - /** - * The namespace where the Bridge class is taken from. - * - * In the current implementation, two separate shared libraries are created for the GPU/CUDA and CPU/C++ implementations. - * Actually, two shared libraries for GPU and CPU are created for each of the five SIMD implementations on CPUs (none, sse4, avx2, 512y, 512z). - * A single fcreatebridge_ symbol is created in each library with the same name, connected to the appropriate Bridge on CPU or GPU. - * The Fortran MadEvent code is always the same: the choice whether to use a CPU or GPU implementation is done by linking the appropriate library. - * As the names of the two CPU/GPU libraries are the same in the five SIMD implementations, the choice of SIMD is done by setting LD_LIBRARY_PATH. - * - * In a future implementation, a single heterogeneous shared library may be created, with the same interface. - * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. - * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. - */ -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif - - /** - * The floating point precision used in Fortran arrays. - * This is presently hardcoded to double precision (REAL*8). - */ - using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays - //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays - - /** - * Create a Bridge and return its pointer. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param nevtF the pointer to the number of events in the Fortran arrays - * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - */ - void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) - { -#ifdef MGONGPUCPP_GPUIMPL - GpuRuntime::setUp(); -#endif - // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) - // FIXME: disable OMP in Bridge when called from Fortran - *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); - } - - /** - * Delete a Bridge. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - */ - void fbridgedelete_( CppObjectInFortran** ppbridge ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); - delete pbridge; -#ifdef MGONGPUCPP_GPUIMPL - GpuRuntime::tearDown(); -#endif - } - - /** - * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant alphas) - * @param rndhel the pointer to the input random numbers for helicity selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the pointer to the input channels i.e. Feynman diagrams to enhance (1 to n: 0 is an invalid value!) - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void fbridgesequence_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); -#ifdef MGONGPUCPP_GPUIMPL - // Use the device/GPU implementation in the CUDA library - // (there is also a host implementation in this library) - pbridge->gpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); -#else - // Use the host/CPU implementation in the C++ library - // (there is no device implementation in this library) - pbridge->cpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); -#endif - } - - /** - * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++, without multi-channel mode. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant alphas) - * @param rndhel the pointer to the input random numbers for helicity selection - * @param rndcol the pointer to the input random numbers for color selection - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ) - { - //printf("fbridgesequence_nomultichannel_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); - fbridgesequence_( ppbridge, momenta, gs, iflavorVec, rndhel, rndcol, nullptr, mes, selhel, selcol, pgoodHelOnly ); - } - - /** - * Retrieve the number of good helicities for helicity filtering in the Bridge. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param pngoodhel the pointer to the output number of good helicities - * @param pntothel the pointer to the output total number of helicities - */ - void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, - unsigned int* pngoodhel, - unsigned int* pntothel ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgegetngoodhel_: invalid Bridge address" ); - *pngoodhel = pbridge->nGoodHel(); - *pntothel = pbridge->nTotHel(); - } -} diff --git a/madgraph/iolibs/template_files/madmatrix/fbridge.h b/madgraph/iolibs/template_files/madmatrix/fbridge.h deleted file mode 100644 index e676df0a33..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/fbridge.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: Z. Wettersten (Oct 2024) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, A. Thete, A. Valassi (2025). - -#include "Bridge.h" -#include "CPPProcess.h" -#include "GpuRuntime.h" - -#ifndef _FBRIDGE_H_ -#define _FBRIDGE_H_ - -extern "C" -{ -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif - - using FORTRANFPTYPE = double; - - void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ); - - void fbridgedelete_( CppObjectInFortran** ppbridge ); - - void fbridgesequence_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ); - - void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ); - - void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, unsigned int* pngoodhel, unsigned int* pntothel ); -} -#endif // _FBRIDGE_H_ diff --git a/madgraph/iolibs/template_files/madmatrix/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/fsampler.cc deleted file mode 100644 index 8a4468fb3c..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/fsampler.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Feb 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "mgOnGpuConfig.h" - -#include "Bridge.h" -#include "CPPProcess.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" - -//-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - template - class Sampler final : public CppObjectInFortran - { - public: - // Constructor - // @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran arrays - // @param nparF (NEXTERNAL, nexternal.inc) number of external particles in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) - // @param np4F number of momenta components, usually 4, in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) - Sampler( int nevtF, int nparF, int np4F ); - // Destructor - virtual ~Sampler() {} - // Delete copy/move constructors and assignment operators - Sampler( const Sampler& ) = delete; - Sampler( Sampler&& ) = delete; - Sampler& operator=( const Sampler& ) = delete; - Sampler& operator=( Sampler&& ) = delete; - // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta - void samplerHostSequence( FORTRANFPTYPE* fortranMomenta ); - private: - const int m_nevt; // The number of events in each iteration - int m_iiter; // The iteration counter (for random number seeding) -#ifndef MGONGPUCPP_GPUIMPL - HostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers - HostBufferMomenta m_hstMomenta; // Memory buffers for momenta - HostBufferWeights m_hstWeights; // Memory buffers for sampling weights -#else - PinnedHostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers - PinnedHostBufferMomenta m_hstMomenta; // Memory buffers for momenta - PinnedHostBufferWeights m_hstWeights; // Memory buffers for sampling weights -#endif - std::unique_ptr m_prnk; // The appropriate RandomNumberKernel - std::unique_ptr m_prsk; // The appropriate SamplingKernel - // HARDCODED DEFAULTS - static constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) - }; - - template - Sampler::Sampler( int nevtF, int nparF, int np4F ) - : m_nevt( nevtF ) - , m_iiter( 0 ) - , m_hstRndmom( nevtF ) - , m_hstMomenta( nevtF ) - , m_hstWeights( nevtF ) - , m_prnk( new CommonRandomNumberKernel( m_hstRndmom ) ) - , m_prsk( new RamboSamplingKernelHost( energy, m_hstRndmom, m_hstMomenta, m_hstWeights, nevtF ) ) - { - if( nparF != CPPProcess::npar ) throw std::runtime_error( "Sampler constructor: npar mismatch" ); - if( np4F != CPPProcess::np4 ) throw std::runtime_error( "Sampler constructor: np4 mismatch" ); - std::cout << "WARNING! Instantiate host Sampler (nevt=" << m_nevt << ")" << std::endl; - } - - // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta - template - void Sampler::samplerHostSequence( FORTRANFPTYPE* fortranMomenta ) - { - std::cout << "Iteration #" << m_iiter + 1 << std::endl; - // === STEP 1 OF 3 - // --- 1a. Seed rnd generator (to get same results on host and device in curand) - // [NB This should not be necessary using the host API: "Generation functions - // can be called multiple times on the same generator to generate successive - // blocks of results. For pseudorandom generators, multiple calls to generation - // functions will yield the same result as a single call with a large size."] - // *** NB! REMEMBER THAT THE FORTRAN SAMPLER ALWAYS USES COMMON RANDOM NUMBERS! *** - constexpr unsigned long long seed = 20200805; - m_prnk->seedGenerator( seed + m_iiter ); - m_iiter++; - // --- 1b. Generate all relevant numbers to build nevt events (i.e. nevt phase space points) on the host - m_prnk->generateRnarray(); - //std::cout << "Got random numbers" << std::endl; - // === STEP 2 OF 3 - // --- 2a. Fill in momenta of initial state particles on the device - m_prsk->getMomentaInitial(); - //std::cout << "Got initial momenta" << std::endl; - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - m_prsk->getMomentaFinal(); - //std::cout << "Got final momenta" << std::endl; - // --- 2c. TransposeC2F - hst_transposeMomentaC2F( m_hstMomenta.data(), fortranMomenta, m_nevt ); - } -} - -//-------------------------------------------------------------------------- - -extern "C" -{ -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif - - /** - * The floating point precision used in Fortran arrays. - * This is presently hardcoded to double precision (REAL*8). - */ - using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays - //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays - - /** - * Create a Sampler and return its pointer. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - * @param nevtF the pointer to the number of events in the Fortran arrays - * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - */ - void fsamplercreate_( CppObjectInFortran** ppsampler, const int* pnevtF, const int* pnparF, const int* pnp4F ) - { - *ppsampler = new Sampler( *pnevtF, *pnparF, *pnp4F ); - } - - /** - * Delete a Sampler. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - */ - void fsamplerdelete_( CppObjectInFortran** ppsampler ) - { - Sampler* psampler = dynamic_cast*>( *ppsampler ); - if( psampler == 0 ) throw std::runtime_error( "fsamplerdelete_: invalid Sampler address" ); - delete psampler; - } - - /** - * Execute the matrix-element calculation "sequence" via a Sampler on GPU/CUDA or CUDA/C++. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param mes the pointer to the output matrix elements - */ - void fsamplersequence_( CppObjectInFortran** ppsampler, FORTRANFPTYPE* momenta ) - { - Sampler* psampler = dynamic_cast*>( *ppsampler ); - if( psampler == 0 ) throw std::runtime_error( "fsamplersequence_: invalid Sampler address" ); - // Use the host/CPU implementation (there is no device implementation) - psampler->samplerHostSequence( momenta ); - } -} - -//-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/processConfig.h b/madgraph/iolibs/template_files/madmatrix/processConfig.h deleted file mode 100644 index bfaf20065b..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/processConfig.h +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (May 2025) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MG5_CONFIG_%(processid_uppercase)s_H -#define MG5_CONFIG_%(processid_uppercase)s_H 1 - -namespace processConfig { - - constexpr int ndiagrams = %(ndiagrams)d; - -} - -#endif // MG5_CONFIG_%(processid_uppercase)s_H diff --git a/madgraph/iolibs/template_files/madmatrix/process_cc.inc b/madgraph/iolibs/template_files/madmatrix/process_cc.inc index 4dff7e869d..fae1b79c55 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_cc.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_cc.inc @@ -27,7 +27,6 @@ #include "MemoryAccessMomenta.h" #include "MemoryAccessWavefunctions.h" #include "color_sum.h" -#include "processConfig.h" #include "MemoryAccessDenominators.h" #include "MemoryAccessNumerators.h" diff --git a/madgraph/iolibs/template_files/madmatrix/process_sigmaKin_function.inc b/madgraph/iolibs/template_files/madmatrix/process_sigmaKin_function.inc index 227301a6ea..7e7a754092 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_sigmaKin_function.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_sigmaKin_function.inc @@ -17,7 +17,7 @@ gpuMemset( allMEs, 0, nevt * sizeof( fptype ) ); gpuMemset( ghelAllJamps, 0, cNGoodHel * ncolor * mgOnGpu::nx2 * nevt * sizeof( fptype ) ); gpuMemset( colAllJamp2s, 0, ncolor * nevt * sizeof( fptype ) ); - gpuMemset( ghelAllNumerators, 0, cNGoodHel * processConfig::ndiagrams * nevt * sizeof( fptype ) ); + gpuMemset( ghelAllNumerators, 0, cNGoodHel * ProcessData::ndiagrams * nevt * sizeof( fptype ) ); gpuMemset( ghelAllDenominators, 0, cNGoodHel * nevt * sizeof( fptype ) ); gpuMemset( ghelAllMEs, 0, cNGoodHel * nevt * sizeof( fptype ) ); #else @@ -29,11 +29,11 @@ fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); MEs_sv = fptype_sv{ 0 }; - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ProcessData::ndiagrams ); fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); - for( int i = 0; i < processConfig::ndiagrams; ++i ) + for( int i = 0; i < ProcessData::ndiagrams; ++i ) { numerators_sv[i] = fptype_sv{ 0 }; } @@ -60,7 +60,7 @@ { const int ihel = cGoodHel[ighel]; fptype* hAllJamps = ghelAllJamps + ighel * nevt; // HACK: bypass DeviceAccessJamp (consistent with layout defined there) - fptype* hAllNumerators = ghelAllNumerators + ighel * nevt * processConfig::ndiagrams; + fptype* hAllNumerators = ghelAllNumerators + ighel * nevt * ProcessData::ndiagrams; fptype* hAllDenominators = ghelAllDenominators + ighel * nevt; gpuLaunchKernelStream( calculate_jamps, gpublocks, gputhreads, ghelStreams[ighel], ihel, allmomenta, allcouplings, iflavorVec, hAllJamps, storeChannelWeights, hAllNumerators, hAllDenominators, colAllJamp2s, nevt, false ); } @@ -192,14 +192,14 @@ for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) { if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - normalization += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + normalization += allNumerators[ievt / neppV * neppV * ProcessData::ndiagrams + ichan * neppV + ieppV %% neppV]; } channelIdVec[ieppV] = mgOnGpu::nchannels; for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) { if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - numerator_sum += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + numerator_sum += allNumerators[ievt / neppV * neppV * ProcessData::ndiagrams + ichan * neppV + ieppV %% neppV]; if( allrnddiagram[ievt] < numerator_sum / normalization ) { @@ -295,7 +295,7 @@ if( mulChannelWeight && allChannelIds != nullptr ) // fix segfault #892 (not 'channelIds[0] != 0') { const unsigned int channelId = getChannelId( allChannelIds, ievt0, false ); - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ProcessData::ndiagrams ); fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); diff --git a/madgraph/iolibs/template_files/madmatrix/runTest.cc b/madgraph/iolibs/template_files/madmatrix/runTest.cc index 369d7197cc..bebb64dfc2 100644 --- a/madgraph/iolibs/template_files/madmatrix/runTest.cc +++ b/madgraph/iolibs/template_files/madmatrix/runTest.cc @@ -19,7 +19,7 @@ #include "RamboSamplingKernels.h" #include "RandomNumberKernels.h" #include "coloramps.h" -#include "epoch_process_id.h" +#include "ProcessData.h" #include diff --git a/madgraph/iolibs/template_files/madmatrix/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/testmisc.cc index 74c4902dc5..168d7b3c32 100644 --- a/madgraph/iolibs/template_files/madmatrix/testmisc.cc +++ b/madgraph/iolibs/template_files/madmatrix/testmisc.cc @@ -12,7 +12,7 @@ #include "mgOnGpuVectors.h" #include "constexpr_math.h" -#include "epoch_process_id.h" +#include "ProcessData.h" #include "valgrind.h" #include diff --git a/madgraph/iolibs/template_files/madmatrix/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/testxxx.cc index 4fcf7d26b6..d8ef3c9262 100644 --- a/madgraph/iolibs/template_files/madmatrix/testxxx.cc +++ b/madgraph/iolibs/template_files/madmatrix/testxxx.cc @@ -14,7 +14,7 @@ #include "MemoryAccessMomenta.h" #include "MemoryAccessWavefunctions.h" #include "MemoryBuffers.h" -#include "epoch_process_id.h" +#include "ProcessData.h" #include diff --git a/madmatrix/model_handling.py b/madmatrix/model_handling.py index 38eda7c593..8bd313e0bc 100644 --- a/madmatrix/model_handling.py +++ b/madmatrix/model_handling.py @@ -1730,8 +1730,6 @@ def generate_process_files(self): """Generate mgOnGpuConfig.h, CPPProcess.cc, CPPProcess.h, check_sa.cc, gXXX.cu links""" ###misc.sprint('Entering OneProcessExporterMadMatrix.generate_process_files') self.edit_mgonGPU() - self.edit_processidfile() # AV new file (NB this is Sigma-specific, should not be a symlink to Subprocesses) - self.edit_processConfig() # sub process specific, not to be symlinked from the Subprocesses directory self.edit_colorsum() # AV new file (NB this is Sigma-specific, should not be a symlink to Subprocesses) self.edit_coloramps() super().generate_process_files() @@ -1767,6 +1765,8 @@ def edit_processdata(self): replace_dict['nipc'] = self._nipc replace_dict['nipf'] = self._nipf replace_dict['ndpf'] = self._ndpf + replace_dict['processid'] = self.name + replace_dict['processid_uppercase'] = self.name.upper() ff = open(pjoin(self.path, 'ProcessData.h'), 'w') ff.write(template % replace_dict) ff.close() @@ -1848,18 +1848,6 @@ def edit_mgonGPU(self): ff.write(template % replace_dict) ff.close() - # AV - new method - def edit_processidfile(self): - """Generate epoch_process_id.h""" - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_processidfile') - template = open(pjoin(self.template_path,'madmatrix','epoch_process_id.h'),'r').read() - replace_dict = {} - replace_dict['processid'] = self.name - replace_dict['processid_uppercase'] = self.name.upper() - ff = open(pjoin(self.path, 'epoch_process_id.h'),'w') - ff.write(template % replace_dict) - ff.close() - # generate process specific color matrix data - algo is backend owned def edit_colorsum(self): """Generate ColorMatrixData.h""" @@ -1872,17 +1860,6 @@ def edit_colorsum(self): ff.write(template % replace_dict) ff.close() - def edit_processConfig(self): - """Generate process_config.h""" - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_processConfig') - template = open(pjoin(self.template_path,'madmatrix','processConfig.h'),'r').read() - replace_dict = {} - replace_dict['ndiagrams'] = len(self.matrix_elements[0].get('diagrams')) - replace_dict['processid_uppercase'] = self.name.upper() - ff = open(pjoin(self.path, 'processConfig.h'),'w') - ff.write(template % replace_dict) - ff.close() - # AV - new method def edit_coloramps(self): """Generate coloramps.h""" From f552ba4f66ff3de22e7a5ee29ba96cfa5c87a549 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 5 Aug 2026 23:41:19 +0200 Subject: [PATCH 11/25] All color information in ColorData.h more prune for standalone_mg7 output --- .../template_files/madmatrix/ColorData.h | 53 +++++++++++++++ .../madmatrix/ColorMatrixData.h | 23 ------- .../template_files/madmatrix/ProcessTables.h | 2 +- .../madmatrix/backend/cpu/SigmaKin.cc | 2 +- .../madmatrix/backend/cpu/color_sum.cc | 2 +- .../madmatrix/backend/gpu/SigmaKin.cc | 2 +- .../madmatrix/backend/gpu/color_sum.cc | 2 +- .../madmatrix/backend/simd/SigmaKin.cc | 2 +- .../madmatrix/backend/simd/color_sum.cc | 2 +- .../template_files/madmatrix/coloramps.h | 68 ------------------- .../template_files/madmatrix/process_cc.inc | 2 +- .../template_files/madmatrix/runTest.cc | 2 +- madmatrix/model_handling.py | 27 +++----- madmatrix/output.py | 11 +-- 14 files changed, 77 insertions(+), 123 deletions(-) create mode 100644 madgraph/iolibs/template_files/madmatrix/ColorData.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/ColorMatrixData.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/coloramps.h diff --git a/madgraph/iolibs/template_files/madmatrix/ColorData.h b/madgraph/iolibs/template_files/madmatrix/ColorData.h new file mode 100644 index 0000000000..c0a51b9996 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/ColorData.h @@ -0,0 +1,53 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Process-specific color structure, generated once per subprocess: the +// normalized color matrix (for color_sum_cpu/color_sum_gpu) and the +// diagram/channel/config maps for multichannel color selection. + +#ifndef COLORDATA_H +#define COLORDATA_H 1 + +#include "mgOnGpuConfig.h" +#include "ProcessData.h" + +namespace ColorMatrixData +{ + constexpr int ncolor = ProcessData::ncolor; + +%(color_matrix_lines)s +} + +namespace mgOnGpu +{ + // Diagram: C-indexed [0,ndiagrams). Channel (channelId): F-indexed [1,nchannels], + // not all diagrams have one (#919); channelId-1 indexes channel2iconfig. Config + // (iconfig): F-indexed [1,nconfigSDE]; iconfig-1 indexes icolamp. + constexpr unsigned int nchannels = %(nb_diag)i; // may be < ndiagrams, see #919 + static_assert( nchannels <= ProcessData::ndiagrams, "nchannels should be <= ndiagrams" ); // #910 #919 + + // Map channel (C-indexed) to iconfig (F-indexed); -1 = no associated iconfig (#917) + __device__ constexpr int channel2iconfig[%(nb_diag)i] = { +%(channelc2iconfig_lines)s + }; + + // Host copy of channel2iconfig (channel2iconfig itself is device-only on GPU, #917) +#ifndef MGONGPUCPP_GPUIMPL + constexpr const int* hostChannel2iconfig = channel2iconfig; +#else + constexpr int hostChannel2iconfig[%(nb_diag)i] = { +%(channelc2iconfig_lines)s + }; +#endif + + // #configs with an associated iconfig for single-diagram enhancement (#917) + constexpr unsigned int nconfigSDE = %(nb_channel)s; + + // Map iconfig (C-indexed) to the mask of allowed colors + __device__ constexpr bool icolamp[%(nb_channel)s][%(nb_color)s] = { +%(is_LC)s + }; +} + +#endif // COLORDATA_H diff --git a/madgraph/iolibs/template_files/madmatrix/ColorMatrixData.h b/madgraph/iolibs/template_files/madmatrix/ColorMatrixData.h deleted file mode 100644 index 7a8f7e7ca6..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/ColorMatrixData.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Integrated with the MadGraph7 project in Feb 2026. -// -// Process-specific color matrix, generated once per subprocess. Kept as a -// header (not a .cc) so the backend-owned color_sum_cpu/color_sum_gpu -// (backend/{cpu,simd,gpu}/color_sum.cc) can #include it and still constexpr- -// evaluate the normalized color matrix at compile time. - -#ifndef COLORMATRIXDATA_H -#define COLORMATRIXDATA_H 1 - -#include "mgOnGpuConfig.h" -#include "ProcessData.h" - -namespace ColorMatrixData -{ - constexpr int ncolor = ProcessData::ncolor; - -%(color_matrix_lines)s -} - -#endif // COLORMATRIXDATA_H diff --git a/madgraph/iolibs/template_files/madmatrix/ProcessTables.h b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h index 10378fb62a..5b294e36ad 100644 --- a/madgraph/iolibs/template_files/madmatrix/ProcessTables.h +++ b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h @@ -5,7 +5,7 @@ // Process-specific compile-time data tables, generated once per subprocess, // for backend-owned code (backend/{cpu,simd,gpu}/SigmaKin.cc) that can't take // this data as a runtime parameter without losing constexpr-ness. Unlike -// ProcessData.h these are arrays, not scalars, and unlike ColorMatrixData.h +// ProcessData.h these are arrays, not scalars, and unlike ColorData.h // there's no backend-conditional algorithm consuming them directly - it's // pulled in via ProcessTables::name from backend-owned function bodies. // diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc index 9700f35338..31ad05f0f0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc @@ -24,7 +24,7 @@ #include "MemoryAccessNumerators.h" #include "MemoryAccessWavefunctions.h" #include "color_sum.h" -#include "coloramps.h" +#include "ColorData.h" namespace mg5amcCpu { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc index 94bd2733f7..2d614f4533 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc @@ -8,7 +8,7 @@ #include "mgOnGpuConfig.h" -#include "ColorMatrixData.h" // P1-generated: colorMatrix/colorDenom +#include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps #include "MemoryAccessMatrixElements.h" namespace mg5amcCpu diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc index 5ca72c78e4..ee826fb32e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc @@ -24,7 +24,7 @@ #include "MemoryAccessNumerators.h" #include "MemoryAccessWavefunctions.h" #include "color_sum.h" -#include "coloramps.h" +#include "ColorData.h" namespace mg5amcGpu { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc index 7d8dd2c31a..00d5c5dfe9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc @@ -8,7 +8,7 @@ #include "mgOnGpuConfig.h" -#include "ColorMatrixData.h" // P1-generated: colorMatrix/colorDenom +#include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps #include "MemoryAccessMatrixElements.h" namespace mg5amcGpu diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc index 6bb2eae817..021ef4662c 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc @@ -24,7 +24,7 @@ #include "MemoryAccessNumerators.h" #include "MemoryAccessWavefunctions.h" #include "color_sum.h" -#include "coloramps.h" +#include "ColorData.h" namespace mg5amcCpu { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc index 8ec6f8045a..29830ddacb 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc @@ -8,7 +8,7 @@ #include "mgOnGpuConfig.h" -#include "ColorMatrixData.h" // P1-generated: colorMatrix/colorDenom +#include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps #include "MemoryAccessMatrixElements.h" namespace mg5amcCpu diff --git a/madgraph/iolibs/template_files/madmatrix/coloramps.h b/madgraph/iolibs/template_files/madmatrix/coloramps.h deleted file mode 100644 index 027f1aa44e..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/coloramps.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: O. Mattelaer, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef COLORAMPS_H -#define COLORAMPS_H 1 - -#include "CPPProcess.h" - - -namespace mgOnGpu -{ - // Summary of numbering and indexing conventions for the relevant concepts (see issue #826 and PR #852) - // - Diagram number (no variable) in [0, N_diagrams-1]: all values are allowed (N_diagrams distinct values) - // It follows the same C-style indexing of MadSpace - // => this number is displayed for information before each block of code in CPPProcess.cc - // - Channel number ("channelId" in C, CHANNEL_ID in F) in [1, N_channels]: not all values are allowed (N_config <= N_channels <= N_diagrams distinct values) - // *** NB channelId is a diagram number: but ALL diagrams > N_channels, and also some < N_channels, do not have an associated SDE config number (#919) *** - // => this number (with F indexing as in ps/pdf output) is passed around as an API argument between cudacpp functions - // Note: the old API passes around a single CHANNEL_ID (and uses CHANNEL_ID=0 to indicate no-multichannel mode, but this is not used in coloramps.h), - // while the new API passes around an array of CHANNEL_ID's (and uses a NULL array pointer to indicate no-multichannel mode) - // - Channel number in C indexing: "channelID - 1" - // => this number (with C indexing) is used as the index of the channel2iconfig array below - // - Config number ("iconfig" in C, ICONFIG in F) in [1, N_config]: all values are allowed (N_config <= N_channels <= N_diagrams distinct values) - // - Config number in C indexing: "iconfig - 1" - // => this number (with C indexing) is used as the index of the icolamp array below - - // The number of channels in the channel2iconfig array below - // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** - constexpr unsigned int nchannels = %(nb_diag)i; -#ifdef MGONGPUCPP_GPUIMPL - static_assert( nchannels <= mg5amcGpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 -#else - static_assert( nchannels <= mg5amcCpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 -#endif - - // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) - // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) - // This array has N_diagrams elements, but only N_config <= N_diagrams valid values (iconfig>0) - // (NB: this array is created on the host in C++ code and on the device in GPU code, but a host copy is also needed in runTest #917) - __device__ constexpr int channel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed -%(channelc2iconfig_lines)s - }; - - // Host copy of the channel2iconfig array (this is needed in runTest #917) -#ifndef MGONGPUCPP_GPUIMPL - constexpr const int* hostChannel2iconfig = channel2iconfig; -#else - constexpr int hostChannel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed -%(channelc2iconfig_lines)s - }; -#endif - - // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) - constexpr unsigned int nconfigSDE = %(nb_channel)s; - - // Map iconfig to the mask of allowed colors (e.g. "colormask = icolamp[iconfig - 1]": input index uses C indexing) - // This array has N_config <= N_diagrams elements - // (NB: this array is created on the host in C++ code and on the device in GPU code) - __device__ constexpr bool icolamp[%(nb_channel)s][%(nb_color)s] = { // note: a trailing comma in the initializer list is allowed -%(is_LC)s - }; - -} - -#endif // COLORAMPS_H diff --git a/madgraph/iolibs/template_files/madmatrix/process_cc.inc b/madgraph/iolibs/template_files/madmatrix/process_cc.inc index fae1b79c55..bd78feb4c2 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_cc.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_cc.inc @@ -30,7 +30,7 @@ #include "MemoryAccessDenominators.h" #include "MemoryAccessNumerators.h" -#include "coloramps.h" +#include "ColorData.h" #include #include diff --git a/madgraph/iolibs/template_files/madmatrix/runTest.cc b/madgraph/iolibs/template_files/madmatrix/runTest.cc index bebb64dfc2..91b2f17755 100644 --- a/madgraph/iolibs/template_files/madmatrix/runTest.cc +++ b/madgraph/iolibs/template_files/madmatrix/runTest.cc @@ -18,7 +18,7 @@ #include "MemoryBuffers.h" #include "RamboSamplingKernels.h" #include "RandomNumberKernels.h" -#include "coloramps.h" +#include "ColorData.h" #include "ProcessData.h" #include diff --git a/madmatrix/model_handling.py b/madmatrix/model_handling.py index 8bd313e0bc..db927c0e1d 100644 --- a/madmatrix/model_handling.py +++ b/madmatrix/model_handling.py @@ -1730,8 +1730,7 @@ def generate_process_files(self): """Generate mgOnGpuConfig.h, CPPProcess.cc, CPPProcess.h, check_sa.cc, gXXX.cu links""" ###misc.sprint('Entering OneProcessExporterMadMatrix.generate_process_files') self.edit_mgonGPU() - self.edit_colorsum() # AV new file (NB this is Sigma-specific, should not be a symlink to Subprocesses) - self.edit_coloramps() + self.edit_colordata() # AV new file (NB this is Sigma-specific, should not be a symlink to Subprocesses) super().generate_process_files() # needs to be after get_matrix_element_calls to have nwf ready self.edit_processdata() @@ -1848,21 +1847,14 @@ def edit_mgonGPU(self): ff.write(template % replace_dict) ff.close() - # generate process specific color matrix data - algo is backend owned - def edit_colorsum(self): - """Generate ColorMatrixData.h""" - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_colorsum') - template = open(pjoin(self.template_path,'madmatrix','ColorMatrixData.h'),'r').read() + # generate process specific color matrix + channel/config maps - algo is backend owned + def edit_colordata(self): + """Generate ColorData.h""" + ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_colordata') + template = open(pjoin(self.template_path,'madmatrix','ColorData.h'),'r').read() replace_dict = {} # Extract color matrix again (this was also in get_matrix_single_process called within get_all_sigmaKin_lines) replace_dict['color_matrix_lines'] = self.get_color_matrix_lines(self.matrix_elements[0]) - ff = open(pjoin(self.path, 'ColorMatrixData.h'),'w') - ff.write(template % replace_dict) - ff.close() - - # AV - new method - def edit_coloramps(self): - """Generate coloramps.h""" # we don't sort self.multi_channel_map, and we rely on MadSpace sorting # so, diagrams there may be unsorted @@ -1871,12 +1863,7 @@ def edit_coloramps(self): for config in config_subproc_map_C: config_subproc_map.append([c+1 for c in config]) - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_coloramps') - template = open(pjoin(self.template_path,'madmatrix','coloramps.h'),'r').read() - ff = open(pjoin(self.path, 'coloramps.h'),'w') # The following five lines from OneProcessExporterCPP.get_sigmaKin_lines (using OneProcessExporterCPP.get_icolamp_lines) - replace_dict={} - iconfig_to_diag = {} diag_to_iconfig = {} iconfig = 0 @@ -1922,6 +1909,8 @@ def edit_coloramps(self): icolamp_text += text % (iconfigc+1, iconfig_to_diag[iconfigc+1]-1) # diag - 1 is to follow MadSpace indexing icolamp.append(icolamp_text) replace_dict['is_LC'] = '\n'.join(icolamp) + + ff = open(pjoin(self.path, 'ColorData.h'),'w') ff.write(template % replace_dict) ff.close() diff --git a/madmatrix/output.py b/madmatrix/output.py index 03885b813a..0cbcd2033e 100644 --- a/madmatrix/output.py +++ b/madmatrix/output.py @@ -88,8 +88,11 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): ]), # Backend-owned skeleton files live only under backend// now # (see backend_variants below); only genuinely backend-agnostic files - # (no backend/ counterpart) are copied flat into SubProcesses/. - 'SubProcesses': relative_path_list(madmatrix_templates, ['nvtx.h', 'umami.h', 'rambo.h']), + # (no backend/ counterpart) are copied flat into SubProcesses/. umami.h + # is the only one needed outside standalone mode too (it's the header + # for backend//umami.cc's UMAMI API); nvtx.h/rambo.h are + # standalone-driver-only (see _standalone_extra_files below). + 'SubProcesses': relative_path_list(madmatrix_templates, ['umami.h']), # run_card.toml is generated in finalize() (ProcessExporterMG7.create_run_card) # from the template, not copied verbatim. # Default cards for the optional post-processing tools @@ -122,7 +125,7 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): # single top-level backend// dir via the Makefile's INCFLAGS/vpath # (see BACKENDDIR in madmatrix.mk). Only files with no backend/ counterpart # - genuinely backend-agnostic - stay here. - to_link_in_P = ['nvtx.h', 'umami.h', 'rambo.h'] + to_link_in_P = ['umami.h'] template_src_make = pjoin(madmatrix_templates, 'madmatrix_src.mk') template_Sub_make = pjoin(madmatrix_templates, 'madmatrix.mk') @@ -209,7 +212,7 @@ class ProcessExporterMadMatrixStandalone(ProcessExporterMadMatrix): template_Sub_make = pjoin(ProcessExporterMadMatrix.madmatrix_templates, 'madmatrix_standalone.mk') # Standalone-only template files needed to build check_sa.exe - _standalone_extra_files = ['check_sa.cc', + _standalone_extra_files = ['check_sa.cc', 'nvtx.h', 'rambo.h', 'RamboSamplingKernels.cc', 'RamboSamplingKernels.h', 'CommonRandomNumberKernel.cc', 'CommonRandomNumbers.h', 'RandomNumberKernels.h', From a35d5f367a08e3ceb42c9bb8b0f77b8ceeb3abee Mon Sep 17 00:00:00 2001 From: stloufra Date: Thu, 6 Aug 2026 10:37:54 +0200 Subject: [PATCH 12/25] More prune delete the Gpu* files from cpu and simd. More prune of death code and #ifdefs --- .../madmatrix/CommonRandomNumberKernel.cc | 4 + .../madmatrix/RamboSamplingKernels.cc | 2 + .../backend/cpu/CrossSectionKernels.cc | 2 +- .../backend/cpu/CrossSectionKernels.h | 37 -- .../madmatrix/backend/cpu/GpuAbstraction.h | 168 -------- .../madmatrix/backend/cpu/GpuRuntime.h | 29 -- .../backend/cpu/MatrixElementKernels.cc | 2 +- .../backend/cpu/MatrixElementKernels.h | 1 - .../madmatrix/backend/cpu/MemoryBuffers.h | 24 -- .../madmatrix/backend/cpu/SigmaKin.cc | 3 +- .../madmatrix/backend/cpu/color_sum.h | 1 - .../madmatrix/backend/cpu/mgOnGpuConfig.h | 192 ++-------- .../madmatrix/backend/cpu/mgOnGpuCxtypes.h | 361 +----------------- .../madmatrix/backend/cpu/umami.cc | 1 - .../backend/gpu/CrossSectionKernels.h | 37 -- .../madmatrix/backend/gpu/GpuAbstraction.h | 2 +- .../backend/gpu/MatrixElementKernels.cc | 4 +- .../madmatrix/backend/gpu/MemoryBuffers.h | 23 -- .../madmatrix/backend/gpu/SigmaKin.cc | 4 +- .../madmatrix/backend/gpu/mgOnGpuConfig.h | 16 +- .../madmatrix/backend/gpu/mgOnGpuCxtypes.h | 14 +- .../backend/simd/CrossSectionKernels.cc | 2 +- .../backend/simd/CrossSectionKernels.h | 37 -- .../madmatrix/backend/simd/GpuAbstraction.h | 168 -------- .../madmatrix/backend/simd/GpuRuntime.h | 29 -- .../backend/simd/MatrixElementKernels.cc | 2 +- .../backend/simd/MatrixElementKernels.h | 1 - .../madmatrix/backend/simd/MemoryBuffers.h | 24 -- .../madmatrix/backend/simd/SigmaKin.cc | 3 +- .../madmatrix/backend/simd/color_sum.h | 1 - .../madmatrix/backend/simd/mgOnGpuConfig.h | 192 ++-------- .../madmatrix/backend/simd/mgOnGpuCxtypes.h | 361 +----------------- .../madmatrix/backend/simd/umami.cc | 1 - .../template_files/madmatrix/check_sa.cc | 2 + .../template_files/madmatrix/process_cc.inc | 1 - .../template_files/madmatrix/process_h.inc | 1 - 36 files changed, 108 insertions(+), 1644 deletions(-) delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuAbstraction.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/GpuAbstraction.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h diff --git a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc index 16671b5686..cdac738384 100644 --- a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc @@ -4,8 +4,12 @@ // Further modified by: J. Teig, A. Valassi (2021-2024). // Integrated with the MadGraph7 project in Feb 2026. +#include "mgOnGpuConfig.h" + #include "CommonRandomNumbers.h" +#ifdef MGONGPUCPP_GPUIMPL #include "GpuAbstraction.h" +#endif #include "MemoryBuffers.h" #include "RandomNumberKernels.h" diff --git a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc index d40b1b4174..024c3ee2ee 100644 --- a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc @@ -6,7 +6,9 @@ #include "RamboSamplingKernels.h" +#ifdef MGONGPUCPP_GPUIMPL #include "GpuRuntime.h" +#endif #include "MemoryAccessMomenta.h" #include "MemoryAccessRandomNumbers.h" #include "MemoryAccessWeights.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc index 861bca8940..019eb2e978 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc @@ -6,11 +6,11 @@ #include "CrossSectionKernels.h" -#include "GpuAbstraction.h" #include "MemoryAccessMatrixElements.h" #include "MemoryAccessWeights.h" #include "MemoryBuffers.h" +#include #include // ****************************************************************************************** diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h index 9fb3c3fce2..4ac3ded803 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h @@ -92,43 +92,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- - /* -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating the calculation of event statistics on a GPU device - class CrossSectionKernelDevice : public CrossSectionKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats, // output: event statistics - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~CrossSectionKernelDevice(){} - - // Reset gpublocks and gputhreads - void setGrid( const size_t gpublocks, const size_t gputhreads ); - - // Update event statistics - void updateEventStatistics( const bool debug=false ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - - }; -#endif - */ //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuAbstraction.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuAbstraction.h deleted file mode 100644 index 6f709dfe9a..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuAbstraction.h +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: J. Teig (Jul 2023) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2020-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MG5AMC_GPUABSTRACTION_H -#define MG5AMC_GPUABSTRACTION_H 1 - -#include "mgOnGpuConfig.h" - -#include - -//-------------------------------------------------------------------------- - -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) - -#ifndef MGONGPU_HAS_NO_BLAS -#include "cublas_v2.h" -#endif - -#define gpuError_t cudaError_t -#define gpuPeekAtLastError cudaPeekAtLastError -#define gpuGetErrorString cudaGetErrorString -#define gpuSuccess cudaSuccess - -#define gpuMallocHost( ptr, size ) checkGpu( cudaMallocHost( ptr, size ) ) -#define gpuMalloc( ptr, size ) checkGpu( cudaMalloc( ptr, size ) ) - -#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( cudaMemcpy( dstData, srcData, srcBytes, func ) ) -#define gpuMemset( data, value, bytes ) checkGpu( cudaMemset( data, value, bytes ) ) -#define gpuMemcpyHostToDevice cudaMemcpyHostToDevice -#define gpuMemcpyDeviceToHost cudaMemcpyDeviceToHost -#define gpuMemcpyDeviceToDevice cudaMemcpyDeviceToDevice -#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( cudaMemcpyToSymbol( type1, type2, size ) ) - -#define gpuFree( ptr ) checkGpu( cudaFree( ptr ) ) -#define gpuFreeHost( ptr ) checkGpu( cudaFreeHost( ptr ) ) - -#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( cudaGetSymbolAddress( devPtr, symbol ) ) - -#define gpuSetDevice cudaSetDevice -#define gpuDeviceSynchronize cudaDeviceSynchronize -#define gpuDeviceReset cudaDeviceReset - -#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) -//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> -#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) -#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) - -#define gpuStream_t cudaStream_t -#define gpuStreamCreate( pStream ) checkGpu( cudaStreamCreate( pStream ) ) -#define gpuStreamDestroy( stream ) checkGpu( cudaStreamDestroy( stream ) ) -#define gpuMallocAsync( ptr, size, stream ) checkGpu( cudaMallocAsync( ptr, size, stream ) ) -#define gpuFreeAsync( ptr, stream ) checkGpu( cudaFreeAsync( ptr, stream ) ) -#define gpuStreamSynchronize( stream ) checkGpu( cudaStreamSynchronize( stream ) ) - -#define gpuBlasStatus_t cublasStatus_t -#define GPUBLAS_STATUS_SUCCESS CUBLAS_STATUS_SUCCESS -#ifndef MGONGPU_HAS_NO_BLAS -#define gpuBlasHandle_t cublasHandle_t -#else -#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds -#endif -#define gpuBlasCreate cublasCreate -#define gpuBlasDestroy cublasDestroy -#define gpuBlasSetStream cublasSetStream - -#define gpuBlasSaxpy cublasSaxpy -#define gpuBlasSdot cublasSdot -#define gpuBlasSgemv cublasSgemv -#define gpuBlasSgemm cublasSgemm -#define gpuBlasSgemmStridedBatched cublasSgemmStridedBatched -#define gpuBlasDaxpy cublasDaxpy -#define gpuBlasDdot cublasDdot -#define gpuBlasDgemv cublasDgemv -#define gpuBlasDgemm cublasDgemm -#define gpuBlasDgemmStridedBatched cublasDgemmStridedBatched -#define GPUBLAS_OP_N CUBLAS_OP_N -#define GPUBLAS_OP_T CUBLAS_OP_T - -//-------------------------------------------------------------------------- - -#elif defined __HIPCC__ - -#ifndef MGONGPU_HAS_NO_BLAS -#include "hipblas/hipblas.h" -#endif - -#define gpuError_t hipError_t -#define gpuPeekAtLastError hipPeekAtLastError -#define gpuGetErrorString hipGetErrorString -#define gpuSuccess hipSuccess - -#define gpuMallocHost( ptr, size ) checkGpu( hipHostMalloc( ptr, size ) ) // HostMalloc better -#define gpuMalloc( ptr, size ) checkGpu( hipMalloc( ptr, size ) ) - -#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( hipMemcpy( dstData, srcData, srcBytes, func ) ) -#define gpuMemset( data, value, bytes ) checkGpu( hipMemset( data, value, bytes ) ) -#define gpuMemcpyHostToDevice hipMemcpyHostToDevice -#define gpuMemcpyDeviceToHost hipMemcpyDeviceToHost -#define gpuMemcpyDeviceToDevice hipMemcpyDeviceToDevice -#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( hipMemcpyToSymbol( type1, type2, size ) ) - -#define gpuFree( ptr ) checkGpu( hipFree( ptr ) ) -#define gpuFreeHost( ptr ) checkGpu( hipHostFree( ptr ) ) - -#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( hipGetSymbolAddress( devPtr, symbol ) ) - -#define gpuSetDevice hipSetDevice -#define gpuDeviceSynchronize hipDeviceSynchronize -#define gpuDeviceReset hipDeviceReset - -#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) -//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> -#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) -#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) - -#define gpuStream_t hipStream_t -#define gpuStreamCreate( pStream ) checkGpu( hipStreamCreate( pStream ) ) -#define gpuStreamDestroy( stream ) checkGpu( hipStreamDestroy( stream ) ) -#define gpuMallocAsync( ptr, size, stream ) checkGpu( hipMallocAsync( ptr, size, stream ) ) -#define gpuFreeAsync( ptr, stream ) checkGpu( hipFreeAsync( ptr, stream ) ) -#define gpuStreamSynchronize( stream ) checkGpu( hipStreamSynchronize( stream ) ) - -#define gpuBlasStatus_t hipblasStatus_t -#define GPUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS -#ifndef MGONGPU_HAS_NO_BLAS -#define gpuBlasHandle_t hipblasHandle_t -#else -#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds -#endif -#define gpuBlasCreate hipblasCreate -#define gpuBlasDestroy hipblasDestroy -#define gpuBlasSetStream hipblasSetStream - -#define gpuBlasSaxpy hipblasSaxpy -#define gpuBlasSdot hipblasSdot -#define gpuBlasSgemv hipblasSgemv -#define gpuBlasSgemm hipblasSgemm -#define gpuBlasSgemmStridedBatched hipblasSgemmStridedBatched -#define gpuBlasDaxpy hipblasDaxpy -#define gpuBlasDdot hipblasDdot -#define gpuBlasDgemv hipblasDgemv -#define gpuBlasDgemm hipblasDgemm -#define gpuBlasDgemmStridedBatched hipblasDgemmStridedBatched -#define GPUBLAS_OP_N HIPBLAS_OP_N -#define GPUBLAS_OP_T HIPBLAS_OP_T - -#endif - -//-------------------------------------------------------------------------- - -#ifdef MGONGPU_FPTYPE2_FLOAT -#define gpuBlasTaxpy gpuBlasSaxpy -#define gpuBlasTdot gpuBlasSdot -#define gpuBlasTgemv gpuBlasSgemv -#define gpuBlasTgemm gpuBlasSgemm -#define gpuBlasTgemmStridedBatched gpuBlasSgemmStridedBatched -#else -#define gpuBlasTaxpy gpuBlasDaxpy -#define gpuBlasTdot gpuBlasDdot -#define gpuBlasTgemv gpuBlasDgemv -#define gpuBlasTgemm gpuBlasDgemm -#define gpuBlasTgemmStridedBatched gpuBlasDgemmStridedBatched -#endif - -#endif // MG5AMC_GPUABSTRACTION_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h deleted file mode 100644 index 7366509821..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/GpuRuntime.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: J. Teig (Jun 2023, based on earlier work by S. Roiser) for the MG5aMC CUDACPP plugin. -// Further modified by: O. Mattelaer, S. Roiser, J. Teig, A. Valassi, Z. Wettersten (2020-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MG5AMC_GPURUNTIME_H -#define MG5AMC_GPURUNTIME_H 1 - -// MG5AMC on GPU uses the CUDA runtime API, not the lower level CUDA driver API -// See https://docs.nvidia.com/cuda/cuda-runtime-api/driver-vs-runtime-api.html#driver-vs-runtime-api - -#include "GpuAbstraction.h" - -#include - -//-------------------------------------------------------------------------- - -// See https://stackoverflow.com/a/14038590 - -//-------------------------------------------------------------------------- - - -//-------------------------------------------------------------------------- - - -//-------------------------------------------------------------------------- - -#endif // MG5AMC_GPURUNTIME_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc index 4d8d01c60e..661fb88ffe 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc @@ -8,10 +8,10 @@ #include "ProcessData.h" #include "SigmaKin.h" -#include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" +#include #include // for fetestexcept #include #include diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h index 58cd323e39..1693d61c42 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h @@ -9,7 +9,6 @@ #include "mgOnGpuConfig.h" -#include "GpuAbstraction.h" #include "MemoryBuffers.h" #include diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h index 4d7edc9f0d..3bb7b591e7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h @@ -12,7 +12,6 @@ #include "mgOnGpuCxtypes.h" #include "ProcessData.h" -#include "GpuRuntime.h" #include "Parameters.h" #include @@ -161,29 +160,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- - /* - // A base class encapsulating a memory buffer with ONE fptype per event - typedef BufferBase BufferOneFp; - - // The size (number of elements) per event in a memory buffer with ONE fptype per event - constexpr size_t sizePerEventOneFp = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer with ONE fptype per event - typedef HostBuffer HostBufferOneFp; -#else - // A class encapsulating a CUDA pinned host buffer for gs - typedef PinnedHostBuffer PinnedHostBufferOneFp; - // A class encapsulating a CUDA device buffer for gs - typedef DeviceBuffer DeviceBufferOneFp; -#endif - - // Memory buffers for Gs (related to the event-by-event strength of running coupling constant alphas QCD) - typedef BufferOneFp BufferGs; - typedef HostBufferOneFp HostBufferGs; - typedef PinnedHostBufferOneFp PinnedHostBufferGs; - typedef DeviceBufferOneFp DeviceBufferGs; - */ //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc index 31ad05f0f0..4a0d3eaaa9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc @@ -11,7 +11,6 @@ #include "CPPProcess.h" // ProcessData.h, Parameters.h, HelAmps_.h transitively #include "ProcessTables.h" -#include "GpuRuntime.h" #include "MemoryAccessAmplitudes.h" #include "MemoryAccessChannelIds.h" #include "MemoryAccessCouplings.h" @@ -26,6 +25,8 @@ #include "color_sum.h" #include "ColorData.h" +#include + namespace mg5amcCpu { using namespace ProcessData; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h index c88f717621..8f4000a93a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h @@ -12,7 +12,6 @@ #include "mgOnGpuVectors.h" #include "ProcessData.h" -#include "GpuAbstraction.h" namespace mg5amcCpu { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h index dd13522513..668894b961 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h @@ -7,151 +7,51 @@ #ifndef MGONGPUCONFIG_H #define MGONGPUCONFIG_H 1 -// Is this a GPU (CUDA, HIP) or CPU implementation? -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#define MGONGPUCPP_GPUIMPL cuda -#elif defined __HIPCC__ -#define MGONGPUCPP_GPUIMPL hip -#include "hip/hip_runtime.h" // needed for blockDim, blockIdx, threadIdx: better in mgOnGpuConfig.h than in GpuAbstraction.h -#else -#undef MGONGPUCPP_GPUIMPL -#endif - -// Make sure that __HIP_PLATFORM_NVIDIA__ is undefined -// (__HIP_PLATFORM_AMD__ is defined by hipcc or in HiprandRandomNumberKernel.cc) -#undef __HIP_PLATFORM_NVIDIA__ // disable hiprand for NVidia (curand) - -// ** NB1 Throughputs (e.g. 6.8E8) are events/sec for "./gcheck.exe -p 65536 128 12" -// ** NB2 Baseline on b7g47n0004 fluctuates (probably depends on load on other VMs) - -// Choose if curand is supported for generating random numbers -// For HIP, by default, do not allow curand to be used (hiprand or common random numbers will be used instead) -// For both CUDA and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_CURAND -// (there exist CUDA installations, e.g. using the HPC package, which do not include curand - see PR #784 and #785) -#if defined __HIPCC__ -#define MGONGPU_HAS_NO_CURAND 1 -#else -//#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -//#undef MGONGPU_HAS_NO_CURAND // default -////#define MGONGPU_HAS_NO_CURAND 1 -//#else -//#undef MGONGPU_HAS_NO_CURAND // default -////#define MGONGPU_HAS_NO_CURAND 1 -//#endif -#endif - -// Choose if hiprand is supported for generating random numbers -// For CUDA, by default, do not allow hiprand to be used (curand or common random numbers will be used instead) -// For both HIP and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_HIPRAND -// (there may exist HIP installations which do not include hiprand?) -#if defined __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#define MGONGPU_HAS_NO_HIPRAND 1 -#else -//#ifdef __HIPCC__ -//#undef MGONGPU_HAS_NO_HIPRAND // default -////#define MGONGPU_HAS_NO_HIPRAND 1 -//#else -//#undef MGONGPU_HAS_NO_HIPRAND // default -////#define MGONGPU_HAS_NO_HIPRAND 1 -//#endif -#endif +// cpu backend: always built with a plain host compiler, never nvcc/hipcc, so the +// GPU-backend selection macro (see gpu/mgOnGpuConfig.h) is deliberately never +// defined here - single-file, all-backend headers rely on that to pick branches. // Choose floating point precision (for everything but color algebra #537) -// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE_FLOAT, nothing happens (issue #167) +// If set from outside with e.g. -DMGONGPU_FPTYPE_FLOAT, nothing happens (#167) #if not defined MGONGPU_FPTYPE_DOUBLE and not defined MGONGPU_FPTYPE_FLOAT -// Floating point precision (CHOOSE ONLY ONE) #define MGONGPU_FPTYPE_DOUBLE 1 // default //#define MGONGPU_FPTYPE_FLOAT 1 // 2x faster #endif // Choose floating point precision (for color algebra alone #537) -// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE2_FLOAT, nothing happens (issue #167) #if not defined MGONGPU_FPTYPE2_DOUBLE and not defined MGONGPU_FPTYPE2_FLOAT -// Floating point precision (CHOOSE ONLY ONE) #define MGONGPU_FPTYPE2_DOUBLE 1 // default //#define MGONGPU_FPTYPE2_FLOAT 1 // 2x faster #endif -// Choose whether to inline all HelAmps functions -// This optimization can gain almost a factor 4 in C++, similar to -flto (issue #229) -// By default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_INLINE_HELAMPS -//#undef MGONGPU_INLINE_HELAMPS // default -////#define MGONGPU_INLINE_HELAMPS 1 +// Choose whether to inline all HelAmps functions (can gain ~4x, issue #229) +// By default off; set from outside with -DMGONGPU_INLINE_HELAMPS +//#define MGONGPU_INLINE_HELAMPS 1 -// Choose whether to hardcode the cIPD physics parameters rather than reading them from user cards -// This optimization can gain 20% in CUDA in eemumu (issue #39) -// By default, do not hardcode, but allow this macro to be set from outside with e.g. -DMGONGPU_HARDCODE_PARAM -// ** NB: The option to use hardcoded cIPD physics parameters is supported again even now when alphas is running (#373) -// ** NB: Note however that it now only refers to cIPD parameters (cIPC parameters are always accessed through global memory) -//#undef MGONGPU_HARDCODE_PARAM // default -////#define MGONGPU_HARDCODE_PARAM 1 +// Choose whether to hardcode cIPD physics parameters instead of reading user cards +// By default off; set from outside with -DMGONGPU_HARDCODE_PARAM +//#define MGONGPU_HARDCODE_PARAM 1 -/* clang-format off */ -// Complex type in CUDA: thrust or cucomplex or cxsmpl (CHOOSE ONLY ONE) -// (NB THIS IS MGONGPU_*CU*CXTYPE_xxx) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#define MGONGPU_CUCXTYPE_THRUST 1 // default (~1.15E9/double, ~3.2E9/float) -//#define MGONGPU_CUCXTYPE_CUCOMPLEX 1 // ~10 percent slower (1.03E9/double, ~2.8E9/float) -//#define MGONGPU_CUCXTYPE_CXSMPL 1 // ~10 percent slower (1.00E9/double, ~2.9E9/float) +// Complex type in C++: cxsmpl by default, or std::complex (CHOOSE ONLY ONE) +//#define MGONGPU_CPPCXTYPE_STDCOMPLEX 1 // ~8% slower on float, same on double +#define MGONGPU_CPPCXTYPE_CXSMPL 1 // default -// Complex type in HIP: cxsmpl (ONLY ONE OPTION POSSIBLE? #810) -// (NB THIS IS MGONGPU_*HIP*CXTYPE_xxx) -#elif defined __HIPCC__ -#define MGONGPU_HIPCXTYPE_CXSMPL 1 // default for HIP - -// Complex type in C++: std::complex or cxsmpl (CHOOSE ONLY ONE) -// (NB THIS IS MGONGPU_*CPP*CXTYPE_xxx) -#else -//#define MGONGPU_CPPCXTYPE_STDCOMPLEX 1 // ~8 percent slower on float, same on double (5.1E6/double, 9.4E6/float) -#define MGONGPU_CPPCXTYPE_CXSMPL 1 // new default (5.1E6/double, 10.2E6/float) -#endif - -// Choose if cuBLAS and hipBLAS are supported for generating random numbers -// For both CUDA and HIP, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_BLAS -// (there may exist CUDA/HIP installations, e.g. using the HPC package, which do not include cuBLAS/hipBLAS?) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -//#undef MGONGPU_HAS_NO_BLAS // default -////#define MGONGPU_HAS_NO_BLAS 1 -#elif defined __HIPCC__ -//#undef MGONGPU_HAS_NO_BLAS // default -////#define MGONGPU_HAS_NO_BLAS 1 -#else +// No BLAS on the cpu backend (cuBLAS/hipBLAS are GPU-only) #define MGONGPU_HAS_NO_BLAS 1 -#endif -// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#undef MGONGPU_NSIGHT_DEBUG // default in CUDA -//#define MGONGPU_NSIGHT_DEBUG 1 // CURRENTLY NO LONGER SUPPORTED! -#else -#undef MGONGPU_NSIGHT_DEBUG // only option in HIP or C++ -#endif /* clang-format on */ +// nsight compute (ncu) debugging is CUDA-only; always off here +#undef MGONGPU_NSIGHT_DEBUG -// SANITY CHECKS (floating point precision for everything but color algebra #537) +// SANITY CHECKS #if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE_FLOAT -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE_DOUBLE or defined MGONGPU_FPTYPE_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE_DOUBLE or MGONGPU_FPTYPE_FLOAT #endif - -// SANITY CHECKS (floating point precision for color algebra alone #537) #if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE2_DOUBLE or defined MGONGPU_FPTYPE2_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE2_DOUBLE or MGONGPU_FPTYPE2_FLOAT #endif #if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE_FLOAT #error You cannot use double precision for color algebra and single precision elsewhere #endif - -// SANITY CHECKS (CUDA complex number implementation) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#if defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CUCOMPLEX -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX for CUDA -#elif defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CXSMPL -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CXSMPL for CUDA -#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX and defined MGONGPU_CUCXTYPE_CXSMPL -#error You must CHOOSE (ONE AND) ONLY ONE OF MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL for CUDA -#endif -#endif - -// SANITY CHECKS (C++ complex number implementation) #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX and defined MGONGPU_CPPCXTYPE_CXSMPL #error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL for C++ #endif @@ -159,9 +59,6 @@ // NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) namespace mgOnGpu { - - // --- Type definitions - // Floating point type (for everything but color algebra #537): fptype #if defined MGONGPU_FPTYPE_DOUBLE typedef double fptype; // double precision (8 bytes, fp64) @@ -176,25 +73,14 @@ namespace mgOnGpu typedef float fptype2; // single precision (4 bytes, fp32) #endif - // --- Platform-specific software implementation details - - // Maximum number of blocks per grid - // ** NB Some arrays of pointers will be allocated statically to fit all these blocks - // ** (the actual memory for each block will then be allocated dynamically only for existing blocks) - //const int nbpgMAX = 2048; - // Maximum number of threads per block - //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" // Alignment requirement for using reinterpret_cast with SIMD vectorized code - // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) - // Only needed for C++ code but can be enforced also in NVCC builds of C++ code using CUDA>=11.2 and C++17 (#318, #319, #333) - constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) - + constexpr int cppAlign = 64; // 64-byte i.e. 512-bit } -// Expose typedefs and operators outside the namespace +// Expose typedefs outside the namespace using mgOnGpu::fptype; using mgOnGpu::fptype2; @@ -204,81 +90,63 @@ using mgOnGpu::fptype2; #endif // C++ SIMD vectorization width (this will be used to set neppV) -#if defined __AVX512VL__ && defined MGONGPU_PVW512 // C++ "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#if defined __AVX512VL__ && defined MGONGPU_PVW512 // "512z" AVX512 512-bit: 8 (DOUBLE) or 16 (FLOAT) #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 8 #else #define MGONGPU_CPPSIMD 16 #endif -#elif defined __AVX512VL__ // C++ "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#elif defined __AVX512VL__ // "512y" AVX512 256-bit: 4 (DOUBLE) or 8 (FLOAT) [gcc default] #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 4 #else #define MGONGPU_CPPSIMD 8 #endif -#elif defined __AVX2__ // C++ "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#elif defined __AVX2__ // "avx2" 256-bit: 4 (DOUBLE) or 8 (FLOAT) [clang default] #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 4 #else #define MGONGPU_CPPSIMD 8 #endif -#elif defined __SSE4_2__ // C++ "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [Power9 default] +#elif defined __SSE4_2__ // "sse4" SSE4.2 128-bit: 2 (DOUBLE) or 4 (FLOAT) [Power9 default] #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 2 #else #define MGONGPU_CPPSIMD 4 #endif -#elif defined __ARM_NEON // C++ "sse4" ARM NEON (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [ARM default] +#elif defined __ARM_NEON // ARM NEON 128-bit: 2 (DOUBLE) or 4 (FLOAT) [ARM default] #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 2 #else #define MGONGPU_CPPSIMD 4 #endif -#else // C++ "none" i.e. no SIMD +#else // "none" i.e. no SIMD #undef MGONGPU_CPPSIMD #endif -/* clang-format off */ -// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation [NB: CURRENTLY NO LONGER SUPPORTED!] -// Arguments (not used so far): text is __FUNCTION__, code is 0 (start) or 1 (end) -//#if defined __CUDACC__ && defined MGONGPU_NSIGHT_DEBUG // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -//#define mgDebugDeclare() __shared__ float mgDebugCounter[mgOnGpu::ntpbMAX]; -//#define mgDebugInitialise() { mgDebugCounter[threadIdx.x] = 0; } -//#define mgDebug( code, text ) { mgDebugCounter[threadIdx.x] += 1; } -//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%f\n", mgDebugCounter[threadIdx.x] ); } -//#else +// No-op debug macros (nsight-based debugging is CUDA-only, unused here) #define mgDebugDeclare() /*noop*/ #define mgDebugInitialise() /*noop*/ #define mgDebug( code, text ) /*noop*/ #define mgDebugFinalise() /*noop*/ -//#endif /* clang-format on */ // Define empty CUDA/HIP declaration specifiers for C++ #define __global__ #define __host__ #define __device__ -// For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two (https://stackoverflow.com/a/108360) +// For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two inline constexpr bool ispoweroftwo( int n ) { return ( n > 0 ) && !( n & ( n - 1 ) ); } -// Compiler version support (#96): require nvcc from CUDA >= 11.2, e.g. to use C++17 (see #333) -#ifdef __NVCC__ -#if( __CUDACC_VER_MAJOR__ < 11 ) || ( __CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ < 2 ) -#error Unsupported CUDA version: please use CUDA >= 11.2 -#endif -#endif - -// Compiler version support (#96): require clang >= 11 +// Compiler version support (#96) #if defined __clang__ #if( __clang_major__ < 11 ) #error Unsupported clang version: please use clang >= 11 #endif -// Compiler version support (#96): require gcc >= 9.3, e.g. for some OMP issues (see #269) -// [NB skip this check for the gcc toolchain below clang or icx (TEMPORARY? #355)] #elif defined __GNUC__ #if( __GNUC__ < 9 ) || ( __GNUC__ == 9 && __GNUC_MINOR__ < 3 ) #error Unsupported gcc version: please gcc >= 9.3 diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h index 900a496b7d..dea5dcf2d1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h @@ -19,94 +19,12 @@ #include -// Complex type in cuda: thrust or cucomplex or cxsmpl -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#if defined MGONGPU_CUCXTYPE_THRUST -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) -#include -#pragma clang diagnostic pop -#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX -#include -#elif not defined MGONGPU_CUCXTYPE_CXSMPL -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL -#endif -// Complex type in HIP: cxsmpl -#elif defined __HIPCC__ -#if not defined MGONGPU_HIPCXTYPE_CXSMPL -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_HIPCXTYPE_CXSMPL -#endif -#else -// Complex type in c++ or HIP: std::complex or cxsmpl +// Complex type in c++: std::complex or cxsmpl #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX #include #elif not defined MGONGPU_CPPCXTYPE_CXSMPL #error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL #endif -#endif - -//========================================================================== -// COMPLEX TYPES: INSTRUMENTED CUCOMPLEX CLASS (cucomplex) -//========================================================================== - -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#if defined MGONGPU_CUCXTYPE_CUCOMPLEX -namespace mg5amcGpu -{ -#if defined MGONGPU_FPTYPE_DOUBLE - class cucomplex - { - public: - __host__ __device__ cucomplex( const double& r = 0, const double& i = 0 ) - : m_ri( make_cuDoubleComplex( r, i ) ) {} - __host__ __device__ constexpr cucomplex( const cuDoubleComplex& ri ) - : m_ri( ri ) {} - //__host__ __device__ operator cuDoubleComplex&() { return m_ri; } - __host__ __device__ constexpr operator cuDoubleComplex() const { return m_ri; } - __host__ __device__ double real() const { return cuCreal( m_ri ); } - __host__ __device__ double imag() const { return cuCimag( m_ri ); } - inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) - { - m_ri = cuCadd( m_ri, c ); - return *this; - } - inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) - { - m_ri = cuCsub( m_ri, c ); - return *this; - } - private: - cuDoubleComplex m_ri; - }; -#elif defined MGONGPU_FPTYPE_FLOAT - class cucomplex - { - public: - __host__ __device__ cucomplex( const float& r = 0, const float& i = 0 ) - : m_ri( make_cuFloatComplex( r, i ) ) {} - __host__ __device__ constexpr cucomplex( const cuFloatComplex& ri ) - : m_ri( ri ) {} - //__host__ __device__ operator cuFloatComplex&() { return m_ri; } - __host__ __device__ constexpr operator cuFloatComplex() const { return m_ri; } - __host__ __device__ float real() const { return cuCrealf( m_ri ); } - __host__ __device__ float imag() const { return cuCimagf( m_ri ); } - inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) - { - m_ri = cuCaddf( m_ri, c ); - return *this; - } - inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) - { - m_ri = cuCsubf( m_ri, c ); - return *this; - } - private: - cuFloatComplex m_ri; - }; -#endif -} -#endif -#endif //========================================================================== // COMPLEX TYPES: SIMPLE COMPLEX CLASS (cxsmpl) @@ -135,16 +53,8 @@ namespace mgOnGpu /* clang-format off */ __host__ __device__ constexpr const FP& real() const { return m_real; } __host__ __device__ constexpr const FP& imag() const { return m_imag; } template __host__ __device__ constexpr operator cxsmpl() const { return cxsmpl( m_real, m_imag ); } -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#ifdef MGONGPU_CUCXTYPE_THRUST - template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } -#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX - __host__ __device__ constexpr operator mg5amcGpu::cucomplex() const { return mg5amcGpu::cucomplex( m_real, m_imag ); } -#endif -#else #ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX template __host__ __device__ constexpr operator std::complex() const { return std::complex( m_real, m_imag ); } -#endif #endif private: FP m_real, m_imag; // RI @@ -292,24 +202,13 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) namespace mg5amcCpu { // --- Type definitions (complex type: cxtype) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#if defined MGONGPU_CUCXTYPE_THRUST - typedef thrust::complex cxtype; -#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX - typedef cucomplex cxtype; -#else - typedef cxsmpl cxtype; -#endif -#else // c++ #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX typedef std::complex cxtype; #else typedef cxsmpl cxtype; -#endif #endif // SANITY CHECK: memory access may be based on casts of fptype[2] to cxtype (e.g. for wavefunctions) @@ -324,13 +223,12 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) namespace mg5amcCpu { -#if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL +#if defined MGONGPU_CPPCXTYPE_CXSMPL //------------------------------ - // CUDA or C++ - using cxsmpl + // C++ - using cxsmpl //------------------------------ inline __host__ __device__ cxtype @@ -369,258 +267,11 @@ namespace mg5amcCpu return cxmake( c.real(), c.imag() ); } -#endif // #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL - - //========================================================================== - -#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST // cuda + thrust (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) - - //------------------------------ - // CUDA - using thrust::complex - //------------------------------ - - inline __host__ __device__ cxtype - cxmake( const fptype& r, const fptype& i ) - { - return cxtype( r, i ); // thrust::complex constructor - } - - inline __host__ __device__ fptype - cxreal( const cxtype& c ) - { - return c.real(); // thrust::complex::real() - } - - inline __host__ __device__ fptype - cximag( const cxtype& c ) - { - return c.imag(); // thrust::complex::imag() - } - - inline __host__ __device__ cxtype - cxconj( const cxtype& c ) - { - return conj( c ); // conj( thrust::complex ) - } - - inline __host__ __device__ const cxtype& - cxmake( const cxtype& c ) - { - return c; - } - -#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST - - //========================================================================== - -#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX // cuda + cucomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) - - //------------------------------ - // CUDA - using cuComplex - //------------------------------ - -#if defined MGONGPU_FPTYPE_DOUBLE // cuda + cucomplex + double - - //+++++++++++++++++++++++++ - // cuDoubleComplex ONLY - //+++++++++++++++++++++++++ - - inline __host__ __device__ cxtype - cxmake( const fptype& r, const fptype& i ) - { - return make_cuDoubleComplex( r, i ); - } - - inline __host__ __device__ fptype - cxreal( const cxtype& c ) - { - return cuCreal( c ); // returns by value - } - - inline __host__ __device__ fptype - cximag( const cxtype& c ) - { - return cuCimag( c ); // returns by value - } - - inline __host__ __device__ cxtype - operator+( const cxtype& a, const cxtype& b ) - { - return cuCadd( a, b ); - } - - inline __host__ __device__ cxtype - operator-( const cxtype& a, const cxtype& b ) - { - return cuCsub( a, b ); - } - - inline __host__ __device__ cxtype - operator*( const cxtype& a, const cxtype& b ) - { - return cuCmul( a, b ); - } - - inline __host__ __device__ cxtype - operator/( const cxtype& a, const cxtype& b ) - { - return cuCdiv( a, b ); - } - - inline __host__ std::ostream& - operator<<( std::ostream& out, const cxtype& c ) - { - //out << std::complex( cxreal( c ), cximag( c ) ); - out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma - return out; - } - -#elif defined MGONGPU_FPTYPE_FLOAT // cuda + cucomplex + float - - //+++++++++++++++++++++++++ - // cuFloatComplex ONLY - //+++++++++++++++++++++++++ - - inline __host__ __device__ cxtype - cxmake( const fptype& r, const fptype& i ) - { - return make_cuFloatComplex( r, i ); - } - - inline __host__ __device__ fptype - cxreal( const cxtype& c ) - { - return cuCrealf( c ); // returns by value - } - - inline __host__ __device__ fptype - cximag( const cxtype& c ) - { - return cuCimagf( c ); // returns by value - } - - inline __host__ __device__ cxtype - operator+( const cxtype& a, const cxtype& b ) - { - return cuCaddf( a, b ); - } - - inline __host__ __device__ cxtype - operator-( const cxtype& a, const cxtype& b ) - { - return cuCsubf( a, b ); - } - - inline __host__ __device__ cxtype - operator*( const cxtype& a, const cxtype& b ) - { - return cuCmulf( a, b ); - } - - inline __host__ __device__ cxtype - operator/( const cxtype& a, const cxtype& b ) - { - return cuCdivf( a, b ); - } - - inline __host__ cxtype // NOT __device__ - cxmake( const std::complex& c ) // std::complex to cucomplex (cast double-to-float) - { - return cxmake( (fptype)c.real(), (fptype)c.imag() ); - } - - inline __host__ std::ostream& - operator<<( std::ostream& out, const cxtype& c ) - { - //out << std::complex( cxreal( c ), cximag( c ) ); - out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma - return out; - } - -#endif - - //+++++++++++++++++++++++++ - // cuDoubleComplex OR - // cuFloatComplex - //+++++++++++++++++++++++++ - - inline __host__ __device__ cxtype - operator+( const cxtype a ) - { - return a; - } - - inline __host__ __device__ cxtype - operator-( const cxtype& a ) - { - return cxmake( -cxreal( a ), -cximag( a ) ); - } - - inline __host__ __device__ cxtype - operator+( const fptype& a, const cxtype& b ) - { - return cxmake( a, 0 ) + b; - } - - inline __host__ __device__ cxtype - operator-( const fptype& a, const cxtype& b ) - { - return cxmake( a, 0 ) - b; - } - - inline __host__ __device__ cxtype - operator*( const fptype& a, const cxtype& b ) - { - return cxmake( a, 0 ) * b; - } - - inline __host__ __device__ cxtype - operator/( const fptype& a, const cxtype& b ) - { - return cxmake( a, 0 ) / b; - } - - inline __host__ __device__ cxtype - operator+( const cxtype& a, const fptype& b ) - { - return a + cxmake( b, 0 ); - } - - inline __host__ __device__ cxtype - operator-( const cxtype& a, const fptype& b ) - { - return a - cxmake( b, 0 ); - } - - inline __host__ __device__ cxtype - operator*( const cxtype& a, const fptype& b ) - { - return a * cxmake( b, 0 ); - } - - inline __host__ __device__ cxtype - operator/( const cxtype& a, const fptype& b ) - { - return a / cxmake( b, 0 ); - } - - inline __host__ __device__ cxtype - cxconj( const cxtype& c ) - { - return cxmake( cxreal( c ), -cximag( c ) ); - } - - inline __host__ cxtype // NOT __device__ - cxmake( const std::complex& c ) // std::complex to cucomplex (float-to-float or double-to-double) - { - return cxmake( c.real(), c.imag() ); - } - -#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX +#endif // #if defined MGONGPU_CPPCXTYPE_CXSMPL //========================================================================== -#if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX // c++/hip + stdcomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX //------------------------------ // C++ - using std::complex @@ -664,7 +315,7 @@ namespace mg5amcCpu } #endif -#endif // #if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX +#endif // #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc index 7fcdb3d91f..cefa01b48f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc @@ -9,7 +9,6 @@ #include "ProcessData.h" #include "CPPProcess.h" // needed to construct/initProc the process object (umami_initialize) #include "SigmaKin.h" // sigmaKin_getGoodHel/setGoodHel -#include "GpuRuntime.h" #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h index 9f2a179e8b..c70dff0143 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h @@ -92,43 +92,6 @@ namespace mg5amcGpu //-------------------------------------------------------------------------- - /* -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating the calculation of event statistics on a GPU device - class CrossSectionKernelDevice : public CrossSectionKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats, // output: event statistics - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~CrossSectionKernelDevice(){} - - // Reset gpublocks and gputhreads - void setGrid( const size_t gpublocks, const size_t gputhreads ); - - // Update event statistics - void updateEventStatistics( const bool debug=false ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - - }; -#endif - */ //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h index 6f709dfe9a..5462a3eeeb 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h @@ -13,7 +13,7 @@ //-------------------------------------------------------------------------- -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #ifndef MGONGPU_HAS_NO_BLAS #include "cublas_v2.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc index 7c91a9929e..207a17cc8d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc @@ -234,7 +234,7 @@ namespace mg5amcGpu #endif } #ifndef MGONGPU_HAS_NO_BLAS -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ // Analyse environment variable CUDACPP_RUNTIME_CUBLASTF32TENSOR const char* blasEnv2 = getenv( "CUDACPP_RUNTIME_CUBLASTF32TENSOR" ); if( blasEnv2 && std::string( blasEnv2 ) != "" ) @@ -307,7 +307,7 @@ namespace mg5amcGpu if( m_blasColorSum ) { checkGpuBlas( gpuBlasCreate( &m_blasHandle ) ); -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ if( m_blasTf32Tensor ) checkGpuBlas( cublasSetMathMode( m_blasHandle, CUBLAS_TF32_TENSOR_OP_MATH ) ); // enable TF32 tensor cores #endif diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h index fa4b2a72a1..796b49ed20 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h @@ -180,29 +180,6 @@ namespace mg5amcGpu //-------------------------------------------------------------------------- - /* - // A base class encapsulating a memory buffer with ONE fptype per event - typedef BufferBase BufferOneFp; - - // The size (number of elements) per event in a memory buffer with ONE fptype per event - constexpr size_t sizePerEventOneFp = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer with ONE fptype per event - typedef HostBuffer HostBufferOneFp; -#else - // A class encapsulating a CUDA pinned host buffer for gs - typedef PinnedHostBuffer PinnedHostBufferOneFp; - // A class encapsulating a CUDA device buffer for gs - typedef DeviceBuffer DeviceBufferOneFp; -#endif - - // Memory buffers for Gs (related to the event-by-event strength of running coupling constant alphas QCD) - typedef BufferOneFp BufferGs; - typedef HostBufferOneFp HostBufferGs; - typedef PinnedHostBufferOneFp PinnedHostBufferGs; - typedef DeviceBufferOneFp DeviceBufferGs; - */ //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc index ee826fb32e..4fa881936f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc @@ -180,7 +180,7 @@ namespace mg5amcGpu { constexpr size_t nxcoup = ndcoup + nIPC; // both dependent and independent couplings const fptype* allCOUPs[nxcoup]; -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #pragma nv_diagnostic push #pragma nv_diag_suppress 186 // e.g. <> #endif @@ -188,7 +188,7 @@ namespace mg5amcGpu allCOUPs[idcoup] = CD_ACCESS::idcoupAccessBufferConst( allcouplings, idcoup ); // dependent couplings, vary event-by-event for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) allCOUPs[ndcoup + iicoup] = CI_ACCESS::iicoupAccessBufferConst( cIPC, iicoup ); // independent couplings, fixed for all events -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #pragma nv_diagnostic pop #endif // CUDA kernels take input/output buffers with momenta/MEs for all events diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h index bc3607d686..9110dc483f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h @@ -8,7 +8,7 @@ #define MGONGPUCONFIG_H 1 // Is this a GPU (CUDA, HIP) or CPU implementation? -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #define MGONGPUCPP_GPUIMPL cuda #elif defined __HIPCC__ #define MGONGPUCPP_GPUIMPL hip @@ -31,7 +31,7 @@ #if defined __HIPCC__ #define MGONGPU_HAS_NO_CURAND 1 #else -//#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#ifdef __CUDACC__ // this must be __CUDACC__ //#undef MGONGPU_HAS_NO_CURAND // default ////#define MGONGPU_HAS_NO_CURAND 1 //#else @@ -44,7 +44,7 @@ // For CUDA, by default, do not allow hiprand to be used (curand or common random numbers will be used instead) // For both HIP and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_HIPRAND // (there may exist HIP installations which do not include hiprand?) -#if defined __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#if defined __CUDACC__ // this must be __CUDACC__ #define MGONGPU_HAS_NO_HIPRAND 1 #else //#ifdef __HIPCC__ @@ -89,7 +89,7 @@ /* clang-format off */ // Complex type in CUDA: thrust or cucomplex or cxsmpl (CHOOSE ONLY ONE) // (NB THIS IS MGONGPU_*CU*CXTYPE_xxx) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #define MGONGPU_CUCXTYPE_THRUST 1 // default (~1.15E9/double, ~3.2E9/float) //#define MGONGPU_CUCXTYPE_CUCOMPLEX 1 // ~10 percent slower (1.03E9/double, ~2.8E9/float) //#define MGONGPU_CUCXTYPE_CXSMPL 1 // ~10 percent slower (1.00E9/double, ~2.9E9/float) @@ -109,7 +109,7 @@ // Choose if cuBLAS and hipBLAS are supported for generating random numbers // For both CUDA and HIP, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_BLAS // (there may exist CUDA/HIP installations, e.g. using the HPC package, which do not include cuBLAS/hipBLAS?) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ //#undef MGONGPU_HAS_NO_BLAS // default ////#define MGONGPU_HAS_NO_BLAS 1 #elif defined __HIPCC__ @@ -120,7 +120,7 @@ #endif // CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #undef MGONGPU_NSIGHT_DEBUG // default in CUDA //#define MGONGPU_NSIGHT_DEBUG 1 // CURRENTLY NO LONGER SUPPORTED! #else @@ -141,7 +141,7 @@ #endif // SANITY CHECKS (CUDA complex number implementation) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #if defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CUCOMPLEX #error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX for CUDA #elif defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CXSMPL @@ -205,7 +205,7 @@ using mgOnGpu::fptype2; /* clang-format off */ // CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation [NB: CURRENTLY NO LONGER SUPPORTED!] // Arguments (not used so far): text is __FUNCTION__, code is 0 (start) or 1 (end) -//#if defined __CUDACC__ && defined MGONGPU_NSIGHT_DEBUG // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +//#if defined __CUDACC__ && defined MGONGPU_NSIGHT_DEBUG // this must be __CUDACC__ //#define mgDebugDeclare() __shared__ float mgDebugCounter[mgOnGpu::ntpbMAX]; //#define mgDebugInitialise() { mgDebugCounter[threadIdx.x] = 0; } //#define mgDebug( code, text ) { mgDebugCounter[threadIdx.x] += 1; } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h index c185d2e14d..1cd951672e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h @@ -20,7 +20,7 @@ #include // Complex type in cuda: thrust or cucomplex or cxsmpl -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #if defined MGONGPU_CUCXTYPE_THRUST #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) @@ -49,7 +49,7 @@ // COMPLEX TYPES: INSTRUMENTED CUCOMPLEX CLASS (cucomplex) //========================================================================== -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #if defined MGONGPU_CUCXTYPE_CUCOMPLEX namespace mg5amcGpu { @@ -135,7 +135,7 @@ namespace mgOnGpu /* clang-format off */ __host__ __device__ constexpr const FP& real() const { return m_real; } __host__ __device__ constexpr const FP& imag() const { return m_imag; } template __host__ __device__ constexpr operator cxsmpl() const { return cxsmpl( m_real, m_imag ); } -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #ifdef MGONGPU_CUCXTYPE_THRUST template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } #elif defined MGONGPU_CUCXTYPE_CUCOMPLEX @@ -296,7 +296,7 @@ namespace mg5amcGpu namespace mg5amcGpu { // --- Type definitions (complex type: cxtype) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #if defined MGONGPU_CUCXTYPE_THRUST typedef thrust::complex cxtype; #elif defined MGONGPU_CUCXTYPE_CUCOMPLEX @@ -373,7 +373,7 @@ namespace mg5amcGpu //========================================================================== -#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST // cuda + thrust (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) +#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST // cuda + thrust (this must be __CUDACC__) //------------------------------ // CUDA - using thrust::complex @@ -413,7 +413,7 @@ namespace mg5amcGpu //========================================================================== -#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX // cuda + cucomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) +#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX // cuda + cucomplex (this must be __CUDACC__) //------------------------------ // CUDA - using cuComplex @@ -620,7 +620,7 @@ namespace mg5amcGpu //========================================================================== -#if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX // c++/hip + stdcomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) +#if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX // c++/hip + stdcomplex (this must be __CUDACC__) //------------------------------ // C++ - using std::complex diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc index 861bca8940..019eb2e978 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc @@ -6,11 +6,11 @@ #include "CrossSectionKernels.h" -#include "GpuAbstraction.h" #include "MemoryAccessMatrixElements.h" #include "MemoryAccessWeights.h" #include "MemoryBuffers.h" +#include #include // ****************************************************************************************** diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h index 9fb3c3fce2..4ac3ded803 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h @@ -92,43 +92,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- - /* -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating the calculation of event statistics on a GPU device - class CrossSectionKernelDevice : public CrossSectionKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats, // output: event statistics - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~CrossSectionKernelDevice(){} - - // Reset gpublocks and gputhreads - void setGrid( const size_t gpublocks, const size_t gputhreads ); - - // Update event statistics - void updateEventStatistics( const bool debug=false ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - - }; -#endif - */ //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuAbstraction.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuAbstraction.h deleted file mode 100644 index 6f709dfe9a..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuAbstraction.h +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: J. Teig (Jul 2023) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2020-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MG5AMC_GPUABSTRACTION_H -#define MG5AMC_GPUABSTRACTION_H 1 - -#include "mgOnGpuConfig.h" - -#include - -//-------------------------------------------------------------------------- - -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) - -#ifndef MGONGPU_HAS_NO_BLAS -#include "cublas_v2.h" -#endif - -#define gpuError_t cudaError_t -#define gpuPeekAtLastError cudaPeekAtLastError -#define gpuGetErrorString cudaGetErrorString -#define gpuSuccess cudaSuccess - -#define gpuMallocHost( ptr, size ) checkGpu( cudaMallocHost( ptr, size ) ) -#define gpuMalloc( ptr, size ) checkGpu( cudaMalloc( ptr, size ) ) - -#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( cudaMemcpy( dstData, srcData, srcBytes, func ) ) -#define gpuMemset( data, value, bytes ) checkGpu( cudaMemset( data, value, bytes ) ) -#define gpuMemcpyHostToDevice cudaMemcpyHostToDevice -#define gpuMemcpyDeviceToHost cudaMemcpyDeviceToHost -#define gpuMemcpyDeviceToDevice cudaMemcpyDeviceToDevice -#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( cudaMemcpyToSymbol( type1, type2, size ) ) - -#define gpuFree( ptr ) checkGpu( cudaFree( ptr ) ) -#define gpuFreeHost( ptr ) checkGpu( cudaFreeHost( ptr ) ) - -#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( cudaGetSymbolAddress( devPtr, symbol ) ) - -#define gpuSetDevice cudaSetDevice -#define gpuDeviceSynchronize cudaDeviceSynchronize -#define gpuDeviceReset cudaDeviceReset - -#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) -//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> -#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) -#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) - -#define gpuStream_t cudaStream_t -#define gpuStreamCreate( pStream ) checkGpu( cudaStreamCreate( pStream ) ) -#define gpuStreamDestroy( stream ) checkGpu( cudaStreamDestroy( stream ) ) -#define gpuMallocAsync( ptr, size, stream ) checkGpu( cudaMallocAsync( ptr, size, stream ) ) -#define gpuFreeAsync( ptr, stream ) checkGpu( cudaFreeAsync( ptr, stream ) ) -#define gpuStreamSynchronize( stream ) checkGpu( cudaStreamSynchronize( stream ) ) - -#define gpuBlasStatus_t cublasStatus_t -#define GPUBLAS_STATUS_SUCCESS CUBLAS_STATUS_SUCCESS -#ifndef MGONGPU_HAS_NO_BLAS -#define gpuBlasHandle_t cublasHandle_t -#else -#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds -#endif -#define gpuBlasCreate cublasCreate -#define gpuBlasDestroy cublasDestroy -#define gpuBlasSetStream cublasSetStream - -#define gpuBlasSaxpy cublasSaxpy -#define gpuBlasSdot cublasSdot -#define gpuBlasSgemv cublasSgemv -#define gpuBlasSgemm cublasSgemm -#define gpuBlasSgemmStridedBatched cublasSgemmStridedBatched -#define gpuBlasDaxpy cublasDaxpy -#define gpuBlasDdot cublasDdot -#define gpuBlasDgemv cublasDgemv -#define gpuBlasDgemm cublasDgemm -#define gpuBlasDgemmStridedBatched cublasDgemmStridedBatched -#define GPUBLAS_OP_N CUBLAS_OP_N -#define GPUBLAS_OP_T CUBLAS_OP_T - -//-------------------------------------------------------------------------- - -#elif defined __HIPCC__ - -#ifndef MGONGPU_HAS_NO_BLAS -#include "hipblas/hipblas.h" -#endif - -#define gpuError_t hipError_t -#define gpuPeekAtLastError hipPeekAtLastError -#define gpuGetErrorString hipGetErrorString -#define gpuSuccess hipSuccess - -#define gpuMallocHost( ptr, size ) checkGpu( hipHostMalloc( ptr, size ) ) // HostMalloc better -#define gpuMalloc( ptr, size ) checkGpu( hipMalloc( ptr, size ) ) - -#define gpuMemcpy( dstData, srcData, srcBytes, func ) checkGpu( hipMemcpy( dstData, srcData, srcBytes, func ) ) -#define gpuMemset( data, value, bytes ) checkGpu( hipMemset( data, value, bytes ) ) -#define gpuMemcpyHostToDevice hipMemcpyHostToDevice -#define gpuMemcpyDeviceToHost hipMemcpyDeviceToHost -#define gpuMemcpyDeviceToDevice hipMemcpyDeviceToDevice -#define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( hipMemcpyToSymbol( type1, type2, size ) ) - -#define gpuFree( ptr ) checkGpu( hipFree( ptr ) ) -#define gpuFreeHost( ptr ) checkGpu( hipHostFree( ptr ) ) - -#define gpuGetSymbolAddress( devPtr, symbol ) checkGpu( hipGetSymbolAddress( devPtr, symbol ) ) - -#define gpuSetDevice hipSetDevice -#define gpuDeviceSynchronize hipDeviceSynchronize -#define gpuDeviceReset hipDeviceReset - -#define gpuLaunchKernel( kernel, blocks, threads, ... ) kernel<<>>( __VA_ARGS__ ) -//#define gpuLaunchKernelSharedMem( kernel, blocks, threads, sharedMem, ... ) kernel<<>>( __VA_> -#define gpuLaunchKernelStream( kernel, blocks, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) -#define gpuLaunchKernel2D( kernel, blocks_x, blocks_y, threads, stream, ... ) kernel<<>>( __VA_ARGS__ ) - -#define gpuStream_t hipStream_t -#define gpuStreamCreate( pStream ) checkGpu( hipStreamCreate( pStream ) ) -#define gpuStreamDestroy( stream ) checkGpu( hipStreamDestroy( stream ) ) -#define gpuMallocAsync( ptr, size, stream ) checkGpu( hipMallocAsync( ptr, size, stream ) ) -#define gpuFreeAsync( ptr, stream ) checkGpu( hipFreeAsync( ptr, stream ) ) -#define gpuStreamSynchronize( stream ) checkGpu( hipStreamSynchronize( stream ) ) - -#define gpuBlasStatus_t hipblasStatus_t -#define GPUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS -#ifndef MGONGPU_HAS_NO_BLAS -#define gpuBlasHandle_t hipblasHandle_t -#else -#define gpuBlasHandle_t void // hack to keep the same API also in noBLAS builds -#endif -#define gpuBlasCreate hipblasCreate -#define gpuBlasDestroy hipblasDestroy -#define gpuBlasSetStream hipblasSetStream - -#define gpuBlasSaxpy hipblasSaxpy -#define gpuBlasSdot hipblasSdot -#define gpuBlasSgemv hipblasSgemv -#define gpuBlasSgemm hipblasSgemm -#define gpuBlasSgemmStridedBatched hipblasSgemmStridedBatched -#define gpuBlasDaxpy hipblasDaxpy -#define gpuBlasDdot hipblasDdot -#define gpuBlasDgemv hipblasDgemv -#define gpuBlasDgemm hipblasDgemm -#define gpuBlasDgemmStridedBatched hipblasDgemmStridedBatched -#define GPUBLAS_OP_N HIPBLAS_OP_N -#define GPUBLAS_OP_T HIPBLAS_OP_T - -#endif - -//-------------------------------------------------------------------------- - -#ifdef MGONGPU_FPTYPE2_FLOAT -#define gpuBlasTaxpy gpuBlasSaxpy -#define gpuBlasTdot gpuBlasSdot -#define gpuBlasTgemv gpuBlasSgemv -#define gpuBlasTgemm gpuBlasSgemm -#define gpuBlasTgemmStridedBatched gpuBlasSgemmStridedBatched -#else -#define gpuBlasTaxpy gpuBlasDaxpy -#define gpuBlasTdot gpuBlasDdot -#define gpuBlasTgemv gpuBlasDgemv -#define gpuBlasTgemm gpuBlasDgemm -#define gpuBlasTgemmStridedBatched gpuBlasDgemmStridedBatched -#endif - -#endif // MG5AMC_GPUABSTRACTION_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h deleted file mode 100644 index 7366509821..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/GpuRuntime.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: J. Teig (Jun 2023, based on earlier work by S. Roiser) for the MG5aMC CUDACPP plugin. -// Further modified by: O. Mattelaer, S. Roiser, J. Teig, A. Valassi, Z. Wettersten (2020-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MG5AMC_GPURUNTIME_H -#define MG5AMC_GPURUNTIME_H 1 - -// MG5AMC on GPU uses the CUDA runtime API, not the lower level CUDA driver API -// See https://docs.nvidia.com/cuda/cuda-runtime-api/driver-vs-runtime-api.html#driver-vs-runtime-api - -#include "GpuAbstraction.h" - -#include - -//-------------------------------------------------------------------------- - -// See https://stackoverflow.com/a/14038590 - -//-------------------------------------------------------------------------- - - -//-------------------------------------------------------------------------- - - -//-------------------------------------------------------------------------- - -#endif // MG5AMC_GPURUNTIME_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc index 4d8d01c60e..661fb88ffe 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc @@ -8,10 +8,10 @@ #include "ProcessData.h" #include "SigmaKin.h" -#include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" +#include #include // for fetestexcept #include #include diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h index 58cd323e39..1693d61c42 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h @@ -9,7 +9,6 @@ #include "mgOnGpuConfig.h" -#include "GpuAbstraction.h" #include "MemoryBuffers.h" #include diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h index 4d7edc9f0d..3bb7b591e7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h @@ -12,7 +12,6 @@ #include "mgOnGpuCxtypes.h" #include "ProcessData.h" -#include "GpuRuntime.h" #include "Parameters.h" #include @@ -161,29 +160,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- - /* - // A base class encapsulating a memory buffer with ONE fptype per event - typedef BufferBase BufferOneFp; - - // The size (number of elements) per event in a memory buffer with ONE fptype per event - constexpr size_t sizePerEventOneFp = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer with ONE fptype per event - typedef HostBuffer HostBufferOneFp; -#else - // A class encapsulating a CUDA pinned host buffer for gs - typedef PinnedHostBuffer PinnedHostBufferOneFp; - // A class encapsulating a CUDA device buffer for gs - typedef DeviceBuffer DeviceBufferOneFp; -#endif - - // Memory buffers for Gs (related to the event-by-event strength of running coupling constant alphas QCD) - typedef BufferOneFp BufferGs; - typedef HostBufferOneFp HostBufferGs; - typedef PinnedHostBufferOneFp PinnedHostBufferGs; - typedef DeviceBufferOneFp DeviceBufferGs; - */ //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc index 021ef4662c..060bf06d1a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc @@ -11,7 +11,6 @@ #include "CPPProcess.h" // ProcessData.h, Parameters.h, HelAmps_.h transitively #include "ProcessTables.h" -#include "GpuRuntime.h" #include "MemoryAccessAmplitudes.h" #include "MemoryAccessChannelIds.h" #include "MemoryAccessCouplings.h" @@ -26,6 +25,8 @@ #include "color_sum.h" #include "ColorData.h" +#include + namespace mg5amcCpu { using namespace ProcessData; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h index c88f717621..8f4000a93a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h @@ -12,7 +12,6 @@ #include "mgOnGpuVectors.h" #include "ProcessData.h" -#include "GpuAbstraction.h" namespace mg5amcCpu { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h index dd13522513..2dad35d57b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h @@ -7,151 +7,51 @@ #ifndef MGONGPUCONFIG_H #define MGONGPUCONFIG_H 1 -// Is this a GPU (CUDA, HIP) or CPU implementation? -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#define MGONGPUCPP_GPUIMPL cuda -#elif defined __HIPCC__ -#define MGONGPUCPP_GPUIMPL hip -#include "hip/hip_runtime.h" // needed for blockDim, blockIdx, threadIdx: better in mgOnGpuConfig.h than in GpuAbstraction.h -#else -#undef MGONGPUCPP_GPUIMPL -#endif - -// Make sure that __HIP_PLATFORM_NVIDIA__ is undefined -// (__HIP_PLATFORM_AMD__ is defined by hipcc or in HiprandRandomNumberKernel.cc) -#undef __HIP_PLATFORM_NVIDIA__ // disable hiprand for NVidia (curand) - -// ** NB1 Throughputs (e.g. 6.8E8) are events/sec for "./gcheck.exe -p 65536 128 12" -// ** NB2 Baseline on b7g47n0004 fluctuates (probably depends on load on other VMs) - -// Choose if curand is supported for generating random numbers -// For HIP, by default, do not allow curand to be used (hiprand or common random numbers will be used instead) -// For both CUDA and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_CURAND -// (there exist CUDA installations, e.g. using the HPC package, which do not include curand - see PR #784 and #785) -#if defined __HIPCC__ -#define MGONGPU_HAS_NO_CURAND 1 -#else -//#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -//#undef MGONGPU_HAS_NO_CURAND // default -////#define MGONGPU_HAS_NO_CURAND 1 -//#else -//#undef MGONGPU_HAS_NO_CURAND // default -////#define MGONGPU_HAS_NO_CURAND 1 -//#endif -#endif - -// Choose if hiprand is supported for generating random numbers -// For CUDA, by default, do not allow hiprand to be used (curand or common random numbers will be used instead) -// For both HIP and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_HIPRAND -// (there may exist HIP installations which do not include hiprand?) -#if defined __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#define MGONGPU_HAS_NO_HIPRAND 1 -#else -//#ifdef __HIPCC__ -//#undef MGONGPU_HAS_NO_HIPRAND // default -////#define MGONGPU_HAS_NO_HIPRAND 1 -//#else -//#undef MGONGPU_HAS_NO_HIPRAND // default -////#define MGONGPU_HAS_NO_HIPRAND 1 -//#endif -#endif +// simd backend: always built with a plain host compiler, never nvcc/hipcc, so the +// GPU-backend selection macro (see gpu/mgOnGpuConfig.h) is deliberately never +// defined here - single-file, all-backend headers rely on that to pick branches. // Choose floating point precision (for everything but color algebra #537) -// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE_FLOAT, nothing happens (issue #167) +// If set from outside with e.g. -DMGONGPU_FPTYPE_FLOAT, nothing happens (#167) #if not defined MGONGPU_FPTYPE_DOUBLE and not defined MGONGPU_FPTYPE_FLOAT -// Floating point precision (CHOOSE ONLY ONE) #define MGONGPU_FPTYPE_DOUBLE 1 // default //#define MGONGPU_FPTYPE_FLOAT 1 // 2x faster #endif // Choose floating point precision (for color algebra alone #537) -// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE2_FLOAT, nothing happens (issue #167) #if not defined MGONGPU_FPTYPE2_DOUBLE and not defined MGONGPU_FPTYPE2_FLOAT -// Floating point precision (CHOOSE ONLY ONE) #define MGONGPU_FPTYPE2_DOUBLE 1 // default //#define MGONGPU_FPTYPE2_FLOAT 1 // 2x faster #endif -// Choose whether to inline all HelAmps functions -// This optimization can gain almost a factor 4 in C++, similar to -flto (issue #229) -// By default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_INLINE_HELAMPS -//#undef MGONGPU_INLINE_HELAMPS // default -////#define MGONGPU_INLINE_HELAMPS 1 +// Choose whether to inline all HelAmps functions (can gain ~4x, issue #229) +// By default off; set from outside with -DMGONGPU_INLINE_HELAMPS +//#define MGONGPU_INLINE_HELAMPS 1 -// Choose whether to hardcode the cIPD physics parameters rather than reading them from user cards -// This optimization can gain 20% in CUDA in eemumu (issue #39) -// By default, do not hardcode, but allow this macro to be set from outside with e.g. -DMGONGPU_HARDCODE_PARAM -// ** NB: The option to use hardcoded cIPD physics parameters is supported again even now when alphas is running (#373) -// ** NB: Note however that it now only refers to cIPD parameters (cIPC parameters are always accessed through global memory) -//#undef MGONGPU_HARDCODE_PARAM // default -////#define MGONGPU_HARDCODE_PARAM 1 +// Choose whether to hardcode cIPD physics parameters instead of reading user cards +// By default off; set from outside with -DMGONGPU_HARDCODE_PARAM +//#define MGONGPU_HARDCODE_PARAM 1 -/* clang-format off */ -// Complex type in CUDA: thrust or cucomplex or cxsmpl (CHOOSE ONLY ONE) -// (NB THIS IS MGONGPU_*CU*CXTYPE_xxx) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#define MGONGPU_CUCXTYPE_THRUST 1 // default (~1.15E9/double, ~3.2E9/float) -//#define MGONGPU_CUCXTYPE_CUCOMPLEX 1 // ~10 percent slower (1.03E9/double, ~2.8E9/float) -//#define MGONGPU_CUCXTYPE_CXSMPL 1 // ~10 percent slower (1.00E9/double, ~2.9E9/float) +// Complex type in C++: cxsmpl by default, or std::complex (CHOOSE ONLY ONE) +//#define MGONGPU_CPPCXTYPE_STDCOMPLEX 1 // ~8% slower on float, same on double +#define MGONGPU_CPPCXTYPE_CXSMPL 1 // default -// Complex type in HIP: cxsmpl (ONLY ONE OPTION POSSIBLE? #810) -// (NB THIS IS MGONGPU_*HIP*CXTYPE_xxx) -#elif defined __HIPCC__ -#define MGONGPU_HIPCXTYPE_CXSMPL 1 // default for HIP - -// Complex type in C++: std::complex or cxsmpl (CHOOSE ONLY ONE) -// (NB THIS IS MGONGPU_*CPP*CXTYPE_xxx) -#else -//#define MGONGPU_CPPCXTYPE_STDCOMPLEX 1 // ~8 percent slower on float, same on double (5.1E6/double, 9.4E6/float) -#define MGONGPU_CPPCXTYPE_CXSMPL 1 // new default (5.1E6/double, 10.2E6/float) -#endif - -// Choose if cuBLAS and hipBLAS are supported for generating random numbers -// For both CUDA and HIP, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_BLAS -// (there may exist CUDA/HIP installations, e.g. using the HPC package, which do not include cuBLAS/hipBLAS?) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -//#undef MGONGPU_HAS_NO_BLAS // default -////#define MGONGPU_HAS_NO_BLAS 1 -#elif defined __HIPCC__ -//#undef MGONGPU_HAS_NO_BLAS // default -////#define MGONGPU_HAS_NO_BLAS 1 -#else +// No BLAS on the simd backend (cuBLAS/hipBLAS are GPU-only) #define MGONGPU_HAS_NO_BLAS 1 -#endif -// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#undef MGONGPU_NSIGHT_DEBUG // default in CUDA -//#define MGONGPU_NSIGHT_DEBUG 1 // CURRENTLY NO LONGER SUPPORTED! -#else -#undef MGONGPU_NSIGHT_DEBUG // only option in HIP or C++ -#endif /* clang-format on */ +// nsight compute (ncu) debugging is CUDA-only; always off here +#undef MGONGPU_NSIGHT_DEBUG -// SANITY CHECKS (floating point precision for everything but color algebra #537) +// SANITY CHECKS #if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE_FLOAT -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE_DOUBLE or defined MGONGPU_FPTYPE_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE_DOUBLE or MGONGPU_FPTYPE_FLOAT #endif - -// SANITY CHECKS (floating point precision for color algebra alone #537) #if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE2_DOUBLE or defined MGONGPU_FPTYPE2_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE2_DOUBLE or MGONGPU_FPTYPE2_FLOAT #endif #if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE_FLOAT #error You cannot use double precision for color algebra and single precision elsewhere #endif - -// SANITY CHECKS (CUDA complex number implementation) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#if defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CUCOMPLEX -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX for CUDA -#elif defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CXSMPL -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CXSMPL for CUDA -#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX and defined MGONGPU_CUCXTYPE_CXSMPL -#error You must CHOOSE (ONE AND) ONLY ONE OF MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL for CUDA -#endif -#endif - -// SANITY CHECKS (C++ complex number implementation) #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX and defined MGONGPU_CPPCXTYPE_CXSMPL #error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL for C++ #endif @@ -159,9 +59,6 @@ // NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) namespace mgOnGpu { - - // --- Type definitions - // Floating point type (for everything but color algebra #537): fptype #if defined MGONGPU_FPTYPE_DOUBLE typedef double fptype; // double precision (8 bytes, fp64) @@ -176,25 +73,14 @@ namespace mgOnGpu typedef float fptype2; // single precision (4 bytes, fp32) #endif - // --- Platform-specific software implementation details - - // Maximum number of blocks per grid - // ** NB Some arrays of pointers will be allocated statically to fit all these blocks - // ** (the actual memory for each block will then be allocated dynamically only for existing blocks) - //const int nbpgMAX = 2048; - // Maximum number of threads per block - //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" // Alignment requirement for using reinterpret_cast with SIMD vectorized code - // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) - // Only needed for C++ code but can be enforced also in NVCC builds of C++ code using CUDA>=11.2 and C++17 (#318, #319, #333) - constexpr int cppAlign = 64; // alignment requirement for SIMD vectorization (64-byte i.e. 512-bit) - + constexpr int cppAlign = 64; // 64-byte i.e. 512-bit } -// Expose typedefs and operators outside the namespace +// Expose typedefs outside the namespace using mgOnGpu::fptype; using mgOnGpu::fptype2; @@ -204,81 +90,63 @@ using mgOnGpu::fptype2; #endif // C++ SIMD vectorization width (this will be used to set neppV) -#if defined __AVX512VL__ && defined MGONGPU_PVW512 // C++ "512z" AVX512 with 512 width (512-bit ie 64-byte): 8 (DOUBLE) or 16 (FLOAT) +#if defined __AVX512VL__ && defined MGONGPU_PVW512 // "512z" AVX512 512-bit: 8 (DOUBLE) or 16 (FLOAT) #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 8 #else #define MGONGPU_CPPSIMD 16 #endif -#elif defined __AVX512VL__ // C++ "512y" AVX512 with 256 width (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [gcc DEFAULT] +#elif defined __AVX512VL__ // "512y" AVX512 256-bit: 4 (DOUBLE) or 8 (FLOAT) [gcc default] #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 4 #else #define MGONGPU_CPPSIMD 8 #endif -#elif defined __AVX2__ // C++ "avx2" AVX2 (256-bit ie 32-byte): 4 (DOUBLE) or 8 (FLOAT) [clang DEFAULT] +#elif defined __AVX2__ // "avx2" 256-bit: 4 (DOUBLE) or 8 (FLOAT) [clang default] #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 4 #else #define MGONGPU_CPPSIMD 8 #endif -#elif defined __SSE4_2__ // C++ "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [Power9 default] +#elif defined __SSE4_2__ // "sse4" SSE4.2 128-bit: 2 (DOUBLE) or 4 (FLOAT) [Power9 default] #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 2 #else #define MGONGPU_CPPSIMD 4 #endif -#elif defined __ARM_NEON // C++ "sse4" ARM NEON (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [ARM default] +#elif defined __ARM_NEON // ARM NEON 128-bit: 2 (DOUBLE) or 4 (FLOAT) [ARM default] #ifdef MGONGPU_FPTYPE_DOUBLE #define MGONGPU_CPPSIMD 2 #else #define MGONGPU_CPPSIMD 4 #endif -#else // C++ "none" i.e. no SIMD +#else // "none" i.e. no SIMD #undef MGONGPU_CPPSIMD #endif -/* clang-format off */ -// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation [NB: CURRENTLY NO LONGER SUPPORTED!] -// Arguments (not used so far): text is __FUNCTION__, code is 0 (start) or 1 (end) -//#if defined __CUDACC__ && defined MGONGPU_NSIGHT_DEBUG // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -//#define mgDebugDeclare() __shared__ float mgDebugCounter[mgOnGpu::ntpbMAX]; -//#define mgDebugInitialise() { mgDebugCounter[threadIdx.x] = 0; } -//#define mgDebug( code, text ) { mgDebugCounter[threadIdx.x] += 1; } -//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%f\n", mgDebugCounter[threadIdx.x] ); } -//#else +// No-op debug macros (nsight-based debugging is CUDA-only, unused here) #define mgDebugDeclare() /*noop*/ #define mgDebugInitialise() /*noop*/ #define mgDebug( code, text ) /*noop*/ #define mgDebugFinalise() /*noop*/ -//#endif /* clang-format on */ // Define empty CUDA/HIP declaration specifiers for C++ #define __global__ #define __host__ #define __device__ -// For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two (https://stackoverflow.com/a/108360) +// For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two inline constexpr bool ispoweroftwo( int n ) { return ( n > 0 ) && !( n & ( n - 1 ) ); } -// Compiler version support (#96): require nvcc from CUDA >= 11.2, e.g. to use C++17 (see #333) -#ifdef __NVCC__ -#if( __CUDACC_VER_MAJOR__ < 11 ) || ( __CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ < 2 ) -#error Unsupported CUDA version: please use CUDA >= 11.2 -#endif -#endif - -// Compiler version support (#96): require clang >= 11 +// Compiler version support (#96) #if defined __clang__ #if( __clang_major__ < 11 ) #error Unsupported clang version: please use clang >= 11 #endif -// Compiler version support (#96): require gcc >= 9.3, e.g. for some OMP issues (see #269) -// [NB skip this check for the gcc toolchain below clang or icx (TEMPORARY? #355)] #elif defined __GNUC__ #if( __GNUC__ < 9 ) || ( __GNUC__ == 9 && __GNUC_MINOR__ < 3 ) #error Unsupported gcc version: please gcc >= 9.3 diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h index 900a496b7d..dea5dcf2d1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h @@ -19,94 +19,12 @@ #include -// Complex type in cuda: thrust or cucomplex or cxsmpl -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#if defined MGONGPU_CUCXTYPE_THRUST -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) -#include -#pragma clang diagnostic pop -#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX -#include -#elif not defined MGONGPU_CUCXTYPE_CXSMPL -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL -#endif -// Complex type in HIP: cxsmpl -#elif defined __HIPCC__ -#if not defined MGONGPU_HIPCXTYPE_CXSMPL -#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_HIPCXTYPE_CXSMPL -#endif -#else -// Complex type in c++ or HIP: std::complex or cxsmpl +// Complex type in c++: std::complex or cxsmpl #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX #include #elif not defined MGONGPU_CPPCXTYPE_CXSMPL #error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL #endif -#endif - -//========================================================================== -// COMPLEX TYPES: INSTRUMENTED CUCOMPLEX CLASS (cucomplex) -//========================================================================== - -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#if defined MGONGPU_CUCXTYPE_CUCOMPLEX -namespace mg5amcGpu -{ -#if defined MGONGPU_FPTYPE_DOUBLE - class cucomplex - { - public: - __host__ __device__ cucomplex( const double& r = 0, const double& i = 0 ) - : m_ri( make_cuDoubleComplex( r, i ) ) {} - __host__ __device__ constexpr cucomplex( const cuDoubleComplex& ri ) - : m_ri( ri ) {} - //__host__ __device__ operator cuDoubleComplex&() { return m_ri; } - __host__ __device__ constexpr operator cuDoubleComplex() const { return m_ri; } - __host__ __device__ double real() const { return cuCreal( m_ri ); } - __host__ __device__ double imag() const { return cuCimag( m_ri ); } - inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) - { - m_ri = cuCadd( m_ri, c ); - return *this; - } - inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) - { - m_ri = cuCsub( m_ri, c ); - return *this; - } - private: - cuDoubleComplex m_ri; - }; -#elif defined MGONGPU_FPTYPE_FLOAT - class cucomplex - { - public: - __host__ __device__ cucomplex( const float& r = 0, const float& i = 0 ) - : m_ri( make_cuFloatComplex( r, i ) ) {} - __host__ __device__ constexpr cucomplex( const cuFloatComplex& ri ) - : m_ri( ri ) {} - //__host__ __device__ operator cuFloatComplex&() { return m_ri; } - __host__ __device__ constexpr operator cuFloatComplex() const { return m_ri; } - __host__ __device__ float real() const { return cuCrealf( m_ri ); } - __host__ __device__ float imag() const { return cuCimagf( m_ri ); } - inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) - { - m_ri = cuCaddf( m_ri, c ); - return *this; - } - inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) - { - m_ri = cuCsubf( m_ri, c ); - return *this; - } - private: - cuFloatComplex m_ri; - }; -#endif -} -#endif -#endif //========================================================================== // COMPLEX TYPES: SIMPLE COMPLEX CLASS (cxsmpl) @@ -135,16 +53,8 @@ namespace mgOnGpu /* clang-format off */ __host__ __device__ constexpr const FP& real() const { return m_real; } __host__ __device__ constexpr const FP& imag() const { return m_imag; } template __host__ __device__ constexpr operator cxsmpl() const { return cxsmpl( m_real, m_imag ); } -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#ifdef MGONGPU_CUCXTYPE_THRUST - template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } -#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX - __host__ __device__ constexpr operator mg5amcGpu::cucomplex() const { return mg5amcGpu::cucomplex( m_real, m_imag ); } -#endif -#else #ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX template __host__ __device__ constexpr operator std::complex() const { return std::complex( m_real, m_imag ); } -#endif #endif private: FP m_real, m_imag; // RI @@ -292,24 +202,13 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) namespace mg5amcCpu { // --- Type definitions (complex type: cxtype) -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#if defined MGONGPU_CUCXTYPE_THRUST - typedef thrust::complex cxtype; -#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX - typedef cucomplex cxtype; -#else - typedef cxsmpl cxtype; -#endif -#else // c++ #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX typedef std::complex cxtype; #else typedef cxsmpl cxtype; -#endif #endif // SANITY CHECK: memory access may be based on casts of fptype[2] to cxtype (e.g. for wavefunctions) @@ -324,13 +223,12 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) namespace mg5amcCpu { -#if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL +#if defined MGONGPU_CPPCXTYPE_CXSMPL //------------------------------ - // CUDA or C++ - using cxsmpl + // C++ - using cxsmpl //------------------------------ inline __host__ __device__ cxtype @@ -369,258 +267,11 @@ namespace mg5amcCpu return cxmake( c.real(), c.imag() ); } -#endif // #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL - - //========================================================================== - -#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST // cuda + thrust (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) - - //------------------------------ - // CUDA - using thrust::complex - //------------------------------ - - inline __host__ __device__ cxtype - cxmake( const fptype& r, const fptype& i ) - { - return cxtype( r, i ); // thrust::complex constructor - } - - inline __host__ __device__ fptype - cxreal( const cxtype& c ) - { - return c.real(); // thrust::complex::real() - } - - inline __host__ __device__ fptype - cximag( const cxtype& c ) - { - return c.imag(); // thrust::complex::imag() - } - - inline __host__ __device__ cxtype - cxconj( const cxtype& c ) - { - return conj( c ); // conj( thrust::complex ) - } - - inline __host__ __device__ const cxtype& - cxmake( const cxtype& c ) - { - return c; - } - -#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST - - //========================================================================== - -#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX // cuda + cucomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) - - //------------------------------ - // CUDA - using cuComplex - //------------------------------ - -#if defined MGONGPU_FPTYPE_DOUBLE // cuda + cucomplex + double - - //+++++++++++++++++++++++++ - // cuDoubleComplex ONLY - //+++++++++++++++++++++++++ - - inline __host__ __device__ cxtype - cxmake( const fptype& r, const fptype& i ) - { - return make_cuDoubleComplex( r, i ); - } - - inline __host__ __device__ fptype - cxreal( const cxtype& c ) - { - return cuCreal( c ); // returns by value - } - - inline __host__ __device__ fptype - cximag( const cxtype& c ) - { - return cuCimag( c ); // returns by value - } - - inline __host__ __device__ cxtype - operator+( const cxtype& a, const cxtype& b ) - { - return cuCadd( a, b ); - } - - inline __host__ __device__ cxtype - operator-( const cxtype& a, const cxtype& b ) - { - return cuCsub( a, b ); - } - - inline __host__ __device__ cxtype - operator*( const cxtype& a, const cxtype& b ) - { - return cuCmul( a, b ); - } - - inline __host__ __device__ cxtype - operator/( const cxtype& a, const cxtype& b ) - { - return cuCdiv( a, b ); - } - - inline __host__ std::ostream& - operator<<( std::ostream& out, const cxtype& c ) - { - //out << std::complex( cxreal( c ), cximag( c ) ); - out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma - return out; - } - -#elif defined MGONGPU_FPTYPE_FLOAT // cuda + cucomplex + float - - //+++++++++++++++++++++++++ - // cuFloatComplex ONLY - //+++++++++++++++++++++++++ - - inline __host__ __device__ cxtype - cxmake( const fptype& r, const fptype& i ) - { - return make_cuFloatComplex( r, i ); - } - - inline __host__ __device__ fptype - cxreal( const cxtype& c ) - { - return cuCrealf( c ); // returns by value - } - - inline __host__ __device__ fptype - cximag( const cxtype& c ) - { - return cuCimagf( c ); // returns by value - } - - inline __host__ __device__ cxtype - operator+( const cxtype& a, const cxtype& b ) - { - return cuCaddf( a, b ); - } - - inline __host__ __device__ cxtype - operator-( const cxtype& a, const cxtype& b ) - { - return cuCsubf( a, b ); - } - - inline __host__ __device__ cxtype - operator*( const cxtype& a, const cxtype& b ) - { - return cuCmulf( a, b ); - } - - inline __host__ __device__ cxtype - operator/( const cxtype& a, const cxtype& b ) - { - return cuCdivf( a, b ); - } - - inline __host__ cxtype // NOT __device__ - cxmake( const std::complex& c ) // std::complex to cucomplex (cast double-to-float) - { - return cxmake( (fptype)c.real(), (fptype)c.imag() ); - } - - inline __host__ std::ostream& - operator<<( std::ostream& out, const cxtype& c ) - { - //out << std::complex( cxreal( c ), cximag( c ) ); - out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma - return out; - } - -#endif - - //+++++++++++++++++++++++++ - // cuDoubleComplex OR - // cuFloatComplex - //+++++++++++++++++++++++++ - - inline __host__ __device__ cxtype - operator+( const cxtype a ) - { - return a; - } - - inline __host__ __device__ cxtype - operator-( const cxtype& a ) - { - return cxmake( -cxreal( a ), -cximag( a ) ); - } - - inline __host__ __device__ cxtype - operator+( const fptype& a, const cxtype& b ) - { - return cxmake( a, 0 ) + b; - } - - inline __host__ __device__ cxtype - operator-( const fptype& a, const cxtype& b ) - { - return cxmake( a, 0 ) - b; - } - - inline __host__ __device__ cxtype - operator*( const fptype& a, const cxtype& b ) - { - return cxmake( a, 0 ) * b; - } - - inline __host__ __device__ cxtype - operator/( const fptype& a, const cxtype& b ) - { - return cxmake( a, 0 ) / b; - } - - inline __host__ __device__ cxtype - operator+( const cxtype& a, const fptype& b ) - { - return a + cxmake( b, 0 ); - } - - inline __host__ __device__ cxtype - operator-( const cxtype& a, const fptype& b ) - { - return a - cxmake( b, 0 ); - } - - inline __host__ __device__ cxtype - operator*( const cxtype& a, const fptype& b ) - { - return a * cxmake( b, 0 ); - } - - inline __host__ __device__ cxtype - operator/( const cxtype& a, const fptype& b ) - { - return a / cxmake( b, 0 ); - } - - inline __host__ __device__ cxtype - cxconj( const cxtype& c ) - { - return cxmake( cxreal( c ), -cximag( c ) ); - } - - inline __host__ cxtype // NOT __device__ - cxmake( const std::complex& c ) // std::complex to cucomplex (float-to-float or double-to-double) - { - return cxmake( c.real(), c.imag() ); - } - -#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX +#endif // #if defined MGONGPU_CPPCXTYPE_CXSMPL //========================================================================== -#if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX // c++/hip + stdcomplex (this must be __CUDACC__ and not MGONGPUCPP_GPUIMPL) +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX //------------------------------ // C++ - using std::complex @@ -664,7 +315,7 @@ namespace mg5amcCpu } #endif -#endif // #if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX +#endif // #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc index 7fcdb3d91f..cefa01b48f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc @@ -9,7 +9,6 @@ #include "ProcessData.h" #include "CPPProcess.h" // needed to construct/initProc the process object (umami_initialize) #include "SigmaKin.h" // sigmaKin_getGoodHel/setGoodHel -#include "GpuRuntime.h" #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/check_sa.cc index 50af5a947c..bd1cd4c303 100644 --- a/madgraph/iolibs/template_files/madmatrix/check_sa.cc +++ b/madgraph/iolibs/template_files/madmatrix/check_sa.cc @@ -26,8 +26,10 @@ #include "mgOnGpuConfig.h" #include "CPPProcess.h" +#ifdef MGONGPUCPP_GPUIMPL #include "GpuAbstraction.h" #include "GpuRuntime.h" +#endif #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" #include "RamboSamplingKernels.h" diff --git a/madgraph/iolibs/template_files/madmatrix/process_cc.inc b/madgraph/iolibs/template_files/madmatrix/process_cc.inc index bd78feb4c2..77d8880750 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_cc.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_cc.inc @@ -15,7 +15,6 @@ #include "mgOnGpuConfig.h" -#include "GpuRuntime.h" %(hel_amps_h)s #include "MemoryAccessAmplitudes.h" #include "MemoryAccessIflavorVec.h" diff --git a/madgraph/iolibs/template_files/madmatrix/process_h.inc b/madgraph/iolibs/template_files/madmatrix/process_h.inc index 1396a8fa61..91469ce8d4 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_h.inc @@ -18,7 +18,6 @@ #include "mgOnGpuVectors.h" -#include "GpuAbstraction.h" #include "Parameters.h" #include "ProcessData.h" #include "SigmaKin.h" // for the setters CPPProcess's constructor/initProc call From 065d87a02ebc40beb3049c33a46a0e2f85c63f63 Mon Sep 17 00:00:00 2001 From: stloufra Date: Thu, 6 Aug 2026 11:13:00 +0200 Subject: [PATCH 13/25] Delete mgOn* from src --- .../iolibs/template_files/madmatrix/madmatrix.mk | 3 ++- .../iolibs/template_files/madmatrix/madmatrix_src.mk | 3 ++- madmatrix/model_handling.py | 1 - madmatrix/output.py | 3 +-- madmatrix/trex.py | 12 +++++++++--- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk index 70254f32cd..033473bbf5 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk @@ -383,7 +383,7 @@ export GPUSUFFIX # Export BACKEND (resolved from cppauto above if needed; used e.g. to name the common library) export BACKEND -# Map BACKEND to its backend/ source subdirectory +# Map BACKEND to its backend/ source subdirectory ifneq ($(GPUCC),) override BACKENDDIR = gpu else ifeq ($(BACKEND),cppnone) @@ -391,6 +391,7 @@ else ifeq ($(BACKEND),cppnone) else override BACKENDDIR = simd endif +export BACKENDDIR #------------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix_src.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix_src.mk index 8b13f7f4e1..ba88f6b923 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix_src.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix_src.mk @@ -12,7 +12,8 @@ SHELL := /bin/bash #------------------------------------------------------------------------------- #=== Configure common compiler flags for CUDA and C++ -INCFLAGS = -I. +# BACKENDDIR has been exported from the subprocess Makefile +INCFLAGS = -I. -I../backend/$(BACKENDDIR) #------------------------------------------------------------------------------- diff --git a/madmatrix/model_handling.py b/madmatrix/model_handling.py index db927c0e1d..7f37f29d46 100644 --- a/madmatrix/model_handling.py +++ b/madmatrix/model_handling.py @@ -1729,7 +1729,6 @@ def get_process_info_lines(self, matrix_element): def generate_process_files(self): """Generate mgOnGpuConfig.h, CPPProcess.cc, CPPProcess.h, check_sa.cc, gXXX.cu links""" ###misc.sprint('Entering OneProcessExporterMadMatrix.generate_process_files') - self.edit_mgonGPU() self.edit_colordata() # AV new file (NB this is Sigma-specific, should not be a symlink to Subprocesses) super().generate_process_files() # needs to be after get_matrix_element_calls to have nwf ready diff --git a/madmatrix/output.py b/madmatrix/output.py index 0cbcd2033e..f26239066e 100644 --- a/madmatrix/output.py +++ b/madmatrix/output.py @@ -83,8 +83,7 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): from_template = {'.': relative_path_list(home_path, ['COPYRIGHT', 'COPYING', 'COPYING.LESSER']), 'src': relative_path_list(madmatrix_templates, [ - 'mgOnGpuFptypes.h', 'mgOnGpuCxtypes.h', 'mgOnGpuVectors.h', - 'constexpr_math.h', 'read_slha.h', 'read_slha.cc' + 'read_slha.h', 'read_slha.cc' ]), # Backend-owned skeleton files live only under backend// now # (see backend_variants below); only genuinely backend-agnostic files diff --git a/madmatrix/trex.py b/madmatrix/trex.py index 1efeba2849..f903569fb6 100644 --- a/madmatrix/trex.py +++ b/madmatrix/trex.py @@ -187,6 +187,8 @@ def edit_rwgt_runner(self): def generate_process_files(self): """Generate mgOnGpuConfig.h, CPPProcess.cc, CPPProcess.h, check_sa.cc, gXXX.cu links""" super().generate_process_files() + # for TREX for now no /backend + self.edit_mgonGPU() self.edit_rwgt_header() self.edit_rwgt_runner() @@ -202,9 +204,13 @@ class TREX_ProcessExporter(output.ProcessExporterMadMatrix): r = PLUGINDIR + '/MadtRex/template_files/' m = PLUGINDIR + '/MadtRex/makefiles/' from_template = dict(output.ProcessExporterMadMatrix.from_template) - from_template['src'] = from_template['src'] + [t+'librex.so', t+'libtearex.so', - t+'Rex.h', t+'teaRex.h', - r+'rwgt_instance.h', r+'rwgt_instance.cc'] + # for TREX for now no /backend + from_template['src'] = from_template['src'] + output.relative_path_list( + output.ProcessExporterMadMatrix.madmatrix_templates, + ['mgOnGpuFptypes.h', 'mgOnGpuCxtypes.h', 'mgOnGpuVectors.h', 'constexpr_math.h'] + ) + [t+'librex.so', t+'libtearex.so', + t+'Rex.h', t+'teaRex.h', + r+'rwgt_instance.h', r+'rwgt_instance.cc'] from_template['SubProcesses'] = from_template['SubProcesses'] + [m+'cudacpp_driver.mk', r+'rwgt_instance.h', t+'Rex.h', t+'teaRex.h'] From d557e727767c79f3586d167ac8a541b7f5aa268c Mon Sep 17 00:00:00 2001 From: stloufra Date: Thu, 6 Aug 2026 19:17:59 +0200 Subject: [PATCH 14/25] Shrink CPPProcess fpeEnable(), getCompiler() move in constexpr_math.h and mgOnGpuConfig.h tHel/tFlavors/flavorPDGs in ProcessData getCompiler had zero calls -> dropped mass/tIPD/tIPC setup (non-hardcoded branch) is now a generated pointer-to-member table read by generic gatherFptype/ gatherCxtype helpers in Parameters.h, instead of one assignment per parameter. The MGONGPU_HARDCODE_PARAM branch same TODO --- .../template_files/madmatrix/ProcessData.h | 8 ++ .../madmatrix/backend/cpu/constexpr_math.h | 23 ++++++ .../madmatrix/backend/cpu/mgOnGpuConfig.h | 68 +++++++++++++++++ .../madmatrix/backend/gpu/constexpr_math.h | 23 ++++++ .../madmatrix/backend/gpu/mgOnGpuConfig.h | 67 +++++++++++++++++ .../madmatrix/backend/simd/constexpr_math.h | 23 ++++++ .../madmatrix/backend/simd/mgOnGpuConfig.h | 68 +++++++++++++++++ .../madmatrix/cpp_model_parameters_h.inc | 14 ++++ .../template_files/madmatrix/process_cc.inc | 41 ---------- .../madmatrix/process_class.inc | 3 - .../process_function_definitions.inc | 74 ------------------- madmatrix/model_handling.py | 50 ++++++++----- 12 files changed, 325 insertions(+), 137 deletions(-) diff --git a/madgraph/iolibs/template_files/madmatrix/ProcessData.h b/madgraph/iolibs/template_files/madmatrix/ProcessData.h index d00063291b..6f17fe9858 100644 --- a/madgraph/iolibs/template_files/madmatrix/ProcessData.h +++ b/madgraph/iolibs/template_files/madmatrix/ProcessData.h @@ -32,6 +32,14 @@ namespace ProcessData constexpr int nIPC = %(nipc)d; constexpr int nIPF = %(nipf)d; constexpr int nDPF = %(ndpf)d; + + // Helicities for the process [NB do keep 'static' for this constexpr array, see issue #283] + // *** NB There is no automatic check yet that these are in the same order as Fortran! #569 *** +%(thel_lines)s + + // Host-side flavor table: single source of truth for PDG ids (used by both + // CPPProcess's constructor copy into cFlavors and CPPProcess::flavorPDG). +%(tflavors_lines)s } // Process identification for test/debug tooling. Must stay #define (not diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h index 356f034c92..c671aa5db7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h @@ -10,8 +10,10 @@ #include "mgOnGpuConfig.h" #include +#include // for feenableexcept, fegetexcept and FE_XXX #include #include +#include #include // SWITCH ON/OFF DEBUGGING @@ -326,6 +328,27 @@ namespace mg5amcCpu { return constexpr_atan_quad( x ); } + + // Enable FPE traps (see #701, #733, #831 - except on MacOS where feenableexcept is not defined #730) + // [NB1: Fortran default is -ffpe-trap=none, i.e. FPE traps are not enabled, https://gcc.gnu.org/onlinedocs/gfortran/Debugging-Options.html] + // [NB2: Fortran default is -ffpe-summary=invalid,zero,overflow,underflow,denormal, i.e. warn at the end on STOP] + inline void + fpeEnable() + { + static bool first = true; // FIXME: quick and dirty hack to do this only once (can be removed when separate C++/CUDA builds are implemented) + if( !first ) return; + first = false; +#ifndef __APPLE__ // on MacOS feenableexcept is not defined #730 + constexpr bool enableFPE = true; // this is hardcoded and no longer controlled by getenv( "CUDACPP_RUNTIME_ENABLEFPE" ) + if( enableFPE ) + { + std::cout << "INFO: The following Floating Point Exceptions will cause SIGFPE program aborts: FE_DIVBYZERO, FE_INVALID, FE_OVERFLOW" << std::endl; + feenableexcept( FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW ); // new strategy #831 (do not enable FE_UNDERFLOW) + } +#else + //std::cout << "INFO: Keep default SIGFPE settings because feenableexcept is not available on MacOS" << std::endl; +#endif + } } #endif // constexpr_math_h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h index 668894b961..683a80f6c4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h @@ -7,6 +7,9 @@ #ifndef MGONGPUCONFIG_H #define MGONGPUCONFIG_H 1 +#include +#include + // cpu backend: always built with a plain host compiler, never nvcc/hipcc, so the // GPU-backend selection macro (see gpu/mgOnGpuConfig.h) is deliberately never // defined here - single-file, all-backend headers rely on that to pick branches. @@ -78,6 +81,71 @@ namespace mgOnGpu // Alignment requirement for using reinterpret_cast with SIMD vectorized code constexpr int cppAlign = 64; // 64-byte i.e. 512-bit + + // Retrieve the compiler that was used to build this module + inline std::string + getCompiler() + { + std::stringstream out; + // HIP version (HIPCC) + // [Use __HIPCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'hipcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'hipcc -dM -E -x hip -I ../../src CPPProcess.cc | grep HIP'] +#ifdef __HIPCC__ +#if defined HIP_VERSION_MAJOR && defined HIP_VERSION_MINOR && defined HIP_VERSION_PATCH + out << "hipcc " << HIP_VERSION_MAJOR << "." << HIP_VERSION_MINOR << "." << HIP_VERSION_PATCH; +#else + out << "hipcc UNKNOWN"; +#endif + out << " ("; +#endif + // CUDA version (NVCC) + // [Use __NVCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'nvcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'nvcc --compiler-options -dM -E dummy.c | grep CUDA': see https://stackoverflow.com/a/53713712] +#ifdef __NVCC__ +#if defined __CUDACC_VER_MAJOR__ && defined __CUDACC_VER_MINOR__ && defined __CUDACC_VER_BUILD__ + out << "nvcc " << __CUDACC_VER_MAJOR__ << "." << __CUDACC_VER_MINOR__ << "." << __CUDACC_VER_BUILD__; +#else + out << "nvcc UNKNOWN"; +#endif + out << " ("; +#endif + // ICX version (either as CXX or as host compiler inside NVCC) +#if defined __INTEL_COMPILER +#error "icc is no longer supported: please use icx" +#elif defined __INTEL_LLVM_COMPILER // alternative: __INTEL_CLANG_COMPILER + out << "icx " << __INTEL_LLVM_COMPILER; +#ifdef __NVCC__ + out << ", "; +#else + out << " ("; +#endif +#endif + // CLANG version (either as CXX or as host compiler inside NVCC or inside ICX) +#if defined __clang__ +#if defined __clang_major__ && defined __clang_minor__ && defined __clang_patchlevel__ +#ifdef __APPLE__ + out << "Apple clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#else + out << "clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#endif +#else + out << "clang UNKNOWKN"; +#endif +#else + // GCC version (either as CXX or as host compiler inside NVCC) +#if defined __GNUC__ && defined __GNUC_MINOR__ && defined __GNUC_PATCHLEVEL__ + out << "gcc " << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__; +#else + out << "gcc UNKNOWKN"; +#endif +#endif +#if defined __HIPCC__ or defined __NVCC__ or defined __INTEL_LLVM_COMPILER + out << ")"; +#endif + return out.str(); + } } // Expose typedefs outside the namespace diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h index 6864e7d7b0..6219b4e0df 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h @@ -10,8 +10,10 @@ #include "mgOnGpuConfig.h" #include +#include // for feenableexcept, fegetexcept and FE_XXX #include #include +#include #include // SWITCH ON/OFF DEBUGGING @@ -326,6 +328,27 @@ namespace mg5amcGpu { return constexpr_atan_quad( x ); } + + // Enable FPE traps (see #701, #733, #831 - except on MacOS where feenableexcept is not defined #730) + // [NB1: Fortran default is -ffpe-trap=none, i.e. FPE traps are not enabled, https://gcc.gnu.org/onlinedocs/gfortran/Debugging-Options.html] + // [NB2: Fortran default is -ffpe-summary=invalid,zero,overflow,underflow,denormal, i.e. warn at the end on STOP] + inline void + fpeEnable() + { + static bool first = true; // FIXME: quick and dirty hack to do this only once (can be removed when separate C++/CUDA builds are implemented) + if( !first ) return; + first = false; +#ifndef __APPLE__ // on MacOS feenableexcept is not defined #730 + constexpr bool enableFPE = true; // this is hardcoded and no longer controlled by getenv( "CUDACPP_RUNTIME_ENABLEFPE" ) + if( enableFPE ) + { + std::cout << "INFO: The following Floating Point Exceptions will cause SIGFPE program aborts: FE_DIVBYZERO, FE_INVALID, FE_OVERFLOW" << std::endl; + feenableexcept( FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW ); // new strategy #831 (do not enable FE_UNDERFLOW) + } +#else + //std::cout << "INFO: Keep default SIGFPE settings because feenableexcept is not available on MacOS" << std::endl; +#endif + } } #endif // constexpr_math_h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h index 9110dc483f..9cecf1e8fb 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h @@ -7,6 +7,9 @@ #ifndef MGONGPUCONFIG_H #define MGONGPUCONFIG_H 1 +#include +#include + // Is this a GPU (CUDA, HIP) or CPU implementation? #ifdef __CUDACC__ // this must be __CUDACC__ #define MGONGPUCPP_GPUIMPL cuda @@ -188,6 +191,70 @@ namespace mgOnGpu // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) // Only needed for C++ code but can be enforced also in NVCC builds of C++ code using CUDA>=11.2 and C++17 (#318, #319, #333) + // Retrieve the compiler that was used to build this module + inline std::string + getCompiler() + { + std::stringstream out; + // HIP version (HIPCC) + // [Use __HIPCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'hipcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'hipcc -dM -E -x hip -I ../../src CPPProcess.cc | grep HIP'] +#ifdef __HIPCC__ +#if defined HIP_VERSION_MAJOR && defined HIP_VERSION_MINOR && defined HIP_VERSION_PATCH + out << "hipcc " << HIP_VERSION_MAJOR << "." << HIP_VERSION_MINOR << "." << HIP_VERSION_PATCH; +#else + out << "hipcc UNKNOWN"; +#endif + out << " ("; +#endif + // CUDA version (NVCC) + // [Use __NVCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'nvcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'nvcc --compiler-options -dM -E dummy.c | grep CUDA': see https://stackoverflow.com/a/53713712] +#ifdef __NVCC__ +#if defined __CUDACC_VER_MAJOR__ && defined __CUDACC_VER_MINOR__ && defined __CUDACC_VER_BUILD__ + out << "nvcc " << __CUDACC_VER_MAJOR__ << "." << __CUDACC_VER_MINOR__ << "." << __CUDACC_VER_BUILD__; +#else + out << "nvcc UNKNOWN"; +#endif + out << " ("; +#endif + // ICX version (either as CXX or as host compiler inside NVCC) +#if defined __INTEL_COMPILER +#error "icc is no longer supported: please use icx" +#elif defined __INTEL_LLVM_COMPILER // alternative: __INTEL_CLANG_COMPILER + out << "icx " << __INTEL_LLVM_COMPILER; +#ifdef __NVCC__ + out << ", "; +#else + out << " ("; +#endif +#endif + // CLANG version (either as CXX or as host compiler inside NVCC or inside ICX) +#if defined __clang__ +#if defined __clang_major__ && defined __clang_minor__ && defined __clang_patchlevel__ +#ifdef __APPLE__ + out << "Apple clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#else + out << "clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#endif +#else + out << "clang UNKNOWKN"; +#endif +#else + // GCC version (either as CXX or as host compiler inside NVCC) +#if defined __GNUC__ && defined __GNUC_MINOR__ && defined __GNUC_PATCHLEVEL__ + out << "gcc " << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__; +#else + out << "gcc UNKNOWKN"; +#endif +#endif +#if defined __HIPCC__ or defined __NVCC__ or defined __INTEL_LLVM_COMPILER + out << ")"; +#endif + return out.str(); + } } // Expose typedefs and operators outside the namespace diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h index 356f034c92..c671aa5db7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h @@ -10,8 +10,10 @@ #include "mgOnGpuConfig.h" #include +#include // for feenableexcept, fegetexcept and FE_XXX #include #include +#include #include // SWITCH ON/OFF DEBUGGING @@ -326,6 +328,27 @@ namespace mg5amcCpu { return constexpr_atan_quad( x ); } + + // Enable FPE traps (see #701, #733, #831 - except on MacOS where feenableexcept is not defined #730) + // [NB1: Fortran default is -ffpe-trap=none, i.e. FPE traps are not enabled, https://gcc.gnu.org/onlinedocs/gfortran/Debugging-Options.html] + // [NB2: Fortran default is -ffpe-summary=invalid,zero,overflow,underflow,denormal, i.e. warn at the end on STOP] + inline void + fpeEnable() + { + static bool first = true; // FIXME: quick and dirty hack to do this only once (can be removed when separate C++/CUDA builds are implemented) + if( !first ) return; + first = false; +#ifndef __APPLE__ // on MacOS feenableexcept is not defined #730 + constexpr bool enableFPE = true; // this is hardcoded and no longer controlled by getenv( "CUDACPP_RUNTIME_ENABLEFPE" ) + if( enableFPE ) + { + std::cout << "INFO: The following Floating Point Exceptions will cause SIGFPE program aborts: FE_DIVBYZERO, FE_INVALID, FE_OVERFLOW" << std::endl; + feenableexcept( FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW ); // new strategy #831 (do not enable FE_UNDERFLOW) + } +#else + //std::cout << "INFO: Keep default SIGFPE settings because feenableexcept is not available on MacOS" << std::endl; +#endif + } } #endif // constexpr_math_h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h index 2dad35d57b..966f10492e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h @@ -7,6 +7,9 @@ #ifndef MGONGPUCONFIG_H #define MGONGPUCONFIG_H 1 +#include +#include + // simd backend: always built with a plain host compiler, never nvcc/hipcc, so the // GPU-backend selection macro (see gpu/mgOnGpuConfig.h) is deliberately never // defined here - single-file, all-backend headers rely on that to pick branches. @@ -78,6 +81,71 @@ namespace mgOnGpu // Alignment requirement for using reinterpret_cast with SIMD vectorized code constexpr int cppAlign = 64; // 64-byte i.e. 512-bit + + // Retrieve the compiler that was used to build this module + inline std::string + getCompiler() + { + std::stringstream out; + // HIP version (HIPCC) + // [Use __HIPCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'hipcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'hipcc -dM -E -x hip -I ../../src CPPProcess.cc | grep HIP'] +#ifdef __HIPCC__ +#if defined HIP_VERSION_MAJOR && defined HIP_VERSION_MINOR && defined HIP_VERSION_PATCH + out << "hipcc " << HIP_VERSION_MAJOR << "." << HIP_VERSION_MINOR << "." << HIP_VERSION_PATCH; +#else + out << "hipcc UNKNOWN"; +#endif + out << " ("; +#endif + // CUDA version (NVCC) + // [Use __NVCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'nvcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'nvcc --compiler-options -dM -E dummy.c | grep CUDA': see https://stackoverflow.com/a/53713712] +#ifdef __NVCC__ +#if defined __CUDACC_VER_MAJOR__ && defined __CUDACC_VER_MINOR__ && defined __CUDACC_VER_BUILD__ + out << "nvcc " << __CUDACC_VER_MAJOR__ << "." << __CUDACC_VER_MINOR__ << "." << __CUDACC_VER_BUILD__; +#else + out << "nvcc UNKNOWN"; +#endif + out << " ("; +#endif + // ICX version (either as CXX or as host compiler inside NVCC) +#if defined __INTEL_COMPILER +#error "icc is no longer supported: please use icx" +#elif defined __INTEL_LLVM_COMPILER // alternative: __INTEL_CLANG_COMPILER + out << "icx " << __INTEL_LLVM_COMPILER; +#ifdef __NVCC__ + out << ", "; +#else + out << " ("; +#endif +#endif + // CLANG version (either as CXX or as host compiler inside NVCC or inside ICX) +#if defined __clang__ +#if defined __clang_major__ && defined __clang_minor__ && defined __clang_patchlevel__ +#ifdef __APPLE__ + out << "Apple clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#else + out << "clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#endif +#else + out << "clang UNKNOWKN"; +#endif +#else + // GCC version (either as CXX or as host compiler inside NVCC) +#if defined __GNUC__ && defined __GNUC_MINOR__ && defined __GNUC_PATCHLEVEL__ + out << "gcc " << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__; +#else + out << "gcc UNKNOWKN"; +#endif +#endif +#if defined __HIPCC__ or defined __NVCC__ or defined __INTEL_LLVM_COMPILER + out << ")"; +#endif + return out.str(); + } } // Expose typedefs outside the namespace diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc index c13f8ebe2f..75b642878f 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc @@ -112,6 +112,20 @@ namespace mg5amcCpu static Parameters* instance; }; + // Generic readers for initProc: dereference an array of pointer-to-Parameters-member + // (a per-process compile-time table) into a matching array of tIPD/tIPC values. + template + inline void gatherFptype( const Parameters* pars, double Parameters::* const ( &members )[N], fptype ( &out )[N] ) + { + for( int i = 0; i < N; ++i ) out[i] = (fptype)( pars->*members[i] ); + } + + template + inline void gatherCxtype( const Parameters* pars, cxtype Parameters::* const ( &members )[N], cxtype ( &out )[N] ) + { + for( int i = 0; i < N; ++i ) out[i] = cxmake( pars->*members[i] ); + } + } // end namespace mg5amcGpu/mg5amcCpu #else%(eftwarn1)s diff --git a/madgraph/iolibs/template_files/madmatrix/process_cc.inc b/madgraph/iolibs/template_files/madmatrix/process_cc.inc index 77d8880750..33b938494a 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_cc.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_cc.inc @@ -33,8 +33,6 @@ #include #include -#include // for feenableexcept, fegetexcept and FE_XXX -#include // for FLT_MIN #include #include #include @@ -46,44 +44,5 @@ //-------------------------------------------------------------------------- -// Enable FPE traps (see #701, #733, #831 - except on MacOS where feenableexcept is not defined #730) -// [NB1: Fortran default is -ffpe-trap=none, i.e. FPE traps are not enabled, https://gcc.gnu.org/onlinedocs/gfortran/Debugging-Options.html] -// [NB2: Fortran default is -ffpe-summary=invalid,zero,overflow,underflow,denormal, i.e. warn at the end on STOP] -inline void -fpeEnable() -{ - static bool first = true; // FIXME: quick and dirty hack to do this only once (can be removed when separate C++/CUDA builds are implemented) - if( !first ) return; - first = false; -#ifndef __APPLE__ // on MacOS feenableexcept is not defined #730 - //int fpes = fegetexcept(); - //std::cout << "fpeEnable: analyse fegetexcept()=" << fpes << std::endl; - //std::cout << "fpeEnable: FE_DIVBYZERO is" << ( ( fpes & FE_DIVBYZERO ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_INEXACT is" << ( ( fpes & FE_INEXACT ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_INVALID is" << ( ( fpes & FE_INVALID ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_OVERFLOW is" << ( ( fpes & FE_OVERFLOW ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_UNDERFLOW is" << ( ( fpes & FE_UNDERFLOW ) ? " " : " NOT " ) << "enabled" << std::endl; - constexpr bool enableFPE = true; // this is hardcoded and no longer controlled by getenv( "CUDACPP_RUNTIME_ENABLEFPE" ) - if( enableFPE ) - { - std::cout << "INFO: The following Floating Point Exceptions will cause SIGFPE program aborts: FE_DIVBYZERO, FE_INVALID, FE_OVERFLOW" << std::endl; - feenableexcept( FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW ); // new strategy #831 (do not enable FE_UNDERFLOW) - //fpes = fegetexcept(); - //std::cout << "fpeEnable: analyse fegetexcept()=" << fpes << std::endl; - //std::cout << "fpeEnable: FE_DIVBYZERO is" << ( ( fpes & FE_DIVBYZERO ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_INEXACT is" << ( ( fpes & FE_INEXACT ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_INVALID is" << ( ( fpes & FE_INVALID ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_OVERFLOW is" << ( ( fpes & FE_OVERFLOW ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_UNDERFLOW is" << ( ( fpes & FE_UNDERFLOW ) ? " " : " NOT " ) << "enabled" << std::endl; - } - else - { - //std::cout << "INFO: Do not enable SIGFPE traps for Floating Point Exceptions" << std::endl; - } -#else - //std::cout << "INFO: Keep default SIGFPE settings because feenableexcept is not available on MacOS" << std::endl; -#endif -} - %(process_function_definitions)s //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/process_class.inc b/madgraph/iolibs/template_files/madmatrix/process_class.inc index 44a853d4bb..990f935f18 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_class.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_class.inc @@ -25,9 +25,6 @@ // Initialize process (read model parameters from file) virtual void initProc( const std::string& param_card_name ); - // Retrieve the compiler that was used to build this module - static const std::string getCompiler(); - // PDG id for a given flavor combination index and particle index (0-based). // Useful for tools that need to enumerate flavor combinations without going through UMAMI. static int flavorPDG( int iflavor, int ipar ); diff --git a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc index 09acde6a72..266f724528 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc @@ -27,10 +27,6 @@ namespace mg5amcCpu { using namespace ProcessData; - // Host-side flavor table: single source of truth for PDG ids (used by both the - // constructor copy into cFlavors and the public CPPProcess::flavorPDG accessor). -%(all_flavors)s - //-------------------------------------------------------------------------- CPPProcess::CPPProcess( bool verbose, @@ -42,9 +38,6 @@ namespace mg5amcCpu #endif , m_masses() { - // Helicities for the process [NB do keep 'static' for this constexpr array, see issue #283] - // *** NB There is no automatic check yet that these are in the same order as Fortran! #569 *** -%(all_helicities)s setHelicitiesAndFlavors( &tHel[0][0], &tFlavors[0][0] ); // Enable SIGFPE traps for Floating Point Exceptions @@ -125,71 +118,4 @@ namespace mg5amcCpu #endif //-------------------------------------------------------------------------- - - // Retrieve the compiler that was used to build this module - const std::string - CPPProcess::getCompiler() - { - std::stringstream out; - // HIP version (HIPCC) - // [Use __HIPCC__ instead of MGONGPUCPP_GPUIMPL here!] - // [This tests if 'hipcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] - // [Check 'hipcc -dM -E -x hip -I ../../src CPPProcess.cc | grep HIP'] -#ifdef __HIPCC__ -#if defined HIP_VERSION_MAJOR && defined HIP_VERSION_MINOR && defined HIP_VERSION_PATCH - out << "hipcc " << HIP_VERSION_MAJOR << "." << HIP_VERSION_MINOR << "." << HIP_VERSION_PATCH; -#else - out << "hipcc UNKNOWN"; -#endif - out << " ("; -#endif - // CUDA version (NVCC) - // [Use __NVCC__ instead of MGONGPUCPP_GPUIMPL here!] - // [This tests if 'nvcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] - // [Check 'nvcc --compiler-options -dM -E dummy.c | grep CUDA': see https://stackoverflow.com/a/53713712] -#ifdef __NVCC__ -#if defined __CUDACC_VER_MAJOR__ && defined __CUDACC_VER_MINOR__ && defined __CUDACC_VER_BUILD__ - out << "nvcc " << __CUDACC_VER_MAJOR__ << "." << __CUDACC_VER_MINOR__ << "." << __CUDACC_VER_BUILD__; -#else - out << "nvcc UNKNOWN"; -#endif - out << " ("; -#endif - // ICX version (either as CXX or as host compiler inside NVCC) -#if defined __INTEL_COMPILER -#error "icc is no longer supported: please use icx" -#elif defined __INTEL_LLVM_COMPILER // alternative: __INTEL_CLANG_COMPILER - out << "icx " << __INTEL_LLVM_COMPILER; -#ifdef __NVCC__ - out << ", "; -#else - out << " ("; -#endif -#endif - // CLANG version (either as CXX or as host compiler inside NVCC or inside ICX) -#if defined __clang__ -#if defined __clang_major__ && defined __clang_minor__ && defined __clang_patchlevel__ -#ifdef __APPLE__ - out << "Apple clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; -#else - out << "clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; -#endif -#else - out << "clang UNKNOWKN"; -#endif -#else - // GCC version (either as CXX or as host compiler inside NVCC) -#if defined __GNUC__ && defined __GNUC_MINOR__ && defined __GNUC_PATCHLEVEL__ - out << "gcc " << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__; -#else - out << "gcc UNKNOWKN"; -#endif -#endif -#if defined __HIPCC__ or defined __NVCC__ or defined __INTEL_LLVM_COMPILER - out << ")"; -#endif - return out.str(); - } - - //-------------------------------------------------------------------------- } // end namespace diff --git a/madmatrix/model_handling.py b/madmatrix/model_handling.py index 7f37f29d46..5b84bfe19a 100644 --- a/madmatrix/model_handling.py +++ b/madmatrix/model_handling.py @@ -1539,7 +1539,7 @@ def get_process_class_definitions(self, write=True): def get_process_function_definitions(self, write=True): """The complete class definition for the process""" replace_dict = super().get_process_function_definitions(write=False) # defines replace_dict['initProc_lines'] - replace_dict['hardcoded_initProc_lines'] = replace_dict['initProc_lines'].replace( 'm_pars->', 'Parameters::') + replace_dict['hardcoded_initProc_lines'] = self.get_hardcoded_initProc_lines(self.matrix_elements[0]) replace_dict['nparams'] = len(self.params2order) replace_dict['coupling_list'] = ' ' replace_dict['hel_amps_cc'] = '#include \"HelAmps_%s.cc\"' % self.model_name # AV @@ -1572,8 +1572,12 @@ def get_process_function_definitions(self, write=True): self._ndpf = len(flv_couplings_dep) if len(coupling_indep) > 0: - replace_dict['cipcassign'] = 'const cxtype tIPC[nIPC] = { cxmake( m_pars->%s ) };\n setIndependentCouplings( tIPC );'\ - % ( ' ), cxmake( m_pars->'.join(coupling_indep) ) # AV only indep! + replace_dict['cipcassign'] = ('static constexpr cxtype Parameters::* const cIPC_members[nIPC] = {\n' + ' &Parameters::' + ',\n &Parameters::'.join(coupling_indep) + '\n' + ' };\n' + ' cxtype tIPC[nIPC];\n' + ' gatherCxtype( m_pars, cIPC_members, tIPC );\n' + ' setIndependentCouplings( tIPC );') coup_str_hrd = 'const cxtype tIPC[nIPC] = { cxmake( Parameters::%s ) };\n setIndependentCouplings( tIPC );'\ % ( ' ), cxmake( Parameters::'.join(coupling_indep) ) replace_dict['cipchrdassign'] = coup_str_hrd @@ -1582,8 +1586,12 @@ def get_process_function_definitions(self, write=True): replace_dict['cipchrdassign'] = '//const cxtype tIPC[0] = { ... }; // nIPC=0' if len(params) > 0: - replace_dict['cipdassign'] = 'const fptype tIPD[nIPD] = { (fptype)m_pars->%s };\n setIndependentParams( tIPD );'\ - %( ', (fptype)m_pars->'.join(params) ) + replace_dict['cipdassign'] = ('static constexpr double Parameters::* const cIPD_members[nIPD] = {\n' + ' &Parameters::' + ',\n &Parameters::'.join(params) + '\n' + ' };\n' + ' fptype tIPD[nIPD];\n' + ' gatherFptype( m_pars, cIPD_members, tIPD );\n' + ' setIndependentParams( tIPD );') replace_dict['cipdhrdassign'] = 'const fptype tIPD[nIPD] = { (fptype)Parameters::%s };\n setIndependentParams( tIPD );'\ %( ', (fptype)Parameters::'.join(params) ) else: @@ -1632,11 +1640,6 @@ def get_process_function_definitions(self, write=True): if( Parameters::nBsmIndepParam > 0 ) setBsmIndepParam( Parameters::mdl_bsmIndepParam, Parameters::nBsmIndepParam ); #endif''' - replace_dict['all_helicities'] = self.get_helicity_matrix(self.matrix_elements[0]) - replace_dict['all_helicities'] = replace_dict['all_helicities'] .replace('helicities', 'tHel') - replace_dict['all_flavors'] = self.get_flavor_matrix(self.matrix_elements[0]) - replace_dict['all_flavors'] = replace_dict['all_flavors'].replace('flavors', 'tFlavors') - file = self.read_template_file(self.process_definition_template) % replace_dict # HACK! ignore write=False case if len(params) == 0: # remove cIPD from OpenMP pragma (issue #349) file_lines = file.split('\n') @@ -1765,6 +1768,8 @@ def edit_processdata(self): replace_dict['ndpf'] = self._ndpf replace_dict['processid'] = self.name replace_dict['processid_uppercase'] = self.name.upper() + replace_dict['thel_lines'] = self.get_helicity_matrix(me).replace('helicities', 'tHel') + replace_dict['tflavors_lines'] = self.get_flavor_matrix(me).replace('flavors', 'tFlavors') ff = open(pjoin(self.path, 'ProcessData.h'), 'w') ff.write(template % replace_dict) ff.close() @@ -1976,24 +1981,31 @@ def get_color_matrix_lines(self, matrix_element): # AV - replace the export_cpp.OneProcessExporterCPP method (improve formatting) def get_initProc_lines(self, matrix_element, color_amplitudes): - """Get initProc_lines for function definition for CPPProcess::initProc""" - initProc_lines = [] - initProc_lines.append('// Set external particle masses for this matrix element') + """initProc_lines for CPPProcess::initProc (non-hardcoded branch): a generated pointer-to-member table read by gatherFptype() (see Parameters.h).""" + masses = [part.get('mass') for part in matrix_element.get_external_wavefunctions()] + return ('// Set external particle masses for this matrix element\n' + ' static constexpr double Parameters::* const massMembers[npar] = {\n' + ' &Parameters::' + ',\n &Parameters::'.join(masses) + '\n' + ' };\n' + ' fptype tMasses[npar];\n' + ' gatherFptype( m_pars, massMembers, tMasses );\n' + ' m_masses.assign( tMasses, tMasses + npar );') + + def get_hardcoded_initProc_lines(self, matrix_element): + """initProc_lines for CPPProcess::initProc, MGONGPU_HARDCODE_PARAM branch: Parameters has no instance here, so this stays imperative.""" + initProc_lines = ['// Set external particle masses for this matrix element'] for part in matrix_element.get_external_wavefunctions(): - ###initProc_lines.append('mME.push_back(pars->%s);' % part.get('mass')) - initProc_lines.append(' m_masses.push_back( m_pars->%s );' % part.get('mass')) # AV - ###for i, colamp in enumerate(color_amplitudes): - ### initProc_lines.append('jamp2_sv[%d] = new double[%d];' % (i, len(colamp))) # AV - this was commented out already + initProc_lines.append(' m_masses.push_back( Parameters::%s );' % part.get('mass')) return '\n'.join(initProc_lines) # AV - replace the export_cpp.OneProcessExporterCPP method (fix helicity order and improve formatting) def get_helicity_matrix(self, matrix_element): """Return the Helicity matrix definition lines for this matrix element""" - helicity_line = ' static constexpr short helicities[ncomb][npar] = {\n '; # AV (this is tHel) + helicity_line = ' static constexpr short helicities[ncomb][npar] = {\n '; # AV (this is tHel) helicity_line_list = [] for helicities in matrix_element.get_helicity_matrix(allow_reverse=True): # AV was False: different order in Fortran and cudacpp! #569 helicity_line_list.append( '{ ' + ', '.join(['%d'] * len(helicities)) % tuple(helicities) + ' }' ) # AV - return helicity_line + ',\n '.join(helicity_line_list) + ' };' # AV + return helicity_line + ',\n '.join(helicity_line_list) + ' };' # AV def get_flavor_matrix(self, matrix_element): """Return the flavor matrix definition lines for this matrix element""" From a919bcd51fff2e521941c17d12d3ea67a05afc12 Mon Sep 17 00:00:00 2001 From: stloufra Date: Mon, 17 Aug 2026 18:02:57 +0200 Subject: [PATCH 15/25] Uni namspace and bug fixed unified namespace madgraph instead of mg5amcGpu/mg5amcCpu. Fix for some functions depending on vector lane size --- .../madmatrix/CommonRandomNumberKernel.cc | 6 +-- .../madmatrix/CurandRandomNumberKernel.cc | 6 +-- .../madmatrix/HiprandRandomNumberKernel.cc | 6 +-- .../template_files/madmatrix/MadgraphTest.h | 6 +-- .../template_files/madmatrix/ProcessTables.h | 8 +--- .../madmatrix/RamboSamplingKernels.cc | 6 +-- .../madmatrix/RamboSamplingKernels.h | 6 +-- .../madmatrix/RandomNumberKernels.h | 6 +-- .../backend/cpu/CrossSectionKernels.cc | 2 +- .../backend/cpu/CrossSectionKernels.h | 2 +- .../madmatrix/backend/cpu/EventStatistics.h | 2 +- .../backend/cpu/MatrixElementKernels.cc | 4 +- .../backend/cpu/MatrixElementKernels.h | 2 +- .../backend/cpu/MemoryAccessAmplitudes.h | 6 +-- .../backend/cpu/MemoryAccessChannelIds.h | 6 +-- .../backend/cpu/MemoryAccessCouplings.h | 10 ++--- .../backend/cpu/MemoryAccessCouplingsFixed.h | 6 +-- .../backend/cpu/MemoryAccessDenominators.h | 6 +-- .../madmatrix/backend/cpu/MemoryAccessGs.h | 6 +-- .../backend/cpu/MemoryAccessIflavorVec.h | 6 +-- .../backend/cpu/MemoryAccessMatrixElements.h | 6 +-- .../backend/cpu/MemoryAccessMomenta.h | 6 +-- .../backend/cpu/MemoryAccessNumerators.h | 6 +-- .../backend/cpu/MemoryAccessVectors.h | 2 +- .../backend/cpu/MemoryAccessWavefunctions.h | 6 +-- .../backend/cpu/MemoryAccessWeights.h | 6 +-- .../madmatrix/backend/cpu/MemoryBuffers.h | 2 +- .../madmatrix/backend/cpu/SigmaKin.cc | 3 +- .../madmatrix/backend/cpu/SigmaKin.h | 2 +- .../madmatrix/backend/cpu/color_sum.cc | 2 +- .../madmatrix/backend/cpu/color_sum.h | 2 +- .../madmatrix/backend/cpu/constexpr_math.h | 3 +- .../madmatrix/backend/cpu/mgOnGpuCxtypes.h | 14 +++---- .../madmatrix/backend/cpu/mgOnGpuFptypes.h | 6 +-- .../madmatrix/backend/cpu/mgOnGpuVectors.h | 41 ++++++++++++++++--- .../madmatrix/backend/cpu/umami.cc | 4 +- .../backend/gpu/CrossSectionKernels.cc | 4 +- .../backend/gpu/CrossSectionKernels.h | 2 +- .../madmatrix/backend/gpu/EventStatistics.h | 2 +- .../madmatrix/backend/gpu/GpuRuntime.h | 2 +- .../backend/gpu/MatrixElementKernels.cc | 4 +- .../backend/gpu/MatrixElementKernels.h | 2 +- .../backend/gpu/MemoryAccessAmplitudes.h | 6 +-- .../backend/gpu/MemoryAccessChannelIds.h | 6 +-- .../backend/gpu/MemoryAccessCouplings.h | 10 ++--- .../backend/gpu/MemoryAccessCouplingsFixed.h | 6 +-- .../backend/gpu/MemoryAccessDenominators.h | 6 +-- .../madmatrix/backend/gpu/MemoryAccessGs.h | 6 +-- .../backend/gpu/MemoryAccessIflavorVec.h | 6 +-- .../backend/gpu/MemoryAccessMatrixElements.h | 6 +-- .../backend/gpu/MemoryAccessMomenta.h | 6 +-- .../backend/gpu/MemoryAccessNumerators.h | 6 +-- .../backend/gpu/MemoryAccessWavefunctions.h | 6 +-- .../backend/gpu/MemoryAccessWeights.h | 6 +-- .../madmatrix/backend/gpu/MemoryBuffers.h | 2 +- .../madmatrix/backend/gpu/SigmaKin.cc | 2 +- .../madmatrix/backend/gpu/SigmaKin.h | 2 +- .../madmatrix/backend/gpu/color_sum.cc | 2 +- .../madmatrix/backend/gpu/color_sum.h | 2 +- .../madmatrix/backend/gpu/constexpr_math.h | 4 +- .../madmatrix/backend/gpu/mgOnGpuCxtypes.h | 22 +++++----- .../madmatrix/backend/gpu/mgOnGpuFptypes.h | 6 +-- .../madmatrix/backend/gpu/mgOnGpuVectors.h | 10 ++--- .../madmatrix/backend/gpu/umami.cc | 4 +- .../backend/simd/CrossSectionKernels.cc | 2 +- .../backend/simd/CrossSectionKernels.h | 2 +- .../madmatrix/backend/simd/EventStatistics.h | 2 +- .../backend/simd/MatrixElementKernels.cc | 4 +- .../backend/simd/MatrixElementKernels.h | 2 +- .../backend/simd/MemoryAccessAmplitudes.h | 6 +-- .../backend/simd/MemoryAccessChannelIds.h | 10 ++--- .../backend/simd/MemoryAccessCouplings.h | 14 +++---- .../backend/simd/MemoryAccessCouplingsFixed.h | 6 +-- .../backend/simd/MemoryAccessDenominators.h | 6 +-- .../madmatrix/backend/simd/MemoryAccessGs.h | 14 +++---- .../backend/simd/MemoryAccessIflavorVec.h | 10 ++--- .../backend/simd/MemoryAccessMatrixElements.h | 10 ++--- .../backend/simd/MemoryAccessMomenta.h | 14 +++---- .../backend/simd/MemoryAccessNumerators.h | 6 +-- .../backend/simd/MemoryAccessVectors.h | 2 +- .../backend/simd/MemoryAccessWavefunctions.h | 6 +-- .../backend/simd/MemoryAccessWeights.h | 6 +-- .../madmatrix/backend/simd/MemoryBuffers.h | 2 +- .../madmatrix/backend/simd/SigmaKin.cc | 3 +- .../madmatrix/backend/simd/SigmaKin.h | 2 +- .../madmatrix/backend/simd/color_sum.cc | 2 +- .../madmatrix/backend/simd/color_sum.h | 2 +- .../madmatrix/backend/simd/constexpr_math.h | 4 +- .../madmatrix/backend/simd/mgOnGpuCxtypes.h | 14 +++---- .../madmatrix/backend/simd/mgOnGpuFptypes.h | 6 +-- .../madmatrix/backend/simd/mgOnGpuVectors.h | 35 +++++++++++++--- .../madmatrix/backend/simd/umami.cc | 4 +- .../template_files/madmatrix/check_sa.cc | 6 +-- .../template_files/madmatrix/constexpr_math.h | 8 +--- .../madmatrix/cpp_hel_amps_h.inc | 6 +-- .../madmatrix/cpp_model_parameters_cc.inc | 6 +-- .../madmatrix/cpp_model_parameters_h.inc | 30 ++++---------- .../template_files/madmatrix/massless_rambo.h | 6 +-- .../template_files/madmatrix/mgOnGpuCxtypes.h | 38 +++++------------ .../template_files/madmatrix/mgOnGpuFptypes.h | 8 ++-- .../template_files/madmatrix/mgOnGpuVectors.h | 18 +++----- .../template_files/madmatrix/process_cc.inc | 1 + .../process_function_definitions.inc | 7 +--- .../template_files/madmatrix/process_h.inc | 6 +-- .../template_files/madmatrix/runTest.cc | 6 +-- .../template_files/madmatrix/testmisc.cc | 14 ++----- .../template_files/madmatrix/testxxx.cc | 22 +++------- 107 files changed, 342 insertions(+), 403 deletions(-) diff --git a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc index cdac738384..8aba05355a 100644 --- a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc @@ -15,11 +15,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc index 0c8e6e6445..098328a876 100644 --- a/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc @@ -24,11 +24,7 @@ inline void assertCurand( curandStatus_t code, const char *file, int line, bool } #endif /* clang-format on */ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_CURAND diff --git a/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc index b806da2e09..a16f15796a 100644 --- a/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc @@ -28,11 +28,7 @@ inline void assertHiprand( hiprandStatus_t code, const char *file, int line, boo } #endif /* clang-format on */ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_HIPRAND diff --git a/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h index 12b7d45adf..2ab9b17432 100644 --- a/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h +++ b/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h @@ -27,11 +27,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -using mg5amcGpu::CPPProcess; -#else -using mg5amcCpu::CPPProcess; -#endif +using madgraph::CPPProcess; namespace { diff --git a/madgraph/iolibs/template_files/madmatrix/ProcessTables.h b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h index 5b294e36ad..01105bec46 100644 --- a/madgraph/iolibs/template_files/madmatrix/ProcessTables.h +++ b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h @@ -10,7 +10,7 @@ // pulled in via ProcessTables::name from backend-owned function bodies. // // Namespace-wrapped (unlike ProcessData.h) because it needs FLV_COUPLING, -// which is itself backend-namespaced (mg5amcCpu::/mg5amcGpu::, see Parameters.h). +// which is itself backend-namespaced (madgraph::/madgraph::, see Parameters.h). #ifndef PROCESSTABLES_H #define PROCESSTABLES_H 1 @@ -19,11 +19,7 @@ #include "ProcessData.h" #include "Parameters.h" // for FLV_COUPLING::max_flavor -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { namespace ProcessTables { diff --git a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc index 024c3ee2ee..aa75a0c5ad 100644 --- a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc @@ -18,11 +18,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h index 6244d55618..80a17aefac 100644 --- a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h @@ -13,11 +13,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h index 0fb880bb36..ee11017750 100644 --- a/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h @@ -18,11 +18,7 @@ struct curandGenerator_st; struct rocrand_generator_base_type; typedef rocrand_generator_base_type hiprandGenerator_st; -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc index 019eb2e978..3ad7c5d1ca 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc @@ -79,7 +79,7 @@ debug_me_is_abnormal( const fptype& me, size_t ievtALL ) //============================================================================ -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h index 4ac3ded803..261f7953d9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h @@ -14,7 +14,7 @@ //============================================================================ -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h index 1be807a9bd..5eee6de231 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h @@ -17,7 +17,7 @@ #include #include -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc index 661fb88ffe..171f815ac5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc @@ -18,7 +18,7 @@ //============================================================================ -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- @@ -143,7 +143,7 @@ namespace mg5amcCpu //============================================================================ -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h index 1693d61c42..b74a7e19cb 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h @@ -14,7 +14,7 @@ #include #include -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h index 9cd0e794c0..7a58f2fc92 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_AMPLITUDES 1 -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -156,6 +156,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h index 1963d3dfff..2c301811ec 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -110,6 +110,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h index 97435bd038..ab5f7de4c5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h @@ -15,8 +15,8 @@ #include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM #include "MemoryBuffers.h" // for HostBufferCouplings::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -212,9 +212,9 @@ namespace mg5amcCpu constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madgraph::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast #endif } */ @@ -257,6 +257,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h index fd1e1abadf..7ce36f4c6f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h @@ -14,8 +14,8 @@ //#include "MemoryAccessHelpers.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -80,6 +80,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h index 01611fe92d..10046c1293 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h index 9faa088a45..2739a0bd80 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -148,6 +148,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h index 1903b43ed4..af8d37bf6f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h @@ -12,8 +12,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -109,6 +109,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h index 7f9cd4c11c..6175a16d1b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -131,6 +131,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h index b7c175fe51..245a2ba8ba 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h @@ -13,8 +13,8 @@ #include "MemoryAccessHelpers.h" #include "MemoryAccessVectors.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -204,6 +204,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h index 42356061e7..efa609aee1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h index db9addfb21..6a3c642c56 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h @@ -11,7 +11,7 @@ #include "mgOnGpuVectors.h" -namespace mg5amcCpu // this is only needed for CPU SIMD vectorization +namespace madgraph // this is only needed for CPU SIMD vectorization { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h index 672a66bc09..66d7f35ac9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -161,6 +161,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h index 0a82fe49d1..12df545d92 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h @@ -11,8 +11,8 @@ #include "MemoryAccessHelpers.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -141,6 +141,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h index 3bb7b591e7..83f316223b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h @@ -16,7 +16,7 @@ #include -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc index 4a0d3eaaa9..2b7bf5fad3 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc @@ -26,8 +26,9 @@ #include "ColorData.h" #include +#include -namespace mg5amcCpu +namespace madgraph { using namespace ProcessData; using namespace ProcessTables; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h index 725ff931d2..79d1a5d6e7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h @@ -14,7 +14,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuCxtypes.h" // for cxtype -namespace mg5amcCpu +namespace madgraph { __global__ void computeDependentCouplings( const fptype* allgs, diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc index 2d614f4533..2b04b4e8c9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc @@ -11,7 +11,7 @@ #include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps #include "MemoryAccessMatrixElements.h" -namespace mg5amcCpu +namespace madgraph { using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h index 8f4000a93a..b0abc3d030 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h @@ -13,7 +13,7 @@ #include "ProcessData.h" -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h index c671aa5db7..8adb8a7cda 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h @@ -29,8 +29,7 @@ #define CONSTEXPRMATHVAR constexpr #endif -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +namespace madgraph { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h index dea5dcf2d1..ed7ec2fdf0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h @@ -73,7 +73,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -namespace mg5amcCpu +namespace madgraph { template inline __host__ std::ostream& @@ -202,7 +202,7 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -namespace mg5amcCpu +namespace madgraph { // --- Type definitions (complex type: cxtype) #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX @@ -223,7 +223,7 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -namespace mg5amcCpu +namespace madgraph { #if defined MGONGPU_CPPCXTYPE_CXSMPL @@ -331,14 +331,14 @@ namespace mg5amcCpu return cxmake( c.real(), c.imag() ); } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph //========================================================================== // COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined @@ -373,7 +373,7 @@ namespace mg5amcCpu return out; } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h index fa465e7658..62e454648c 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h @@ -12,8 +12,8 @@ #include #include -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //========================================================================== @@ -46,6 +46,6 @@ namespace mg5amcCpu //========================================================================== -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h index c464174d5f..61c7c4bb6f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h @@ -32,8 +32,8 @@ //#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) #endif -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { const int neppV = 1; @@ -54,8 +54,8 @@ namespace mg5amcCpu //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // Printout to stream for user defined types @@ -199,6 +199,16 @@ namespace mg5amcCpu } return out; */ +#if MGONGPU_CPPSIMD == 2 + fptype2_v out = + { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v2[0], (fptype2)v2[1] }; +#elif MGONGPU_CPPSIMD == 4 + fptype2_v out = + { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v1[2], (fptype2)v1[3], (fptype2)v2[0], (fptype2)v2[1], (fptype2)v2[2], (fptype2)v2[3] }; +#elif MGONGPU_CPPSIMD == 8 + fptype2_v out = + { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v1[2], (fptype2)v1[3], (fptype2)v1[4], (fptype2)v1[5], (fptype2)v1[6], (fptype2)v1[7], (fptype2)v2[0], (fptype2)v2[1], (fptype2)v2[2], (fptype2)v2[3], (fptype2)v2[4], (fptype2)v2[5], (fptype2)v2[6], (fptype2)v2[7] }; +#endif return out; } @@ -212,6 +222,16 @@ namespace mg5amcCpu out[ieppV] = v[ieppV]; } */ +#if MGONGPU_CPPSIMD == 2 + fptype_v out = + { (fptype)v[0], (fptype)v[1] }; +#elif MGONGPU_CPPSIMD == 4 + fptype_v out = + { (fptype)v[0], (fptype)v[1], (fptype)v[2], (fptype)v[3] }; +#elif MGONGPU_CPPSIMD == 8 + fptype_v out = + { (fptype)v[0], (fptype)v[1], (fptype)v[2], (fptype)v[3], (fptype)v[4], (fptype)v[5], (fptype)v[6], (fptype)v[7] }; +#endif return out; } @@ -225,9 +245,20 @@ namespace mg5amcCpu out[ieppV] = v[ieppV+neppV]; } */ +#if MGONGPU_CPPSIMD == 2 + fptype_v out = + { (fptype)v[2], (fptype)v[3] }; +#elif MGONGPU_CPPSIMD == 4 + fptype_v out = + { (fptype)v[4], (fptype)v[5], (fptype)v[6], (fptype)v[7] }; +#elif MGONGPU_CPPSIMD == 8 + fptype_v out = + { (fptype)v[8], (fptype)v[9], (fptype)v[10], (fptype)v[11], (fptype)v[12], (fptype)v[13], (fptype)v[14], (fptype)v[15] }; +#endif return out; } + #endif // #if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT @@ -258,6 +289,6 @@ namespace mg5amcCpu //========================================================================== -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MGONGPUVECTORS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc index cefa01b48f..cedf05b15b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc @@ -17,7 +17,7 @@ #include #include -using namespace mg5amcCpu; +using namespace madgraph; namespace { @@ -301,7 +301,7 @@ extern "C" } HostBufferBase momenta( rounded_count * ProcessData::npar * 4 ); - HostBufferBase couplings( rounded_count * mg5amcCpu::Parameters_dependentCouplings::ndcoup * 2 ); + HostBufferBase couplings( rounded_count * madgraph::Parameters_dependentCouplings::ndcoup * 2 ); HostBufferBase g_s( rounded_count ); HostBufferBase helicity_random( rounded_count ); HostBufferBase color_random( rounded_count ); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc index 53430fe8d4..e70fd6ae1b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc @@ -79,7 +79,7 @@ debug_me_is_abnormal( const fptype& me, size_t ievtALL ) //============================================================================ -namespace mg5amcGpu +namespace madgraph { //-------------------------------------------------------------------------- @@ -183,7 +183,7 @@ namespace mg5amcGpu //============================================================================ -namespace mg5amcGpu +namespace madgraph { /* diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h index c70dff0143..261f7953d9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h @@ -14,7 +14,7 @@ //============================================================================ -namespace mg5amcGpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h index c8a25e2369..5eee6de231 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h @@ -17,7 +17,7 @@ #include #include -namespace mg5amcGpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h index ed9887e045..c6aca5328a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h @@ -43,7 +43,7 @@ inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, boo //-------------------------------------------------------------------------- -namespace mg5amcGpu +namespace madgraph { // Instantiate a GpuRuntime at the beginnining of the application's main to // invoke gpuSetDevice(0) in the constructor and book a gpuDeviceReset() call in the destructor diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc index 207a17cc8d..c79f407484 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc @@ -18,7 +18,7 @@ //============================================================================ -namespace mg5amcGpu +namespace madgraph { //-------------------------------------------------------------------------- @@ -146,7 +146,7 @@ namespace mg5amcGpu //============================================================================ -namespace mg5amcGpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h index 3be385f40e..c4e7db8a21 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h @@ -16,7 +16,7 @@ #include #include -namespace mg5amcGpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h index d0b9d95642..7a58f2fc92 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_AMPLITUDES 1 -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -156,6 +156,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h index 95ab36a00a..2c301811ec 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -110,6 +110,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h index 8b191722a1..ab5f7de4c5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h @@ -15,8 +15,8 @@ #include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM #include "MemoryBuffers.h" // for HostBufferCouplings::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -212,9 +212,9 @@ namespace mg5amcGpu constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madgraph::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast #endif } */ @@ -257,6 +257,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h index 43dff43510..7ce36f4c6f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h @@ -14,8 +14,8 @@ //#include "MemoryAccessHelpers.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -80,6 +80,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h index b7aa50c871..10046c1293 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h index faf362d745..2739a0bd80 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -148,6 +148,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h index dafb4b4fcb..af8d37bf6f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h @@ -12,8 +12,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -109,6 +109,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h index 17a1f33460..6175a16d1b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -131,6 +131,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h index 0f76adb03c..0230ea8a1a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h @@ -13,8 +13,8 @@ #include "MemoryAccessHelpers.h" #include "MemoryAccessVectors.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -205,6 +205,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h index e69d9e82fc..efa609aee1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h index f023a5ecc0..66d7f35ac9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -161,6 +161,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h index fd36d9be67..12df545d92 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h @@ -11,8 +11,8 @@ #include "MemoryAccessHelpers.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -141,6 +141,6 @@ namespace mg5amcGpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h index 796b49ed20..ed1f5ec02b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h @@ -17,7 +17,7 @@ #include -namespace mg5amcGpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc index 4fa881936f..a7d9814b12 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc @@ -26,7 +26,7 @@ #include "color_sum.h" #include "ColorData.h" -namespace mg5amcGpu +namespace madgraph { using namespace ProcessData; using namespace ProcessTables; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h index c6254c7184..90e730a620 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h @@ -16,7 +16,7 @@ #include "mgOnGpuVectors.h" // for fptype_sv #include "GpuAbstraction.h" // for gpuBlasHandle_t, gpuStream_t -namespace mg5amcGpu +namespace madgraph { __global__ void computeDependentCouplings( const fptype* allgs, diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc index 00d5c5dfe9..584e805d9c 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc @@ -11,7 +11,7 @@ #include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps #include "MemoryAccessMatrixElements.h" -namespace mg5amcGpu +namespace madgraph { using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h index 8d9fea5577..0517d1c79e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h @@ -14,7 +14,7 @@ #include "ProcessData.h" #include "GpuAbstraction.h" -namespace mg5amcGpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h index 6219b4e0df..e3438e00b2 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h @@ -29,8 +29,8 @@ #define CONSTEXPRMATHVAR constexpr #endif -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h index 1cd951672e..abec0fb3d8 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h @@ -51,7 +51,7 @@ #ifdef __CUDACC__ // this must be __CUDACC__ #if defined MGONGPU_CUCXTYPE_CUCOMPLEX -namespace mg5amcGpu +namespace madgraph { #if defined MGONGPU_FPTYPE_DOUBLE class cucomplex @@ -139,7 +139,7 @@ namespace mgOnGpu /* clang-format off */ #ifdef MGONGPU_CUCXTYPE_THRUST template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } #elif defined MGONGPU_CUCXTYPE_CUCOMPLEX - __host__ __device__ constexpr operator mg5amcGpu::cucomplex() const { return mg5amcGpu::cucomplex( m_real, m_imag ); } + __host__ __device__ constexpr operator madgraph::cucomplex() const { return madgraph::cucomplex( m_real, m_imag ); } #endif #else #ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX @@ -163,7 +163,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -namespace mg5amcGpu +namespace madgraph { template inline __host__ std::ostream& @@ -292,8 +292,8 @@ namespace mg5amcGpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // --- Type definitions (complex type: cxtype) #ifdef __CUDACC__ // this must be __CUDACC__ @@ -324,8 +324,8 @@ namespace mg5amcGpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL @@ -680,14 +680,14 @@ namespace mg5amcGpu return cxmake( c.real(), c.imag() ); } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph //========================================================================== // COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined @@ -722,7 +722,7 @@ namespace mg5amcGpu return out; } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h index 34c6536323..e14cbcfc42 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h @@ -12,8 +12,8 @@ #include #include -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //========================================================================== @@ -66,6 +66,6 @@ namespace mg5amcGpu //========================================================================== -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h index b2e3b61b8d..7747658900 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h @@ -32,8 +32,8 @@ //#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) #endif -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { const int neppV = 1; @@ -54,8 +54,8 @@ namespace mg5amcGpu //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcGpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //========================================================================== @@ -128,6 +128,6 @@ namespace mg5amcGpu //========================================================================== -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MGONGPUVECTORS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc index ed8c9fd301..834917a151 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc @@ -18,7 +18,7 @@ #include #include -using namespace mg5amcGpu; +using namespace madgraph; namespace { @@ -347,7 +347,7 @@ extern "C" int *helicity_index, *color_index; unsigned int *flavor_indices, *diagram_index; - std::size_t n_coup = mg5amcGpu::Parameters_dependentCouplings::ndcoup; + std::size_t n_coup = madgraph::Parameters_dependentCouplings::ndcoup; std::array, 16> ptrs_and_sizes = {{ {reinterpret_cast(&momenta), rounded_count * ProcessData::npar * 4 * sizeof( fptype )}, {reinterpret_cast(&couplings), rounded_count * n_coup * 2 * sizeof( fptype )}, diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc index 019eb2e978..3ad7c5d1ca 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc @@ -79,7 +79,7 @@ debug_me_is_abnormal( const fptype& me, size_t ievtALL ) //============================================================================ -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h index 4ac3ded803..261f7953d9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h @@ -14,7 +14,7 @@ //============================================================================ -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h index 1be807a9bd..5eee6de231 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h @@ -17,7 +17,7 @@ #include #include -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc index 661fb88ffe..171f815ac5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc @@ -18,7 +18,7 @@ //============================================================================ -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- @@ -143,7 +143,7 @@ namespace mg5amcCpu //============================================================================ -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h index 1693d61c42..b74a7e19cb 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h @@ -14,7 +14,7 @@ #include #include -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h index 9cd0e794c0..7a58f2fc92 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_AMPLITUDES 1 -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -156,6 +156,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h index afdca9d788..115ade33e2 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -100,9 +100,9 @@ namespace mg5amcCpu { const unsigned int& out = kernelAccessConst_s( buffer ); // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madgraph::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madgraph::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } }; @@ -113,6 +113,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h index 00de06527a..dc69aff815 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h @@ -15,8 +15,8 @@ #include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM #include "MemoryBuffers.h" // for HostBufferCouplings::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -189,9 +189,9 @@ namespace mg5amcCpu constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madgraph::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) @@ -218,9 +218,9 @@ namespace mg5amcCpu constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madgraph::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast #endif } */ @@ -263,6 +263,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h index fd1e1abadf..7ce36f4c6f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h @@ -14,8 +14,8 @@ //#include "MemoryAccessHelpers.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -80,6 +80,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h index 01611fe92d..10046c1293 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h index 484a48116a..de5955cedf 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -116,9 +116,9 @@ namespace mg5amcCpu { fptype& out = kernelAccess_s( buffer ); // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madgraph::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) @@ -141,9 +141,9 @@ namespace mg5amcCpu { const fptype& out = kernelAccessConst_s( buffer ); // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madgraph::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } }; @@ -154,6 +154,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h index fcd1d2d724..159d55c30e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h @@ -12,8 +12,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -99,9 +99,9 @@ namespace mg5amcCpu { const unsigned int& out = kernelAccessConst_s( buffer ); // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madgraph::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madgraph::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } }; @@ -112,6 +112,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h index 8f18847d56..40f0e7e106 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -116,9 +116,9 @@ namespace mg5amcCpu { fptype& out = kernelAccess_s( buffer ); // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madgraph::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) @@ -134,6 +134,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h index b591a33624..0f36b4a3c4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h @@ -13,8 +13,8 @@ #include "MemoryAccessHelpers.h" #include "MemoryAccessVectors.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -202,21 +202,21 @@ namespace mg5amcCpu //static bool first=true; if( first ){ std::cout << "WARNING! assume aligned AOSOA, skip check" << std::endl; first=false; } // SLOWER (5.06E6) // FASTEST? (5.09E6 in eemumu 512y) // This assumes alignment for momenta1d without checking - causes segmentation fault in reinterpret_cast if not aligned! - return mg5amcCpu::fptypevFromAlignedArray( out ); // use reinterpret_cast + return madgraph::fptypevFromAlignedArray( out ); // use reinterpret_cast } else if( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ) { //static bool first=true; if( first ){ std::cout << "WARNING! aligned AOSOA, reinterpret cast" << std::endl; first=false; } // SLOWER (5.00E6) // DEFAULT! A tiny bit (<1%) slower because of the alignment check (5.07E6 in eemumu 512y) // This explicitly checks buffer alignment to avoid segmentation faults in reinterpret_cast - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } else { //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWER (4.93E6) // A bit (1%) slower (5.05E6 in eemumu 512y) // This does not require buffer alignment, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 - return mg5amcCpu::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) + return madgraph::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) } } else @@ -228,7 +228,7 @@ namespace mg5amcCpu auto decoderIeppv = [buffer, ip4, ipar]( int ieppV ) -> const fptype& { return MemoryAccessMomenta::ieventAccessIp4IparConst( buffer, ievt0 + ieppV, ip4, ipar ); }; - return mg5amcCpu::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) + return madgraph::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) } } @@ -248,6 +248,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h index 42356061e7..efa609aee1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h index 628ae29cc4..8c8ece0f56 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h @@ -11,7 +11,7 @@ #include "mgOnGpuVectors.h" -namespace mg5amcCpu // this is only needed for CPU SIMD vectorization +namespace madgraph // this is only needed for CPU SIMD vectorization { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h index 672a66bc09..66d7f35ac9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -161,6 +161,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h index 0a82fe49d1..12df545d92 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h @@ -11,8 +11,8 @@ #include "MemoryAccessHelpers.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //---------------------------------------------------------------------------- @@ -141,6 +141,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h index 3bb7b591e7..83f316223b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h @@ -16,7 +16,7 @@ #include -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc index 060bf06d1a..ebe384b31a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc @@ -26,8 +26,9 @@ #include "ColorData.h" #include +#include -namespace mg5amcCpu +namespace madgraph { using namespace ProcessData; using namespace ProcessTables; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h index 725ff931d2..79d1a5d6e7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h @@ -14,7 +14,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuCxtypes.h" // for cxtype -namespace mg5amcCpu +namespace madgraph { __global__ void computeDependentCouplings( const fptype* allgs, diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc index 29830ddacb..4936784da6 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc @@ -11,7 +11,7 @@ #include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps #include "MemoryAccessMatrixElements.h" -namespace mg5amcCpu +namespace madgraph { using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h index 8f4000a93a..b0abc3d030 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h @@ -13,7 +13,7 @@ #include "ProcessData.h" -namespace mg5amcCpu +namespace madgraph { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h index c671aa5db7..e3438e00b2 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h @@ -29,8 +29,8 @@ #define CONSTEXPRMATHVAR constexpr #endif -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h index dea5dcf2d1..ed7ec2fdf0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h @@ -73,7 +73,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -namespace mg5amcCpu +namespace madgraph { template inline __host__ std::ostream& @@ -202,7 +202,7 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -namespace mg5amcCpu +namespace madgraph { // --- Type definitions (complex type: cxtype) #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX @@ -223,7 +223,7 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -namespace mg5amcCpu +namespace madgraph { #if defined MGONGPU_CPPCXTYPE_CXSMPL @@ -331,14 +331,14 @@ namespace mg5amcCpu return cxmake( c.real(), c.imag() ); } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph //========================================================================== // COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined @@ -373,7 +373,7 @@ namespace mg5amcCpu return out; } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h index fa465e7658..62e454648c 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h @@ -12,8 +12,8 @@ #include #include -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { //========================================================================== @@ -46,6 +46,6 @@ namespace mg5amcCpu //========================================================================== -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h index ce5a07fb19..e5ead4b329 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h @@ -32,8 +32,8 @@ //#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) #endif -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { const int neppV = MGONGPU_CPPSIMD; @@ -146,8 +146,8 @@ namespace mg5amcCpu //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -namespace mg5amcCpu +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // Printout to stream for user defined types @@ -713,8 +713,16 @@ namespace mg5amcCpu } return out; */ +#if MGONGPU_CPPSIMD == 2 fptype2_v out = { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v2[0], (fptype2)v2[1] }; +#elif MGONGPU_CPPSIMD == 4 + fptype2_v out = + { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v1[2], (fptype2)v1[3], (fptype2)v2[0], (fptype2)v2[1], (fptype2)v2[2], (fptype2)v2[3] }; +#elif MGONGPU_CPPSIMD == 8 + fptype2_v out = + { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v1[2], (fptype2)v1[3], (fptype2)v1[4], (fptype2)v1[5], (fptype2)v1[6], (fptype2)v1[7], (fptype2)v2[0], (fptype2)v2[1], (fptype2)v2[2], (fptype2)v2[3], (fptype2)v2[4], (fptype2)v2[5], (fptype2)v2[6], (fptype2)v2[7] }; +#endif return out; } @@ -728,8 +736,16 @@ namespace mg5amcCpu out[ieppV] = v[ieppV]; } */ +#if MGONGPU_CPPSIMD == 2 fptype_v out = { (fptype)v[0], (fptype)v[1] }; +#elif MGONGPU_CPPSIMD == 4 + fptype_v out = + { (fptype)v[0], (fptype)v[1], (fptype)v[2], (fptype)v[3] }; +#elif MGONGPU_CPPSIMD == 8 + fptype_v out = + { (fptype)v[0], (fptype)v[1], (fptype)v[2], (fptype)v[3], (fptype)v[4], (fptype)v[5], (fptype)v[6], (fptype)v[7] }; +#endif return out; } @@ -743,11 +759,20 @@ namespace mg5amcCpu out[ieppV] = v[ieppV+neppV]; } */ +#if MGONGPU_CPPSIMD == 2 fptype_v out = { (fptype)v[2], (fptype)v[3] }; +#elif MGONGPU_CPPSIMD == 4 + fptype_v out = + { (fptype)v[4], (fptype)v[5], (fptype)v[6], (fptype)v[7] }; +#elif MGONGPU_CPPSIMD == 8 + fptype_v out = + { (fptype)v[8], (fptype)v[9], (fptype)v[10], (fptype)v[11], (fptype)v[12], (fptype)v[13], (fptype)v[14], (fptype)v[15] }; +#endif return out; } + #endif // #if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT @@ -778,6 +803,6 @@ namespace mg5amcCpu //========================================================================== -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MGONGPUVECTORS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc index cefa01b48f..cedf05b15b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc @@ -17,7 +17,7 @@ #include #include -using namespace mg5amcCpu; +using namespace madgraph; namespace { @@ -301,7 +301,7 @@ extern "C" } HostBufferBase momenta( rounded_count * ProcessData::npar * 4 ); - HostBufferBase couplings( rounded_count * mg5amcCpu::Parameters_dependentCouplings::ndcoup * 2 ); + HostBufferBase couplings( rounded_count * madgraph::Parameters_dependentCouplings::ndcoup * 2 ); HostBufferBase g_s( rounded_count ); HostBufferBase helicity_random( rounded_count ); HostBufferBase color_random( rounded_count ); diff --git a/madgraph/iolibs/template_files/madmatrix/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/check_sa.cc index bd1cd4c303..fc5d4611f8 100644 --- a/madgraph/iolibs/template_files/madmatrix/check_sa.cc +++ b/madgraph/iolibs/template_files/madmatrix/check_sa.cc @@ -58,11 +58,7 @@ namespace { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif + using namespace madgraph; // Fixed physics inputs fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV diff --git a/madgraph/iolibs/template_files/madmatrix/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/constexpr_math.h index 32c1bbb963..362ad19c4f 100644 --- a/madgraph/iolibs/template_files/madmatrix/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/constexpr_math.h @@ -27,12 +27,8 @@ #define CONSTEXPRMATHVAR constexpr #endif -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc b/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc index 8dde0fff4b..932bfadc94 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc @@ -27,11 +27,7 @@ //#include //#include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { // ALOHA-style object for easy flavor consolidation and non-template API diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc index 4650fc76c8..68e91b1a78 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc @@ -16,11 +16,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else -using namespace mg5amcCpu; -#endif +using namespace madgraph; #ifndef MGONGPU_HARDCODE_PARAM diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc index 75b642878f..2daa801795 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc @@ -32,12 +32,8 @@ #include "read_slha.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // Define FLV_COUPLING struct struct FLV_COUPLING { @@ -126,19 +122,15 @@ namespace mg5amcCpu for( int i = 0; i < N; ++i ) out[i] = cxmake( pars->*members[i] ); } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #else%(eftwarn1)s #include #include -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // Hardcoded constexpr physics parameters namespace Parameters // keep the same name rather than HardcodedParameters for simplicity @@ -170,18 +162,14 @@ namespace mg5amcCpu %(hasbsmip)s__device__ constexpr double mdl_bsmIndepParam[nBsmIndepParam] = { %(bsmip)s }; } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { namespace Parameters_dependentCouplings { @@ -271,7 +259,7 @@ namespace mg5amcCpu } #pragma GCC diagnostic pop -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/massless_rambo.h index bcd3990bed..ce3e6f19ef 100644 --- a/madgraph/iolibs/template_files/madmatrix/massless_rambo.h +++ b/madgraph/iolibs/template_files/madmatrix/massless_rambo.h @@ -22,11 +22,7 @@ #include // Simplified rambo version for 2 to N (with N>=2) processes with massless particles -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { namespace massless_rambo { diff --git a/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h index d6e1d02e05..a26a0a5a22 100644 --- a/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h @@ -51,7 +51,7 @@ #ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) #if defined MGONGPU_CUCXTYPE_CUCOMPLEX -namespace mg5amcGpu +namespace madgraph { #if defined MGONGPU_FPTYPE_DOUBLE class cucomplex @@ -139,7 +139,7 @@ namespace mgOnGpu /* clang-format off */ #ifdef MGONGPU_CUCXTYPE_THRUST template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } #elif defined MGONGPU_CUCXTYPE_CUCOMPLEX - __host__ __device__ constexpr operator mg5amcGpu::cucomplex() const { return mg5amcGpu::cucomplex( m_real, m_imag ); } + __host__ __device__ constexpr operator madgraph::cucomplex() const { return madgraph::cucomplex( m_real, m_imag ); } #endif #else #ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX @@ -163,11 +163,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { template inline __host__ std::ostream& @@ -296,12 +292,8 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // --- Type definitions (complex type: cxtype) #ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) @@ -332,12 +324,8 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL @@ -692,18 +680,14 @@ namespace mg5amcCpu return cxmake( c.real(), c.imag() ); } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph //========================================================================== // COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined @@ -738,7 +722,7 @@ namespace mg5amcCpu return out; } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h index a2489d05d5..e6bc6de847 100644 --- a/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h @@ -12,11 +12,11 @@ #include #include -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) #ifdef MGONGPUCPP_GPUIMPL // cuda -namespace mg5amcGpu +namespace madgraph #else -namespace mg5amcCpu +namespace madgraph #endif { //========================================================================== @@ -97,6 +97,6 @@ namespace mg5amcCpu //========================================================================== -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h index 74d93f05a3..966d60141d 100644 --- a/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h @@ -32,12 +32,8 @@ //#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) #endif -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { #ifdef MGONGPU_CPPSIMD @@ -156,12 +152,8 @@ namespace mg5amcCpu //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { #ifndef MGONGPUCPP_GPUIMPL @@ -927,6 +919,6 @@ namespace mg5amcCpu //========================================================================== -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madgraph #endif // MGONGPUVECTORS_H diff --git a/madgraph/iolibs/template_files/madmatrix/process_cc.inc b/madgraph/iolibs/template_files/madmatrix/process_cc.inc index 33b938494a..bf5f79e7c8 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_cc.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_cc.inc @@ -30,6 +30,7 @@ #include "MemoryAccessDenominators.h" #include "MemoryAccessNumerators.h" #include "ColorData.h" +#include "ProcessTables.h" #include #include diff --git a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc index 266f724528..2e20301ea7 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc @@ -19,13 +19,10 @@ // physics-parameter/coupling setup (initProc), and the flavor accessor. //========================================================================== -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { using namespace ProcessData; + using namespace ProcessTables; //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/process_h.inc b/madgraph/iolibs/template_files/madmatrix/process_h.inc index 91469ce8d4..8558062b9d 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_h.inc @@ -27,11 +27,7 @@ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { %(process_class_definitions)s //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/runTest.cc b/madgraph/iolibs/template_files/madmatrix/runTest.cc index 91b2f17755..2a6ea357ac 100644 --- a/madgraph/iolibs/template_files/madmatrix/runTest.cc +++ b/madgraph/iolibs/template_files/madmatrix/runTest.cc @@ -23,11 +23,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else -using namespace mg5amcCpu; -#endif +using namespace madgraph; struct CUDA_CPU_TestBase : public TestDriverBase { diff --git a/madgraph/iolibs/template_files/madmatrix/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/testmisc.cc index 168d7b3c32..6cb9da9558 100644 --- a/madgraph/iolibs/template_files/madmatrix/testmisc.cc +++ b/madgraph/iolibs/template_files/madmatrix/testmisc.cc @@ -31,12 +31,8 @@ #define XTESTID( s ) TESTID( s ) -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +namespace madgraph { #ifdef MGONGPU_CPPSIMD /* clang-format off */ #define EXPECT_TRUE_sv( cond ) { bool_v mask( cond ); EXPECT_TRUE( maskand( mask ) ); } @@ -65,11 +61,7 @@ namespace mg5amcCpu TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif + using namespace madgraph; //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/testxxx.cc index d8ef3c9262..6d694951ad 100644 --- a/madgraph/iolibs/template_files/madmatrix/testxxx.cc +++ b/madgraph/iolibs/template_files/madmatrix/testxxx.cc @@ -33,11 +33,7 @@ #define XTESTID( s ) TESTID( s ) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madgraph { std::string fpeHandlerMessage = "unknown"; int fpeHandlerIevt = -1; @@ -54,11 +50,7 @@ namespace mg5amcCpu TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif + using namespace madgraph; #ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); #endif @@ -73,9 +65,9 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV // Fill in the input momenta #ifdef MGONGPUCPP_GPUIMPL - mg5amcGpu::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] + madgraph::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] #else - mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] + madgraph::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] #endif /* clang-format off */ // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! const fptype par0[np4 * nevt] = // AOS[nevt][np4] @@ -317,11 +309,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { for( int ievt = 0; ievt < nevt; ievt++ ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif + using namespace madgraph; if( debug ) { std::cout << std::endl; From f6b8c3cbc8701a09c7741b8bb02685a6e182a84b Mon Sep 17 00:00:00 2001 From: stloufra Date: Tue, 18 Aug 2026 11:39:42 +0200 Subject: [PATCH 16/25] createNormalizedColorMatrix noop for CPU to get rid off more ifdefs by having noop instance for cpu and simd --- .../iolibs/template_files/madmatrix/ProcessTables.h | 2 +- .../template_files/madmatrix/backend/cpu/color_sum.h | 3 +++ .../template_files/madmatrix/backend/simd/color_sum.h | 3 +++ .../madmatrix/process_function_definitions.inc | 10 ++-------- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/madgraph/iolibs/template_files/madmatrix/ProcessTables.h b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h index 01105bec46..569966da85 100644 --- a/madgraph/iolibs/template_files/madmatrix/ProcessTables.h +++ b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h @@ -10,7 +10,7 @@ // pulled in via ProcessTables::name from backend-owned function bodies. // // Namespace-wrapped (unlike ProcessData.h) because it needs FLV_COUPLING, -// which is itself backend-namespaced (madgraph::/madgraph::, see Parameters.h). +// which is itself namespace-wrapped too (madgraph::, see Parameters.h). #ifndef PROCESSTABLES_H #define PROCESSTABLES_H 1 diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h index b0abc3d030..1417988a26 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h @@ -17,6 +17,9 @@ namespace madgraph { //-------------------------------------------------------------------------- + // No-op on cpu/simd: the normalized color matrix is already a compile-time + // constexpr there (see color_sum.cc); only gpu needs a runtime push to device memory. + inline void createNormalizedColorMatrix() {} //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h index b0abc3d030..1417988a26 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h @@ -17,6 +17,9 @@ namespace madgraph { //-------------------------------------------------------------------------- + // No-op on cpu/simd: the normalized color matrix is already a compile-time + // constexpr there (see color_sum.cc); only gpu needs a runtime push to device memory. + inline void createNormalizedColorMatrix() {} //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc index 2e20301ea7..20f0e4736b 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc @@ -78,10 +78,7 @@ namespace madgraph //m_pars->printDependentCouplings(); // now computed event-by-event (running alphas #373) } %(initProc_lines)s -#ifdef MGONGPUCPP_GPUIMPL - // Create the normalized color matrix in device memory - createNormalizedColorMatrix(); -#endif + createNormalizedColorMatrix(); // no-op on cpu/simd, pushes to device memory on gpu // Read physics parameters like masses and couplings from user configuration files, // then hand them to the backend-owned storage via the setters (see SigmaKin.cc). %(cipdassign)s @@ -103,10 +100,7 @@ namespace madgraph //Parameters::printDependentCouplings(); // now computed event-by-event (running alphas #373) } %(hardcoded_initProc_lines)s -#ifdef MGONGPUCPP_GPUIMPL - // Create the normalized color matrix in device memory - createNormalizedColorMatrix(); -#endif + createNormalizedColorMatrix(); // no-op on cpu/simd, pushes to device memory on gpu %(cipdhrdassign)s %(cipchrdassign)s %(cipfhrdassign)s From 37990e30b790b1941382a051edb0c914dd8da522 Mon Sep 17 00:00:00 2001 From: stloufra Date: Tue, 18 Aug 2026 13:19:41 +0200 Subject: [PATCH 17/25] hostChannel2iconfig only in runTest.cc --- .../template_files/madmatrix/ColorData.h | 9 ------ .../madmatrix/backend/gpu/GpuAbstraction.h | 2 ++ .../template_files/madmatrix/runTest.cc | 29 ++++++++++++++++++- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/madgraph/iolibs/template_files/madmatrix/ColorData.h b/madgraph/iolibs/template_files/madmatrix/ColorData.h index c0a51b9996..bd0c804db4 100644 --- a/madgraph/iolibs/template_files/madmatrix/ColorData.h +++ b/madgraph/iolibs/template_files/madmatrix/ColorData.h @@ -32,15 +32,6 @@ namespace mgOnGpu %(channelc2iconfig_lines)s }; - // Host copy of channel2iconfig (channel2iconfig itself is device-only on GPU, #917) -#ifndef MGONGPUCPP_GPUIMPL - constexpr const int* hostChannel2iconfig = channel2iconfig; -#else - constexpr int hostChannel2iconfig[%(nb_diag)i] = { -%(channelc2iconfig_lines)s - }; -#endif - // #configs with an associated iconfig for single-diagram enhancement (#917) constexpr unsigned int nconfigSDE = %(nb_channel)s; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h index 5462a3eeeb..3dd375ecf7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h @@ -33,6 +33,7 @@ #define gpuMemcpyDeviceToHost cudaMemcpyDeviceToHost #define gpuMemcpyDeviceToDevice cudaMemcpyDeviceToDevice #define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( cudaMemcpyToSymbol( type1, type2, size ) ) +#define gpuMemcpyFromSymbol( type1, type2, size ) checkGpu( cudaMemcpyFromSymbol( type1, type2, size ) ) #define gpuFree( ptr ) checkGpu( cudaFree( ptr ) ) #define gpuFreeHost( ptr ) checkGpu( cudaFreeHost( ptr ) ) @@ -101,6 +102,7 @@ #define gpuMemcpyDeviceToHost hipMemcpyDeviceToHost #define gpuMemcpyDeviceToDevice hipMemcpyDeviceToDevice #define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( hipMemcpyToSymbol( type1, type2, size ) ) +#define gpuMemcpyFromSymbol( type1, type2, size ) checkGpu( hipMemcpyFromSymbol( type1, type2, size ) ) #define gpuFree( ptr ) checkGpu( hipFree( ptr ) ) #define gpuFreeHost( ptr ) checkGpu( hipHostFree( ptr ) ) diff --git a/madgraph/iolibs/template_files/madmatrix/runTest.cc b/madgraph/iolibs/template_files/madmatrix/runTest.cc index 2a6ea357ac..8cd3a9a01c 100644 --- a/madgraph/iolibs/template_files/madmatrix/runTest.cc +++ b/madgraph/iolibs/template_files/madmatrix/runTest.cc @@ -9,6 +9,11 @@ #include "mgOnGpuConfig.h" +#ifdef MGONGPUCPP_GPUIMPL +#include "GpuAbstraction.h" +#include "GpuRuntime.h" // for checkGpu(), used by gpuMemcpyFromSymbol() +#endif + #include "CPPProcess.h" #include "MadgraphTest.h" #include "MatrixElementKernels.h" @@ -25,6 +30,28 @@ using namespace madgraph; +// Host-accessible copy of mgOnGpu::channel2iconfig, needed only by setChannelIds() +#ifndef MGONGPUCPP_GPUIMPL +inline const int* +getHostChannel2iconfig() +{ + return mgOnGpu::channel2iconfig; +} +#else +inline const int* +getHostChannel2iconfig() +{ + static int hostCopy[mgOnGpu::nchannels]; + static bool first = true; + if( first ) + { + first = false; + gpuMemcpyFromSymbol( hostCopy, mgOnGpu::channel2iconfig, mgOnGpu::nchannels * sizeof( int ) ); + } + return hostCopy; +} +#endif + struct CUDA_CPU_TestBase : public TestDriverBase { static constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout @@ -60,7 +87,7 @@ struct CUDA_CPU_TestBase : public TestDriverBase //for( unsigned int idiagram = 1; idiagram < CPPProcess::ndiagrams; idiagram++ ) // two bugs #920 and #919 for( unsigned int idiagram = 0; idiagram < mgOnGpu::nchannels; idiagram++ ) // fix #920 and work around #919 { - if( mgOnGpu::hostChannel2iconfig[idiagram] == iconfig ) + if( getHostChannel2iconfig()[idiagram] == iconfig ) { channelId = idiagram + 1; // fix #917 (NB add +1 because channelId uses F indexing) break; From b8324d7bcfd2702db055f494fb258ac026de77d6 Mon Sep 17 00:00:00 2001 From: stloufra Date: Tue, 18 Aug 2026 13:55:47 +0200 Subject: [PATCH 18/25] Clean the MGONGPU_CPPSIMD from /backend/cpu --- .../madmatrix/backend/cpu/mgOnGpuConfig.h | 35 +------ .../madmatrix/backend/cpu/mgOnGpuVectors.h | 97 +------------------ 2 files changed, 2 insertions(+), 130 deletions(-) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h index 683a80f6c4..c7a12b182d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h @@ -156,41 +156,8 @@ using mgOnGpu::fptype2; #ifdef MGONGPU_NOARMNEON #undef __ARM_NEON #endif - -// C++ SIMD vectorization width (this will be used to set neppV) -#if defined __AVX512VL__ && defined MGONGPU_PVW512 // "512z" AVX512 512-bit: 8 (DOUBLE) or 16 (FLOAT) -#ifdef MGONGPU_FPTYPE_DOUBLE -#define MGONGPU_CPPSIMD 8 -#else -#define MGONGPU_CPPSIMD 16 -#endif -#elif defined __AVX512VL__ // "512y" AVX512 256-bit: 4 (DOUBLE) or 8 (FLOAT) [gcc default] -#ifdef MGONGPU_FPTYPE_DOUBLE -#define MGONGPU_CPPSIMD 4 -#else -#define MGONGPU_CPPSIMD 8 -#endif -#elif defined __AVX2__ // "avx2" 256-bit: 4 (DOUBLE) or 8 (FLOAT) [clang default] -#ifdef MGONGPU_FPTYPE_DOUBLE -#define MGONGPU_CPPSIMD 4 -#else -#define MGONGPU_CPPSIMD 8 -#endif -#elif defined __SSE4_2__ // "sse4" SSE4.2 128-bit: 2 (DOUBLE) or 4 (FLOAT) [Power9 default] -#ifdef MGONGPU_FPTYPE_DOUBLE -#define MGONGPU_CPPSIMD 2 -#else -#define MGONGPU_CPPSIMD 4 -#endif -#elif defined __ARM_NEON // ARM NEON 128-bit: 2 (DOUBLE) or 4 (FLOAT) [ARM default] -#ifdef MGONGPU_FPTYPE_DOUBLE -#define MGONGPU_CPPSIMD 2 -#else -#define MGONGPU_CPPSIMD 4 -#endif -#else // "none" i.e. no SIMD #undef MGONGPU_CPPSIMD -#endif + // No-op debug macros (nsight-based debugging is CUDA-only, unused here) #define mgDebugDeclare() /*noop*/ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h index 61c7c4bb6f..f216bf7b65 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h @@ -84,19 +84,6 @@ namespace madgraph */ -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - inline std::ostream& - operator<<( std::ostream& out, const fptype2_v& v ) - { - out << "{ " << v[0]; - for( int i = 1; i < neppV2; i++ ) out << ", " << v[i]; - out << " }"; - return out; - } -#endif - - - //-------------------------------------------------------------------------- /* @@ -178,89 +165,7 @@ namespace madgraph // Functions and operators for fptype2_v -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - - inline fptype2_v - fpvmerge( const fptype_v& v1, const fptype_v& v2 ) - { - // This code is not very efficient! It makes mixed precision FFV/color not faster than double on C++ (#537). - // I considered various alternatives, including - // - in gcc12 and clang, __builtin_shufflevector (works with different vector lengths, BUT the same fptype...) - // - casting vector(4)double to vector(4)float and then assigning via reinterpret_cast... but how to do the cast? - // Probably the best solution is intrinsics? - // - see https://stackoverflow.com/questions/5139363 - // - see https://stackoverflow.com/questions/54518744 - /* - fptype2_v out; - for( int ieppV = 0; ieppV < neppV; ieppV++ ) - { - out[ieppV] = v1[ieppV]; - out[ieppV+neppV] = v2[ieppV]; - } - return out; - */ -#if MGONGPU_CPPSIMD == 2 - fptype2_v out = - { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v2[0], (fptype2)v2[1] }; -#elif MGONGPU_CPPSIMD == 4 - fptype2_v out = - { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v1[2], (fptype2)v1[3], (fptype2)v2[0], (fptype2)v2[1], (fptype2)v2[2], (fptype2)v2[3] }; -#elif MGONGPU_CPPSIMD == 8 - fptype2_v out = - { (fptype2)v1[0], (fptype2)v1[1], (fptype2)v1[2], (fptype2)v1[3], (fptype2)v1[4], (fptype2)v1[5], (fptype2)v1[6], (fptype2)v1[7], (fptype2)v2[0], (fptype2)v2[1], (fptype2)v2[2], (fptype2)v2[3], (fptype2)v2[4], (fptype2)v2[5], (fptype2)v2[6], (fptype2)v2[7] }; -#endif - return out; - } - - inline fptype_v - fpvsplit0( const fptype2_v& v ) - { - /* - fptype_v out = {}; // see #594 - for( int ieppV = 0; ieppV < neppV; ieppV++ ) - { - out[ieppV] = v[ieppV]; - } - */ -#if MGONGPU_CPPSIMD == 2 - fptype_v out = - { (fptype)v[0], (fptype)v[1] }; -#elif MGONGPU_CPPSIMD == 4 - fptype_v out = - { (fptype)v[0], (fptype)v[1], (fptype)v[2], (fptype)v[3] }; -#elif MGONGPU_CPPSIMD == 8 - fptype_v out = - { (fptype)v[0], (fptype)v[1], (fptype)v[2], (fptype)v[3], (fptype)v[4], (fptype)v[5], (fptype)v[6], (fptype)v[7] }; -#endif - return out; - } - - inline fptype_v - fpvsplit1( const fptype2_v& v ) - { - /* - fptype_v out = {}; // see #594 - for( int ieppV = 0; ieppV < neppV; ieppV++ ) - { - out[ieppV] = v[ieppV+neppV]; - } - */ -#if MGONGPU_CPPSIMD == 2 - fptype_v out = - { (fptype)v[2], (fptype)v[3] }; -#elif MGONGPU_CPPSIMD == 4 - fptype_v out = - { (fptype)v[4], (fptype)v[5], (fptype)v[6], (fptype)v[7] }; -#elif MGONGPU_CPPSIMD == 8 - fptype_v out = - { (fptype)v[8], (fptype)v[9], (fptype)v[10], (fptype)v[11], (fptype)v[12], (fptype)v[13], (fptype)v[14], (fptype)v[15] }; -#endif - return out; - } - - -#endif // #if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - + // Keeps living only on the SIMD /backend //========================================================================== From a2164295606f25e3459dbd5aa991e45547a5154c Mon Sep 17 00:00:00 2001 From: stloufra Date: Tue, 18 Aug 2026 15:18:12 +0200 Subject: [PATCH 19/25] Getting rid of the mgOnGpuVectors on cpu/gpu Conditionaly included for the simd Needed stuff for the cpu/gpu in mgOnGpuCxtypes --- .../backend/cpu/MemoryAccessCouplings.h | 22 -- .../backend/cpu/MemoryAccessCouplingsFixed.h | 1 - .../backend/cpu/MemoryAccessVectors.h | 2 +- .../madmatrix/backend/cpu/color_sum.h | 2 +- .../madmatrix/backend/cpu/mgOnGpuCxtypes.h | 49 +++++ .../madmatrix/backend/cpu/mgOnGpuVectors.h | 199 ------------------ .../backend/gpu/MemoryAccessCouplings.h | 22 -- .../backend/gpu/MemoryAccessCouplingsFixed.h | 1 - .../backend/gpu/MemoryAccessVectors.h | 2 +- .../madmatrix/backend/gpu/SigmaKin.h | 3 +- .../madmatrix/backend/gpu/color_sum.h | 2 +- .../madmatrix/backend/gpu/mgOnGpuCxtypes.h | 53 +++++ .../madmatrix/backend/gpu/mgOnGpuVectors.h | 133 ------------ .../madmatrix/cpp_hel_amps_h.inc | 6 +- .../madmatrix/cpp_model_parameters_h.inc | 4 +- .../template_files/madmatrix/process_h.inc | 6 +- 16 files changed, 120 insertions(+), 387 deletions(-) delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h delete mode 100644 madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h index ab5f7de4c5..293f7d13f1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h @@ -197,28 +197,6 @@ namespace madgraph return kernelAccessIx2( const_cast( buffer ), ix2 ); } - /* - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] - static __host__ __device__ inline const fptype_sv& - kernelAccessIx2Const( const fptype* buffer, - const int ix2 ) - { - const fptype& out = kernelAccessIx2Const_s( buffer, ix2 ); -#ifndef MGONGPU_CPPSIMD - return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays - constexpr int neppC = MemoryAccessCouplingsBase::neppC; - static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( madgraph::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif - } - */ - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) // [Signature (non const, SCALAR OR VECTOR) ===> cxtype_sv_ref kernelAccess( fptype* buffer ) <===] static __host__ __device__ inline cxtype_sv_ref diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h index 7ce36f4c6f..11a65e19ff 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h @@ -10,7 +10,6 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuCxtypes.h" -#include "mgOnGpuVectors.h" //#include "MemoryAccessHelpers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h index 6a3c642c56..c0546a735b 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "mgOnGpuVectors.h" +#include "mgOnGpuCxtypes.h" namespace madgraph // this is only needed for CPU SIMD vectorization { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h index 1417988a26..defeaef7ac 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "mgOnGpuVectors.h" +#include "mgOnGpuCxtypes.h" #include "ProcessData.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h index ed7ec2fdf0..179bf50db9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h @@ -373,6 +373,55 @@ namespace madgraph return out; } + //-------------------------------------------------------------------------- + + // all needed from mgOnGpuVectors.h for cpu + const int neppV = 1; + +#ifndef MGONGPU_CPPCXTYPE_CXSMPL // operator<< for cxsmpl has already been defined! + inline std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + out << "[" << cxreal( c ) << "," << cximag( c ) << "]"; + return out; + } +#endif + + inline fptype + fpternary( const bool& mask, const fptype& a, const fptype& b ) + { + return ( mask ? a : b ); + } + + inline cxtype + cxternary( const bool& mask, const cxtype& a, const cxtype& b ) + { + return ( mask ? a : b ); + } + + inline bool + maskand( const bool& mask ) + { + return mask; + } + + //vector is scalar + typedef bool bool_sv; + typedef fptype fptype_sv; + typedef fptype2 fptype2_sv; + typedef unsigned int uint_sv; + typedef cxtype cxtype_sv; + typedef cxtype_ref cxtype_sv_ref; + + //vector is scalar + inline cxtype cxzero_sv() { return cxtype( 0, 0 ); } + + inline __host__ __device__ fptype_sv + cxabs2( const cxtype_sv& c ) + { + return cxreal( c ) * cxreal( c ) + cximag( c ) * cximag( c ); + } + } // end namespace madgraph //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h deleted file mode 100644 index f216bf7b65..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuVectors.h +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Nov 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Roiser, A. Valassi, Z. Wettersten (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MGONGPUVECTORS_H -#define MGONGPUVECTORS_H 1 - -#include "mgOnGpuCxtypes.h" -#include "mgOnGpuFptypes.h" - -#include - -//========================================================================== - -//------------------------------ -// Vector types - C++ -//------------------------------ - -#ifdef __clang__ -// If set: return a pair of (fptype&, fptype&) by non-const reference in cxtype_v::operator[] -// This is forbidden in clang ("non-const reference cannot bind to vector element") -// See also https://stackoverflow.com/questions/26554829 -//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // clang test (compilation fails also on clang 12.0, issue #182) -#undef MGONGPU_HAS_CPPCXTYPEV_BRK // clang default -#elif defined __INTEL_COMPILER -//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // icc default? -#undef MGONGPU_HAS_CPPCXTYPEV_BRK // icc test -#else -#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // gcc default -//#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) -#endif - -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph -{ - - const int neppV = 1; - -} - -//-------------------------------------------------------------------------- - -// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) -// DO NOT expose typedefs outside the namespace -//using mgOnGpu::neppV; -//#ifdef MGONGPU_CPPSIMD -//using mgOnGpu::fptype_v; -//using mgOnGpu::fptype2_v; -//using mgOnGpu::cxtype_v; -//using mgOnGpu::bool_v; -//#endif - -//========================================================================== - -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph -{ - - // Printout to stream for user defined types - -#ifndef MGONGPU_CPPCXTYPE_CXSMPL // operator<< for cxsmpl has already been defined! - inline std::ostream& - operator<<( std::ostream& out, const cxtype& c ) - { - out << "[" << cxreal( c ) << "," << cximag( c ) << "]"; - //out << cxreal(c) << "+i" << cximag(c); - return out; - } -#endif - - /* -#ifdef MGONGPU_CPPSIMD - inline std::ostream& - operator<<( std::ostream& out, const bool_v& v ) - { - out << "{ " << v[0]; - for ( int i=1; i( buffer ), ix2 ); } - /* - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) - // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] - static __host__ __device__ inline const fptype_sv& - kernelAccessIx2Const( const fptype* buffer, - const int ix2 ) - { - const fptype& out = kernelAccessIx2Const_s( buffer, ix2 ); -#ifndef MGONGPU_CPPSIMD - return out; -#else - // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays - constexpr int neppC = MemoryAccessCouplingsBase::neppC; - static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( madgraph::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif - } - */ - // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) // [Signature (non const, SCALAR OR VECTOR) ===> cxtype_sv_ref kernelAccess( fptype* buffer ) <===] static __host__ __device__ inline cxtype_sv_ref diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h index 7ce36f4c6f..11a65e19ff 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h @@ -10,7 +10,6 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuCxtypes.h" -#include "mgOnGpuVectors.h" //#include "MemoryAccessHelpers.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h index 4db1875156..f21f90987c 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "mgOnGpuVectors.h" +#include "mgOnGpuCxtypes.h" #endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h index 90e730a620..034a2d5898 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h @@ -12,8 +12,7 @@ #define SIGMAKIN_H 1 #include "mgOnGpuConfig.h" -#include "mgOnGpuCxtypes.h" // for cxtype -#include "mgOnGpuVectors.h" // for fptype_sv +#include "mgOnGpuCxtypes.h" // for cxtype, fptype_sv #include "GpuAbstraction.h" // for gpuBlasHandle_t, gpuStream_t namespace madgraph diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h index 0517d1c79e..54322cd990 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "mgOnGpuVectors.h" +#include "mgOnGpuCxtypes.h" #include "ProcessData.h" #include "GpuAbstraction.h" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h index abec0fb3d8..b834c7ddb7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h @@ -722,6 +722,59 @@ namespace madgraph return out; } + //-------------------------------------------------------------------------- + + // all needed from mgOnGpuVectors.h for gpu + const int neppV = 1; + + // Printout to std::cout for user defined types + inline __host__ __device__ void + print( const fptype& f ) + { + printf( "%f\n", f ); + } + inline __host__ __device__ void + print( const cxtype& c ) + { + printf( "[%f, %f]\n", cxreal( c ), cximag( c ) ); + } + + inline __host__ __device__ fptype + fpternary( const bool& mask, const fptype& a, const fptype& b ) + { + return ( mask ? a : b ); + } + + inline __host__ __device__ cxtype + cxternary( const bool& mask, const cxtype& a, const cxtype& b ) + { + return ( mask ? a : b ); + } + + inline __host__ __device__ bool + maskand( const bool& mask ) + { + return mask; + } + + //vector is scalar on gpu + typedef bool bool_sv; + typedef fptype fptype_sv; + typedef fptype2 fptype2_sv; + typedef unsigned int uint_sv; + typedef cxtype cxtype_sv; + typedef cxtype_ref cxtype_sv_ref; + + //vector is scalar on gpu + inline __host__ __device__ cxtype cxzero_sv() { return cxtype( 0, 0 ); } + + // Functions and operators for cxtype_sv + inline __host__ __device__ fptype_sv + cxabs2( const cxtype_sv& c ) + { + return cxreal( c ) * cxreal( c ) + cximag( c ) * cximag( c ); + } + } // end namespace madgraph //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h deleted file mode 100644 index 7747658900..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuVectors.h +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Nov 2020) for the MG5aMC CUDACPP plugin. -// Further modified by: S. Roiser, A. Valassi, Z. Wettersten (2020-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MGONGPUVECTORS_H -#define MGONGPUVECTORS_H 1 - -#include "mgOnGpuCxtypes.h" -#include "mgOnGpuFptypes.h" - -#include - -//========================================================================== - -//------------------------------ -// Vector types - C++ -//------------------------------ - -#ifdef __clang__ -// If set: return a pair of (fptype&, fptype&) by non-const reference in cxtype_v::operator[] -// This is forbidden in clang ("non-const reference cannot bind to vector element") -// See also https://stackoverflow.com/questions/26554829 -//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // clang test (compilation fails also on clang 12.0, issue #182) -#undef MGONGPU_HAS_CPPCXTYPEV_BRK // clang default -#elif defined __INTEL_COMPILER -//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // icc default? -#undef MGONGPU_HAS_CPPCXTYPEV_BRK // icc test -#else -#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // gcc default -//#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) -#endif - -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph -{ - - const int neppV = 1; - -} - -//-------------------------------------------------------------------------- - -// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) -// DO NOT expose typedefs outside the namespace -//using mgOnGpu::neppV; -//#ifdef MGONGPU_CPPSIMD -//using mgOnGpu::fptype_v; -//using mgOnGpu::fptype2_v; -//using mgOnGpu::cxtype_v; -//using mgOnGpu::bool_v; -//#endif - -//========================================================================== - -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph -{ - - //========================================================================== - - - //------------------------------ - // Vector types - CUDA - //------------------------------ - - // Printout to std::cout for user defined types - inline __host__ __device__ void - print( const fptype& f ) - { - printf( "%f\n", f ); - } - inline __host__ __device__ void - print( const cxtype& c ) - { - printf( "[%f, %f]\n", cxreal( c ), cximag( c ) ); - } - - /* - inline __host__ __device__ const cxtype& - cxvmake( const cxtype& c ) - { - return c; - } - */ - - inline __host__ __device__ fptype - fpternary( const bool& mask, const fptype& a, const fptype& b ) - { - return ( mask ? a : b ); - } - - inline __host__ __device__ cxtype - cxternary( const bool& mask, const cxtype& a, const cxtype& b ) - { - return ( mask ? a : b ); - } - - inline __host__ __device__ bool - maskand( const bool& mask ) - { - return mask; - } - - - //========================================================================== - - // Scalar-or-vector types: scalar in CUDA, vector or scalar in C++ - typedef bool bool_sv; - typedef fptype fptype_sv; - typedef fptype2 fptype2_sv; - typedef unsigned int uint_sv; - typedef cxtype cxtype_sv; - typedef cxtype_ref cxtype_sv_ref; - - // Scalar-or-vector zeros: scalar in CUDA, vector or scalar in C++ - inline __host__ __device__ cxtype cxzero_sv(){ return cxtype( 0, 0 ); } - - //========================================================================== - - // Functions and operators for cxtype_sv - inline __host__ __device__ fptype_sv - cxabs2( const cxtype_sv& c ) - { - return cxreal( c ) * cxreal( c ) + cximag( c ) * cximag( c ); - } - - //========================================================================== - -} // end namespace madgraph - -#endif // MGONGPUVECTORS_H diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc b/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc index 932bfadc94..334ef032b3 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc @@ -17,7 +17,11 @@ #include "mgOnGpuConfig.h" -#include "mgOnGpuVectors.h" +#if __has_include("mgOnGpuVectors.h") // simd only +#include "mgOnGpuVectors.h" +#else +#include "mgOnGpuCxtypes.h" +#endif #include "Parameters.h" diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc index 2daa801795..7b58ce656a 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc @@ -17,7 +17,9 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuCxtypes.h" -#include "mgOnGpuVectors.h" +#if __has_include("mgOnGpuVectors.h") // simd only +#include "mgOnGpuVectors.h" +#endif #include "constexpr_math.h" diff --git a/madgraph/iolibs/template_files/madmatrix/process_h.inc b/madgraph/iolibs/template_files/madmatrix/process_h.inc index 8558062b9d..cfcde6c3e2 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_h.inc @@ -16,7 +16,11 @@ #include "mgOnGpuConfig.h" -#include "mgOnGpuVectors.h" +#if __has_include("mgOnGpuVectors.h") // simd only +#include "mgOnGpuVectors.h" +#else +#include "mgOnGpuCxtypes.h" +#endif #include "Parameters.h" #include "ProcessData.h" From c6c8140cc1eb74ea19fc632622dfeefb342cac30 Mon Sep 17 00:00:00 2001 From: stloufra Date: Tue, 18 Aug 2026 15:50:42 +0200 Subject: [PATCH 20/25] Death code in mgOnGpuVectors on SIMD del --- .../madmatrix/backend/simd/mgOnGpuVectors.h | 59 +------------------ 1 file changed, 3 insertions(+), 56 deletions(-) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h index e5ead4b329..a6e006171d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h @@ -134,16 +134,6 @@ namespace madgraph //-------------------------------------------------------------------------- -// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) -// DO NOT expose typedefs outside the namespace -//using mgOnGpu::neppV; -//#ifdef MGONGPU_CPPSIMD -//using mgOnGpu::fptype_v; -//using mgOnGpu::fptype2_v; -//using mgOnGpu::cxtype_v; -//using mgOnGpu::bool_v; -//#endif - //========================================================================== // NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) @@ -162,19 +152,6 @@ namespace madgraph } #endif - /* -#ifdef MGONGPU_CPPSIMD - inline std::ostream& - operator<<( std::ostream& out, const bool_v& v ) - { - out << "{ " << v[0]; - for ( int i=1; i Date: Wed, 19 Aug 2026 13:26:45 +0200 Subject: [PATCH 21/25] namespace madmatrix change --- .../madmatrix/CommonRandomNumberKernel.cc | 2 +- .../madmatrix/CurandRandomNumberKernel.cc | 2 +- .../madmatrix/HiprandRandomNumberKernel.cc | 2 +- .../template_files/madmatrix/MadgraphTest.h | 2 +- .../template_files/madmatrix/ProcessTables.h | 4 ++-- .../madmatrix/RamboSamplingKernels.cc | 2 +- .../madmatrix/RamboSamplingKernels.h | 2 +- .../madmatrix/RandomNumberKernels.h | 2 +- .../backend/cpu/CrossSectionKernels.cc | 2 +- .../backend/cpu/CrossSectionKernels.h | 2 +- .../madmatrix/backend/cpu/EventStatistics.h | 2 +- .../backend/cpu/MatrixElementKernels.cc | 4 ++-- .../backend/cpu/MatrixElementKernels.h | 2 +- .../backend/cpu/MemoryAccessAmplitudes.h | 6 ++--- .../backend/cpu/MemoryAccessChannelIds.h | 6 ++--- .../backend/cpu/MemoryAccessCouplings.h | 6 ++--- .../backend/cpu/MemoryAccessCouplingsFixed.h | 6 ++--- .../backend/cpu/MemoryAccessDenominators.h | 6 ++--- .../madmatrix/backend/cpu/MemoryAccessGs.h | 6 ++--- .../backend/cpu/MemoryAccessIflavorVec.h | 6 ++--- .../backend/cpu/MemoryAccessMatrixElements.h | 6 ++--- .../backend/cpu/MemoryAccessMomenta.h | 6 ++--- .../backend/cpu/MemoryAccessNumerators.h | 6 ++--- .../backend/cpu/MemoryAccessVectors.h | 2 +- .../backend/cpu/MemoryAccessWavefunctions.h | 6 ++--- .../backend/cpu/MemoryAccessWeights.h | 6 ++--- .../madmatrix/backend/cpu/MemoryBuffers.h | 2 +- .../madmatrix/backend/cpu/SigmaKin.cc | 2 +- .../madmatrix/backend/cpu/SigmaKin.h | 2 +- .../madmatrix/backend/cpu/color_sum.cc | 2 +- .../madmatrix/backend/cpu/color_sum.h | 2 +- .../madmatrix/backend/cpu/constexpr_math.h | 2 +- .../madmatrix/backend/cpu/mgOnGpuCxtypes.h | 14 ++++++------ .../madmatrix/backend/cpu/mgOnGpuFptypes.h | 6 ++--- .../madmatrix/backend/cpu/umami.cc | 4 ++-- .../backend/gpu/CrossSectionKernels.cc | 4 ++-- .../backend/gpu/CrossSectionKernels.h | 2 +- .../madmatrix/backend/gpu/EventStatistics.h | 2 +- .../madmatrix/backend/gpu/GpuRuntime.h | 2 +- .../backend/gpu/MatrixElementKernels.cc | 4 ++-- .../backend/gpu/MatrixElementKernels.h | 2 +- .../backend/gpu/MemoryAccessAmplitudes.h | 6 ++--- .../backend/gpu/MemoryAccessChannelIds.h | 6 ++--- .../backend/gpu/MemoryAccessCouplings.h | 6 ++--- .../backend/gpu/MemoryAccessCouplingsFixed.h | 6 ++--- .../backend/gpu/MemoryAccessDenominators.h | 6 ++--- .../madmatrix/backend/gpu/MemoryAccessGs.h | 6 ++--- .../backend/gpu/MemoryAccessIflavorVec.h | 6 ++--- .../backend/gpu/MemoryAccessMatrixElements.h | 6 ++--- .../backend/gpu/MemoryAccessMomenta.h | 6 ++--- .../backend/gpu/MemoryAccessNumerators.h | 6 ++--- .../backend/gpu/MemoryAccessWavefunctions.h | 6 ++--- .../backend/gpu/MemoryAccessWeights.h | 6 ++--- .../madmatrix/backend/gpu/MemoryBuffers.h | 2 +- .../madmatrix/backend/gpu/SigmaKin.cc | 2 +- .../madmatrix/backend/gpu/SigmaKin.h | 2 +- .../madmatrix/backend/gpu/color_sum.cc | 2 +- .../madmatrix/backend/gpu/color_sum.h | 2 +- .../madmatrix/backend/gpu/constexpr_math.h | 4 ++-- .../madmatrix/backend/gpu/mgOnGpuCxtypes.h | 22 +++++++++---------- .../madmatrix/backend/gpu/mgOnGpuFptypes.h | 6 ++--- .../madmatrix/backend/gpu/umami.cc | 4 ++-- .../backend/simd/CrossSectionKernels.cc | 2 +- .../backend/simd/CrossSectionKernels.h | 2 +- .../madmatrix/backend/simd/EventStatistics.h | 2 +- .../backend/simd/MatrixElementKernels.cc | 4 ++-- .../backend/simd/MatrixElementKernels.h | 2 +- .../backend/simd/MemoryAccessAmplitudes.h | 6 ++--- .../backend/simd/MemoryAccessChannelIds.h | 10 ++++----- .../backend/simd/MemoryAccessCouplings.h | 14 ++++++------ .../backend/simd/MemoryAccessCouplingsFixed.h | 6 ++--- .../backend/simd/MemoryAccessDenominators.h | 6 ++--- .../madmatrix/backend/simd/MemoryAccessGs.h | 14 ++++++------ .../backend/simd/MemoryAccessIflavorVec.h | 10 ++++----- .../backend/simd/MemoryAccessMatrixElements.h | 10 ++++----- .../backend/simd/MemoryAccessMomenta.h | 14 ++++++------ .../backend/simd/MemoryAccessNumerators.h | 6 ++--- .../backend/simd/MemoryAccessVectors.h | 2 +- .../backend/simd/MemoryAccessWavefunctions.h | 6 ++--- .../backend/simd/MemoryAccessWeights.h | 6 ++--- .../madmatrix/backend/simd/MemoryBuffers.h | 2 +- .../madmatrix/backend/simd/SigmaKin.cc | 2 +- .../madmatrix/backend/simd/SigmaKin.h | 2 +- .../madmatrix/backend/simd/color_sum.cc | 2 +- .../madmatrix/backend/simd/color_sum.h | 2 +- .../madmatrix/backend/simd/constexpr_math.h | 4 ++-- .../madmatrix/backend/simd/mgOnGpuCxtypes.h | 14 ++++++------ .../madmatrix/backend/simd/mgOnGpuFptypes.h | 6 ++--- .../madmatrix/backend/simd/mgOnGpuVectors.h | 10 ++++----- .../madmatrix/backend/simd/umami.cc | 4 ++-- .../template_files/madmatrix/check_sa.cc | 2 +- .../template_files/madmatrix/constexpr_math.h | 4 ++-- .../madmatrix/cpp_hel_amps_h.inc | 2 +- .../madmatrix/cpp_model_parameters_cc.inc | 2 +- .../madmatrix/cpp_model_parameters_h.inc | 18 +++++++-------- .../template_files/madmatrix/massless_rambo.h | 2 +- .../template_files/madmatrix/mgOnGpuCxtypes.h | 22 +++++++++---------- .../template_files/madmatrix/mgOnGpuFptypes.h | 8 +++---- .../template_files/madmatrix/mgOnGpuVectors.h | 10 ++++----- .../process_function_definitions.inc | 2 +- .../template_files/madmatrix/process_h.inc | 2 +- .../template_files/madmatrix/runTest.cc | 2 +- .../template_files/madmatrix/testmisc.cc | 6 ++--- .../template_files/madmatrix/testxxx.cc | 10 ++++----- 104 files changed, 268 insertions(+), 268 deletions(-) diff --git a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc index 8aba05355a..4ca90fea38 100644 --- a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc @@ -15,7 +15,7 @@ #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc index 098328a876..da8ba6d7e2 100644 --- a/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc @@ -24,7 +24,7 @@ inline void assertCurand( curandStatus_t code, const char *file, int line, bool } #endif /* clang-format on */ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_CURAND diff --git a/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc index a16f15796a..1b2f2b2ec9 100644 --- a/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc @@ -28,7 +28,7 @@ inline void assertHiprand( hiprandStatus_t code, const char *file, int line, boo } #endif /* clang-format on */ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_HIPRAND diff --git a/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h index 2ab9b17432..690d67b68f 100644 --- a/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h +++ b/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h @@ -27,7 +27,7 @@ #include #include -using madgraph::CPPProcess; +using madmatrix::CPPProcess; namespace { diff --git a/madgraph/iolibs/template_files/madmatrix/ProcessTables.h b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h index 569966da85..93455ba807 100644 --- a/madgraph/iolibs/template_files/madmatrix/ProcessTables.h +++ b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h @@ -10,7 +10,7 @@ // pulled in via ProcessTables::name from backend-owned function bodies. // // Namespace-wrapped (unlike ProcessData.h) because it needs FLV_COUPLING, -// which is itself namespace-wrapped too (madgraph::, see Parameters.h). +// which is itself namespace-wrapped too (madmatrix::, see Parameters.h). #ifndef PROCESSTABLES_H #define PROCESSTABLES_H 1 @@ -19,7 +19,7 @@ #include "ProcessData.h" #include "Parameters.h" // for FLV_COUPLING::max_flavor -namespace madgraph +namespace madmatrix { namespace ProcessTables { diff --git a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc index aa75a0c5ad..2505402c53 100644 --- a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc @@ -18,7 +18,7 @@ #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h index 80a17aefac..4e0253a852 100644 --- a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h @@ -13,7 +13,7 @@ #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h index ee11017750..afbab3d408 100644 --- a/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h @@ -18,7 +18,7 @@ struct curandGenerator_st; struct rocrand_generator_base_type; typedef rocrand_generator_base_type hiprandGenerator_st; -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc index 3ad7c5d1ca..29efce440d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc @@ -79,7 +79,7 @@ debug_me_is_abnormal( const fptype& me, size_t ievtALL ) //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h index 261f7953d9..59a4c098ec 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h @@ -14,7 +14,7 @@ //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h index 5eee6de231..df62a74918 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h @@ -17,7 +17,7 @@ #include #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc index 171f815ac5..3aeeadd9e8 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc @@ -18,7 +18,7 @@ //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- @@ -143,7 +143,7 @@ namespace madgraph //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h index b74a7e19cb..e7cf857b47 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h @@ -14,7 +14,7 @@ #include #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h index 7a58f2fc92..669ed8ead5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_AMPLITUDES 1 -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -156,6 +156,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h index 2c301811ec..0b1c1162a3 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -110,6 +110,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h index 293f7d13f1..491c6f8a5a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h @@ -15,8 +15,8 @@ #include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM #include "MemoryBuffers.h" // for HostBufferCouplings::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -235,6 +235,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h index 11a65e19ff..3eadf6f393 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h @@ -13,8 +13,8 @@ //#include "MemoryAccessHelpers.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -79,6 +79,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h index 10046c1293..9e5b87e674 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h index 2739a0bd80..ce27a06de0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -148,6 +148,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h index af8d37bf6f..a6e5c356e4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h @@ -12,8 +12,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -109,6 +109,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h index 6175a16d1b..505312e236 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -131,6 +131,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h index 245a2ba8ba..78e3bdb14d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h @@ -13,8 +13,8 @@ #include "MemoryAccessHelpers.h" #include "MemoryAccessVectors.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -204,6 +204,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h index efa609aee1..6df3b20cf4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h index c0546a735b..45546ddeb3 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h @@ -11,7 +11,7 @@ #include "mgOnGpuCxtypes.h" -namespace madgraph // this is only needed for CPU SIMD vectorization +namespace madmatrix // this is only needed for CPU SIMD vectorization { diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h index 66d7f35ac9..9406a9d75a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -161,6 +161,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h index 12df545d92..eece092fa3 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h @@ -11,8 +11,8 @@ #include "MemoryAccessHelpers.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -141,6 +141,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h index 83f316223b..88e90aeafb 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h @@ -16,7 +16,7 @@ #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc index 2b7bf5fad3..8e28b1ac95 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc @@ -28,7 +28,7 @@ #include #include -namespace madgraph +namespace madmatrix { using namespace ProcessData; using namespace ProcessTables; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h index 79d1a5d6e7..81956d459d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h @@ -14,7 +14,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuCxtypes.h" // for cxtype -namespace madgraph +namespace madmatrix { __global__ void computeDependentCouplings( const fptype* allgs, diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc index 2b04b4e8c9..a94caa8869 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc @@ -11,7 +11,7 @@ #include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps #include "MemoryAccessMatrixElements.h" -namespace madgraph +namespace madmatrix { using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h index defeaef7ac..8654492051 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h @@ -13,7 +13,7 @@ #include "ProcessData.h" -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h index 8adb8a7cda..c1b4df74c4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h @@ -29,7 +29,7 @@ #define CONSTEXPRMATHVAR constexpr #endif -namespace madgraph +namespace madmatrix { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h index 179bf50db9..4e4b4b987a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h @@ -73,7 +73,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -namespace madgraph +namespace madmatrix { template inline __host__ std::ostream& @@ -202,7 +202,7 @@ namespace madgraph // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -namespace madgraph +namespace madmatrix { // --- Type definitions (complex type: cxtype) #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX @@ -223,7 +223,7 @@ namespace madgraph // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -namespace madgraph +namespace madmatrix { #if defined MGONGPU_CPPCXTYPE_CXSMPL @@ -331,14 +331,14 @@ namespace madgraph return cxmake( c.real(), c.imag() ); } -} // end namespace madgraph +} // end namespace madmatrix //========================================================================== // COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined @@ -422,7 +422,7 @@ namespace madgraph return cxreal( c ) * cxreal( c ) + cximag( c ) * cximag( c ); } -} // end namespace madgraph +} // end namespace madmatrix //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h index 62e454648c..d6fe740b8f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h @@ -12,8 +12,8 @@ #include #include -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //========================================================================== @@ -46,6 +46,6 @@ namespace madgraph //========================================================================== -} // end namespace madgraph +} // end namespace madmatrix #endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc index cedf05b15b..ac462561b0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc @@ -17,7 +17,7 @@ #include #include -using namespace madgraph; +using namespace madmatrix; namespace { @@ -301,7 +301,7 @@ extern "C" } HostBufferBase momenta( rounded_count * ProcessData::npar * 4 ); - HostBufferBase couplings( rounded_count * madgraph::Parameters_dependentCouplings::ndcoup * 2 ); + HostBufferBase couplings( rounded_count * madmatrix::Parameters_dependentCouplings::ndcoup * 2 ); HostBufferBase g_s( rounded_count ); HostBufferBase helicity_random( rounded_count ); HostBufferBase color_random( rounded_count ); diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc index e70fd6ae1b..d0cf3a97a2 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc @@ -79,7 +79,7 @@ debug_me_is_abnormal( const fptype& me, size_t ievtALL ) //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- @@ -183,7 +183,7 @@ namespace madgraph //============================================================================ -namespace madgraph +namespace madmatrix { /* diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h index 261f7953d9..59a4c098ec 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h @@ -14,7 +14,7 @@ //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h index 5eee6de231..df62a74918 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h @@ -17,7 +17,7 @@ #include #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h index c6aca5328a..6aa7df2374 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h @@ -43,7 +43,7 @@ inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, boo //-------------------------------------------------------------------------- -namespace madgraph +namespace madmatrix { // Instantiate a GpuRuntime at the beginnining of the application's main to // invoke gpuSetDevice(0) in the constructor and book a gpuDeviceReset() call in the destructor diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc index c79f407484..121aab5a60 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc @@ -18,7 +18,7 @@ //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- @@ -146,7 +146,7 @@ namespace madgraph //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h index c4e7db8a21..aa406c60d1 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h @@ -16,7 +16,7 @@ #include #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h index 7a58f2fc92..669ed8ead5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_AMPLITUDES 1 -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -156,6 +156,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h index 2c301811ec..0b1c1162a3 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -110,6 +110,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h index 293f7d13f1..491c6f8a5a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h @@ -15,8 +15,8 @@ #include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM #include "MemoryBuffers.h" // for HostBufferCouplings::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -235,6 +235,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h index 11a65e19ff..3eadf6f393 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h @@ -13,8 +13,8 @@ //#include "MemoryAccessHelpers.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -79,6 +79,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h index 10046c1293..9e5b87e674 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h index 2739a0bd80..ce27a06de0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -148,6 +148,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h index af8d37bf6f..a6e5c356e4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h @@ -12,8 +12,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -109,6 +109,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h index 6175a16d1b..505312e236 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -131,6 +131,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h index 0230ea8a1a..10fdc00c8e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h @@ -13,8 +13,8 @@ #include "MemoryAccessHelpers.h" #include "MemoryAccessVectors.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -205,6 +205,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h index efa609aee1..6df3b20cf4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h index 66d7f35ac9..9406a9d75a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -161,6 +161,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h index 12df545d92..eece092fa3 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h @@ -11,8 +11,8 @@ #include "MemoryAccessHelpers.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -141,6 +141,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h index ed1f5ec02b..33227eadfa 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h @@ -17,7 +17,7 @@ #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc index a7d9814b12..280b298441 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc @@ -26,7 +26,7 @@ #include "color_sum.h" #include "ColorData.h" -namespace madgraph +namespace madmatrix { using namespace ProcessData; using namespace ProcessTables; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h index 034a2d5898..4461cdef50 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h @@ -15,7 +15,7 @@ #include "mgOnGpuCxtypes.h" // for cxtype, fptype_sv #include "GpuAbstraction.h" // for gpuBlasHandle_t, gpuStream_t -namespace madgraph +namespace madmatrix { __global__ void computeDependentCouplings( const fptype* allgs, diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc index 584e805d9c..3a35a0a72d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc @@ -11,7 +11,7 @@ #include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps #include "MemoryAccessMatrixElements.h" -namespace madgraph +namespace madmatrix { using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h index 54322cd990..df434100d2 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h @@ -14,7 +14,7 @@ #include "ProcessData.h" #include "GpuAbstraction.h" -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h index e3438e00b2..30eea25afa 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h @@ -29,8 +29,8 @@ #define CONSTEXPRMATHVAR constexpr #endif -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h index b834c7ddb7..f819da5783 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h @@ -51,7 +51,7 @@ #ifdef __CUDACC__ // this must be __CUDACC__ #if defined MGONGPU_CUCXTYPE_CUCOMPLEX -namespace madgraph +namespace madmatrix { #if defined MGONGPU_FPTYPE_DOUBLE class cucomplex @@ -139,7 +139,7 @@ namespace mgOnGpu /* clang-format off */ #ifdef MGONGPU_CUCXTYPE_THRUST template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } #elif defined MGONGPU_CUCXTYPE_CUCOMPLEX - __host__ __device__ constexpr operator madgraph::cucomplex() const { return madgraph::cucomplex( m_real, m_imag ); } + __host__ __device__ constexpr operator madmatrix::cucomplex() const { return madmatrix::cucomplex( m_real, m_imag ); } #endif #else #ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX @@ -163,7 +163,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -namespace madgraph +namespace madmatrix { template inline __host__ std::ostream& @@ -292,8 +292,8 @@ namespace madgraph // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // --- Type definitions (complex type: cxtype) #ifdef __CUDACC__ // this must be __CUDACC__ @@ -324,8 +324,8 @@ namespace madgraph // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL @@ -680,14 +680,14 @@ namespace madgraph return cxmake( c.real(), c.imag() ); } -} // end namespace madgraph +} // end namespace madmatrix //========================================================================== // COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined @@ -775,7 +775,7 @@ namespace madgraph return cxreal( c ) * cxreal( c ) + cximag( c ) * cximag( c ); } -} // end namespace madgraph +} // end namespace madmatrix //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h index e14cbcfc42..3a101094dd 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h @@ -12,8 +12,8 @@ #include #include -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //========================================================================== @@ -66,6 +66,6 @@ namespace madgraph //========================================================================== -} // end namespace madgraph +} // end namespace madmatrix #endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc index 834917a151..de9470c1f9 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc @@ -18,7 +18,7 @@ #include #include -using namespace madgraph; +using namespace madmatrix; namespace { @@ -347,7 +347,7 @@ extern "C" int *helicity_index, *color_index; unsigned int *flavor_indices, *diagram_index; - std::size_t n_coup = madgraph::Parameters_dependentCouplings::ndcoup; + std::size_t n_coup = madmatrix::Parameters_dependentCouplings::ndcoup; std::array, 16> ptrs_and_sizes = {{ {reinterpret_cast(&momenta), rounded_count * ProcessData::npar * 4 * sizeof( fptype )}, {reinterpret_cast(&couplings), rounded_count * n_coup * 2 * sizeof( fptype )}, diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc index 3ad7c5d1ca..29efce440d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc @@ -79,7 +79,7 @@ debug_me_is_abnormal( const fptype& me, size_t ievtALL ) //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h index 261f7953d9..59a4c098ec 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h @@ -14,7 +14,7 @@ //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h index 5eee6de231..df62a74918 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h @@ -17,7 +17,7 @@ #include #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc index 171f815ac5..3aeeadd9e8 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc @@ -18,7 +18,7 @@ //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- @@ -143,7 +143,7 @@ namespace madgraph //============================================================================ -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h index b74a7e19cb..e7cf857b47 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h @@ -14,7 +14,7 @@ #include #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h index 7a58f2fc92..669ed8ead5 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_AMPLITUDES 1 -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -156,6 +156,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h index 115ade33e2..f2939a8ca7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -100,9 +100,9 @@ namespace madgraph { const unsigned int& out = kernelAccessConst_s( buffer ); // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( madgraph::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return madgraph::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madmatrix::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } }; @@ -113,6 +113,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h index dc69aff815..347570a22e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h @@ -15,8 +15,8 @@ #include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM #include "MemoryBuffers.h" // for HostBufferCouplings::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -189,9 +189,9 @@ namespace madgraph constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( madgraph::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) @@ -218,9 +218,9 @@ namespace madgraph constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( madgraph::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast #endif } */ @@ -263,6 +263,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h index 7ce36f4c6f..df36536afe 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h @@ -14,8 +14,8 @@ //#include "MemoryAccessHelpers.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -80,6 +80,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h index 10046c1293..9e5b87e674 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h index de5955cedf..9dc2aba11e 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -116,9 +116,9 @@ namespace madgraph { fptype& out = kernelAccess_s( buffer ); // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( madgraph::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) @@ -141,9 +141,9 @@ namespace madgraph { const fptype& out = kernelAccessConst_s( buffer ); // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( madgraph::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } }; @@ -154,6 +154,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h index 159d55c30e..c64ce4b307 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h @@ -12,8 +12,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -99,9 +99,9 @@ namespace madgraph { const unsigned int& out = kernelAccessConst_s( buffer ); // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( madgraph::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return madgraph::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madmatrix::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } }; @@ -112,6 +112,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h index 40f0e7e106..7dd8626eb7 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h @@ -13,8 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -116,9 +116,9 @@ namespace madgraph { fptype& out = kernelAccess_s( buffer ); // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( madgraph::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) @@ -134,6 +134,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h index 0f36b4a3c4..c1ed427a36 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h @@ -13,8 +13,8 @@ #include "MemoryAccessHelpers.h" #include "MemoryAccessVectors.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -202,21 +202,21 @@ namespace madgraph //static bool first=true; if( first ){ std::cout << "WARNING! assume aligned AOSOA, skip check" << std::endl; first=false; } // SLOWER (5.06E6) // FASTEST? (5.09E6 in eemumu 512y) // This assumes alignment for momenta1d without checking - causes segmentation fault in reinterpret_cast if not aligned! - return madgraph::fptypevFromAlignedArray( out ); // use reinterpret_cast + return madmatrix::fptypevFromAlignedArray( out ); // use reinterpret_cast } else if( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ) { //static bool first=true; if( first ){ std::cout << "WARNING! aligned AOSOA, reinterpret cast" << std::endl; first=false; } // SLOWER (5.00E6) // DEFAULT! A tiny bit (<1%) slower because of the alignment check (5.07E6 in eemumu 512y) // This explicitly checks buffer alignment to avoid segmentation faults in reinterpret_cast - return madgraph::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } else { //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWER (4.93E6) // A bit (1%) slower (5.05E6 in eemumu 512y) // This does not require buffer alignment, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 - return madgraph::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) + return madmatrix::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) } } else @@ -228,7 +228,7 @@ namespace madgraph auto decoderIeppv = [buffer, ip4, ipar]( int ieppV ) -> const fptype& { return MemoryAccessMomenta::ieventAccessIp4IparConst( buffer, ievt0 + ieppV, ip4, ipar ); }; - return madgraph::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) + return madmatrix::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) } } @@ -248,6 +248,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h index efa609aee1..6df3b20cf4 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h @@ -9,8 +9,8 @@ #include "MemoryAccessGs.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -22,6 +22,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h index 8c8ece0f56..fb96f9ab26 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h @@ -11,7 +11,7 @@ #include "mgOnGpuVectors.h" -namespace madgraph // this is only needed for CPU SIMD vectorization +namespace madmatrix // this is only needed for CPU SIMD vectorization { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h index 66d7f35ac9..9406a9d75a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h @@ -15,8 +15,8 @@ #define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -161,6 +161,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h index 12df545d92..eece092fa3 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h @@ -11,8 +11,8 @@ #include "MemoryAccessHelpers.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -141,6 +141,6 @@ namespace madgraph //---------------------------------------------------------------------------- -} // end namespace madgraph +} // end namespace madmatrix #endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h index 83f316223b..88e90aeafb 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h @@ -16,7 +16,7 @@ #include -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc index ebe384b31a..84679743c8 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc @@ -28,7 +28,7 @@ #include #include -namespace madgraph +namespace madmatrix { using namespace ProcessData; using namespace ProcessTables; diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h index 79d1a5d6e7..81956d459d 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h @@ -14,7 +14,7 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuCxtypes.h" // for cxtype -namespace madgraph +namespace madmatrix { __global__ void computeDependentCouplings( const fptype* allgs, diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc index 4936784da6..15b28a305a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc @@ -11,7 +11,7 @@ #include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps #include "MemoryAccessMatrixElements.h" -namespace madgraph +namespace madmatrix { using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h index 1417988a26..f86a9a6c8a 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h @@ -13,7 +13,7 @@ #include "ProcessData.h" -namespace madgraph +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h index e3438e00b2..30eea25afa 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h @@ -29,8 +29,8 @@ #define CONSTEXPRMATHVAR constexpr #endif -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h index ed7ec2fdf0..5a7f09d0cc 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h @@ -73,7 +73,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -namespace madgraph +namespace madmatrix { template inline __host__ std::ostream& @@ -202,7 +202,7 @@ namespace madgraph // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -namespace madgraph +namespace madmatrix { // --- Type definitions (complex type: cxtype) #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX @@ -223,7 +223,7 @@ namespace madgraph // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -namespace madgraph +namespace madmatrix { #if defined MGONGPU_CPPCXTYPE_CXSMPL @@ -331,14 +331,14 @@ namespace madgraph return cxmake( c.real(), c.imag() ); } -} // end namespace madgraph +} // end namespace madmatrix //========================================================================== // COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined @@ -373,7 +373,7 @@ namespace madgraph return out; } -} // end namespace madgraph +} // end namespace madmatrix //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h index 62e454648c..d6fe740b8f 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h @@ -12,8 +12,8 @@ #include #include -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { //========================================================================== @@ -46,6 +46,6 @@ namespace madgraph //========================================================================== -} // end namespace madgraph +} // end namespace madmatrix #endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h index a6e006171d..14bf7ea0bc 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h @@ -32,8 +32,8 @@ //#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) #endif -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { const int neppV = MGONGPU_CPPSIMD; @@ -136,8 +136,8 @@ namespace madgraph //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // Printout to stream for user defined types @@ -750,6 +750,6 @@ namespace madgraph //========================================================================== -} // end namespace madgraph +} // end namespace madmatrix #endif // MGONGPUVECTORS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc index cedf05b15b..ac462561b0 100644 --- a/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/umami.cc @@ -17,7 +17,7 @@ #include #include -using namespace madgraph; +using namespace madmatrix; namespace { @@ -301,7 +301,7 @@ extern "C" } HostBufferBase momenta( rounded_count * ProcessData::npar * 4 ); - HostBufferBase couplings( rounded_count * madgraph::Parameters_dependentCouplings::ndcoup * 2 ); + HostBufferBase couplings( rounded_count * madmatrix::Parameters_dependentCouplings::ndcoup * 2 ); HostBufferBase g_s( rounded_count ); HostBufferBase helicity_random( rounded_count ); HostBufferBase color_random( rounded_count ); diff --git a/madgraph/iolibs/template_files/madmatrix/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/check_sa.cc index fc5d4611f8..7b41f4a30e 100644 --- a/madgraph/iolibs/template_files/madmatrix/check_sa.cc +++ b/madgraph/iolibs/template_files/madmatrix/check_sa.cc @@ -58,7 +58,7 @@ namespace { - using namespace madgraph; + using namespace madmatrix; // Fixed physics inputs fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV diff --git a/madgraph/iolibs/template_files/madmatrix/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/constexpr_math.h index 362ad19c4f..03e9ffaf5c 100644 --- a/madgraph/iolibs/template_files/madmatrix/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/constexpr_math.h @@ -27,8 +27,8 @@ #define CONSTEXPRMATHVAR constexpr #endif -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc b/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc index 334ef032b3..144f2384ed 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc @@ -31,7 +31,7 @@ //#include //#include -namespace madgraph +namespace madmatrix { // ALOHA-style object for easy flavor consolidation and non-template API diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc index 68e91b1a78..c1cebdbd02 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc @@ -16,7 +16,7 @@ #include #include -using namespace madgraph; +using namespace madmatrix; #ifndef MGONGPU_HARDCODE_PARAM diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc index 7b58ce656a..e6db352680 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc @@ -34,8 +34,8 @@ #include "read_slha.h" -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // Define FLV_COUPLING struct struct FLV_COUPLING { @@ -124,15 +124,15 @@ namespace madgraph for( int i = 0; i < N; ++i ) out[i] = cxmake( pars->*members[i] ); } -} // end namespace madgraph +} // end namespace madmatrix #else%(eftwarn1)s #include #include -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // Hardcoded constexpr physics parameters namespace Parameters // keep the same name rather than HardcodedParameters for simplicity @@ -164,14 +164,14 @@ namespace madgraph %(hasbsmip)s__device__ constexpr double mdl_bsmIndepParam[nBsmIndepParam] = { %(bsmip)s }; } -} // end namespace madgraph +} // end namespace madmatrix #endif //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { namespace Parameters_dependentCouplings { @@ -261,7 +261,7 @@ namespace madgraph } #pragma GCC diagnostic pop -} // end namespace madgraph +} // end namespace madmatrix //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/massless_rambo.h index ce3e6f19ef..4f64d1aead 100644 --- a/madgraph/iolibs/template_files/madmatrix/massless_rambo.h +++ b/madgraph/iolibs/template_files/madmatrix/massless_rambo.h @@ -22,7 +22,7 @@ #include // Simplified rambo version for 2 to N (with N>=2) processes with massless particles -namespace madgraph +namespace madmatrix { namespace massless_rambo { diff --git a/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h index a26a0a5a22..c966c17c40 100644 --- a/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h @@ -51,7 +51,7 @@ #ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) #if defined MGONGPU_CUCXTYPE_CUCOMPLEX -namespace madgraph +namespace madmatrix { #if defined MGONGPU_FPTYPE_DOUBLE class cucomplex @@ -139,7 +139,7 @@ namespace mgOnGpu /* clang-format off */ #ifdef MGONGPU_CUCXTYPE_THRUST template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } #elif defined MGONGPU_CUCXTYPE_CUCOMPLEX - __host__ __device__ constexpr operator madgraph::cucomplex() const { return madgraph::cucomplex( m_real, m_imag ); } + __host__ __device__ constexpr operator madmatrix::cucomplex() const { return madmatrix::cucomplex( m_real, m_imag ); } #endif #else #ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX @@ -163,7 +163,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -namespace madgraph +namespace madmatrix { template inline __host__ std::ostream& @@ -292,8 +292,8 @@ namespace madgraph // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // --- Type definitions (complex type: cxtype) #ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) @@ -324,8 +324,8 @@ namespace madgraph // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL @@ -680,14 +680,14 @@ namespace madgraph return cxmake( c.real(), c.imag() ); } -} // end namespace madgraph +} // end namespace madmatrix //========================================================================== // COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined @@ -722,7 +722,7 @@ namespace madgraph return out; } -} // end namespace madgraph +} // end namespace madmatrix //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h index e6bc6de847..0bc805cf72 100644 --- a/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h @@ -12,11 +12,11 @@ #include #include -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) +//One namespace. Split ber backend. #ifdef MGONGPUCPP_GPUIMPL // cuda -namespace madgraph +namespace madmatrix #else -namespace madgraph +namespace madmatrix #endif { //========================================================================== @@ -97,6 +97,6 @@ namespace madgraph //========================================================================== -} // end namespace madgraph +} // end namespace madmatrix #endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h index 966d60141d..8a13174491 100644 --- a/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h @@ -32,8 +32,8 @@ //#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) #endif -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { #ifdef MGONGPU_CPPSIMD @@ -152,8 +152,8 @@ namespace madgraph //========================================================================== -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { #ifndef MGONGPUCPP_GPUIMPL @@ -919,6 +919,6 @@ namespace madgraph //========================================================================== -} // end namespace madgraph +} // end namespace madmatrix #endif // MGONGPUVECTORS_H diff --git a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc index 20f0e4736b..37ff7ec830 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc @@ -19,7 +19,7 @@ // physics-parameter/coupling setup (initProc), and the flavor accessor. //========================================================================== -namespace madgraph +namespace madmatrix { using namespace ProcessData; using namespace ProcessTables; diff --git a/madgraph/iolibs/template_files/madmatrix/process_h.inc b/madgraph/iolibs/template_files/madmatrix/process_h.inc index cfcde6c3e2..04668e1c35 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_h.inc @@ -31,7 +31,7 @@ //-------------------------------------------------------------------------- -namespace madgraph +namespace madmatrix { %(process_class_definitions)s //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/runTest.cc b/madgraph/iolibs/template_files/madmatrix/runTest.cc index 8cd3a9a01c..4601fa3a8f 100644 --- a/madgraph/iolibs/template_files/madmatrix/runTest.cc +++ b/madgraph/iolibs/template_files/madmatrix/runTest.cc @@ -28,7 +28,7 @@ #include -using namespace madgraph; +using namespace madmatrix; // Host-accessible copy of mgOnGpu::channel2iconfig, needed only by setChannelIds() #ifndef MGONGPUCPP_GPUIMPL diff --git a/madgraph/iolibs/template_files/madmatrix/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/testmisc.cc index 6cb9da9558..7e0b2f5ef0 100644 --- a/madgraph/iolibs/template_files/madmatrix/testmisc.cc +++ b/madgraph/iolibs/template_files/madmatrix/testmisc.cc @@ -31,8 +31,8 @@ #define XTESTID( s ) TESTID( s ) -// NB: the madgraph namespace: types are now split per backend file, not per namespace (see #318 and #725) -namespace madgraph +//One namespace. Split ber backend. +namespace madmatrix { #ifdef MGONGPU_CPPSIMD /* clang-format off */ #define EXPECT_TRUE_sv( cond ) { bool_v mask( cond ); EXPECT_TRUE( maskand( mask ) ); } @@ -61,7 +61,7 @@ namespace madgraph TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) { - using namespace madgraph; + using namespace madmatrix; //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/testxxx.cc index 6d694951ad..b9af47f014 100644 --- a/madgraph/iolibs/template_files/madmatrix/testxxx.cc +++ b/madgraph/iolibs/template_files/madmatrix/testxxx.cc @@ -33,7 +33,7 @@ #define XTESTID( s ) TESTID( s ) -namespace madgraph +namespace madmatrix { std::string fpeHandlerMessage = "unknown"; int fpeHandlerIevt = -1; @@ -50,7 +50,7 @@ namespace madgraph TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { - using namespace madgraph; + using namespace madmatrix; #ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); #endif @@ -65,9 +65,9 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV // Fill in the input momenta #ifdef MGONGPUCPP_GPUIMPL - madgraph::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] + madmatrix::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] #else - madgraph::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] + madmatrix::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] #endif /* clang-format off */ // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! const fptype par0[np4 * nevt] = // AOS[nevt][np4] @@ -309,7 +309,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { for( int ievt = 0; ievt < nevt; ievt++ ) { - using namespace madgraph; + using namespace madmatrix; if( debug ) { std::cout << std::endl; From 5e6989a606f31c99904af4ddfe6158fbc00a7ae2 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 19 Aug 2026 14:14:01 +0200 Subject: [PATCH 22/25] rambo/random move to /src/rambo in standalone --- .../template_files/madmatrix/madmatrix.mk | 9 ++++++--- .../madmatrix/madmatrix_standalone.mk | 3 ++- .../rambo}/CommonRandomNumberKernel.cc | 0 .../{ => src/rambo}/CommonRandomNumbers.h | 0 .../rambo}/CurandRandomNumberKernel.cc | 0 .../rambo}/HiprandRandomNumberKernel.cc | 0 .../{ => src/rambo}/RamboSamplingKernels.cc | 0 .../{ => src/rambo}/RamboSamplingKernels.h | 0 .../{ => src/rambo}/RandomNumberKernels.h | 0 .../{ => src/rambo}/massless_rambo.h | 0 .../madmatrix/{ => src/rambo}/rambo.h | 0 madmatrix/output.py | 19 +++++++++++-------- 12 files changed, 19 insertions(+), 12 deletions(-) rename madgraph/iolibs/template_files/madmatrix/{ => src/rambo}/CommonRandomNumberKernel.cc (100%) rename madgraph/iolibs/template_files/madmatrix/{ => src/rambo}/CommonRandomNumbers.h (100%) rename madgraph/iolibs/template_files/madmatrix/{ => src/rambo}/CurandRandomNumberKernel.cc (100%) rename madgraph/iolibs/template_files/madmatrix/{ => src/rambo}/HiprandRandomNumberKernel.cc (100%) rename madgraph/iolibs/template_files/madmatrix/{ => src/rambo}/RamboSamplingKernels.cc (100%) rename madgraph/iolibs/template_files/madmatrix/{ => src/rambo}/RamboSamplingKernels.h (100%) rename madgraph/iolibs/template_files/madmatrix/{ => src/rambo}/RandomNumberKernels.h (100%) rename madgraph/iolibs/template_files/madmatrix/{ => src/rambo}/massless_rambo.h (100%) rename madgraph/iolibs/template_files/madmatrix/{ => src/rambo}/rambo.h (100%) diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk index 033473bbf5..0469ed6301 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk @@ -436,7 +436,7 @@ endif # The common library name carries the full BACKEND suffix so each vectorisation/GPU variant is distinct. MADMATRIX_COMMONLIB = madmatrix_common_$(BACKEND) LIBFLAGS = -L$(LIBDIR) -l$(MADMATRIX_COMMONLIB) -INCFLAGS += -I$(SRC) +INCFLAGS += -I$(SRC) -I$(SRC)/rambo #------------------------------------------------------------------------------- @@ -724,6 +724,9 @@ objects_lib=$(BUILDDIR)/CPPProcess.o $(BUILDDIR)/color_sum.o $(BUILDDIR)/MatrixE # Backend-owned sources vpath %%.cc ../../backend/$(BACKENDDIR) +# Rambo/random-number sources +vpath %%.cc ../../src/rambo + # Explicitly define the default goal (this is not necessary as it is the first target, which is implicitly the default goal) .DEFAULT_GOAL := all.$(TAG) @@ -769,11 +772,11 @@ endif # incompatible backends (different BACKEND, FPTYPE, etc.) in the same directory. # Use USEBUILDDIR=1 to build for multiple backends simultaneously without cleaning. ifeq ($(GPUCC),) -$(BUILDDIR)/%%.o : %%.cc *.h ../../backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) +$(BUILDDIR)/%%.o : %%.cc *.h ../../backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(SRC)/rambo/*.h $(BUILDDIR)/.build.$(TAG) @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(INCFLAGS) $(CXXFLAGS) -c $< -o $@ else -$(BUILDDIR)/%%.o : %%.cc *.h ../../backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) +$(BUILDDIR)/%%.o : %%.cc *.h ../../backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(SRC)/rambo/*.h $(BUILDDIR)/.build.$(TAG) @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi $(GPUCC) $(CPPFLAGS) $(INCFLAGS) $(GPUFLAGS) -c -x $(GPULANGUAGE) $< -o $@ endif diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk index 618cb9e42a..505776c3d3 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk @@ -19,7 +19,8 @@ include madmatrix.mk #=== Standalone driver (check_sa.exe) configuration # Standalone-only object files (compiled via the generic %%.o pattern rule from -# madmatrix.mk). +# madmatrix.mk; RamboSamplingKernels.cc/CommonRandomNumberKernel.cc are found +# in ../../src/rambo/ via the vpath in madmatrix.mk override standalone_objects = $(BUILDDIR)/RamboSamplingKernels.o \ $(BUILDDIR)/CommonRandomNumberKernel.o \ $(BUILDDIR)/check_sa.o diff --git a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/src/rambo/CommonRandomNumberKernel.cc similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc rename to madgraph/iolibs/template_files/madmatrix/src/rambo/CommonRandomNumberKernel.cc diff --git a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/src/rambo/CommonRandomNumbers.h similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/CommonRandomNumbers.h rename to madgraph/iolibs/template_files/madmatrix/src/rambo/CommonRandomNumbers.h diff --git a/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/src/rambo/CurandRandomNumberKernel.cc similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc rename to madgraph/iolibs/template_files/madmatrix/src/rambo/CurandRandomNumberKernel.cc diff --git a/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/src/rambo/HiprandRandomNumberKernel.cc similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc rename to madgraph/iolibs/template_files/madmatrix/src/rambo/HiprandRandomNumberKernel.cc diff --git a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/src/rambo/RamboSamplingKernels.cc similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc rename to madgraph/iolibs/template_files/madmatrix/src/rambo/RamboSamplingKernels.cc diff --git a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/src/rambo/RamboSamplingKernels.h similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h rename to madgraph/iolibs/template_files/madmatrix/src/rambo/RamboSamplingKernels.h diff --git a/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/src/rambo/RandomNumberKernels.h similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h rename to madgraph/iolibs/template_files/madmatrix/src/rambo/RandomNumberKernels.h diff --git a/madgraph/iolibs/template_files/madmatrix/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/src/rambo/massless_rambo.h similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/massless_rambo.h rename to madgraph/iolibs/template_files/madmatrix/src/rambo/massless_rambo.h diff --git a/madgraph/iolibs/template_files/madmatrix/rambo.h b/madgraph/iolibs/template_files/madmatrix/src/rambo/rambo.h similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/rambo.h rename to madgraph/iolibs/template_files/madmatrix/src/rambo/rambo.h diff --git a/madmatrix/output.py b/madmatrix/output.py index f26239066e..e657b9eca3 100644 --- a/madmatrix/output.py +++ b/madmatrix/output.py @@ -89,8 +89,9 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): # (see backend_variants below); only genuinely backend-agnostic files # (no backend/ counterpart) are copied flat into SubProcesses/. umami.h # is the only one needed outside standalone mode too (it's the header - # for backend//umami.cc's UMAMI API); nvtx.h/rambo.h are - # standalone-driver-only (see _standalone_extra_files below). + # for backend//umami.cc's UMAMI API); nvtx.h is + # standalone-driver-only (see _standalone_extra_files below). The + # rambo/random-number sources live once in src/rambo/ 'SubProcesses': relative_path_list(madmatrix_templates, ['umami.h']), # run_card.toml is generated in finalize() (ProcessExporterMG7.create_run_card) # from the template, not copied verbatim. @@ -118,6 +119,11 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): sorted(os.listdir(pjoin(backend_template_dir, _backend_variant)))) del _backend_variant + # Rambo/random-number files copy in src/rambo/ + rambo_template_dir = pjoin(madmatrix_templates, 'src', 'rambo') + from_template['src/rambo'] = relative_path_list( + rambo_template_dir, sorted(os.listdir(rambo_template_dir))) + # Backend-owned skeleton files (GpuRuntime.h, color_sum.{h,cc}, the # MemoryAccess*.h family, MatrixElementKernels/CrossSectionKernels/umami.cc, # etc.) are NOT linked into P* at all: they are compiled straight from the @@ -129,7 +135,7 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): template_src_make = pjoin(madmatrix_templates, 'madmatrix_src.mk') template_Sub_make = pjoin(madmatrix_templates, 'madmatrix.mk') - dirs_to_create = ['bin', 'src', 'lib', 'Cards', 'SubProcesses', + dirs_to_create = ['bin', 'src', 'src/rambo', 'lib', 'Cards', 'SubProcesses', 'backend', 'backend/cpu', 'backend/simd', @@ -211,11 +217,8 @@ class ProcessExporterMadMatrixStandalone(ProcessExporterMadMatrix): template_Sub_make = pjoin(ProcessExporterMadMatrix.madmatrix_templates, 'madmatrix_standalone.mk') # Standalone-only template files needed to build check_sa.exe - _standalone_extra_files = ['check_sa.cc', 'nvtx.h', 'rambo.h', - 'RamboSamplingKernels.cc', 'RamboSamplingKernels.h', - 'CommonRandomNumberKernel.cc', 'CommonRandomNumbers.h', - 'RandomNumberKernels.h', - 'massless_rambo.h', 'timer.h', 'timermap.h'] + symlinked into every P1_*/. + _standalone_extra_files = ['check_sa.cc', 'nvtx.h', 'timer.h', 'timermap.h'] from_template = dict(ProcessExporterMadMatrix.from_template) from_template['SubProcesses'] = (ProcessExporterMadMatrix.from_template['SubProcesses'] From 6b5a265f18b1493be40b1166c364970829e83559 Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 19 Aug 2026 14:20:01 +0200 Subject: [PATCH 23/25] Comment bugfix --- madmatrix/output.py | 1 - 1 file changed, 1 deletion(-) diff --git a/madmatrix/output.py b/madmatrix/output.py index e657b9eca3..5a6c30d25b 100644 --- a/madmatrix/output.py +++ b/madmatrix/output.py @@ -217,7 +217,6 @@ class ProcessExporterMadMatrixStandalone(ProcessExporterMadMatrix): template_Sub_make = pjoin(ProcessExporterMadMatrix.madmatrix_templates, 'madmatrix_standalone.mk') # Standalone-only template files needed to build check_sa.exe - symlinked into every P1_*/. _standalone_extra_files = ['check_sa.cc', 'nvtx.h', 'timer.h', 'timermap.h'] from_template = dict(ProcessExporterMadMatrix.from_template) From 479494c5e7d8b1f58b2816cce33c34b97bf4912f Mon Sep 17 00:00:00 2001 From: stloufra Date: Wed, 19 Aug 2026 15:07:53 +0200 Subject: [PATCH 24/25] BugFix: relinking for before compiled backend when making ccpnone -> cppsse4 -> cppnone rule for check_sa checked that cppnone buildir content is older than the check_sa and did not relink, silently reusing the sse4 lib. now small prerequisity storing last compiled backend solution --- .../template_files/madmatrix/madmatrix_standalone.mk | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk index 505776c3d3..7fd3af109f 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk @@ -25,6 +25,16 @@ override standalone_objects = $(BUILDDIR)/RamboSamplingKernels.o \ $(BUILDDIR)/CommonRandomNumberKernel.o \ $(BUILDDIR)/check_sa.o +# force relink when changing to before compiled backend +CHECK_SA_BACKEND_MARKER = .check_sa_backend +check_sa.exe: $(CHECK_SA_BACKEND_MARKER) +$(CHECK_SA_BACKEND_MARKER): FORCE + @if [ ! -f $(CHECK_SA_BACKEND_MARKER) ] || [ "$$(cat $(CHECK_SA_BACKEND_MARKER) 2>/dev/null)" != "$(BACKEND)" ]; then \ + echo $(BACKEND) > $(CHECK_SA_BACKEND_MARKER); \ + fi +.PHONY: FORCE +FORCE: + # Top-level standalone goal: process lib + standalone driver. .PHONY: standalone_all standalone_all: all.$(TAG) check_sa.exe @@ -50,7 +60,7 @@ clean: clean_standalone .PHONY: clean_standalone clean_standalone: rm -f $(BUILDDIR)/RamboSamplingKernels.o $(BUILDDIR)/CommonRandomNumberKernel.o $(BUILDDIR)/check_sa.o - rm -f check_sa.exe + rm -f check_sa.exe $(CHECK_SA_BACKEND_MARKER) # 'cleanall' from madmatrix.mk also wipes build.* directories, which already # covers our standalone objects when USEBUILDDIR=1. We only need to take care From 7ee6b196e4ffcdabe34f0d4c24eacf93cbde5f1c Mon Sep 17 00:00:00 2001 From: stloufra Date: Thu, 20 Aug 2026 12:28:15 +0200 Subject: [PATCH 25/25] HIP arch auto detect --- madgraph/iolibs/template_files/madmatrix/madmatrix.mk | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk index 0469ed6301..c54be1436a 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk @@ -326,7 +326,13 @@ ifeq ($(BACKEND),cuda) else ifeq ($(BACKEND),hip) # example architecture values MI200:gfx90a, MI350X:gfx942 - MADGRAPH_HIP_ARCHITECTURE ?= gfx942 + # auto detect arch if not set; if fail gfx942 default + ifeq ($(origin MADGRAPH_HIP_ARCHITECTURE),undefined) + MADGRAPH_HIP_ARCHITECTURE := $(shell $(HIP_HOME)/bin/rocm_agent_enumerator 2>/dev/null | grep -v gfx000 | sort -u | head -1) + endif + ifeq ($(MADGRAPH_HIP_ARCHITECTURE),) + MADGRAPH_HIP_ARCHITECTURE := gfx942 + endif # Set GPUCC as $(HIP_HOME)/bin/hipcc (it was already checked above that this exists) GPUCC = $(HIP_HOME)/bin/hipcc XCOMPILERFLAG =