diff --git a/.gitignore b/.gitignore index e601b64224..ccfce81acb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ __pycache__ *.mod *.a PROC* +!madgraph/iolibs/template_files/madmatrix/ProcessData.h +!madgraph/iolibs/template_files/madmatrix/ProcessTables.h cmd* ptest* *.pkl diff --git a/madgraph/iolibs/template_files/madmatrix/Bridge.h b/madgraph/iolibs/template_files/madmatrix/Bridge.h deleted file mode 100644 index 8417e3ad15..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/Bridge.h +++ /dev/null @@ -1,633 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Nov 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2021-2025). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef BRIDGE_H -#define BRIDGE_H 1 - -#include "mgOnGpuConfig.h" - -#include "CPPProcess.h" // for CPPProcess -#include "CrossSectionKernels.h" // for flagAbnormalMEs -#include "MatrixElementKernels.h" // for MatrixElementKernelHost, MatrixElementKernelDevice -#include "MemoryAccessMomenta.h" // for MemoryAccessMomenta::neppM -#include "MemoryBuffers.h" // for HostBufferMomenta, DeviceBufferMomenta etc - -//#ifdef __HIPCC__ -//#include // see -//https://rocm.docs.amd.com/en/docs-5.4.3/CHANGELOG.html#id79 #else #include -// // bypass this completely to ease portability on LUMI #803 #endif - -#include // bypass std::filesystem #803 - -#include -#include -#include -#include -#include -#include -#include - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - /** - * A base class for a class whose pointer is passed between Fortran and C++. - * This is not really necessary, but it allows minimal type checks on all such - * pointers. - */ - struct CppObjectInFortran - { - CppObjectInFortran() {} - virtual ~CppObjectInFortran() {} - }; - - //-------------------------------------------------------------------------- - /** - * A templated class for calling the CUDA/C++ matrix element calculations of the - * event generation workflow. The FORTRANFPTYPE template parameter indicates the - * precision of the Fortran momenta from MadEvent (float or double). The - * precision of the matrix element calculation is hardcoded in the fptype - * typedef in CUDA/C++. - * - * The Fortran momenta passed in are in the form of - * DOUBLE PRECISION P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) - * where the dimensions are , , - * . In memory, this is stored in a way that C reads as an array - * P_MULTI[nevtF][nparF][np4F]. The CUDA/C++ momenta are stored as an - * array[npagM][npar][np4][neppM] with nevt=npagM*neppM. The Bridge is - * configured to store nevt==nevtF events in CUDA/C++. It also checks that - * Fortran and C++ parameters match, nparF==npar and np4F==np4. - * - * The cpu/gpu sequences take FORTRANFPTYPE* (not fptype*) momenta/MEs. - * This allows mixing double in MadEvent Fortran with float in CUDA/C++ - * sigmaKin. In the fcheck_sa.f test, Fortran uses double while CUDA/C++ may use - * double or float. In the check_sa "--bridge" test, everything is implemented - * in fptype (double or float). - */ - template - class Bridge final : public CppObjectInFortran - { - public: - /** - * Constructor - * - * @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran array - * loops (VECSIZE_USED <= VECSIZE_MEMMAX) - * @param nparF (NEXTERNAL, nexternal.inc) number of external particles in - * Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F number of momenta components, usually 4, in Fortran arrays - * (KEPT FOR SANITY CHECKS ONLY) - */ - Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F ); - - /** - * Destructor - */ - virtual ~Bridge() {} - - // Delete copy/move constructors and assignment operators - Bridge( const Bridge& ) = delete; - Bridge( Bridge&& ) = delete; - Bridge& operator=( const Bridge& ) = delete; - Bridge& operator=( Bridge&& ) = delete; - -#ifdef MGONGPUCPP_GPUIMPL - /** - * Set the gpublocks and gputhreads for the gpusequence - throws if evnt != - * gpublocks*gputhreads (this is needed for BridgeKernel tests rather than for - * actual production use in Fortran) - * - * @param gpublocks number of gpublocks - * @param gputhreads number of gputhreads - */ - void set_gpugrid( const int gpublocks, const int gputhreads ); - - /** - * Sequence to be executed for the Cuda matrix element calculation - * - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant - * alphas) - * @param iflavorVec the index of the flavor combination - * @param rndhel the pointer to the input random numbers for helicity - * selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 - * to n - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void gpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); -#else - /** - * Sequence to be executed for the vectorized CPU matrix element calculation - * - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant - * alphas) - * @param iflavorVec the index of the flavor combination - * @param rndhel the pointer to the input random numbers for helicity - * selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the Feynman diagram to enhance in multi-channel mode if 1 - * to n - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void cpu_sequence( const FORTRANFPTYPE* momenta, const FORTRANFPTYPE* gs, const unsigned int* iflavorVec, const FORTRANFPTYPE* rndhel, const FORTRANFPTYPE* rndcol, const unsigned int* channelIds, FORTRANFPTYPE* mes, int* selhel, int* selcol, const bool goodHelOnly = false ); -#endif - - // Return the number of good helicities (-1 initially when they have not yet - // been calculated) - int nGoodHel() const { return m_nGoodHel; } - - // Return the total number of helicities (expose cudacpp ncomb in the Bridge - // interface to Fortran) - constexpr int nTotHel() const { return CPPProcess::ncomb; } - - private: - unsigned int m_nevt; // number of events - int m_nGoodHel; // the number of good helicities (-1 initially when they have - // not yet been calculated) - -#ifdef MGONGPUCPP_GPUIMPL - int m_gputhreads; // number of gpu threads (default set from number of - // events, can be modified) - int m_gpublocks; // number of gpu blocks (default set from number of events, - // can be modified) - DeviceBuffer m_devMomentaF; - DeviceBufferMomenta m_devMomentaC; - DeviceBufferGs m_devGs; - DeviceBufferIflavorVec m_devIflavorVec; - DeviceBufferRndNumHelicity m_devRndHel; - DeviceBufferRndNumColor m_devRndCol; - DeviceBufferMatrixElements m_devMEs; - DeviceBufferSelectedHelicity m_devSelHel; - DeviceBufferSelectedColor m_devSelCol; - DeviceBufferChannelIds m_devChannelIds; - PinnedHostBufferIflavorVec m_hstIflavorVec; - PinnedHostBufferGs m_hstGs; - PinnedHostBufferRndNumHelicity m_hstRndHel; - PinnedHostBufferRndNumColor m_hstRndCol; - PinnedHostBufferMatrixElements m_hstMEs; - PinnedHostBufferSelectedHelicity m_hstSelHel; - PinnedHostBufferSelectedColor m_hstSelCol; - PinnedHostBufferChannelIds m_hstChannelIds; - std::unique_ptr m_pmek; - // static constexpr int s_gputhreadsmin = 16; // minimum number of gpu threads - // (TEST VALUE FOR MADEVENT) - static constexpr int s_gputhreadsmin = - 32; // minimum number of gpu threads (DEFAULT) -#else - HostBufferMomenta m_hstMomentaC; - HostBufferGs m_hstGs; - HostBufferIflavorVec m_hstIflavorVec; - HostBufferRndNumHelicity m_hstRndHel; - HostBufferRndNumColor m_hstRndCol; - HostBufferMatrixElements m_hstMEs; - HostBufferSelectedHelicity m_hstSelHel; - HostBufferSelectedColor m_hstSelCol; - HostBufferChannelIds m_hstChannelIds; - std::unique_ptr m_pmek; -#endif - }; - - //-------------------------------------------------------------------------- - // - // Forward declare transposition methods - // - -#ifdef MGONGPUCPP_GPUIMPL - - template - __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); - -#endif // MGONGPUCPP_GPUIMPL - - template - void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ); - - template - void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ); - - //-------------------------------------------------------------------------- - // - // Implementations of member functions of class Bridge - // - - template - Bridge::Bridge( unsigned int nevtF, unsigned int nparF, unsigned int np4F) - : m_nevt( nevtF ), m_nGoodHel( -1 ) -#ifdef MGONGPUCPP_GPUIMPL - , m_gputhreads( 256 ) // default number of gpu threads - , m_gpublocks( m_nevt / m_gputhreads ) // this ensures m_nevt <= m_gpublocks*m_gputhreads - , m_devMomentaF( m_nevt ) - , m_devMomentaC( m_nevt ) - , m_devIflavorVec( m_nevt ) - , m_devGs( m_nevt ) - , m_devRndHel( m_nevt ) - , m_devRndCol( m_nevt ) - , m_devMEs( m_nevt ) - , m_devSelHel( m_nevt ) - , m_devSelCol( m_nevt ) - , m_devChannelIds( m_nevt ) -#else - , m_hstMomentaC( m_nevt ) -#endif - , m_hstGs( m_nevt ) - , m_hstIflavorVec( m_nevt ) - , m_hstRndHel( m_nevt ) - , m_hstRndCol( m_nevt ) - , m_hstMEs( m_nevt ) - , m_hstSelHel( m_nevt ) - , m_hstSelCol( m_nevt ) - , m_hstChannelIds( m_nevt ) - , m_pmek( nullptr ) - { - if( nparF != CPPProcess::npar ) - throw std::runtime_error( "Bridge constructor: npar mismatch" ); - if( np4F != CPPProcess::np4 ) - throw std::runtime_error( "Bridge constructor: np4 mismatch" ); -#ifdef MGONGPUCPP_GPUIMPL - if( ( m_nevt < s_gputhreadsmin ) || ( m_nevt % s_gputhreadsmin != 0 ) ) - throw std::runtime_error( - "Bridge constructor: nevt should be a multiple of " + - std::to_string( s_gputhreadsmin ) ); - while( m_nevt != m_gpublocks * m_gputhreads ) - { - m_gputhreads /= 2; - if( m_gputhreads < s_gputhreadsmin ) - throw std::logic_error( - "Bridge constructor: FIXME! cannot choose gputhreads" ); // this - // should - // never - // happen! - m_gpublocks = m_nevt / m_gputhreads; - } -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Instantiate device Bridge (nevt=" << m_nevt - << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads - << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" - << std::endl; -#endif - m_pmek.reset( new MatrixElementKernelDevice( - m_devMomentaC, m_devGs, m_devIflavorVec, m_devRndHel, m_devRndCol, m_devChannelIds, m_devMEs, m_devSelHel, m_devSelCol, m_gpublocks, m_gputhreads) ); -#else -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Instantiate host Bridge (nevt=" << m_nevt << ")" - << std::endl; -#endif - m_pmek.reset( new MatrixElementKernelHost( - m_hstMomentaC, m_hstGs, m_hstIflavorVec, m_hstRndHel, m_hstRndCol, m_hstChannelIds, m_hstMEs, m_hstSelHel, m_hstSelCol, m_nevt ) ); -#endif // MGONGPUCPP_GPUIMPL - // Create a process object, read param card and set parameters - // FIXME: the process instance can happily go out of scope because it is only - // needed to read parameters? - // FIXME: the CPPProcess should really be a singleton? what if fbridgecreate - // is called from several Fortran threads? - CPPProcess process( /*verbose=*/false ); - std::string paramCard = - "../Cards/param_card.dat"; // ZW: change default param_card.dat location - // to one dir down - /* -#ifdef __HIPCC__ - if( !std::experimental::filesystem::exists( paramCard ) ) paramCard = "../" + -paramCard; #else if( !std::filesystem::exists( paramCard ) ) paramCard = "../" + -paramCard; #endif - */ - // struct stat dummybuffer; // bypass std::filesystem #803 - // if( !( stat( paramCard.c_str(), &dummyBuffer ) == 0 ) ) paramCard = "../" + - // paramCard; // - auto fileExists = []( std::string& fileName ) - { - struct stat buffer; - return stat( fileName.c_str(), &buffer ) == 0; - }; - size_t paramCardCheck = 2; // ZW: check for paramCard up to 2 directories up - for( size_t k = 0; k < paramCardCheck; ++k ) - { - if( fileExists( paramCard ) ) break; // bypass std::filesystem #803 - paramCard = "../" + paramCard; - } - process.initProc( paramCard ); - } - -#ifdef MGONGPUCPP_GPUIMPL - template - void Bridge::set_gpugrid( const int gpublocks, - const int gputhreads ) - { - if( m_nevt != gpublocks * gputhreads ) - throw std::runtime_error( - "Bridge: gpublocks*gputhreads must equal m_nevt in set_gpugrid" ); - m_gpublocks = gpublocks; - m_gputhreads = gputhreads; -#ifdef MGONGPUCPP_VERBOSE - std::cout << "WARNING! Set grid in Bridge (nevt=" << m_nevt - << ", gpublocks=" << m_gpublocks << ", gputhreads=" << m_gputhreads - << ", gpublocks*gputhreads=" << m_gpublocks * m_gputhreads << ")" - << std::endl; -#endif - m_pmek->setGrid( m_gpublocks, m_gputhreads ); - } -#endif - -#ifdef MGONGPUCPP_GPUIMPL - template - void Bridge::gpu_sequence( const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool goodHelOnly ) - { - constexpr int neppM = MemoryAccessMomenta::neppM; - if constexpr( neppM == 1 && std::is_same_v ) - { - gpuMemcpy( m_devMomentaC.data(), momenta, m_devMomentaC.bytes(), gpuMemcpyHostToDevice ); - } - else - { - gpuMemcpy( m_devMomentaF.data(), momenta, m_devMomentaF.bytes(), gpuMemcpyHostToDevice ); - const int thrPerEvt = - CPPProcess::npar * - CPPProcess::np4; // AV: transpose alg does 1 element per thread (NOT 1 - // event per thread) - // const int thrPerEvt = 1; // AV: try new alg with 1 event per thread... - // this seems slower - gpuLaunchKernel( dev_transposeMomentaF2C, m_gpublocks * thrPerEvt, m_gputhreads, m_devMomentaF.data(), m_devMomentaC.data(), m_nevt ); - } - if constexpr( std::is_same_v ) - { - memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); - } - else - { - std::copy( gs, gs + m_nevt, m_hstGs.data() ); - std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); - std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); - } - const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); - if( useChannelIds ) - memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); - // else ... // no need to initialize m_hstChannel: it is allocated with - // gpuMallocHost and NOT initialized in PinnedHostBufferBase, but it is NOT - // used later on - // initialise iflavorVec - memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); - copyDeviceFromHost( m_devGs, m_hstGs ); - copyDeviceFromHost( m_devRndHel, m_hstRndHel ); - copyDeviceFromHost( m_devRndCol, m_hstRndCol ); - if( useChannelIds ) copyDeviceFromHost( m_devChannelIds, m_hstChannelIds ); - copyDeviceFromHost( m_devIflavorVec, m_hstIflavorVec ); - if( m_nGoodHel < 0 ) - { - m_nGoodHel = m_pmek->computeGoodHelicities(); - if( m_nGoodHel < 0 ) - throw std::runtime_error( - "Bridge gpu_sequence: computeGoodHelicities returned nGoodHel<0" ); - } - if( goodHelOnly ) return; - m_pmek->computeMatrixElements( useChannelIds ); - copyHostFromDevice( m_hstMEs, m_devMEs ); -#ifdef MGONGPUCPP_VERBOSE - flagAbnormalMEs( m_hstMEs.data(), m_nevt ); -#endif - copyHostFromDevice( m_hstSelHel, m_devSelHel ); - copyHostFromDevice( m_hstSelCol, m_devSelCol ); - if constexpr( std::is_same_v ) - { - memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); - memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); - memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); - } - else - { - std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); - std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); - std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); - } - } -#endif - -#ifndef MGONGPUCPP_GPUIMPL - template - void Bridge::cpu_sequence( const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool goodHelOnly ) - { - hst_transposeMomentaF2C( momenta, m_hstMomentaC.data(), m_nevt ); - if constexpr( std::is_same_v ) - { - memcpy( m_hstGs.data(), gs, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndHel.data(), rndhel, m_nevt * sizeof( FORTRANFPTYPE ) ); - memcpy( m_hstRndCol.data(), rndcol, m_nevt * sizeof( FORTRANFPTYPE ) ); - } - else - { - std::copy( gs, gs + m_nevt, m_hstGs.data() ); - std::copy( rndhel, rndhel + m_nevt, m_hstRndHel.data() ); - std::copy( rndcol, rndcol + m_nevt, m_hstRndCol.data() ); - } - const bool useChannelIds = ( channelIds != nullptr ) && ( !goodHelOnly ); - if( useChannelIds ) - memcpy( m_hstChannelIds.data(), channelIds, m_nevt * sizeof( unsigned int ) ); - // else ... // no need to initialize m_hstChannel: it is allocated and default - // initialized in HostBufferBase (and it is not used later on anyway) - // initialise iflavorVec - memcpy( m_hstIflavorVec.data(), iflavorVec, m_nevt * sizeof( unsigned int ) ); - if( m_nGoodHel < 0 ) - { - m_nGoodHel = m_pmek->computeGoodHelicities(); - if( m_nGoodHel < 0 ) - throw std::runtime_error( - "Bridge cpu_sequence: computeGoodHelicities returned nGoodHel<0" ); - } - if( goodHelOnly ) return; - m_pmek->computeMatrixElements( useChannelIds ); -#ifdef MGONGPUCPP_VERBOSE - flagAbnormalMEs( m_hstMEs.data(), m_nevt ); -#endif - if constexpr( std::is_same_v ) - { - memcpy( mes, m_hstMEs.data(), m_hstMEs.bytes() ); - memcpy( selhel, m_hstSelHel.data(), m_hstSelHel.bytes() ); - memcpy( selcol, m_hstSelCol.data(), m_hstSelCol.bytes() ); - } - else - { - std::copy( m_hstMEs.data(), m_hstMEs.data() + m_nevt, mes ); - std::copy( m_hstSelHel.data(), m_hstSelHel.data() + m_nevt, selhel ); - std::copy( m_hstSelCol.data(), m_hstSelCol.data() + m_nevt, selcol ); - } - } -#endif - - //-------------------------------------------------------------------------- - // - // Implementations of transposition methods - // - FORTRAN arrays: P_MULTI(0:3, NEXTERNAL, VECSIZE_USED) ==> - // p_multi[nevtF][nparF][np4F] in C++ (AOS) - // - C++ array: momenta[npagM][npar][np4][neppM] with nevt=npagM*neppM (AOSOA) - // - -#ifdef MGONGPUCPP_GPUIMPL - template - __global__ void dev_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool oldImplementation = true; // default: use old implementation - if constexpr( oldImplementation ) - { - // SR initial implementation - constexpr int part = CPPProcess::npar; - constexpr int mome = CPPProcess::np4; - constexpr int strd = MemoryAccessMomenta::neppM; - int pos = blockDim.x * blockIdx.x + threadIdx.x; - int arrlen = nevt * part * mome; - if( pos < arrlen ) - { - int page_i = pos / ( strd * mome * part ); - int rest_1 = pos % ( strd * mome * part ); - int part_i = rest_1 / ( strd * mome ); - int rest_2 = rest_1 % ( strd * mome ); - int mome_i = rest_2 / strd; - int strd_i = rest_2 % strd; - int inpos = ( page_i * strd + strd_i ) // event number - * ( part * mome ) // event size (pos of event) - + part_i * mome // particle inside event - + mome_i; // momentum inside particle - out[pos] = in[inpos]; // F2C (Fortran to C) - } - } - else - { - // AV attempt another implementation with 1 event per thread: this seems - // slower... F-style: AOS[nevtF][nparF][np4F] C-style: - // AOSOA[npagM][npar][np4][neppM] with nevt=npagM*neppM - constexpr int npar = CPPProcess::npar; - constexpr int np4 = CPPProcess::np4; - constexpr int neppM = MemoryAccessMomenta::neppM; - assert( nevt % neppM == - 0 ); // number of events is not a multiple of neppM??? - int ievt = blockDim.x * blockIdx.x + threadIdx.x; - int ipagM = ievt / neppM; - int ieppM = ievt % neppM; - for( int ip4 = 0; ip4 < np4; ip4++ ) - for( int ipar = 0; ipar < npar; ipar++ ) - { - int cpos = ipagM * npar * np4 * neppM + ipar * np4 * neppM + - ip4 * neppM + ieppM; - int fpos = ievt * npar * np4 + ipar * np4 + ip4; - out[cpos] = in[fpos]; // F2C (Fortran to C) - } - } - } -#endif - - template - void hst_transposeMomenta( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool oldImplementation = false; // default: use new implementation - if constexpr( oldImplementation ) - { - // SR initial implementation - constexpr unsigned int part = CPPProcess::npar; - constexpr unsigned int mome = CPPProcess::np4; - constexpr unsigned int strd = MemoryAccessMomenta::neppM; - unsigned int arrlen = nevt * part * mome; - for( unsigned int pos = 0; pos < arrlen; ++pos ) - { - unsigned int page_i = pos / ( strd * mome * part ); - unsigned int rest_1 = pos % ( strd * mome * part ); - unsigned int part_i = rest_1 / ( strd * mome ); - unsigned int rest_2 = rest_1 % ( strd * mome ); - unsigned int mome_i = rest_2 / strd; - unsigned int strd_i = rest_2 % strd; - unsigned int inpos = ( page_i * strd + strd_i ) // event number - * ( part * mome ) // event size (pos of event) - + part_i * mome // particle inside event - + mome_i; // momentum inside particle - if constexpr( F2C ) // needs c++17 and cuda >=11.2 (#333) - out[pos] = in[inpos]; // F2C (Fortran to C) - else - out[inpos] = in[pos]; // C2F (C to Fortran) - } - } - else - { - // AV attempt another implementation: this is slightly faster (better c++ - // pipelining?) [NB! this is not a transposition, it is an AOS to AOSOA - // conversion: if neppM=1, a memcpy is enough] F-style: - // AOS[nevtF][nparF][np4F] C-style: AOSOA[npagM][npar][np4][neppM] with - // nevt=npagM*neppM - constexpr unsigned int npar = CPPProcess::npar; - constexpr unsigned int np4 = CPPProcess::np4; - constexpr unsigned int neppM = MemoryAccessMomenta::neppM; - if constexpr( neppM == 1 && std::is_same_v ) - { - memcpy( out, in, nevt * npar * np4 * sizeof( Tin ) ); - } - else - { - const unsigned int npagM = nevt / neppM; - assert( nevt % neppM == - 0 ); // number of events is not a multiple of neppM??? - for( unsigned int ipagM = 0; ipagM < npagM; ipagM++ ) - for( unsigned int ip4 = 0; ip4 < np4; ip4++ ) - for( unsigned int ipar = 0; ipar < npar; ipar++ ) - for( unsigned int ieppM = 0; ieppM < neppM; ieppM++ ) - { - unsigned int ievt = ipagM * neppM + ieppM; - unsigned int cpos = ipagM * npar * np4 * neppM + - ipar * np4 * neppM + ip4 * neppM + ieppM; - unsigned int fpos = ievt * npar * np4 + ipar * np4 + ip4; - if constexpr( F2C ) - out[cpos] = in[fpos]; // F2C (Fortran to C) - else - out[fpos] = in[cpos]; // C2F (C to Fortran) - } - } - } - } - - template - void hst_transposeMomentaF2C( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool F2C = true; - hst_transposeMomenta( in, out, nevt ); - } - - template - void hst_transposeMomentaC2F( const Tin* in, Tout* out, const unsigned int nevt ) - { - constexpr bool F2C = false; - hst_transposeMomenta( in, out, nevt ); - } - - //-------------------------------------------------------------------------- -} // namespace mg5amcGpu -#endif // BRIDGE_H diff --git a/madgraph/iolibs/template_files/madmatrix/BridgeKernels.cc b/madgraph/iolibs/template_files/madmatrix/BridgeKernels.cc deleted file mode 100644 index 65c97616d8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/BridgeKernels.cc +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "BridgeKernels.h" - -#include "GpuAbstraction.h" -#include "MemoryAccessMomenta.h" - -#include - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E,px,py,pz) - constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - - //-------------------------------------------------------------------------- - - BridgeKernelBase::BridgeKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( nevt ) - , m_bridge( nevt, npar, np4 ) - { - if( m_momenta.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: momenta must be a host array" ); - if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "BridgeKernelBase: matrixElements must be a host array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "BridgeKernelBase: nevt mismatch with matrixElements" ); - } - - //-------------------------------------------------------------------------- -} - -//============================================================================ - -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu -{ - - //-------------------------------------------------------------------------- - - BridgeKernelHost::BridgeKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: Gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt) - : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, nevt) - , m_fortranMomenta( nevt ) - { - } - - //-------------------------------------------------------------------------- - - void BridgeKernelHost::transposeInputMomentaC2F() - { - hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); - } - - //-------------------------------------------------------------------------- - - int BridgeKernelHost::computeGoodHelicities() - { - constexpr bool goodHelOnly = true; - constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering - m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - return m_bridge.nGoodHel(); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelHost::computeMatrixElements( const bool useChannelIds ) - { - constexpr bool goodHelOnly = false; - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - m_bridge.cpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -{ - - //-------------------------------------------------------------------------- - - BridgeKernelDevice::BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: Gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads) - : BridgeKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol, gpublocks * gputhreads) - , m_fortranMomenta( nevt() ) - , m_gpublocks( gpublocks ) - , m_gputhreads( gputhreads ) - { - if( m_gpublocks == 0 ) throw std::runtime_error( "BridgeKernelDevice: gpublocks must be > 0" ); - if( m_gputhreads == 0 ) throw std::runtime_error( "BridgeKernelDevice: gputhreads must be > 0" ); - m_bridge.set_gpugrid( gpublocks, gputhreads ); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelDevice::transposeInputMomentaC2F() - { - hst_transposeMomentaC2F( m_momenta.data(), m_fortranMomenta.data(), nevt() ); - } - - //-------------------------------------------------------------------------- - - int BridgeKernelDevice::computeGoodHelicities() - { - constexpr bool goodHelOnly = true; - constexpr unsigned int* pChannelIds = nullptr; // disable multi-channel for helicity filtering - m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - return m_bridge.nGoodHel(); - } - - //-------------------------------------------------------------------------- - - void BridgeKernelDevice::computeMatrixElements( const bool useChannelIds ) - { - constexpr bool goodHelOnly = false; - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - m_bridge.gpu_sequence( m_fortranMomenta.data(), m_gs.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), goodHelOnly ); - } - - //-------------------------------------------------------------------------- - -} -#endif - -//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/BridgeKernels.h b/madgraph/iolibs/template_files/madmatrix/BridgeKernels.h deleted file mode 100644 index 2d873486a8..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/BridgeKernels.h +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally originally by: A. Valassi (Jan 2022). -// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef BRIDGEKERNELS_H -#define BRIDGEKERNELS_H 1 - -#include "mgOnGpuConfig.h" - -#include "Bridge.h" -#include "MatrixElementKernels.h" -#include "MemoryBuffers.h" - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - //-------------------------------------------------------------------------- - - // A Bridge wrapper base class encapsulating matrix element calculations on a CPU host - class BridgeKernelBase : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - BridgeKernelBase( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~BridgeKernelBase() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - virtual void transposeInputMomentaC2F() = 0; - - protected: - - // The wrapped bridge - Bridge m_bridge; - }; - - //-------------------------------------------------------------------------- - -#ifndef MGONGPUCPP_GPUIMPL - // A Bridge wrapper class encapsulating matrix element calculations on a CPU host - class BridgeKernelHost final : public BridgeKernelBase - { - public: - - // Constructor from existing input and output buffers - BridgeKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~BridgeKernelHost() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - void transposeInputMomentaC2F() override final; - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // The buffer for the input momenta, transposed to Fortran array indexing - HostBufferMomenta m_fortranMomenta; - }; -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - // A Bridge wrapper class encapsulating matrix element calculations on a GPU device - class BridgeKernelDevice : public BridgeKernelBase - { - public: - - // Constructor from existing input and output buffers - BridgeKernelDevice( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t gpublocks, - const size_t gputhreads); - - // Destructor - virtual ~BridgeKernelDevice() {} - - // Transpose input momenta from C to Fortran before the matrix element calculation in the Bridge - void transposeInputMomentaC2F() override final; - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The buffer for the input momenta, transposed to Fortran array indexing - PinnedHostBufferMomenta m_fortranMomenta; - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - }; -#endif - - //-------------------------------------------------------------------------- -} -#endif // BRIDGEKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/ColorData.h b/madgraph/iolibs/template_files/madmatrix/ColorData.h new file mode 100644 index 0000000000..bd0c804db4 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/ColorData.h @@ -0,0 +1,44 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Process-specific color structure, generated once per subprocess: the +// normalized color matrix (for color_sum_cpu/color_sum_gpu) and the +// diagram/channel/config maps for multichannel color selection. + +#ifndef COLORDATA_H +#define COLORDATA_H 1 + +#include "mgOnGpuConfig.h" +#include "ProcessData.h" + +namespace ColorMatrixData +{ + constexpr int ncolor = ProcessData::ncolor; + +%(color_matrix_lines)s +} + +namespace mgOnGpu +{ + // Diagram: C-indexed [0,ndiagrams). Channel (channelId): F-indexed [1,nchannels], + // not all diagrams have one (#919); channelId-1 indexes channel2iconfig. Config + // (iconfig): F-indexed [1,nconfigSDE]; iconfig-1 indexes icolamp. + constexpr unsigned int nchannels = %(nb_diag)i; // may be < ndiagrams, see #919 + static_assert( nchannels <= ProcessData::ndiagrams, "nchannels should be <= ndiagrams" ); // #910 #919 + + // Map channel (C-indexed) to iconfig (F-indexed); -1 = no associated iconfig (#917) + __device__ constexpr int channel2iconfig[%(nb_diag)i] = { +%(channelc2iconfig_lines)s + }; + + // #configs with an associated iconfig for single-diagram enhancement (#917) + constexpr unsigned int nconfigSDE = %(nb_channel)s; + + // Map iconfig (C-indexed) to the mask of allowed colors + __device__ constexpr bool icolamp[%(nb_channel)s][%(nb_color)s] = { +%(is_LC)s + }; +} + +#endif // COLORDATA_H diff --git a/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h b/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h index 12b7d45adf..690d67b68f 100644 --- a/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h +++ b/madgraph/iolibs/template_files/madmatrix/MadgraphTest.h @@ -27,11 +27,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -using mg5amcGpu::CPPProcess; -#else -using mg5amcCpu::CPPProcess; -#endif +using madmatrix::CPPProcess; namespace { diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/MemoryAccessVectors.h deleted file mode 100644 index 8563d8f2f7..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessVectors.h +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MemoryAccessVectors_H -#define MemoryAccessVectors_H 1 - -#include "mgOnGpuConfig.h" - -#include "mgOnGpuVectors.h" - -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu // this is only needed for CPU SIMD vectorization -{ - -#ifdef MGONGPU_CPPSIMD - //-------------------------------------------------------------------------- - - // Cast one non-const fptype_v reference (one vector of neppV fptype values) from one non-const fptype reference (#435), - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned - inline fptype_v& fptypevFromAlignedArray( fptype& ref ) - { - return *reinterpret_cast( &ref ); - } - - inline uint_v& uintvFromAlignedArray( unsigned int& ref ) - { - return *reinterpret_cast( &ref ); - } - - // Cast one const fptype_v reference (one vector of neppV fptype values) from one const fptype reference, - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned - inline const fptype_v& fptypevFromAlignedArray( const fptype& ref ) - { - return *reinterpret_cast( &ref ); - } - - inline const uint_v& uintvFromAlignedArray( const unsigned int& ref ) - { - return *reinterpret_cast( &ref ); - } - - // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, - // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", but that the arrays are not aligned - inline fptype_v fptypevFromUnalignedArray( const fptype& ref ) - { -#if MGONGPU_CPPSIMD == 2 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (2) - *( &ref + 1 ) }; -#elif MGONGPU_CPPSIMD == 4 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (4) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ) }; -#elif MGONGPU_CPPSIMD == 8 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (8) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ), - *( &ref + 4 ), - *( &ref + 5 ), - *( &ref + 6 ), - *( &ref + 7 ) }; -#elif MGONGPU_CPPSIMD == 16 - return fptype_v{ *( &ref ), // explicit initialization of all array elements (16) - *( &ref + 1 ), - *( &ref + 2 ), - *( &ref + 3 ), - *( &ref + 4 ), - *( &ref + 5 ), - *( &ref + 6 ), - *( &ref + 7 ), - *( &ref + 8 ), - *( &ref + 9 ), - *( &ref + 10 ), - *( &ref + 11 ), - *( &ref + 12 ), - *( &ref + 13 ), - *( &ref + 14 ), - *( &ref + 15 ) }; -#else -#error Internal error! Unknown MGONGPU_CPPSIMD value -#endif - } - - // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, - // with no a priori assumption on how the input fptype array should be decoded - template - inline fptype_v fptypevFromArbitraryArray( Functor decoderIeppv ) - { -#if MGONGPU_CPPSIMD == 2 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (2) - decoderIeppv( 1 ) }; -#elif MGONGPU_CPPSIMD == 4 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (4) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ) }; -#elif MGONGPU_CPPSIMD == 8 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (8) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ), - decoderIeppv( 4 ), - decoderIeppv( 5 ), - decoderIeppv( 6 ), - decoderIeppv( 7 ) }; -#elif MGONGPU_CPPSIMD == 16 - return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (16) - decoderIeppv( 1 ), - decoderIeppv( 2 ), - decoderIeppv( 3 ), - decoderIeppv( 4 ), - decoderIeppv( 5 ), - decoderIeppv( 6 ), - decoderIeppv( 7 ), - decoderIeppv( 8 ), - decoderIeppv( 9 ), - decoderIeppv( 10 ), - decoderIeppv( 11 ), - decoderIeppv( 12 ), - decoderIeppv( 13 ), - decoderIeppv( 14 ), - decoderIeppv( 15 ) }; -#else -#error Internal error! Unknown MGONGPU_CPPSIMD value -#endif - } - - //-------------------------------------------------------------------------- -#endif - -} // end namespace -#endif - -#endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/ProcessData.h b/madgraph/iolibs/template_files/madmatrix/ProcessData.h new file mode 100644 index 0000000000..6f17fe9858 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/ProcessData.h @@ -0,0 +1,51 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Process-specific compile-time constants, generated once per subprocess. +// This is the single source of truth for these values: CPPProcess (P1-generated) +// and the backend-owned driver code (backend/{cpu,simd,gpu}/) both read from here +// instead of duplicating the literals. + +#ifndef PROCESSDATA_H +#define PROCESSDATA_H 1 + +namespace ProcessData +{ + constexpr int np4 = 4; // dimensions of a 4-momentum (E,px,py,pz) + constexpr int nw6 = %(nwave)d; // dimensions of each wavefunction (HELAS KEK 91-11) + constexpr int npari = %(nincoming)d; + constexpr int nparf = %(noutcoming)d; + constexpr int npar = npari + nparf; + constexpr int ncomb = %(nbhel)d; // #helicity combinations + constexpr int ndiagrams = %(ndiagrams)d; + constexpr int ncolor = %(ncolor)d; + constexpr int nmaxflavor = %(nmaxflavor)d; + constexpr int nwf = %(nwf)d; // #wavefunctions = #external (npar) + #internal (see #644) + constexpr int nproc = %(nproc)d; // 2 if this process has a mirror process, else 1 + constexpr int proc_id = %(proc_id)d; + constexpr int helcolDenominators[1] = { %(den_factors)s }; // spin/color/identical-particle denominators + + // SM independent parameters/couplings/flavor-couplings used by this process + // (see #823: nIPC/nIPD/nIPF may vary per P1, unlike nicoup which is model-wide) + constexpr int nIPD = %(nipd)d; + constexpr int nIPC = %(nipc)d; + constexpr int nIPF = %(nipf)d; + constexpr int nDPF = %(ndpf)d; + + // Helicities for the process [NB do keep 'static' for this constexpr array, see issue #283] + // *** NB There is no automatic check yet that these are in the same order as Fortran! #569 *** +%(thel_lines)s + + // Host-side flavor table: single source of truth for PDG ids (used by both + // CPPProcess's constructor copy into cFlavors and CPPProcess::flavorPDG). +%(tflavors_lines)s +} + +// Process identification for test/debug tooling. Must stay #define (not +// constexpr): used for macro token-pasting (test suite names) and +// stringification, e.g. TEST( XTESTID( MG_EPOCH_PROCESS_ID ), ... ). +#define MG_EPOCH_PROCESS_ID %(processid_uppercase)s +#define MG_EPOCH_REFERENCE_FILE_NAME "../../test/ref/dump_CPUTest.%(processid)s.txt" + +#endif // PROCESSDATA_H diff --git a/madgraph/iolibs/template_files/madmatrix/ProcessTables.h b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h new file mode 100644 index 0000000000..93455ba807 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/ProcessTables.h @@ -0,0 +1,47 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Process-specific compile-time data tables, generated once per subprocess, +// for backend-owned code (backend/{cpu,simd,gpu}/SigmaKin.cc) that can't take +// this data as a runtime parameter without losing constexpr-ness. Unlike +// ProcessData.h these are arrays, not scalars, and unlike ColorData.h +// there's no backend-conditional algorithm consuming them directly - it's +// pulled in via ProcessTables::name from backend-owned function bodies. +// +// Namespace-wrapped (unlike ProcessData.h) because it needs FLV_COUPLING, +// which is itself namespace-wrapped too (madmatrix::, see Parameters.h). + +#ifndef PROCESSTABLES_H +#define PROCESSTABLES_H 1 + +#include "mgOnGpuConfig.h" // for __device__ +#include "ProcessData.h" +#include "Parameters.h" // for FLV_COUPLING::max_flavor + +namespace madmatrix +{ + namespace ProcessTables + { + using ProcessData::nDPF; + constexpr int nMF = FLV_COUPLING::max_flavor; // max #merged flavors for any merged particle in the model + + // Dependent (event-by-event, running-alphas) flavor couplings: partner + // indices and the per-flavor idcoup are pure compile-time constants (the + // complex values are gathered per event page in calculate_jamps). +%(cdpfdecl)s + + // Decay-aware identical-particle (broken-)symmetry factor data, shared with + // the Fortran / standalone_cpp exporters (_get_broken_symmetry_data). + constexpr int broken_sym_ncomponents = %(broken_sym_ncomponents)d; + constexpr int broken_sym_nentries = %(broken_sym_nentries)d; + __device__ constexpr int broken_sym_component_starts[broken_sym_ncomponents] = { %(broken_sym_component_starts)s }; + __device__ constexpr int broken_sym_component_ends[broken_sym_ncomponents] = { %(broken_sym_component_ends)s }; + __device__ constexpr int broken_sym_component_old_factors[broken_sym_ncomponents] = { %(broken_sym_component_old_factors)s }; + __device__ constexpr int broken_sym_pid_list[broken_sym_nentries] = { %(broken_sym_pid_list)s }; + __device__ constexpr int broken_sym_block_starts[broken_sym_nentries] = { %(broken_sym_block_starts)s }; + __device__ constexpr int broken_sym_block_lengths[broken_sym_nentries] = { %(broken_sym_block_lengths)s }; + } +} + +#endif // PROCESSTABLES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc new file mode 100644 index 0000000000..29efce440d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.cc @@ -0,0 +1,187 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "CrossSectionKernels.h" + +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessWeights.h" +#include "MemoryBuffers.h" + +#include +#include + +// ****************************************************************************************** +// *** NB: Disabling fast math is essential here, otherwise results are undefined *** +// *** NB: This file CrossSectionKernels.cc IS BUILT WITH -fno-fast-math in the Makefile! *** +// *** NB: Attempts with __attribute__((optimize("-fno-fast-math"))) were unsatisfactory *** +// ****************************************************************************************** + +inline bool +fp_is_nan( const fptype& fp ) +{ + //#pragma clang diagnostic push + //#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) + return std::isnan( fp ); // always false for clang in fast math mode (tautological compare)? + //#pragma clang diagnostic pop +} + +inline bool +fp_is_abnormal( const fptype& fp ) +{ + if( fp_is_nan( fp ) ) return true; + if( fp != fp ) return true; + return false; +} + +inline bool +fp_is_zero( const fptype& fp ) +{ + if( fp == 0 ) return true; + return false; +} + +// See https://en.cppreference.com/w/cpp/numeric/math/FP_categories +inline const char* +fp_show_class( const fptype& fp ) +{ + switch( std::fpclassify( fp ) ) + { + case FP_INFINITE: return "Inf"; + case FP_NAN: return "NaN"; + case FP_NORMAL: return "normal"; + case FP_SUBNORMAL: return "subnormal"; + case FP_ZERO: return "zero"; + default: return "unknown"; + } +} + +inline void +debug_me_is_abnormal( const fptype& me, size_t ievtALL ) +{ + std::cout << "DEBUG[" << ievtALL << "]" + << " ME=" << me + << " fpisabnormal=" << fp_is_abnormal( me ) + << " fpclass=" << fp_show_class( me ) + << " (me==me)=" << ( me == me ) + << " (me==me+1)=" << ( me == me + 1 ) + << " isnan=" << fp_is_nan( me ) + << " isfinite=" << std::isfinite( me ) + << " isnormal=" << std::isnormal( me ) + << " is0=" << ( me == 0 ) + << " is1=" << ( me == 1 ) + << " abs(ME)=" << std::abs( me ) + << " isnan=" << fp_is_nan( std::abs( me ) ) + << std::endl; +} + +//============================================================================ + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ) + { + for( unsigned int ievt = 0; ievt < nevt; ievt++ ) + { + if( fp_is_abnormal( hstMEs[ievt] ) ) + { + std::cout << "WARNING! flagging abnormal ME for ievt=" << ievt << std::endl; + hstMEs[ievt] = std::sqrt( -1. ); + } + } + } + + //-------------------------------------------------------------------------- + + CrossSectionKernelHost::CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t nevt ) + : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) + , NumberOfEvents( nevt ) + { + if( m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: samplingWeights must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: matrixElements must be a host array" ); + if( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with samplingWeights" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelHost::updateEventStatistics( const bool debug ) + { + EventStatistics stats; // new statistics for the new nevt events + // FIRST PASS: COUNT ALL/ABN/ZERO EVENTS, COMPUTE MIN/MAX, COMPUTE REFS AS MEANS OF SIMPLE SUMS + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + const size_t ievtALL = m_iter * nevt() + ievt; + // The following events are abnormal in a run with "-p 2048 256 12 -d" + // - check.exe/commonrand: ME[310744,451171,3007871,3163868,4471038,5473927] with fast math + // - check.exe/curand: ME[578162,1725762,2163579,5407629,5435532,6014690] with fast math + // - gcheck.exe/curand: ME[596016,1446938] with fast math + // Debug NaN/abnormal issues + //if ( ievtALL == 310744 ) // this ME is abnormal both with and without fast math + // debug_me_is_abnormal( me, ievtALL ); + //if ( ievtALL == 5473927 ) // this ME is abnormal only with fast math + // debug_me_is_abnormal( me, ievtALL ); + stats.nevtALL++; + if( fp_is_abnormal( me ) ) + { + if( debug ) // only printed out with "-p -d" (matrixelementALL is not filled without -p) + std::cout << "WARNING! ME[" << ievtALL << "] is NaN/abnormal" << std::endl; + stats.nevtABN++; + continue; + } + if( fp_is_zero( me ) ) stats.nevtZERO++; + stats.minME = std::min( stats.minME, (double)me ); + stats.maxME = std::max( stats.maxME, (double)me ); + stats.minWG = std::min( stats.minWG, (double)wg ); + stats.maxWG = std::max( stats.maxWG, (double)wg ); + stats.sumMEdiff += me; // NB stats.refME is 0 here + stats.sumWGdiff += wg; // NB stats.refWG is 0 here + } + stats.refME = stats.meanME(); // draft ref + stats.refWG = stats.meanWG(); // draft ref + stats.sumMEdiff = 0; + stats.sumWGdiff = 0; + // SECOND PASS: IMPROVE MEANS FROM SUMS OF DIFFS TO PREVIOUS REF, UPDATE REF + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + if( fp_is_abnormal( me ) ) continue; + stats.sumMEdiff += ( me - stats.refME ); + stats.sumWGdiff += ( wg - stats.refWG ); + } + stats.refME = stats.meanME(); // final ref + stats.refWG = stats.meanWG(); // final ref + stats.sumMEdiff = 0; + stats.sumWGdiff = 0; + // THIRD PASS: COMPUTE STDDEV FROM SQUARED SUMS OF DIFFS TO REF + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + if( fp_is_abnormal( me ) ) continue; + stats.sqsMEdiff += std::pow( me - stats.refME, 2 ); + stats.sqsWGdiff += std::pow( wg - stats.refWG, 2 ); + } + // FOURTH PASS: UPDATE THE OVERALL STATS BY ADDING THE NEW STATS + m_stats += stats; + // Increment the iterations counter + m_iter++; + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h similarity index 72% rename from madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.h rename to madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h index d1e29b7999..59a4c098ec 100644 --- a/madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/CrossSectionKernels.h @@ -14,11 +14,7 @@ //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- @@ -96,43 +92,6 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- - /* -#ifdef MGONGPUCPP_GPUIMPL - // A class encapsulating the calculation of event statistics on a GPU device - class CrossSectionKernelDevice : public CrossSectionKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - CrossSectionKernelDevice( const BufferWeights& samplingWeights, // input: sampling weights - const BufferMatrixElements& matrixElements, // input: matrix elements - EventStatistics& stats, // output: event statistics - const size_t gpublocks, - const size_t gputhreads ); - - // Destructor - virtual ~CrossSectionKernelDevice(){} - - // Reset gpublocks and gputhreads - void setGrid( const size_t gpublocks, const size_t gputhreads ); - - // Update event statistics - void updateEventStatistics( const bool debug=false ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return true; } - - private: - - // The number of blocks in the GPU grid - size_t m_gpublocks; - - // The number of threads in the GPU grid - size_t m_gputhreads; - - }; -#endif - */ //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h similarity index 97% rename from madgraph/iolibs/template_files/madmatrix/EventStatistics.h rename to madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h index 4a1fd8464d..df62a74918 100644 --- a/madgraph/iolibs/template_files/madmatrix/EventStatistics.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/EventStatistics.h @@ -9,7 +9,7 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" // for npar (meGeVexponent) +#include "ProcessData.h" // for npar (meGeVexponent) #include #include @@ -17,11 +17,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- @@ -142,7 +138,7 @@ namespace mg5amcCpu void printout( std::ostream& out ) const { const EventStatistics& s = *this; - constexpr int meGeVexponent = -( 2 * CPPProcess::npar - 8 ); + constexpr int meGeVexponent = -( 2 * ProcessData::npar - 8 ); out << s.tag << "NumMatrixElems(notAbnormal) = " << s.nevtOK() << std::endl << std::scientific // fixed format: affects all floats (default precision: 6) << s.tag << "MeanMatrixElemValue = ( " << s.meanME() diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc new file mode 100644 index 0000000000..3aeeadd9e8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.cc @@ -0,0 +1,285 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "MatrixElementKernels.h" + +#include "ProcessData.h" +#include "SigmaKin.h" +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" + +#include +#include // for fetestexcept +#include +#include + +//============================================================================ + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + MatrixElementKernelBase::MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol ) // output: color selection + : m_momenta( momenta ) + , m_gs( gs ) + , m_iflavorVec( iflavorVec ) + , m_rndhel( rndhel ) + , m_rndcol( rndcol ) + , m_channelIds( channelIds ) + , m_matrixElements( matrixElements ) + , m_selhel( selhel ) + , m_selcol( selcol ) +#ifdef MGONGPU_CHANNELID_DEBUG + , m_nevtProcessedByChannel() + , m_tag() +#endif + { + //std::cout << "DEBUG: MatrixElementKernelBase ctor " << this << std::endl; +#ifdef MGONGPU_CHANNELID_DEBUG + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + m_nevtProcessedByChannel[channelId] = 0; +#endif + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelBase::~MatrixElementKernelBase() + { + //std::cout << "DEBUG: MatrixElementKernelBase dtor " << this << std::endl; +#ifdef MGONGPU_CHANNELID_DEBUG + MatrixElementKernelBase::dumpNevtProcessedByChannel(); +#endif +#ifdef MGONGPUCPP_VERBOSE + MatrixElementKernelBase::dumpSignallingFPEs(); +#endif + } + + //-------------------------------------------------------------------------- + +#ifdef MGONGPU_CHANNELID_DEBUG + void MatrixElementKernelBase::updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ) + { + if( pHstChannelIds != nullptr ) + { + //std::cout << "DEBUG " << this << ": not nullptr " << nevt << std::endl; + for( unsigned int ievt = 0; ievt < nevt; ievt++ ) + { + const size_t channelId = pHstChannelIds[ievt]; // Fortran indexing + //assert( channelId > 0 ); + //assert( channelId < ProcessData::ndiagrams ); + m_nevtProcessedByChannel[channelId]++; + } + } + else + { + //std::cout << "DEBUG " << this << ": nullptr " << std::endl; + m_nevtProcessedByChannel[0] += nevt; + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPU_CHANNELID_DEBUG + void MatrixElementKernelBase::dumpNevtProcessedByChannel() + { + size_t nevtProcessed = 0; + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + nevtProcessed += m_nevtProcessedByChannel[channelId]; + std::ostringstream sstr; + sstr << " {"; + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + { + if( m_nevtProcessedByChannel[channelId] > 0 ) + { + if( sstr.str() != " {" ) sstr << ","; + if( channelId == 0 ) + sstr << " no-multichannel"; + else + sstr << " " << channelId; + sstr << " : " << m_nevtProcessedByChannel[channelId]; + } + } + sstr << " }"; + std::cout << "DEBUG: MEK " << this; + if( m_tag != "" ) std::cout << " " << m_tag; + std::cout << " processed " << nevtProcessed << " events across " << ProcessData::ndiagrams << " channels" << sstr.str() << std::endl; + } +#endif + + //-------------------------------------------------------------------------- + + void MatrixElementKernelBase::dumpSignallingFPEs() + { + // New strategy for issue #831: add a final report of FPEs + // Note: normally only underflow will be reported here (inexact is switched off because it would almost always signal; + // divbyzero, invalid and overflow are configured by feenablexcept to send a SIGFPE signal, and are normally fixed in the code) + // Note: this is now called in the individual destructors of MEK classes rather than in that of MatrixElementKernelBase(#837) + std::string fpes; + if( std::fetestexcept( FE_DIVBYZERO ) ) fpes += " FE_DIVBYZERO"; + if( std::fetestexcept( FE_INVALID ) ) fpes += " FE_INVALID"; + if( std::fetestexcept( FE_OVERFLOW ) ) fpes += " FE_OVERFLOW"; + if( std::fetestexcept( FE_UNDERFLOW ) ) fpes += " FE_UNDERFLOW"; + //if( std::fetestexcept( FE_INEXACT ) ) fpes += " FE_INEXACT"; // do not print this out: this would almost always signal! + if( fpes == "" ) + std::cout << "INFO: No Floating Point Exceptions have been reported" << std::endl; + else + std::cerr << "INFO: The following Floating Point Exceptions have been reported:" << fpes << std::endl; + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +namespace madmatrix +{ + + //-------------------------------------------------------------------------- + + MatrixElementKernelHost::MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt ) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( nevt ) + , m_couplings( nevt ) + , m_numerators( nevt * ProcessData::ndiagrams ) + , m_denominators( nevt ) + { + //std::cout << "DEBUG: MatrixElementKernelHost::ctor " << this << std::endl; + if( m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: momenta must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: matrixElements must be a host array" ); + if( m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: channelIds must be a device array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with matrixElements" ); + if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with channelIds" ); + if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with iflavorVec" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MatrixElementKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Fail gently and avoid "Illegal instruction (core dumped)" if the host does not support the SIMD used in the ME calculation + // Note: this prevents a crash on pmpe04 but not on some github CI nodes? + // [NB: SIMD vectorization in mg5amc C++ code is only used in the ME calculation below MatrixElementKernelHost!] + if( !MatrixElementKernelHost::hostSupportsSIMD() ) + throw std::runtime_error( "Host does not support the SIMD implementation of MatrixElementKernelsHost" ); + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelHost::~MatrixElementKernelHost() + { + //std::cout << "DEBUG: MatrixElementKernelBase::dtor " << this << std::endl; + } + + //-------------------------------------------------------------------------- + + int MatrixElementKernelHost::computeGoodHelicities() + { + HostBufferHelicityMask hstIsGoodHel( ProcessData::ncomb ); + // ... 0d1. Compute good helicity mask on the host + computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); + sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_numerators.data(), m_denominators.data(), hstIsGoodHel.data(), nevt() ); + // ... 0d2. Copy good helicity list to static memory on the host + // [FIXME! REMOVE THIS STATIC THAT BREAKS MULTITHREADING?] + return sigmaKin_setGoodHel( hstIsGoodHel.data() ); + } + + //-------------------------------------------------------------------------- + + void MatrixElementKernelHost::computeMatrixElements( const bool useChannelIds ) + { + computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_numerators.data(), m_denominators.data(), nullptr, true, nevt() ); +#ifdef MGONGPU_CHANNELID_DEBUG + //std::cout << "DEBUG: MatrixElementKernelHost::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; + MatrixElementKernelBase::updateNevtProcessedByChannel( pChannelIds, nevt() ); +#endif + } + + //-------------------------------------------------------------------------- + + // Does this host system support the SIMD used in the matrix element calculation? + bool MatrixElementKernelHost::hostSupportsSIMD( const bool verbose ) + { +#if defined __AVX512VL__ + bool known = true; + bool ok = __builtin_cpu_supports( "avx512vl" ); + const std::string tag = "skylake-avx512 (AVX512VL)"; +#elif defined __AVX2__ + bool known = true; + bool ok = __builtin_cpu_supports( "avx2" ); + const std::string tag = "haswell (AVX2)"; +#elif defined __SSE4_2__ +#ifdef __PPC__ + // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html + bool known = true; + bool ok = __builtin_cpu_supports( "vsx" ); + const std::string tag = "powerpc vsx (128bit as in SSE4.2)"; +#elif defined( __x86_64__ ) || defined( __i386__ ) + bool known = true; + bool ok = __builtin_cpu_supports( "sse4.2" ); + const std::string tag = "nehalem (SSE4.2)"; +#else // AV FIXME! Added by OM for Mac, should identify the correct __xxx__ flag that should be targeted + // DM now we have an explicit NEON target for ARM + bool known = false; // __builtin_cpu_supports is not supported + bool ok = true; // this is just an assumption! + const std::string tag = "simd arch not defined"; +#endif +#elif defined __ARM_NEON // consider using __BUILTIN_CPU_SUPPORTS__ + bool known = false; // __builtin_cpu_supports is not supported + // See https://stackoverflow.com/q/62783908 + // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu + bool ok = true; // this is just an assumption! + const std::string tag = "arm neon (128bit as in SSE4.2)"; +#else + bool known = true; + bool ok = true; + const std::string tag = "none"; +#endif + if( verbose ) + { + if( tag == "none" ) + std::cout << "INFO: The application does not require the host to support any AVX feature" << std::endl; + else if( ok && known ) + std::cout << "INFO: The application is built for " << tag << " and the host supports it" << std::endl; + else if( ok ) + std::cout << "WARNING: The application is built for " << tag << " but it is unknown if the host supports it" << std::endl; + else + std::cout << "ERROR! The application is built for " << tag << " but the host does not support it" << std::endl; + } + return ok; + } + + //-------------------------------------------------------------------------- + +} + +//============================================================================ + + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h new file mode 100644 index 0000000000..e7cf857b47 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MatrixElementKernels.h @@ -0,0 +1,158 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MATRIXELEMENTKERNELS_H +#define MATRIXELEMENTKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryBuffers.h" + +#include +#include + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + // A base class encapsulating matrix element calculations on a CPU host or on a GPU device + class MatrixElementKernelBase //: virtual public IMatrixElementKernel + { + protected: + + // Constructor from existing input and output buffers + MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol); // output: color selection + + public: + + // Destructor + virtual ~MatrixElementKernelBase(); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + virtual int computeGoodHelicities() = 0; + + // Compute matrix elements + virtual void computeMatrixElements( const bool useChannelIds ) = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + // Dump signalling FPEs (#831 and #837) + static void dumpSignallingFPEs(); + +#ifdef MGONGPU_CHANNELID_DEBUG + // Add a MEK identifier for the channelId debug printout + void setTagForNevtProcessedByChannel( const std::string& tag ) { m_tag = tag; } + + protected: + // Update number of events processed by channel + void updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ); + + // Dump number of events processed by channel + void dumpNevtProcessedByChannel(); +#endif + + protected: + + // The buffer for the input momenta + const BufferMomenta& m_momenta; + + // The buffer for the gs to calculate the alphaS values + const BufferGs& m_gs; + + // The buffer for the flavor indices for the flavor combination + const BufferIflavorVec& m_iflavorVec; + + // The buffer for the random numbers for helicity selection + const BufferRndNumHelicity& m_rndhel; + + // The buffer for the random numbers for color selection + const BufferRndNumColor& m_rndcol; + + // The buffer for the channel ids for single-diagram enhancement + const BufferChannelIds& m_channelIds; + + // The buffer for the output matrix elements + BufferMatrixElements& m_matrixElements; + + // The buffer for the output helicity selection + BufferSelectedHelicity& m_selhel; + + // The buffer for the output color selection + BufferSelectedColor& m_selcol; + +#ifdef MGONGPU_CHANNELID_DEBUG + // The events-per-channel counter for debugging + std::map m_nevtProcessedByChannel; + + // The tag for events-per-channel debugging + std::string m_tag; +#endif + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating matrix element calculations on a CPU host + class MatrixElementKernelHost final : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~MatrixElementKernelHost(); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // Does this host system support the SIMD used in the matrix element calculation? + // [NB: this is private, SIMD vectorization in mg5amc C++ code is currently only used in the ME calculations below MatrixElementKernelHost!] + static bool hostSupportsSIMD( const bool verbose = false ); // ZW: default verbose false + + private: + + // The buffer for the event-by-event couplings that depends on alphas QCD + HostBufferCouplings m_couplings; + + // The buffer for the event-by-event numerators of multichannel factors + HostBufferNumerators m_numerators; + + // The buffer for the event-by-event denominators of multichannel factors + HostBufferDenominators m_denominators; + }; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- +} +#endif // MATRIXELEMENTKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h similarity index 96% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessAmplitudes.h rename to madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h index f41e40bf82..669ed8ead5 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessAmplitudes.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessAmplitudes.h @@ -15,12 +15,8 @@ #define MGONGPU_TRIVIAL_AMPLITUDES 1 -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -160,6 +156,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h new file mode 100644 index 0000000000..0b1c1162a3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessChannelIds.h @@ -0,0 +1,115 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Dec 2023, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessChannelIds_H +#define MemoryAccessChannelIds_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for channel ids + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessChannelIdsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] + static __host__ __device__ inline unsigned int* + ieventAccessRecord( unsigned int* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline unsigned int& + decodeRecord( unsigned int* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessChannelIds : public MemoryAccessChannelIdsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] + static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessChannelIds + { + public: + + // Expose selected functions from MemoryAccessChannelIds + static constexpr auto ieventAccessRecordConst = MemoryAccessChannelIds::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] + static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] + static __host__ __device__ inline const uint_sv& + kernelAccessConst( const unsigned int* buffer ) + { + const unsigned int& out = kernelAccessConst_s( buffer ); + return out; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessChannelIds HostAccessChannelIds; + typedef KernelAccessChannelIds DeviceAccessChannelIds; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h new file mode 100644 index 0000000000..491c6f8a5a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplings.h @@ -0,0 +1,240 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessCouplings_H +#define MemoryAccessCouplings_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM +#include "MemoryBuffers.h" // for HostBufferCouplings::isaligned + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for couplings + // This implementation uses an AOSOA[npagC][ndcoup][nx2][neppC] "super-buffer" where nevt=npagC*neppC + // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessCouplingsBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the coupling AOSOA memory buffer layout + static constexpr int neppC = MemoryAccessMomentaBase::neppM; // use the same AOSOA striding as for momenta + + // SANITY CHECK: check that neppC is a power of two + static_assert( ispoweroftwo( neppC ), "neppC is not a power of 2" ); + + //-------------------------------------------------------------------------- + // ** NB! A single super-buffer AOSOA[npagC][ndcoup][nx2][neppC] includes data for ndcoup different couplings ** + // ** NB! The ieventAccessRecord and kernelAccess functions refer to the buffer for one individual coupling ** + // ** NB! Use idcoupAccessBuffer to add a fixed offset and locate the buffer for one given individual coupling ** + //-------------------------------------------------------------------------- + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (non-const) ===> fptype* idcoupAccessBuffer( fptype* buffer, const int idcoup ) <===] + // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? + static __host__ __device__ inline fptype* + idcoupAccessBuffer( fptype* buffer, // input "super-buffer" + const int idcoup ) + { + constexpr int ipagC = 0; + constexpr int ieppC = 0; + constexpr int ix2 = 0; + // NB! this effectively adds an offset "idcoup * nx2 * neppC" + return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] + } + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (const) ===> const fptype* idcoupAccessBufferConst( const fptype* buffer, const int idcoup ) <===] + // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? + static __host__ __device__ inline const fptype* + idcoupAccessBufferConst( const fptype* buffer, // input "super-buffer" + const int idcoup ) + { + return idcoupAccessBuffer( const_cast( buffer ), idcoup ); + } + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of couplings that dependent on the running alphas QCD in this specific process + static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagC = ievt / neppC; // #event "C-page" + const int ieppC = ievt % neppC; // #event in the current event C-page + constexpr int idcoup = 0; + constexpr int ix2 = 0; + return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ix2 ) + { + constexpr int ipagC = 0; + constexpr int ieppC = 0; + // NB! the offset "idcoup * nx2 * neppC" has been added in idcoupAccessBuffer + constexpr int idcoup = 0; + return buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC]; // AOSOA[ipagC][idcoup][ix2][ieppC] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessCouplings : public MemoryAccessCouplingsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessCouplings + { + public: + + // Expose selected functions from MemoryAccessCouplingsBase + static constexpr auto idcoupAccessBuffer = MemoryAccessCouplingsBase::idcoupAccessBuffer; + static constexpr auto idcoupAccessBufferConst = MemoryAccessCouplingsBase::idcoupAccessBufferConst; + + // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. + // For dependent (event-by-event, running-alphas) couplings the value is an AOSOA record [nx2][neppC] + // (real and imaginary SIMD lanes), so consecutive flavor slots are nx2*neppC fptype's apart. + static constexpr int flv_stride = MemoryAccessCouplingsBase::neppC * mgOnGpu::nx2; + + // Expose selected functions from MemoryAccessCouplings + static constexpr auto ieventAccessRecordConst = MemoryAccessCouplings::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2_s = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2Const_s = + KernelAccessHelper::template kernelAccessFieldConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccessIx2( fptype* buffer, + const int ix2 ) + { + fptype& out = kernelAccessIx2_s( buffer, ix2 ); + return out; + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessIx2Const( const fptype* buffer, + const int ix2 ) + { + return kernelAccessIx2( const_cast( buffer ), ix2 ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non const, SCALAR OR VECTOR) ===> cxtype_sv_ref kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline cxtype_sv_ref + kernelAccess( fptype* buffer ) + { + /* + fptype_sv& real = kernelAccessIx2( buffer, 0 ); + fptype_sv& imag = kernelAccessIx2( buffer, 1 ); + printf( "C_ACCESS::kernelAccess: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); + return cxtype_sv_ref( real, imag ); + */ + return cxtype_sv_ref( kernelAccessIx2( buffer, 0 ), + kernelAccessIx2( buffer, 1 ) ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] + static __host__ __device__ inline cxtype_sv + kernelAccessConst( const fptype* buffer ) + { + /* + const fptype_sv& real = kernelAccessIx2Const( buffer, 0 ); + const fptype_sv& imag = kernelAccessIx2Const( buffer, 1 ); + printf( "C_ACCESS::kernelAccessConst: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); + return cxtype_sv( real, imag ); + */ + return cxtype_sv( kernelAccessIx2Const( buffer, 0 ), + kernelAccessIx2Const( buffer, 1 ) ); + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessCouplings HostAccessCouplings; + typedef KernelAccessCouplings DeviceAccessCouplings; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h new file mode 100644 index 0000000000..3eadf6f393 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessCouplingsFixed.h @@ -0,0 +1,84 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Apr 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessCouplingsFixed_H +#define MemoryAccessCouplingsFixed_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +//#include "MemoryAccessHelpers.h" + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for fixed couplings + // This implementation uses a STRUCT[ndcoup][nx2] "super-buffer" layout: in practice, the cIPC global array + // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling + // [If many implementations are used, a suffix _Sv1 should be appended to the class name] + class MemoryAccessCouplingsFixedBase //_Sv1 + { + public: + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (const) ===> const fptype* iicoupAccessBufferConst( const fptype* buffer, const int iicoup ) <===] + static __host__ __device__ inline const fptype* + iicoupAccessBufferConst( const fptype* buffer, // input "super-buffer": in practice, the cIPC global array + const int iicoup ) + { + constexpr int ix2 = 0; + // NB! this effectively adds an offset "iicoup * nx2" + return &( buffer[iicoup * nx2 + ix2] ); // STRUCT[idcoup][ix2] + } + + private: + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessCouplingsFixed + { + public: + + // Expose selected functions from MemoryAccessCouplingsFixedBase + static constexpr auto iicoupAccessBufferConst = MemoryAccessCouplingsFixedBase::iicoupAccessBufferConst; + + // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. + // For fixed (independent) couplings the value is a single scalar complex (real,imag): nx2 fptype's, broadcast across the SIMD vector. + static constexpr int flv_stride = mgOnGpu::nx2; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] + static __host__ __device__ inline const cxtype_sv + kernelAccessConst( const fptype* buffer ) + { + // TRIVIAL ACCESS to fixed-couplings buffers! + //return cxmake( fptype_sv{ buffer[0] }, fptype_sv{ buffer[1] } ); // NO! BUG #339! + const fptype_sv r_sv = fptype_sv{ 0 } + buffer[0]; + const fptype_sv i_sv = fptype_sv{ 0 } + buffer[1]; + return cxmake( r_sv, i_sv ); // ugly but effective + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessCouplingsFixed HostAccessCouplingsFixed; + typedef KernelAccessCouplingsFixed DeviceAccessCouplingsFixed; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h similarity index 77% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessDenominators.h rename to madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h index 205952e514..9e5b87e674 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessDenominators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessDenominators.h @@ -9,12 +9,8 @@ #include "MemoryAccessGs.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -26,6 +22,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h new file mode 100644 index 0000000000..ce27a06de0 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessGs.h @@ -0,0 +1,153 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessGs_H +#define MemoryAccessGs_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for Gs + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessGsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessGs : public MemoryAccessGsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessGs + { + public: + + // Expose selected functions from MemoryAccessGs + static constexpr auto ieventAccessRecord = MemoryAccessGs::ieventAccessRecord; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccess( fptype* buffer ) <===] + static constexpr auto kernelAccess_s = + KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non-const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccess( fptype* buffer ) + { + fptype& out = kernelAccess_s( buffer ); + return out; + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (SCALAR OR VECTOR) ===> fptype_sv* kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv* + kernelAccessP( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + static constexpr auto kernelAccessConst_s = + KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccess( const fptype* buffer ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessConst( const fptype* buffer ) + { + const fptype& out = kernelAccessConst_s( buffer ); + return out; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessGs HostAccessGs; + typedef KernelAccessGs DeviceAccessGs; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h new file mode 100644 index 0000000000..525196d770 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessHelpers.h @@ -0,0 +1,152 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessHelpers_H +#define MemoryAccessHelpers_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +//---------------------------------------------------------------------------- + +// A templated helper class that includes the boilerplate code for MemoryAccess classes +template +class MemoryAccessHelper +{ +public: + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = T::ieventAccessRecord; + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline const FT* + ieventAccessRecordConst( const FT* buffer, + const int ievt ) + { + return ieventAccessRecord( const_cast( buffer ), ievt ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + static constexpr auto decodeRecord = T::decodeRecord; + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + decodeRecordConst( const FT* buffer, + Ts... args ) // variadic template + { + return T::decodeRecord( const_cast( buffer ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessField( fptype* buffer, const ievt, Ts... args ) <===] + template + static __host__ __device__ inline FT& + ieventAccessField( FT* buffer, + const int ievt, + Ts... args ) // variadic template + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + return T::decodeRecord( T::ieventAccessRecord( buffer, ievt ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessFieldConst( const fptype* buffer, const ievt, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + ieventAccessFieldConst( const FT* buffer, + const int ievt, + Ts... args ) // variadic template + { + return ieventAccessField( const_cast( buffer ), ievt, args... ); + } +}; + +//---------------------------------------------------------------------------- + +// A templated helper class that includes the boilerplate code for KernelAccess classes +template +class KernelAccessHelper : public MemoryAccessHelper +{ +public: + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non-const) ===> fptype* kernelAccessRecord( fptype* buffer ) <===] + static __host__ __device__ inline FT* + kernelAccessRecord( FT* buffer ) + { + if constexpr( !onDevice ) // requires c++17 also in CUDA (#333) + { + // FIXME #436: clarify that buffer includes all events on device, and only the record for an event subset on host! + // FIXME #436: am I not assuming that the following line is always identical to buffer for all access classes T? + return T::ieventAccessRecord( buffer, 0 ); + } + else + { + throw std::runtime_error( "kernelAccessRecord on device is only implemented in CUDA" ); + } + } + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const) ===> const fptype* kernelAccessRecordConst( const fptype* buffer ) <===] + static __host__ __device__ inline const FT* + kernelAccessRecordConst( const FT* buffer ) + { + return kernelAccessRecord( const_cast( buffer ) ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessField( fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline FT& + kernelAccessField( FT* buffer, + Ts... args ) // variadic template + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + return T::decodeRecord( kernelAccessRecord( buffer ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessFieldConst( const fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + kernelAccessFieldConst( const FT* buffer, + Ts... args ) // variadic template + { + return kernelAccessField( const_cast( buffer ), args... ); + } + + //-------------------------------------------------------------------------- +}; + +#endif // MemoryAccessHelpers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h new file mode 100644 index 0000000000..a6e5c356e4 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessIflavorVec.h @@ -0,0 +1,114 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: D. Massaro (Jan 2026, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessIflavorVec_H +#define MemoryAccessIflavorVec_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for channel ids + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessIflavorVecBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] + static __host__ __device__ inline unsigned int* + ieventAccessRecord( unsigned int* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline unsigned int& + decodeRecord( unsigned int* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessIflavorVec : public MemoryAccessIflavorVecBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] + static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessIflavorVec + { + public: + + // Expose selected functions from MemoryAccessIflavorVec + static constexpr auto ieventAccessRecordConst = MemoryAccessIflavorVec::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] + static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] + static __host__ __device__ inline const uint_sv& + kernelAccessConst( const unsigned int* buffer ) + { + const unsigned int& out = kernelAccessConst_s( buffer ); + return out; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessIflavorVec HostAccessIflavorVec; + typedef KernelAccessIflavorVec DeviceAccessIflavorVec; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h new file mode 100644 index 0000000000..505312e236 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMatrixElements.h @@ -0,0 +1,136 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessMatrixElements_H +#define MemoryAccessMatrixElements_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for matrix elements + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessMatrixElementsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessMatrixElements : public MemoryAccessMatrixElementsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessMatrixElements + { + public: + + // Expose selected functions from MemoryAccessMatrixElements + static constexpr auto ieventAccessRecord = MemoryAccessMatrixElements::ieventAccessRecord; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccess_s( fptype* buffer ) <===] + static constexpr auto kernelAccess_s = + KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( const fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccess( fptype* buffer ) + { + fptype& out = kernelAccess_s( buffer ); + return out; + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + static constexpr auto kernelAccessConst = + KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessMatrixElements HostAccessMatrixElements; + typedef KernelAccessMatrixElements DeviceAccessMatrixElements; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h new file mode 100644 index 0000000000..78e3bdb14d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessMomenta.h @@ -0,0 +1,209 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessMomenta_H +#define MemoryAccessMomenta_H 1 + +#include "mgOnGpuConfig.h" + +#include "ProcessData.h" +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for momenta + // This implementation uses an AOSOA[npagM][npar][np4][neppM] where nevt=npagM*neppM + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessMomentaBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the momenta AOSOA memory buffer layout + // (these are all best kept as a compile-time constants: see issue #23) + // ----------------------------------------------------------------------------------------------- + // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register + // --- This is relevant to ensure faster access to momenta from C++ memory cache lines + // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded + // --- In practice, neppR, neppM and neppV could now (in principle) all be different + // ----------------------------------------------------------------------------------------------- + static constexpr int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) + + // SANITY CHECK: check that neppM is a power of two + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a 4-momentum + static constexpr int np4 = ProcessData::np4; + + // The number of particles in this physics process + static constexpr int npar = ProcessData::npar; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagM = ievt / neppM; // #event "M-page" + const int ieppM = ievt % neppM; // #event in the current event M-page + constexpr int ip4 = 0; + constexpr int ipar = 0; + return &( buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM] ); // AOSOA[ipagM][ipar][ip4][ieppM] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ip4, const int ipar" and rename "Field" as "Ip4Ipar"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ip4, + const int ipar ) + { + constexpr int ipagM = 0; + constexpr int ieppM = 0; + return buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessMomenta : public MemoryAccessMomentaBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto decodeRecordIp4Ipar = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto decodeRecordIp4IparConst = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIp4Ipar( fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + static constexpr auto ieventAccessIp4Ipar = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + // DEFAULT VERSION + static constexpr auto ieventAccessIp4IparConst = + MemoryAccessHelper::template ieventAccessFieldConst; + + /* + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + // DEBUG VERSION WITH PRINTOUTS + static __host__ __device__ inline const fptype& + ieventAccessIp4IparConst( const fptype* buffer, + const int ievt, + const int ip4, + const int ipar ) + { + const fptype& out = MemoryAccessHelper::template ieventAccessFieldConst( buffer, ievt, ip4, ipar ); + printf( "ipar=%2d ip4=%2d ievt=%8d out=%8.3f\n", ipar, ip4, ievt, out ); + return out; + } + */ + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessMomenta + { + public: + + // Expose selected functions from MemoryAccessMomenta + static constexpr auto ieventAccessRecordConst = MemoryAccessMomenta::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIp4Ipar( fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto kernelAccessIp4Ipar = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // DEFAULT VERSION + static constexpr auto kernelAccessIp4IparConst_s = + KernelAccessHelper::template kernelAccessFieldConst; + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // DEBUG VERSION WITH PRINTOUTS + static __host__ __device__ inline const fptype& + kernelAccessIp4IparConst_s( const fptype* buffer, + const int ip4, + const int ipar ) + { + const fptype& out = KernelAccessHelper::template kernelAccessFieldConst( buffer, ip4, ipar ); + printf( "ipar=%2d ip4=%2d ievt='kernel' out=%8.3f\n", ipar, ip4, out ); + return out; + } + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> fptype_sv kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // FIXME? Eventually return by const reference and support aligned arrays only? + // FIXME? Currently return by value to support also unaligned and arbitrary arrays + static __host__ __device__ inline fptype_sv + kernelAccessIp4IparConst( const fptype* buffer, + const int ip4, + const int ipar ) + { + const fptype& out = kernelAccessIp4IparConst_s( buffer, ip4, ipar ); + return out; + } + + // Is this a HostAccess or DeviceAccess class? + // [this is only needed for a warning printout in rambo.h for nparf==1 #358] + static __host__ __device__ inline constexpr bool + isOnDevice() + { + return onDevice; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessMomenta HostAccessMomenta; + typedef KernelAccessMomenta DeviceAccessMomenta; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h similarity index 77% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessNumerators.h rename to madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h index c8d0343b7e..6df3b20cf4 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessNumerators.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessNumerators.h @@ -9,12 +9,8 @@ #include "MemoryAccessGs.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -26,6 +22,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h similarity index 97% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessRandomNumbers.h rename to madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h index 57f1d02081..a20726428b 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessRandomNumbers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessRandomNumbers.h @@ -9,15 +9,9 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "MemoryAccessHelpers.h" -#ifdef MGONGPUCPP_GPUIMPL -using mg5amcGpu::CPPProcess; -#else -using mg5amcCpu::CPPProcess; -#endif - //---------------------------------------------------------------------------- // A class describing the internal layout of memory buffers for random numbers @@ -40,10 +34,10 @@ class MemoryAccessRandomNumbersBase //_AOSOAv1 friend class KernelAccessHelper; // The number of components of a 4-momentum - static constexpr int np4 = CPPProcess::np4; + static constexpr int np4 = ProcessData::np4; // The number of final state particles in this physics process - static constexpr int nparf = CPPProcess::nparf; + static constexpr int nparf = ProcessData::nparf; //-------------------------------------------------------------------------- // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h new file mode 100644 index 0000000000..45546ddeb3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessVectors.h @@ -0,0 +1,20 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessVectors_H +#define MemoryAccessVectors_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +namespace madmatrix // this is only needed for CPU SIMD vectorization +{ + + +} // end namespace + +#endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h similarity index 96% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessWavefunctions.h rename to madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h index 14a37c7fea..9406a9d75a 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessWavefunctions.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWavefunctions.h @@ -15,12 +15,8 @@ #define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -165,6 +161,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h similarity index 96% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessWeights.h rename to madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h index c56527f581..eece092fa3 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessWeights.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryAccessWeights.h @@ -11,12 +11,8 @@ #include "MemoryAccessHelpers.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -145,6 +141,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h new file mode 100644 index 0000000000..88e90aeafb --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/MemoryBuffers.h @@ -0,0 +1,340 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021, based on earlier work by S. Hageboeck) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryBuffers_H +#define MemoryBuffers_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "ProcessData.h" +#include "Parameters.h" + +#include + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + namespace MemoryBuffers + { + // Process-independent compile-time constants + static constexpr size_t np4 = ProcessData::np4; + static constexpr size_t nw6 = ProcessData::nw6; + static constexpr size_t nx2 = mgOnGpu::nx2; + // Process-dependent compile-time constants + static constexpr size_t nparf = ProcessData::nparf; + static constexpr size_t npar = ProcessData::npar; + static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; + static constexpr size_t ncolor = ProcessData::ncolor; + } + + //-------------------------------------------------------------------------- + + // An abstract interface encapsulating a given number of events + class INumberOfEvents + { + public: + virtual ~INumberOfEvents() {} + virtual size_t nevt() const = 0; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating a given number of events + class NumberOfEvents : virtual public INumberOfEvents + { + public: + NumberOfEvents( const size_t nevt ) + : m_nevt( nevt ) {} + virtual ~NumberOfEvents() {} + virtual size_t nevt() const override { return m_nevt; } + private: + const size_t m_nevt; + }; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer (not necessarily an event buffer) + template + class BufferBase : virtual public INumberOfEvents + { + protected: + BufferBase( const size_t size, const bool onDevice ) + : m_size( size ), m_data( nullptr ), m_isOnDevice( onDevice ) {} + public: + virtual ~BufferBase() {} + T* data() { return m_data; } + const T* data() const { return m_data; } + T& operator[]( const size_t index ) { return m_data[index]; } + const T& operator[]( const size_t index ) const { return m_data[index]; } + size_t size() const { return m_size; } + size_t bytes() const { return m_size * sizeof( T ); } + bool isOnDevice() const { return m_isOnDevice; } + virtual size_t nevt() const override { throw std::runtime_error( "This BufferBase is not an event buffer" ); } + protected: + const size_t m_size; + T* m_data; + const bool m_isOnDevice; + }; + + //-------------------------------------------------------------------------- + + constexpr bool HostBufferALIGNED = false; // ismisaligned=false + constexpr bool HostBufferMISALIGNED = true; // ismisaligned=true + + // A class encapsulating a C++ host buffer + template + class HostBufferBase : public BufferBase + { + public: + HostBufferBase( const size_t size ) + : BufferBase( size, false ) + { + if constexpr( !ismisaligned ) + this->m_data = new( std::align_val_t( cppAlign ) ) T[size](); + else + this->m_data = new( std::align_val_t( cppAlign ) ) T[size + 1]() + 1; // TEST MISALIGNMENT! + } + virtual ~HostBufferBase() + { + if constexpr( !ismisaligned ) + ::operator delete[]( this->m_data, std::align_val_t( cppAlign ) ); + else + ::operator delete[]( ( this->m_data ) - 1, std::align_val_t( cppAlign ) ); // TEST MISALIGNMENT! + } + static constexpr bool isaligned() { return !ismisaligned; } + public: + static constexpr size_t cppAlign = mgOnGpu::cppAlign; + }; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + // A class encapsulating a C++ host buffer for a given number of events + template + class HostBuffer : public HostBufferBase, virtual private NumberOfEvents + { + public: + HostBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , HostBufferBase( sizePerEvent * nevt ) + { + //std::cout << "HostBuffer::ctor " << this << " " << nevt << std::endl; + } + virtual ~HostBuffer() + { + //std::cout << "HostBuffer::dtor " << this << std::endl; + } + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for momenta random numbers + typedef BufferBase BufferRndNumMomenta; + + // The size (number of elements) per event in a memory buffer for momenta random numbers + constexpr size_t sizePerEventRndNumMomenta = MemoryBuffers::np4 * MemoryBuffers::nparf; + + // A class encapsulating a C++ host buffer for momenta random numbers + typedef HostBuffer HostBufferRndNumMomenta; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for Gs (related to the event-by-event strength of running coupling constant alphas QCD) + typedef BufferBase BufferGs; + + // The size (number of elements) per event in a memory buffer for Gs + constexpr size_t sizePerEventGs = 1; + + // A class encapsulating a C++ host buffer for gs + typedef HostBuffer HostBufferGs; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for numerators (of the multichannel single-diagram enhancement factors) + typedef BufferBase BufferNumerators; + + // The size (number of elements) per event in a memory buffer for numerators + // (should be equal to the number of diagrams in the process) + constexpr size_t sizePerEventNumerators = ProcessData::ndiagrams; + + // A class encapsulating a C++ host buffer for numerators + typedef HostBuffer HostBufferNumerators; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for denominators (of the multichannel single-diagram enhancement factors) + typedef BufferBase BufferDenominators; + + // The size (number of elements) per event in a memory buffer for denominators + constexpr size_t sizePerEventDenominators = 1; + + // A class encapsulating a C++ host buffer for denominators + typedef HostBuffer HostBufferDenominators; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for couplings that depend on the event-by-event running coupling constant alphas QCD + typedef BufferBase BufferCouplings; + + // The size (number of elements) per event in a memory buffer for random numbers + constexpr size_t sizePerEventCouplings = MemoryBuffers::ndcoup * MemoryBuffers::nx2; + + // A class encapsulating a C++ host buffer for couplings + typedef HostBuffer HostBufferCouplings; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for momenta + typedef BufferBase BufferMomenta; + + // The size (number of elements) per event in a memory buffer for momenta + constexpr size_t sizePerEventMomenta = MemoryBuffers::np4 * MemoryBuffers::npar; + + // A class encapsulating a C++ host buffer for momenta + typedef HostBuffer HostBufferMomenta; + //typedef HostBuffer HostBufferMomenta; // TEST MISALIGNMENT! + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for sampling weights + typedef BufferBase BufferWeights; + + // The size (number of elements) per event in a memory buffer for sampling weights + constexpr size_t sizePerEventWeights = 1; + + // A class encapsulating a C++ host buffer for sampling weights + typedef HostBuffer HostBufferWeights; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for matrix elements + typedef BufferBase BufferMatrixElements; + + // The size (number of elements) per event in a memory buffer for matrix elements + constexpr size_t sizePerEventMatrixElements = 1; + + // A class encapsulating a C++ host buffer for matrix elements + typedef HostBuffer HostBufferMatrixElements; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for the helicity mask + typedef BufferBase BufferHelicityMask; + + // A class encapsulating a C++ host buffer for the helicity mask + typedef HostBufferBase HostBufferHelicityMask; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for wavefunctions + typedef BufferBase BufferWavefunctions; + + // The size (number of elements) per event in a memory buffer for wavefunctions + constexpr size_t sizePerEventWavefunctions = MemoryBuffers::nw6 * MemoryBuffers::nx2; + + // A class encapsulating a C++ host buffer for wavefunctions + typedef HostBuffer HostBufferWavefunctions; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for helicity random numbers + typedef BufferBase BufferRndNumHelicity; + + // The size (number of elements) per event in a memory buffer for helicity random numbers + constexpr size_t sizePerEventRndNumHelicity = 1; + + // A class encapsulating a C++ host buffer for helicity random numbers + typedef HostBuffer HostBufferRndNumHelicity; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for color random numbers + typedef BufferBase BufferRndNumColor; + + // The size (number of elements) per event in a memory buffer for color random numbers + constexpr size_t sizePerEventRndNumColor = 1; + + // A class encapsulating a C++ host buffer for color random numbers + typedef HostBuffer HostBufferRndNumColor; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for channel ids + typedef BufferBase BufferChannelIds; + + // The size (number of elements) per event in a memory buffer for channel ids + constexpr size_t sizePerEventChannelId = 1; + + // A class encapsulating a C++ host buffer for channel ids + typedef HostBuffer HostBufferChannelIds; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for channel ids + typedef BufferBase BufferIflavorVec; + + // The size (number of elements) per event in a memory buffer for channel ids + constexpr size_t sizePerEventIflavorVec = 1; + + // A class encapsulating a C++ host buffer for channel ids + typedef HostBuffer HostBufferIflavorVec; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for helicity selection + typedef BufferBase BufferSelectedHelicity; + + // The size (number of elements) per event in a memory buffer for helicity selection + constexpr size_t sizePerEventSelectedHelicity = 1; + + // A class encapsulating a C++ host buffer for helicity selection + typedef HostBuffer HostBufferSelectedHelicity; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for color selection + typedef BufferBase BufferSelectedColor; + + // The size (number of elements) per event in a memory buffer for color selection + constexpr size_t sizePerEventSelectedColor = 1; + + // A class encapsulating a C++ host buffer for color selection + typedef HostBuffer HostBufferSelectedColor; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- +} + +#endif // MemoryBuffers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc new file mode 100644 index 0000000000..8e28b1ac95 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.cc @@ -0,0 +1,568 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin, calculate_jamps, good-helicity filtering. +// The diagram/vertex-call sequence (helas_calls) is process-specific and +// lives in the P1-generated EvaluateDiagrams.inc, #include'd below. + +#include "SigmaKin.h" + +#include "CPPProcess.h" // ProcessData.h, Parameters.h, HelAmps_.h transitively +#include "ProcessTables.h" + +#include "MemoryAccessAmplitudes.h" +#include "MemoryAccessChannelIds.h" +#include "MemoryAccessCouplings.h" +#include "MemoryAccessCouplingsFixed.h" +#include "MemoryAccessDenominators.h" +#include "MemoryAccessGs.h" +#include "MemoryAccessIflavorVec.h" +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessNumerators.h" +#include "MemoryAccessWavefunctions.h" +#include "color_sum.h" +#include "ColorData.h" + +#include +#include + +namespace madmatrix +{ + using namespace ProcessData; + using namespace ProcessTables; + using Parameters_dependentCouplings::ndcoup; // #couplings that vary event by event (depend on running alphas QCD) + using Parameters_independentCouplings::nicoup; // #couplings that are fixed for all events (do not depend on running alphas QCD) + + // The number of SIMD vectors of events processed by calculate_jamps + constexpr int nParity = 1; + + // Helicity/flavor tables and SM parameter/coupling storage, populated once + // by CPPProcess's constructor/initProc via the setters below. + static short cHel[ncomb][npar]; + static short cFlavors[nmaxflavor][npar]; + static int cNGoodHel; + static int cGoodHel[ncomb]; + static fptype cIPD[nIPD > 0 ? nIPD : 1]; + static fptype cIPC[nIPC > 0 ? nIPC * 2 : 1]; + static int cIPF_partner1[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + static int cIPF_partner2[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + static fptype cIPF_value[ProcessTables::nMF * nIPF * 2 > 0 ? ProcessTables::nMF * nIPF * 2 : 1]; + static double bsmIndepParam[Parameters::nBsmIndepParam > 0 ? Parameters::nBsmIndepParam : 1]; + + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ) + { + memcpy( cHel, tHel, ncomb * npar * sizeof( short ) ); + memcpy( cFlavors, tFlavors, nmaxflavor * npar * sizeof( short ) ); + } + + void setIndependentParams( const fptype* tIPD ) + { + if( nIPD > 0 ) memcpy( cIPD, tIPD, nIPD * sizeof( fptype ) ); + } + + void setIndependentCouplings( const cxtype* tIPC ) + { + if( nIPC > 0 ) memcpy( cIPC, tIPC, nIPC * sizeof( cxtype ) ); + } + + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ) + { + if( nIPF == 0 ) return; + memcpy( cIPF_partner1, tIPF_partner1, nMF * nIPF * sizeof( int ) ); + memcpy( cIPF_partner2, tIPF_partner2, nMF * nIPF * sizeof( int ) ); + memcpy( cIPF_value, tIPF_value, nMF * nIPF * sizeof( cxtype ) ); + } + + void setBsmIndepParam( const double* values, int n ) + { + if( n > 0 ) memcpy( bsmIndepParam, values, n * sizeof( double ) ); + } + + //-------------------------------------------------------------------------- + + __device__ INLINE unsigned int + getChannelId( const unsigned int* allChannelIds, const int ievt00, bool sanityCheckMixedPrecision = true ) + { + unsigned int channelId = 0; // disable multichannel single-diagram enhancement unless allChannelIds != nullptr + using CID_ACCESS = HostAccessChannelIds; // non-trivial access: buffer includes all events + // SCALAR channelId for the whole SIMD neppV2 event page (C++), i.e. one or two neppV event page(s) + // The cudacpp implementation ASSUMES (and checks! #898) that all channelIds are the same in a neppV2 SIMD event page + // **NB! in "mixed" precision, using SIMD, calculate_wavefunctions computes MEs for TWO neppV pages with a single channelId! #924 + if( allChannelIds != nullptr ) + { + // First - and/or only - neppV page of channels (iParity=0 => ievt0 = ievt00 + 0 * neppV) + const unsigned int* channelIds = CID_ACCESS::ieventAccessRecordConst( allChannelIds, ievt00 ); // fix bug #899/#911 + uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) + // NB: channelIds_sv is a scalar in no-SIMD C++ + channelId = channelIds_sv; + assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) + (void)sanityCheckMixedPrecision; // no second SIMD page to cross-check in no-SIMD C++ + } + return channelId; + } + + //-------------------------------------------------------------------------- + + __global__ void + computeDependentCouplings( const fptype* allgs, fptype* allcouplings, const int nevt ) + { + using G_ACCESS = HostAccessGs; + using C_ACCESS = HostAccessCouplings; + for( int ipagV = 0; ipagV < nevt / neppV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + const fptype* gs = MemoryAccessGs::ieventAccessRecordConst( allgs, ievt0 ); + fptype* couplings = MemoryAccessCouplings::ieventAccessRecord( allcouplings, ievt0 ); + G2COUP( gs, couplings, bsmIndepParam ); + } + } + + //-------------------------------------------------------------------------- + + // Evaluate QCD partial amplitudes jamps for this given helicity from Feynman diagrams. + // This function processes a single event "page" or SIMD vector (or for two in "mixed" + // precision mode, nParity=2). Accepts a SCALAR channelId because it is GUARANTEED that + // all events in a SIMD vector have the same channelId #898. + void + calculate_jamps( int ihel, + const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + cxtype_sv* allJamp_sv, + bool storeChannelWeights, + fptype* allNumerators, + fptype* allDenominators, + fptype_sv* jamp2_sv, + const int ievt00 ) + { + using M_ACCESS = HostAccessMomenta; + using W_ACCESS = HostAccessWavefunctions; + using A_ACCESS = HostAccessAmplitudes; + using CD_ACCESS = HostAccessCouplings; + using CI_ACCESS = HostAccessCouplingsFixed; + using F_ACCESS = HostAccessIflavorVec; + using NUM_ACCESS = HostAccessNumerators; + using DEN_ACCESS = HostAccessDenominators; + mgDebug( 0, __FUNCTION__ ); + + fptype_sv pvec_sv[nwf][np4]; + cxtype_sv w_sv[nwf][nw6]; // particle wavefunctions within Feynman diagrams + cxtype_sv amp_sv[1]; // invariant amplitude for one given Feynman diagram + ALOHAOBJ aloha_obj[nwf]; + for( int iwf = 0; iwf < nwf; iwf++ ) aloha_obj[iwf] = ALOHAOBJ{ pvec_sv[iwf], w_sv[iwf] }; + fptype* amp_fp = reinterpret_cast( amp_sv ); + + // jamp: sum (for one event or event page) of the invariant amplitudes for + // all Feynman diagrams in a given color combination + cxtype_sv jamp_sv[ncolor] = {}; // all zeros + + for( int iParity = 0; iParity < nParity; ++iParity ) + { + const int ievt0 = ievt00 + iParity * neppV; + + constexpr size_t nxcoup = ndcoup + nIPC; // both dependent and independent couplings + const fptype* allCOUPs[nxcoup]; + for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) + allCOUPs[idcoup] = CD_ACCESS::idcoupAccessBufferConst( allcouplings, idcoup ); // dependent couplings, vary event-by-event + for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) + allCOUPs[ndcoup + iicoup] = CI_ACCESS::iicoupAccessBufferConst( cIPC, iicoup ); // independent couplings, fixed for all events + // C++ kernels take input/output buffers with momenta/MEs for one specific event (the first in the current event page) + const fptype* momenta = M_ACCESS::ieventAccessRecordConst( allmomenta, ievt0 ); + const fptype* COUPs[nxcoup]; + for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) + COUPs[idcoup] = CD_ACCESS::ieventAccessRecordConst( allCOUPs[idcoup], ievt0 ); // dependent couplings, vary event-by-event + for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) + COUPs[ndcoup + iicoup] = allCOUPs[ndcoup + iicoup]; // independent couplings, fixed for all events + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + // Create an array of views over the Flavor Couplings + FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; + + // Dependent (event-by-event, running-alphas) flavor couplings (Step 3): the per-flavor + // values are NOT baked in (they run per event). Gather the current values of the + // underlying dependent couplings for this event page into an AOSOA buffer dpf_value + // (one nx2*neppC SIMD record per (coupling,flavor) slot, matching CD_ACCESS), then build + // an ordinary value-based view over it. The flavor index is constant across a SIMD lane + // (guaranteed by the phase-space integrator), so each lane gets its own running value + // while sharing the same flavor selection. This is the direct analogue of Fortran's + // FLV_xx%VAL(k)%P => GC_yyy(J). The vertex routines are instantiated with CD_ACCESS so + // get_coupling_def reads dpf_value with the right per-flavor stride (CD_ACCESS::flv_stride). + constexpr int ndpfbuf = ( nDPF > 0 ? nDPF * nMF * CD_ACCESS::flv_stride : 1 ); + alignas( mgOnGpu::cppAlign ) fptype dpf_value[ndpfbuf]{}; + for( int idpf = 0; idpf < nDPF; idpf++ ) + for( int imf = 0; imf < nMF; imf++ ) + { + const int idc = cDPF_idcoup[idpf * nMF + imf]; + if( idc >= 0 ) + CD_ACCESS::kernelAccess( dpf_value + ( idpf * nMF + imf ) * CD_ACCESS::flv_stride ) = + CD_ACCESS::kernelAccessConst( COUPs[idc] ); + } + FLV_COUPLING_ARRAY flvCOUPs_dep{ cDPF_partner1, cDPF_partner2, dpf_value }; + + // Reset color flows (reset jamp_sv) at the beginning of a new event or event page + for( int i = 0; i < ncolor; i++ ) { jamp_sv[i] = cxzero_sv(); } + + // Numerators and denominators for the current event (CUDA) or SIMD event page (C++) + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + // Scalar iflavor for the current event (constant across the SIMD vector) + const unsigned int* iflavor_rec = F_ACCESS::ieventAccessRecordConst( iflavorVec, ievt0 ); + const uint_sv iflavor_sv = F_ACCESS::kernelAccessConst( iflavor_rec ); + const unsigned int iflavor = reinterpret_cast(&iflavor_sv)[0]; +#include "EvaluateDiagrams.inc" + + // *** COLOR CHOICE BELOW *** + // Store the leading color flows for choice of color + if( jamp2_sv ) // disable color choice if nullptr + { + for( int icol = 0; icol < ncolor; icol++ ) + jamp2_sv[ncolor * iParity + icol] += cxabs2( jamp_sv[icol] ); // may underflow #831 + } + + // *** PREPARE OUTPUT JAMPS *** + // In C++, copy the local jamp to the output array passed as function argument + for( int icol = 0; icol < ncolor; icol++ ) + allJamp_sv[iParity * ncolor + icol] = jamp_sv[icol]; + } + // END LOOP ON IPARITY + + mgDebug( 1, __FUNCTION__ ); + return; + } + + //-------------------------------------------------------------------------- + + void + sigmaKin_getGoodHel( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + fptype* allMEs, + fptype* allNumerators, + fptype* allDenominators, + bool* isGoodHel, + const int nevt ) + { + // Allocate arrays at build time to contain at least 16 events (or at least neppV events if neppV>16, e.g. in future VPUs) + constexpr int maxtry0 = std::max( 16, neppV ); // 16, but at least neppV (otherwise the npagV loop does not even start) + assert( nevt >= neppV ); + const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt 1 ) + { + for( int i = broken_sym_component_starts[icomp] - 1; i < broken_sym_component_ends[icomp]; i++ ) + { + if( pid_work[i] == 0 ) + continue; + int n_tot = 1; + for( int j = i + 1; j < broken_sym_component_ends[icomp]; j++ ) + { + if( pid_work[i] != pid_work[j] ) + continue; + bool same_block = ( broken_sym_block_lengths[i] == broken_sym_block_lengths[j] ); + for( int k = 0; same_block && k < broken_sym_block_lengths[i]; k++ ) + { + if( cFlavors[iflavor][broken_sym_block_starts[i] - 1 + k] != cFlavors[iflavor][broken_sym_block_starts[j] - 1 + k] ) + same_block = false; + } + if( same_block ) + { + pid_work[j] = 0; + n_tot = n_tot + 1; + old_factor = old_factor / n_tot; + } + } + } + } + total_factor = total_factor * old_factor; + } + return total_factor; + } + + //-------------------------------------------------------------------------- + // Evaluate |M|^2, part independent of incoming flavour + + void + sigmaKin( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + const fptype* allrndhel, + const fptype* allrndcol, + const unsigned int* allChannelIds, + const fptype* allrnddiagram, + fptype* allMEs, + int* allselhel, + int* allselcol, + fptype* allNumerators, + fptype* allDenominators, + unsigned int* allDiagramIdsOut, + bool mulChannelWeight, + const int nevt ) + { + mgDebugInitialise(); + + // SANITY CHECKS for cudacpp code generation (see issues #272 and #343 and PRs #619, #626, #360, #396 and #754) + { + // nprocesses == 2 may happen for "mirror processes" such as P0_uux_ttx within pp_tt012j (see PR #754) + static_assert( nproc == 1 || nproc == 2, "Assume nprocesses == 1 or 2" ); + static_assert( proc_id == 1, "Assume process_id == 1" ); + } + + using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events + using NUM_ACCESS = HostAccessNumerators; // non-trivial access: buffer includes all events + using DEN_ACCESS = HostAccessDenominators; // non-trivial access: buffer includes all events + + // Reset the "matrix elements" - running sums of |M|^2 over helicities for the given event + const int npagV = nevt / neppV; + for( int ipagV = 0; ipagV < npagV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv = fptype_sv{ 0 }; + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + for( int i = 0; i < ndiagrams; ++i ) + { + numerators_sv[i] = fptype_sv{ 0 }; + } + denominators_sv = fptype_sv{ 0 }; + } + + // HELICITY LOOP: CALCULATE WAVEFUNCTIONS (using precomputed good helicities) + const int npagV2 = npagV; // loop on one SIMD page (neppV events) at a time +#ifdef _OPENMP + // OMP multithreading #575 (NB: tested only with gcc11 so far) +#define _OMPLIST0 allcouplings, allMEs, allmomenta, allrndcol, allrndhel, allselcol, allselhel, cGoodHel, cNGoodHel, npagV2 +#define _OMPLIST1 , allDenominators, allNumerators, allChannelIds, mgOnGpu::icolamp, mgOnGpu::channel2iconfig +#pragma omp parallel for default( none ) shared( _OMPLIST0 _OMPLIST1 ) +#undef _OMPLIST0 +#undef _OMPLIST1 +#endif // _OPENMP + for( int ipagV2 = 0; ipagV2 < npagV2; ++ipagV2 ) + { + const int ievt00 = ipagV2 * neppV; // loop on one SIMD page (neppV events) at a time + // Running sum of partial amplitudes squared for event by event color selection (#402) + fptype_sv jamp2_sv[nParity * ncolor] = {}; + fptype_sv MEs_ighel[ncomb] = {}; // sum of MEs for all good helicities up to ighel (for the first - and/or only - neppV page) + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + const int ihel = cGoodHel[ighel]; + cxtype_sv jamp_sv[nParity * ncolor] = {}; // fixed nasty bug (omitting 'nParity' caused memory corruptions after calling calculate_jamps) + bool storeChannelWeights = allChannelIds != nullptr || allrnddiagram != nullptr; + calculate_jamps( ihel, allmomenta, allcouplings, iflavorVec, jamp_sv, storeChannelWeights, allNumerators, allDenominators, jamp2_sv, ievt00 ); + color_sum_cpu( allMEs, jamp_sv, ievt00 ); + MEs_ighel[ighel] = E_ACCESS::kernelAccess( E_ACCESS::ieventAccessRecord( allMEs, ievt00 ) ); + } + // Event-by-event random choice of helicity #403 + for( int ieppV = 0; ieppV < neppV; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + const bool okhel = allrndhel[ievt] < ( MEs_ighel[ighel] / MEs_ighel[cNGoodHel - 1] ); + if( okhel ) + { + const int ihelF = cGoodHel[ighel] + 1; // NB Fortran [1,ncomb], cudacpp [0,ncomb-1] + allselhel[ievt] = ihelF; + break; + } + } + } + const int vecsize = neppV; + unsigned int channelIdVec[vecsize]; + if( allChannelIds != nullptr ) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + channelIdVec[ieppV] = allChannelIds[ievt]; + } + } + + // Event-by-event random choice of channel + if( allrnddiagram != nullptr ) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + fptype numerator_sum = 0., normalization = 0.; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + normalization += allNumerators[ievt / neppV * neppV * ndiagrams + + ichan * neppV + ieppV % neppV]; + } + channelIdVec[ieppV] = mgOnGpu::nchannels; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + numerator_sum += allNumerators[ievt / neppV * neppV * ndiagrams + + ichan * neppV + ieppV % neppV]; + if( allrnddiagram[ievt] < numerator_sum / normalization ) + { + channelIdVec[ieppV] = ichan + 1; + break; + } + } + allDiagramIdsOut[ievt] = channelIdVec[ieppV]; + } + } + + // Event-by-event random choice of color #402 + if( allChannelIds != nullptr || allrnddiagram != nullptr ) // no event-by-event choice of color if channelId == 0 (fix FPE #783) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + unsigned int channelId = channelIdVec[ieppV]; + if( channelId > mgOnGpu::nchannels ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which is greater than nchannels=%d\n", channelId, mgOnGpu::nchannels ); + assert( channelId <= mgOnGpu::nchannels ); // SANITY CHECK #919 #910 + } + const int iconfig = mgOnGpu::channel2iconfig[channelId - 1]; // map N_diagrams to N_config <= N_diagrams configs (fix LHE color mismatch #856: see also #826, #852, #853) + if( iconfig <= 0 ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which has no associated SDE iconfig\n", channelId ); + assert( iconfig > 0 ); // SANITY CHECK #917 + } + else if( iconfig > (int)mgOnGpu::nconfigSDE ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d (invalid SDE iconfig=%d\n > nconfig=%d)", channelId, iconfig, mgOnGpu::nconfigSDE ); + assert( iconfig <= (int)mgOnGpu::nconfigSDE ); // SANITY CHECK #917 + } + fptype targetamp[ncolor] = { 0 }; + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( icolC == 0 ) + targetamp[icolC] = 0; + else + targetamp[icolC] = targetamp[icolC - 1]; + if( mgOnGpu::icolamp[iconfig - 1][icolC] ) targetamp[icolC] += + jamp2_sv[icolC + ncolor * ( ieppV / neppV )]; + } + const int ievt = ievt00 + ieppV; + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( allrndcol[ievt] < ( targetamp[icolC] / targetamp[ncolor - 1] ) ) + { + allselcol[ievt] = icolC + 1; // NB Fortran [1,ncolor], cudacpp [0,ncolor-1] + break; + } + } + } + } + else + { + for( int ieppV = 0; ieppV < neppV; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + allselcol[ievt] = 0; // no color selected in Fortran range [1,ncolor] if channelId == 0 (see #931) + } + } + } + // *** END OF PART 1b - C++ (loop on event pages) + + // PART 2 - FINALISATION (after calculate_jamps) + // Get the final |M|^2 as an average over helicities/colors of the running sum of |M|^2 over helicities for the given event + for( int ipagV = 0; ipagV < npagV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv = MEs_sv * broken_symmetry_factor( iflavorVec[ievt0] ) / helcolDenominators[0]; + if( mulChannelWeight && allChannelIds != nullptr ) // fix segfault #892 (not 'channelIds[0] != 0') + { + const unsigned int channelId = getChannelId( allChannelIds, ievt0, false ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + MEs_sv *= numerators_sv[channelId - 1] / denominators_sv; + } + } + mgDebugFinalise(); + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h new file mode 100644 index 0000000000..81956d459d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/SigmaKin.h @@ -0,0 +1,64 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin and everything it calls (calculate_jamps, +// good-helicity filtering, color/channel selection). Declared here so +// CPPProcess.cc's constructor/initProc (P1-generated) can call the setters +// that populate this file's storage, and so umami.cc/MatrixElementKernels.cc +// can call sigmaKin/computeDependentCouplings. + +#ifndef SIGMAKIN_H +#define SIGMAKIN_H 1 + +#include "mgOnGpuConfig.h" +#include "mgOnGpuCxtypes.h" // for cxtype + +namespace madmatrix +{ + __global__ void + computeDependentCouplings( const fptype* allgs, + fptype* allcouplings, + const int nevt ); + + void + sigmaKin_getGoodHel( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + fptype* allMEs, + fptype* allNumerators, + fptype* allDenominators, + bool* isGoodHel, + const int nevt ); + + int + sigmaKin_setGoodHel( const bool* isGoodHel ); + + void + sigmaKin( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + const fptype* allrndhel, + const fptype* allrndcol, + const unsigned int* allChannelIds, + const fptype* allrnddiagram, + fptype* allMEs, + int* allselhel, + int* allselcol, + fptype* allNumerators, + fptype* allDenominators, + unsigned int* allDiagramIdsOut, + bool mulChannelWeight, + const int nevt ); + + // Setters: called once by CPPProcess (P1-generated) to populate this file's + // otherwise-internal storage, since it can no longer be written directly + // from a different translation unit. + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ); + void setIndependentParams( const fptype* tIPD ); + void setIndependentCouplings( const cxtype* tIPC ); + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ); + void setBsmIndepParam( const double* values, int n ); +} + +#endif // SIGMAKIN_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc new file mode 100644 index 0000000000..a94caa8869 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.cc @@ -0,0 +1,83 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "color_sum.h" + +#include "mgOnGpuConfig.h" + +#include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps +#include "MemoryAccessMatrixElements.h" + +namespace madmatrix +{ + using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor + + + //-------------------------------------------------------------------------- + + void + color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity + const int ievt0 ) // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) + { + // Pre-compute a constexpr triangular color matrix properly normalized #475 + struct TriangularNormalizedColorMatrix + { + // See https://stackoverflow.com/a/34465458 + __host__ __device__ constexpr TriangularNormalizedColorMatrix() + : value() + { + for( int icol = 0; icol < ncolor; icol++ ) + { + // Diagonal terms + value[icol][icol] = colorMatrix[icol][icol] / colorDenom[icol]; + // Off-diagonal terms + for( int jcol = icol + 1; jcol < ncolor; jcol++ ) + value[icol][jcol] = 2 * colorMatrix[icol][jcol] / colorDenom[icol]; + } + } + fptype2 value[ncolor][ncolor]; + }; + static constexpr auto cf2 = TriangularNormalizedColorMatrix(); + // Use the property that M is a real matrix (see #475): + // we can rewrite the quadratic form (A-iB)(M)(A+iB) as AMA - iBMA + iBMA + BMB = AMA + BMB + // In addition, on C++ use the property that M is symmetric (see #475), + // and also use constexpr to compute "2*" and "/colorDenom[icol]" once and for all at compile time: + // we gain (not a factor 2...) in speed here as we only loop over the up diagonal part of the matrix. + // Strangely, CUDA is slower instead, so keep the old implementation for the moment. + fptype_sv deltaMEs = { 0 }; + const cxtype_sv* jamp_sv = allJamp_sv; + // Loop over icol + for( int icol = 0; icol < ncolor; icol++ ) + { + // Diagonal terms + fptype2_sv jampRi_sv = (fptype2_sv)( cxreal( jamp_sv[icol] ) ); + fptype2_sv jampIi_sv = (fptype2_sv)( cximag( jamp_sv[icol] ) ); + fptype2_sv ztempR_sv = cf2.value[icol][icol] * jampRi_sv; + fptype2_sv ztempI_sv = cf2.value[icol][icol] * jampIi_sv; + // Loop over jcol + for( int jcol = icol + 1; jcol < ncolor; jcol++ ) + { + // Off-diagonal terms + fptype2_sv jampRj_sv = (fptype2_sv)( cxreal( jamp_sv[jcol] ) ); + fptype2_sv jampIj_sv = (fptype2_sv)( cximag( jamp_sv[jcol] ) ); + ztempR_sv += cf2.value[icol][jcol] * jampRj_sv; + ztempI_sv += cf2.value[icol][jcol] * jampIj_sv; + } + fptype2_sv deltaMEs2 = ( jampRi_sv * ztempR_sv + jampIi_sv * ztempI_sv ); // may underflow #831 + deltaMEs += deltaMEs2; + } + // *** STORE THE RESULTS *** + using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv += deltaMEs; // fix #435 + } + + //-------------------------------------------------------------------------- + +} // end namespace diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h new file mode 100644 index 0000000000..8654492051 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/color_sum.h @@ -0,0 +1,43 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef COLOR_SUM_H +#define COLOR_SUM_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "ProcessData.h" + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + // No-op on cpu/simd: the normalized color matrix is already a compile-time + // constexpr there (see color_sum.cc); only gpu needs a runtime push to device memory. + inline void createNormalizedColorMatrix() {} + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + void + color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity + const int ievt0 ); // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- +} + +#endif // COLOR_SUM_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h new file mode 100644 index 0000000000..c1b4df74c4 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/constexpr_math.h @@ -0,0 +1,353 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Feb 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef constexpr_math_h +#define constexpr_math_h 1 + +#include "mgOnGpuConfig.h" + +#include +#include // for feenableexcept, fegetexcept and FE_XXX +#include +#include +#include +#include + +// SWITCH ON/OFF DEBUGGING +#undef CONSTEXPR_MATH_DEBUG // no-debug +//#define CONSTEXPR_MATH_DEBUG 1 // debug + +// FOR DEBUGGING +#ifdef CONSTEXPR_MATH_DEBUG +#define CONSTEXPRMATHFUN inline +#define CONSTEXPRMATHVAR const +#else +#define CONSTEXPRMATHFUN constexpr +#define CONSTEXPRMATHVAR constexpr +#endif + +namespace madmatrix +{ + // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) + constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) + { + return curr == prev ? curr : sqrtNewtonRaphson( xx, 0.5 * ( curr + xx / curr ), curr ); + } + constexpr long double constexpr_sqrt( const long double xx ) + { + return xx >= 0 // && x < std::numeric_limits::infinity() // avoid -Wtautological-constant-compare warning in fast math + ? sqrtNewtonRaphson( xx, xx, 0 ) + : std::numeric_limits::quiet_NaN(); + } + + // SQRT constants + constexpr long double constexpr_sqrt2 = constexpr_sqrt( 2 ); + + // Constexpr implementation of floor (see https://stackoverflow.com/a/66146159) + constexpr int constexpr_floor( const long double xx ) + { + const int i = static_cast( xx ); + return xx < i ? i - 1 : i; + } + + // Constexpr implementation of pow + constexpr long double constexpr_pow( const long double base, const long double exp, const bool requireExpGe0 = false ) + { + // NB(1): this iterative integer implementation of constexpr_pow requires exponent >= 0 + if( requireExpGe0 ) assert( exp >= 0 ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + if( exp < 0 ) return 1. / constexpr_pow( base, -exp, true ); + // NB(2): this iterative integer implementation of constexpr_pow requires an integer exponent, excexpt for special cases (1/2, 1/4) + if( exp == 0.5 ) return constexpr_sqrt( base ); + if( exp == 0.25 ) return constexpr_sqrt( constexpr_sqrt( base ) ); + const int iexp = constexpr_floor( exp ); + assert( static_cast( iexp ) == exp ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + // Iterative implementation of pow if exp is a non negative integer + return iexp == 0 ? 1 : base * constexpr_pow( base, iexp - 1 ); + } + + // PI constants + // NB1: M_PIl from from cmath is not defined on Mac + // NB2: std::numbers::pi needs c++20 but we are still using c++17 + // NB3: I could use my constexpr_atan(1)*4... but a literal is better? + //constexpr long double constexpr_pi = M_PIl; // pi + //constexpr long double constexpr_pi_by_2 = M_PI_2l; // pi/2 + //constexpr long double constexpr_pi_by_4 = M_PI_4l; // pi/4 + constexpr long double constexpr_pi = 3.141592653589793238462643383279502884L; // same as M_PIl in gcc + constexpr long double constexpr_pi_by_2 = 1.570796326794896619231321691639751442L; // same as M_PI_2l in gcc + constexpr long double constexpr_pi_by_4 = 0.785398163397448309615660845819875721L; // same as M_PI_4l in gcc + static_assert( constexpr_pi_by_4 * 4 == constexpr_pi ); + static_assert( constexpr_pi_by_4 * 2 == constexpr_pi_by_2 ); + static_assert( constexpr_pi_by_2 * 2 == constexpr_pi ); + + // Constexpr implementation of sin for 0= constexpr_pi_by_4 ) std::cout << "sinTaylor xx=" << xx << std::endl; +#endif + assert( xx >= 0 && "The argument of sinTaylor is lower than the expected range [0,pi/4)" ); + assert( xx < constexpr_pi_by_4 && "The argument of sinTaylor is higher than the expected range [0,pi/4)" ); + long double sinx = 0; + int ipow = 1; + long double delta = xx; + while( true ) + { + long double sinxlast = sinx; + sinx += delta; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + if( sinx == sinxlast ) break; + // Next iteration + ipow += 2; + delta *= -xx * xx / ( ipow - 1 ) / ipow; + } +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + return sinx; + } + +#ifdef CONSTEXPR_MATH_DEBUG + // Debug printouts for trig functions + inline void debug_constexpr_trig( const long double xx, size_t call ) + { + CONSTEXPRMATHVAR long double xxminuspi = xx - constexpr_pi; + CONSTEXPRMATHVAR long double twopiminusxx = 2 * constexpr_pi - xx; + std::cout << std::setprecision( 40 ); + std::cout << "constexpr_sin_quad call=" << call << std::endl + << " xx=" << xx << std::endl + << " (xx-pi)=" << xxminuspi << std::endl + << " (2pi-xx)=" << twopiminusxx << std::endl; + std::cout << std::setprecision( 6 ); + if( xx < 0 ) // (-inf, 0) + std::cout << " -- case 1 (xx < 0)" << std::endl; + else if( xx == 0 ) // [0] *** NEW + std::cout << " -- case 2 (xx == 0)" << std::endl; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + std::cout << " -- case 3 (xx < pi/4)" << std::endl; + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + std::cout << " -- case 4 (xx == pi/4)" << std::endl; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + std::cout << " -- case 5 (xx < pi/2)" << std::endl; + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + std::cout << " -- case 6 (xx == pi/2)" << std::endl; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + std::cout << " -- case 7 (xx < 3*pi/4)" << std::endl; + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + std::cout << " -- case 8 (xx == 3*pi/4)" << std::endl; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + std::cout << " -- case 9 (xx < pi)" << std::endl; + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + std::cout << " -- case 10 (xx == pi)" << std::endl; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + std::cout << " -- case 11 (xx < 2*pi)" << std::endl; + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + std::cout << " -- case 12 (xx == 2*pi)" << std::endl; + else // (2*pi, +inf) + std::cout << " -- case 13 (xx > 2*pi)" << std::endl; + } +#endif + + // Mapping to [0,2*pi) range (long double signature) + constexpr long double mapIn0to2Pi( const long double xx ) + { + return xx - constexpr_floor( xx / 2 / constexpr_pi ) * 2 * constexpr_pi; + } + + // Constexpr implementation of cos (long double signature) + CONSTEXPRMATHFUN long double constexpr_cos_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 1; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx ), 2 ) ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return sinTaylor( constexpr_pi_by_2 - xx ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 0; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return -sinTaylor( xx - constexpr_pi_by_2 ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return -1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return -constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi - xx ), 2 ) ); + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + return -1; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return constexpr_cos_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 1; + else // (2*pi, +inf) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of cos (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_cos( const double x ) + { + return constexpr_cos_quad( x ); + } + + // Constexpr implementation of sin (long double signature) + CONSTEXPRMATHFUN long double constexpr_sin_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 0; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return sinTaylor( xx ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi_by_2 - xx ), 2 ) ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 1; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx - constexpr_pi_by_2 ), 2 ) ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return sinTaylor( constexpr_pi - xx ); + else if( xx == constexpr_pi ) // [pi] *** NEW (1st fix #903 infinite recursion) + return 0; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return -constexpr_sin_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 0; + else // (2*pi, +inf) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of sin (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_sin( const double x ) + { + return constexpr_sin_quad( x ); + } + + // Constexpr implementation of tan (long double signature) + CONSTEXPRMATHFUN long double constexpr_tan_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } + if( xx < 0 ) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + else if( xx < 2 * constexpr_pi ) // [0, 2*pi) + return constexpr_sin_quad( xx, assume0to2Pi ) / constexpr_cos_quad( xx, assume0to2Pi ); + else // [8/4*pi, +inf) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of tan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_tan( const double x ) + { + return constexpr_tan_quad( x ); + } + + // Constexpr implementation of atan for -1= -1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + assert( xx < 1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + long double atanx = 0; + int ipow = 1; + long double xpow = xx; + while( true ) + { + long double atanxlast = atanx; + atanx += xpow / ipow; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", xpow=" << xpow << ", atanx=" << atanx << std::endl; // for debugging (not constexpr) +#endif + if( atanx == atanxlast ) break; + // Next iteration + ipow += 2; + xpow *= -xx * xx; + } + return atanx; + } + + // Constexpr implementation of atan (long double signature) + CONSTEXPRMATHFUN long double constexpr_atan_quad( const long double xx ) + { + if( xx > 1 ) + return constexpr_pi_by_2 - atanTaylor( 1 / xx ); + else if( xx == 1 ) + return constexpr_pi_by_4; + else if( xx > -1 ) + return atanTaylor( xx ); + else if( xx == -1 ) + return -constexpr_pi_by_4; + else // if( xx < -1 ) + return -constexpr_pi_by_2 - atanTaylor( 1 / xx ); + } + + // Constexpr implementation of atan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_atan( const double x ) + { + return constexpr_atan_quad( x ); + } + + // Enable FPE traps (see #701, #733, #831 - except on MacOS where feenableexcept is not defined #730) + // [NB1: Fortran default is -ffpe-trap=none, i.e. FPE traps are not enabled, https://gcc.gnu.org/onlinedocs/gfortran/Debugging-Options.html] + // [NB2: Fortran default is -ffpe-summary=invalid,zero,overflow,underflow,denormal, i.e. warn at the end on STOP] + inline void + fpeEnable() + { + static bool first = true; // FIXME: quick and dirty hack to do this only once (can be removed when separate C++/CUDA builds are implemented) + if( !first ) return; + first = false; +#ifndef __APPLE__ // on MacOS feenableexcept is not defined #730 + constexpr bool enableFPE = true; // this is hardcoded and no longer controlled by getenv( "CUDACPP_RUNTIME_ENABLEFPE" ) + if( enableFPE ) + { + std::cout << "INFO: The following Floating Point Exceptions will cause SIGFPE program aborts: FE_DIVBYZERO, FE_INVALID, FE_OVERFLOW" << std::endl; + feenableexcept( FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW ); // new strategy #831 (do not enable FE_UNDERFLOW) + } +#else + //std::cout << "INFO: Keep default SIGFPE settings because feenableexcept is not available on MacOS" << std::endl; +#endif + } +} + +#endif // constexpr_math_h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h new file mode 100644 index 0000000000..c7a12b182d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuConfig.h @@ -0,0 +1,191 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jul 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCONFIG_H +#define MGONGPUCONFIG_H 1 + +#include +#include + +// cpu backend: always built with a plain host compiler, never nvcc/hipcc, so the +// GPU-backend selection macro (see gpu/mgOnGpuConfig.h) is deliberately never +// defined here - single-file, all-backend headers rely on that to pick branches. + +// Choose floating point precision (for everything but color algebra #537) +// If set from outside with e.g. -DMGONGPU_FPTYPE_FLOAT, nothing happens (#167) +#if not defined MGONGPU_FPTYPE_DOUBLE and not defined MGONGPU_FPTYPE_FLOAT +#define MGONGPU_FPTYPE_DOUBLE 1 // default +//#define MGONGPU_FPTYPE_FLOAT 1 // 2x faster +#endif + +// Choose floating point precision (for color algebra alone #537) +#if not defined MGONGPU_FPTYPE2_DOUBLE and not defined MGONGPU_FPTYPE2_FLOAT +#define MGONGPU_FPTYPE2_DOUBLE 1 // default +//#define MGONGPU_FPTYPE2_FLOAT 1 // 2x faster +#endif + +// Choose whether to inline all HelAmps functions (can gain ~4x, issue #229) +// By default off; set from outside with -DMGONGPU_INLINE_HELAMPS +//#define MGONGPU_INLINE_HELAMPS 1 + +// Choose whether to hardcode cIPD physics parameters instead of reading user cards +// By default off; set from outside with -DMGONGPU_HARDCODE_PARAM +//#define MGONGPU_HARDCODE_PARAM 1 + +// Complex type in C++: cxsmpl by default, or std::complex (CHOOSE ONLY ONE) +//#define MGONGPU_CPPCXTYPE_STDCOMPLEX 1 // ~8% slower on float, same on double +#define MGONGPU_CPPCXTYPE_CXSMPL 1 // default + +// No BLAS on the cpu backend (cuBLAS/hipBLAS are GPU-only) +#define MGONGPU_HAS_NO_BLAS 1 + +// nsight compute (ncu) debugging is CUDA-only; always off here +#undef MGONGPU_NSIGHT_DEBUG + +// SANITY CHECKS +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE_DOUBLE or MGONGPU_FPTYPE_FLOAT +#endif +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE2_DOUBLE or MGONGPU_FPTYPE2_FLOAT +#endif +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You cannot use double precision for color algebra and single precision elsewhere +#endif +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX and defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL for C++ +#endif + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu +{ + // Floating point type (for everything but color algebra #537): fptype +#if defined MGONGPU_FPTYPE_DOUBLE + typedef double fptype; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE_FLOAT + typedef float fptype; // single precision (4 bytes, fp32) +#endif + + // Floating point type (for color algebra alone #537): fptype2 +#if defined MGONGPU_FPTYPE2_DOUBLE + typedef double fptype2; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE2_FLOAT + typedef float fptype2; // single precision (4 bytes, fp32) +#endif + + // Maximum number of threads per block + const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" + + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + constexpr int cppAlign = 64; // 64-byte i.e. 512-bit + + // Retrieve the compiler that was used to build this module + inline std::string + getCompiler() + { + std::stringstream out; + // HIP version (HIPCC) + // [Use __HIPCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'hipcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'hipcc -dM -E -x hip -I ../../src CPPProcess.cc | grep HIP'] +#ifdef __HIPCC__ +#if defined HIP_VERSION_MAJOR && defined HIP_VERSION_MINOR && defined HIP_VERSION_PATCH + out << "hipcc " << HIP_VERSION_MAJOR << "." << HIP_VERSION_MINOR << "." << HIP_VERSION_PATCH; +#else + out << "hipcc UNKNOWN"; +#endif + out << " ("; +#endif + // CUDA version (NVCC) + // [Use __NVCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'nvcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'nvcc --compiler-options -dM -E dummy.c | grep CUDA': see https://stackoverflow.com/a/53713712] +#ifdef __NVCC__ +#if defined __CUDACC_VER_MAJOR__ && defined __CUDACC_VER_MINOR__ && defined __CUDACC_VER_BUILD__ + out << "nvcc " << __CUDACC_VER_MAJOR__ << "." << __CUDACC_VER_MINOR__ << "." << __CUDACC_VER_BUILD__; +#else + out << "nvcc UNKNOWN"; +#endif + out << " ("; +#endif + // ICX version (either as CXX or as host compiler inside NVCC) +#if defined __INTEL_COMPILER +#error "icc is no longer supported: please use icx" +#elif defined __INTEL_LLVM_COMPILER // alternative: __INTEL_CLANG_COMPILER + out << "icx " << __INTEL_LLVM_COMPILER; +#ifdef __NVCC__ + out << ", "; +#else + out << " ("; +#endif +#endif + // CLANG version (either as CXX or as host compiler inside NVCC or inside ICX) +#if defined __clang__ +#if defined __clang_major__ && defined __clang_minor__ && defined __clang_patchlevel__ +#ifdef __APPLE__ + out << "Apple clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#else + out << "clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#endif +#else + out << "clang UNKNOWKN"; +#endif +#else + // GCC version (either as CXX or as host compiler inside NVCC) +#if defined __GNUC__ && defined __GNUC_MINOR__ && defined __GNUC_PATCHLEVEL__ + out << "gcc " << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__; +#else + out << "gcc UNKNOWKN"; +#endif +#endif +#if defined __HIPCC__ or defined __NVCC__ or defined __INTEL_LLVM_COMPILER + out << ")"; +#endif + return out.str(); + } +} + +// Expose typedefs outside the namespace +using mgOnGpu::fptype; +using mgOnGpu::fptype2; + +// Undefine ARM_NEON (hack for cppnone on Apple silicon ARM) +#ifdef MGONGPU_NOARMNEON +#undef __ARM_NEON +#endif +#undef MGONGPU_CPPSIMD + + +// No-op debug macros (nsight-based debugging is CUDA-only, unused here) +#define mgDebugDeclare() /*noop*/ +#define mgDebugInitialise() /*noop*/ +#define mgDebug( code, text ) /*noop*/ +#define mgDebugFinalise() /*noop*/ + +// Define empty CUDA/HIP declaration specifiers for C++ +#define __global__ +#define __host__ +#define __device__ + +// For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two +inline constexpr bool +ispoweroftwo( int n ) +{ + return ( n > 0 ) && !( n & ( n - 1 ) ); +} + +// Compiler version support (#96) +#if defined __clang__ +#if( __clang_major__ < 11 ) +#error Unsupported clang version: please use clang >= 11 +#endif +#elif defined __GNUC__ +#if( __GNUC__ < 9 ) || ( __GNUC__ == 9 && __GNUC_MINOR__ < 3 ) +#error Unsupported gcc version: please gcc >= 9.3 +#endif +#endif + +#endif // MGONGPUCONFIG_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h new file mode 100644 index 0000000000..4e4b4b987a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuCxtypes.h @@ -0,0 +1,429 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022, based on earlier work by D. Smith) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCXTYPES_H +#define MGONGPUCXTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +#include + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) HEADERS +//========================================================================== + +#include + +// Complex type in c++: std::complex or cxsmpl +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX +#include +#elif not defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL +#endif + +//========================================================================== +// COMPLEX TYPES: SIMPLE COMPLEX CLASS (cxsmpl) +//========================================================================== + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu /* clang-format off */ +{ + // The number of floating point types in a complex type (real, imaginary) + constexpr int nx2 = 2; + + // --- Type definition (simple complex type derived from cxtype_v) + template + class cxsmpl + { + public: + __host__ __device__ constexpr cxsmpl() : m_real( 0 ), m_imag( 0 ) {} + cxsmpl( const cxsmpl& ) = default; + cxsmpl( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl( const FP& r, const FP& i = 0 ) : m_real( r ), m_imag( i ) {} + __host__ __device__ constexpr cxsmpl( const std::complex& c ) : m_real( c.real() ), m_imag( c.imag() ) {} + cxsmpl& operator=( const cxsmpl& ) = default; + cxsmpl& operator=( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl& operator+=( const cxsmpl& c ) { m_real += c.real(); m_imag += c.imag(); return *this; } + __host__ __device__ constexpr cxsmpl& operator-=( const cxsmpl& c ) { m_real -= c.real(); m_imag -= c.imag(); return *this; } + __host__ __device__ constexpr const FP& real() const { return m_real; } + __host__ __device__ constexpr const FP& imag() const { return m_imag; } + template __host__ __device__ constexpr operator cxsmpl() const { return cxsmpl( m_real, m_imag ); } +#ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX + template __host__ __device__ constexpr operator std::complex() const { return std::complex( m_real, m_imag ); } +#endif + private: + FP m_real, m_imag; // RI + }; + + template + constexpr // (NB: now valid code? in the past this failed as "a constexpr function cannot have a nonliteral return type mgOnGpu::cxsmpl") + inline __host__ __device__ cxsmpl + conj( const cxsmpl& c ) + { + return cxsmpl( c.real(), -c.imag() ); + } +} /* clang-format on */ + +// Expose the cxsmpl class outside the namespace +using mgOnGpu::cxsmpl; + +// Printout to stream for user defined types +namespace madmatrix +{ + template + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxsmpl& c ) + { + //out << std::complex( c.real(), c.imag() ); + out << "(" << c.real() << ", " << c.imag() << ")"; // add a space after the comma + return out; + } + + // Operators for cxsmpl + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl a ) + { + return a; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a ) + { + return cxsmpl( -a.real(), -a.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() + b.real(), a.imag() + b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) + b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() - b.real(), a.imag() - b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) - b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const double& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const double& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const cxsmpl& b ) + { + FP bnorm = b.real() * b.real() + b.imag() * b.imag(); + return cxsmpl( ( a.real() * b.real() + a.imag() * b.imag() ) / bnorm, + ( a.imag() * b.real() - a.real() * b.imag() ) / bnorm ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) / b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const FP& b ) + { + return a + cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const FP& b ) + { + return a - cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const FP& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const FP& b ) + { + return a / cxsmpl( b, 0 ); + } +} + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS +//========================================================================== + +namespace madmatrix +{ + // --- Type definitions (complex type: cxtype) +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX + typedef std::complex cxtype; +#else + typedef cxsmpl cxtype; +#endif + + // SANITY CHECK: memory access may be based on casts of fptype[2] to cxtype (e.g. for wavefunctions) + static_assert( sizeof( cxtype ) == mgOnGpu::nx2 * sizeof( fptype ), "sizeof(cxtype) is not 2*sizeof(fptype)" ); +} + +// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) +// DO NOT expose typedefs and operators outside the namespace +//using mgOnGpu::cxtype; + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS +//========================================================================== + +namespace madmatrix +{ +#if defined MGONGPU_CPPCXTYPE_CXSMPL + + //------------------------------ + // C++ - using cxsmpl + //------------------------------ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // cxsmpl constructor + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return c.real(); // cxsmpl::real() + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return c.imag(); // cxsmpl::imag() + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( cxsmpl ) + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +#endif // #if defined MGONGPU_CPPCXTYPE_CXSMPL + + //========================================================================== + +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX + + //------------------------------ + // C++ - using std::complex + //------------------------------ + + inline cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // std::complex constructor + } + + inline fptype + cxreal( const cxtype& c ) + { + return c.real(); // std::complex::real() + } + + inline fptype + cximag( const cxtype& c ) + { + return c.imag(); // std::complex::imag() + } + + inline cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( std::complex ) + } + + inline const cxtype& + cxmake( const cxtype& c ) // std::complex to std::complex (float-to-float or double-to-double) + { + return c; + } + +#if defined MGONGPU_FPTYPE_FLOAT + inline cxtype + cxmake( const std::complex& c ) // std::complex to std::complex (cast double-to-float) + { + return cxmake( (fptype)c.real(), (fptype)c.imag() ); + } +#endif + +#endif // #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX + + //========================================================================== + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +} // end namespace madmatrix + +//========================================================================== +// COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) +//========================================================================== + +//One namespace. Split ber backend. +namespace madmatrix +{ + // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] + // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined + // It is now always included in the code because it is needed also to access an fptype wavefunction buffer as a cxtype + class cxtype_ref + { + public: + cxtype_ref() = delete; + cxtype_ref( const cxtype_ref& ) = delete; + cxtype_ref( cxtype_ref&& ) = default; // copy const refs + __host__ __device__ cxtype_ref( fptype& r, fptype& i ) + : m_preal( &r ), m_pimag( &i ) {} // copy (create from) const refs + cxtype_ref& operator=( const cxtype_ref& ) = delete; + //__host__ __device__ cxtype_ref& operator=( cxtype_ref&& c ) {...} // REMOVED! Should copy refs or copy values? No longer needed in cxternary + __host__ __device__ cxtype_ref& operator=( const cxtype& c ) + { + *m_preal = cxreal( c ); + *m_pimag = cximag( c ); + return *this; + } // copy (assign) non-const values + __host__ __device__ operator cxtype() const { return cxmake( *m_preal, *m_pimag ); } + private: + fptype* const m_preal; // const pointer to non-const fptype R + fptype* const m_pimag; // const pointer to non-const fptype I + }; + + // Printout to stream for user defined types + inline __host__ __device__ std::ostream& + operator<<( std::ostream& out, const cxtype_ref& c ) + { + out << (cxtype)c; + return out; + } + + //-------------------------------------------------------------------------- + + // all needed from mgOnGpuVectors.h for cpu + const int neppV = 1; + +#ifndef MGONGPU_CPPCXTYPE_CXSMPL // operator<< for cxsmpl has already been defined! + inline std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + out << "[" << cxreal( c ) << "," << cximag( c ) << "]"; + return out; + } +#endif + + inline fptype + fpternary( const bool& mask, const fptype& a, const fptype& b ) + { + return ( mask ? a : b ); + } + + inline cxtype + cxternary( const bool& mask, const cxtype& a, const cxtype& b ) + { + return ( mask ? a : b ); + } + + inline bool + maskand( const bool& mask ) + { + return mask; + } + + //vector is scalar + typedef bool bool_sv; + typedef fptype fptype_sv; + typedef fptype2 fptype2_sv; + typedef unsigned int uint_sv; + typedef cxtype cxtype_sv; + typedef cxtype_ref cxtype_sv_ref; + + //vector is scalar + inline cxtype cxzero_sv() { return cxtype( 0, 0 ); } + + inline __host__ __device__ fptype_sv + cxabs2( const cxtype_sv& c ) + { + return cxreal( c ) * cxreal( c ) + cximag( c ) * cximag( c ); + } + +} // end namespace madmatrix + +//========================================================================== + +#endif // MGONGPUCXTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h new file mode 100644 index 0000000000..d6fe740b8f --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/mgOnGpuFptypes.h @@ -0,0 +1,51 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUFPTYPES_H +#define MGONGPUFPTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include +#include + +//One namespace. Split ber backend. +namespace madmatrix +{ + //========================================================================== + + + //========================================================================== + + + //------------------------------ + // Floating point types - C++ + //------------------------------ + + inline const fptype& + fpmax( const fptype& a, const fptype& b ) + { + return std::max( a, b ); + } + + inline const fptype& + fpmin( const fptype& a, const fptype& b ) + { + return std::min( a, b ); + } + + inline fptype + fpsqrt( const fptype& f ) + { + return std::sqrt( f ); + } + + + //========================================================================== + +} // end namespace madmatrix + +#endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc new file mode 100644 index 0000000000..ac462561b0 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/cpu/umami.cc @@ -0,0 +1,448 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: T. Heimel (Nov 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro (2026). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "umami.h" + +#include "ProcessData.h" +#include "CPPProcess.h" // needed to construct/initProc the process object (umami_initialize) +#include "SigmaKin.h" // sigmaKin_getGoodHel/setGoodHel +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" + +#include +#include +#include +#include + +using namespace madmatrix; + +namespace +{ + + void* initialize_impl( + const fptype* momenta, + const fptype* couplings, + const unsigned int* flavor_indices, + fptype* matrix_elements, + fptype* numerators, + fptype* denominators, + std::size_t count ) + { + bool is_good_hel[ProcessData::ncomb]; + sigmaKin_getGoodHel( + momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, + is_good_hel, + count ); + sigmaKin_setGoodHel( is_good_hel ); + return nullptr; + } + + void initialize( + const fptype* momenta, + const fptype* couplings, + const unsigned int* flavor_indices, + fptype* matrix_elements, + fptype* numerators, + fptype* denominators, + std::size_t count ) + { + // static local initialization is called exactly once in a thread-safe way + static void* dummy = initialize_impl( momenta, couplings, flavor_indices, matrix_elements, + numerators, + denominators, + count ); + } + + void + transpose_momenta( const double* momenta_in, fptype* momenta_out, std::size_t i_event_in, std::size_t i_event_out, std::size_t stride ) + { + std::size_t page_size = MemoryAccessMomentaBase::neppM; + std::size_t i_page = i_event_out / page_size; + std::size_t i_vector = i_event_out % page_size; + + for( std::size_t i_part = 0; i_part < ProcessData::npar; ++i_part ) + { + for( std::size_t i_mom = 0; i_mom < 4; ++i_mom ) + { + momenta_out[i_page * ProcessData::npar * 4 * page_size + + i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( ProcessData::npar * i_mom + i_part ) + i_event_in]; + } + } + } + + + struct InterfaceInstance + { + bool initialized = false; + }; + + std::vector g_externalMasses; + +} + +extern "C" +{ + UmamiStatus umami_get_meta( UmamiMetaKey meta_key, void* result ) + { + switch( meta_key ) + { + case UMAMI_META_DEVICE: + { + UmamiDevice& device = *static_cast( result ); + device = UMAMI_DEVICE_CPU; + break; + } + case UMAMI_META_PARTICLE_COUNT: + *static_cast( result ) = ProcessData::npar; + break; + case UMAMI_META_DIAGRAM_COUNT: + *static_cast( result ) = ProcessData::ndiagrams; + break; + case UMAMI_META_HELICITY_COUNT: + *static_cast( result ) = ProcessData::ncomb; + break; + case UMAMI_META_COLOR_COUNT: + return UMAMI_ERROR_UNSUPPORTED_META; + case UMAMI_META_MASSES: + { + if( g_externalMasses.size() != (size_t)ProcessData::npar ) return UMAMI_ERROR_UNINITIALIZED_META; + + for( int ipar = 0; ipar < ProcessData::npar; ++ipar ) + static_cast( result )[ipar] = g_externalMasses[ipar]; + break; + } + default: + return UMAMI_ERROR_UNSUPPORTED_META; + } + return UMAMI_SUCCESS; + } + + UmamiStatus umami_supported_inputs( bool const** supported, int* count ) + { + // MOMENTA, ALPHA_S, FLAVOR_INDEX, RANDOM_COLOR, RANDOM_HELICITY, RANDOM_DIAGRAM, + // HELICITY_INDEX=false, DIAGRAM_INDEX=true, CHANNEL_INDEX=false + static const bool data[UMAMI_INPUT_KEY_COUNT] = { true, true, true, true, true, true, false, true }; + *supported = data; + *count = UMAMI_INPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_required_inputs( bool const** required, int* count ) + { + static const bool data[UMAMI_INPUT_KEY_COUNT] = { true }; // MOMENTA only + *required = data; + *count = UMAMI_INPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_supported_outputs( bool const** supported, int* count ) + { + // MATRIX_ELEMENT, DIAGRAM_AMP2, COLOR_INDEX, HELICITY_INDEX, DIAGRAM_INDEX, GPU_STREAM + static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true }; + *supported = data; + *count = UMAMI_OUTPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_initialize( UmamiHandle* handle, char const* param_card_path ) + { + CPPProcess process; + process.initProc( param_card_path ); + + const std::vector& masses = process.getMasses(); + g_externalMasses.assign( masses.begin(), masses.end() ); + + auto instance = new InterfaceInstance(); + *handle = instance; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_set_parameter( + [[maybe_unused]] UmamiHandle handle, + [[maybe_unused]] char const* name, + [[maybe_unused]] double parameter_real, + [[maybe_unused]] double parameter_imag ) + { + return UMAMI_ERROR_NOT_IMPLEMENTED; + } + + UmamiStatus umami_get_parameter( + [[maybe_unused]] UmamiHandle handle, + [[maybe_unused]] char const* name, + [[maybe_unused]] double* parameter_real, + [[maybe_unused]] double* parameter_imag ) + { + return UMAMI_ERROR_NOT_IMPLEMENTED; + } + + UmamiStatus umami_matrix_element( + UmamiHandle handle, + size_t count, + size_t stride, + size_t offset, + size_t input_count, + UmamiInputKey const* input_keys, + void const* const* inputs, + size_t output_count, + UmamiOutputKey const* output_keys, + void* const* outputs ) + { + const double* momenta_in = nullptr; + const double* alpha_s_in = nullptr; + const unsigned int* flavor_indices_in = nullptr; + const double* random_color_in = nullptr; + const double* random_helicity_in = nullptr; + const double* random_diagram_in = nullptr; + [[maybe_unused]] const int* diagram_in = nullptr; // TODO: unused + + for( std::size_t i = 0; i < input_count; ++i ) + { + const void* input = inputs[i]; + switch( input_keys[i] ) + { + case UMAMI_IN_MOMENTA: + momenta_in = static_cast( input ); + break; + case UMAMI_IN_ALPHA_S: + alpha_s_in = static_cast( input ); + break; + case UMAMI_IN_FLAVOR_INDEX: + flavor_indices_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_COLOR: + random_color_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_HELICITY: + random_helicity_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_DIAGRAM: + random_diagram_in = static_cast( input ); + break; + case UMAMI_IN_HELICITY_INDEX: + return UMAMI_ERROR_UNSUPPORTED_INPUT; + case UMAMI_IN_DIAGRAM_INDEX: + diagram_in = static_cast( input ); + break; + default: + return UMAMI_ERROR_UNSUPPORTED_INPUT; + } + } + if( !momenta_in ) return UMAMI_ERROR_MISSING_INPUT; + + double* m2_out = nullptr; + double* amp2_out = nullptr; + int* diagram_out = nullptr; + int* color_out = nullptr; + int* helicity_out = nullptr; + for( std::size_t i = 0; i < output_count; ++i ) + { + void* output = outputs[i]; + switch( output_keys[i] ) + { + case UMAMI_OUT_MATRIX_ELEMENT: + m2_out = static_cast( output ); + break; + case UMAMI_OUT_DIAGRAM_AMP2: + amp2_out = static_cast( output ); + break; + case UMAMI_OUT_COLOR_INDEX: + color_out = static_cast( output ); + break; + case UMAMI_OUT_HELICITY_INDEX: + helicity_out = static_cast( output ); + break; + case UMAMI_OUT_DIAGRAM_INDEX: + diagram_out = static_cast( output ); + break; + default: + return UMAMI_ERROR_UNSUPPORTED_OUTPUT; + } + } + + constexpr std::size_t vector_size = MemoryAccessMomentaBase::neppM; + // need to round to round to double page size for some reason + constexpr std::size_t page_size2 = 2 * vector_size; + std::vector permutation; + std::size_t rounded_count; + + constexpr std::size_t flavor_count = ProcessData::nmaxflavor; + HostBufferBase flavor_indices( ((count + page_size2 - 1) / page_size2 + flavor_count) * page_size2 ); + bool sort_flavors = vector_size > 1 && flavor_count > 1 && flavor_indices_in; + if ( sort_flavors ) + { + permutation.resize(count); + std::size_t voffset = 0; + std::size_t vector_indices[flavor_count] = {}; + std::size_t vector_counts[flavor_count] = {}; + // determine permutation of inputs such that all entries in a SIMD vector + // have the same flavor index + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + unsigned int flav = flavor_indices_in[i_event + offset]; + auto& vcount = vector_counts[flav]; + auto& vindex = vector_indices[flav]; + if ( vcount == 0 ) + { + vindex = voffset * page_size2; + for ( std::size_t i = 0; i < page_size2; ++i) { + flavor_indices[voffset * page_size2 + i] = flav; + } + voffset += 1; + } + permutation[i_event] = vindex + vcount; + vcount = (vcount + 1) % page_size2; + } + rounded_count = voffset * page_size2; + } else { + rounded_count = ( count + page_size2 - 1 ) / page_size2 * page_size2; + } + + HostBufferBase momenta( rounded_count * ProcessData::npar * 4 ); + HostBufferBase couplings( rounded_count * madmatrix::Parameters_dependentCouplings::ndcoup * 2 ); + HostBufferBase g_s( rounded_count ); + HostBufferBase helicity_random( rounded_count ); + HostBufferBase color_random( rounded_count ); + HostBufferBase diagram_random( rounded_count ); + HostBufferBase matrix_elements( rounded_count ); + HostBufferBase diagram_index( rounded_count ); + HostBufferBase numerators( rounded_count * ProcessData::ndiagrams ); + HostBufferBase denominators( rounded_count ); + HostBufferBase helicity_index( rounded_count ); + HostBufferBase color_index( rounded_count ); + if ( sort_flavors ) { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_sorted = permutation[i_event]; + transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_sorted, stride ); + helicity_random[i_sorted] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; + color_random[i_sorted] = random_color_in ? random_color_in[i_event + offset] : 0.5; + diagram_random[i_sorted] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; + g_s[i_sorted] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + } + } else { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_event, stride ); + helicity_random[i_event] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; + color_random[i_event] = random_color_in ? random_color_in[i_event + offset] : 0.5; + diagram_random[i_event] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; + g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; + } + for ( std::size_t i_event = count; i_event < rounded_count; ++i_event ) { + flavor_indices[i_event] = 0; + } + } + computeDependentCouplings( g_s.data(), couplings.data(), rounded_count ); + + InterfaceInstance* instance = static_cast( handle ); + if( !instance->initialized ) + { + initialize( + momenta.data(), + couplings.data(), + flavor_indices.data(), + matrix_elements.data(), + numerators.data(), + denominators.data(), + rounded_count ); + instance->initialized = true; + } + + sigmaKin( + momenta.data(), + couplings.data(), + flavor_indices.data(), + helicity_random.data(), + color_random.data(), + nullptr, + diagram_random.data(), + matrix_elements.data(), + helicity_index.data(), + color_index.data(), + numerators.data(), + denominators.data(), + diagram_index.data(), + false, + rounded_count ); + + if ( sort_flavors ) + { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_sorted = permutation[i_event]; + std::size_t page_size = MemoryAccessMomentaBase::neppM; + std::size_t i_page = i_sorted / page_size; + std::size_t i_vector = i_sorted % page_size; // vector lane + + double denominator = denominators[i_sorted]; + if( m2_out != nullptr ) + { + m2_out[i_event + offset] = matrix_elements[i_sorted]; + } + if( amp2_out != nullptr ) + { + for( std::size_t i_diag = 0; i_diag < ProcessData::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * ProcessData::ndiagrams + i_diag * page_size + i_vector] / denominator; + } + } + if( diagram_out != nullptr ) + { + diagram_out[i_event + offset] = diagram_index[i_sorted] - 1; + } + if( color_out != nullptr ) + { + color_out[i_event + offset] = color_index[i_sorted] - 1; + } + if( helicity_out != nullptr ) + { + helicity_out[i_event + offset] = helicity_index[i_sorted] - 1; + } + } + } else { + std::size_t page_size = MemoryAccessMomentaBase::neppM; + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_page = i_event / page_size; + std::size_t i_vector = i_event % page_size; + + double denominator = denominators[i_event]; + if( m2_out != nullptr ) + { + m2_out[i_event + offset] = matrix_elements[i_event]; + } + if( amp2_out != nullptr ) + { + for( std::size_t i_diag = 0; i_diag < ProcessData::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * ProcessData::ndiagrams + i_diag * page_size + i_vector] / denominator; + } + } + if( diagram_out != nullptr ) + { + diagram_out[i_event + offset] = diagram_index[i_event] - 1; + } + if( color_out != nullptr ) + { + color_out[i_event + offset] = color_index[i_event] - 1; + } + if( helicity_out != nullptr ) + { + helicity_out[i_event + offset] = helicity_index[i_event] - 1; + } + } + } + return UMAMI_SUCCESS; + } + + UmamiStatus umami_free( UmamiHandle handle ) + { + InterfaceInstance* instance = static_cast( handle ); + delete instance; + return UMAMI_SUCCESS; + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc similarity index 98% rename from madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.cc rename to madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc index 7a3e0e0519..d0cf3a97a2 100644 --- a/madgraph/iolibs/template_files/madmatrix/CrossSectionKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.cc @@ -79,11 +79,7 @@ debug_me_is_abnormal( const fptype& me, size_t ievtALL ) //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- @@ -187,8 +183,7 @@ namespace mg5amcCpu //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu +namespace madmatrix { /* @@ -233,6 +228,5 @@ namespace mg5amcGpu */ } -#endif //============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h new file mode 100644 index 0000000000..59a4c098ec --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/CrossSectionKernels.h @@ -0,0 +1,98 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef CROSSSECTIONKERNELS_H +#define CROSSSECTIONKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "EventStatistics.h" +#include "MemoryBuffers.h" + +//============================================================================ + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + // Helper function for Bridge.h: must be compiled without fast math + // Iterate through all output MEs and replace any NaN/abnormal ones by sqrt(-1) + void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ); + + //-------------------------------------------------------------------------- + + // A base class encapsulating the calculation of event statistics on a CPU host or on a GPU device + class CrossSectionKernelBase //: virtual public ICrossSectionKernel + { + protected: + + // Constructor from existing input and output buffers + CrossSectionKernelBase( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats ) // output: event statistics + : m_samplingWeights( samplingWeights ) + , m_matrixElements( matrixElements ) + , m_stats( stats ) + , m_iter( 0 ) + { + // NB: do not initialise EventStatistics (you may be asked to update an existing result) + } + + public: + + // Destructor + virtual ~CrossSectionKernelBase() {} + + // Update event statistics + virtual void updateEventStatistics( const bool debug = false ) = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The buffer for the sampling weights + const BufferWeights& m_samplingWeights; + + // The buffer for the output matrix elements + const BufferMatrixElements& m_matrixElements; + + // The event statistics + EventStatistics& m_stats; + + // The number of iterations processed so far + size_t m_iter; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating the calculation of event statistics on a CPU host + class CrossSectionKernelHost final : public CrossSectionKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t nevt ); + + // Destructor + virtual ~CrossSectionKernelHost() {} + + // Update event statistics + void updateEventStatistics( const bool debug = false ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + }; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- +} +#endif // CROSSSECTIONKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h new file mode 100644 index 0000000000..df62a74918 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/EventStatistics.h @@ -0,0 +1,171 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef EventStatistics_H +#define EventStatistics_H 1 + +#include "mgOnGpuConfig.h" + +#include "ProcessData.h" // for npar (meGeVexponent) + +#include +#include +#include +#include +#include + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + // The EventStatistics struct is used to accumulate running aggregates of event statistics. + // This will eventually include the process cross section and the process maximum weight: + // one important case of EventStatistics will then be the "gridpack" result set, which is + // the output of the "integration" step and the input to "unweighted event generation" step. + // The current implementation only includes statistics for matrix elements (ME) and sampling weights (WG); + // in first approximation, the process cross section and maximum weight are just the mean ME and maximum ME, + // but eventually the sampling weights WG (e.g. from Rambo) must also be taken into account in the calculation. + // The implementation uses differences to reference values to improve numerical precision. + struct EventStatistics + { + public: + size_t nevtALL; // total number of events used + size_t nevtABN; // number of events used, where ME is abnormal (nevtABN <= nevtALL) + size_t nevtZERO; // number of not-abnormal events used, where ME is zero (nevtZERO <= nevtOK) + double minME; // minimum matrix element + double maxME; // maximum matrix element + double minWG; // minimum sampling weight + double maxWG; // maximum sampling weight + double refME; // "reference" matrix element (normally the current mean) + double refWG; // "reference" sampling weight (normally the current mean) + double sumMEdiff; // sum of diff to ref for matrix element + double sumWGdiff; // sum of diff to ref for sampling weight + double sqsMEdiff; // squared sum of diff to ref for matrix element + double sqsWGdiff; // squared sum of diff to ref for sampling weight + std::string tag; // a text tag for printouts + // Number of events used, where ME is not abnormal + size_t nevtOK() const { return nevtALL - nevtABN; } + // Mean matrix element + // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] + double meanME() const + { + return refME + ( nevtOK() > 0 ? sumMEdiff / nevtOK() : 0 ); + } + // Mean sampling weight + // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] + double meanWG() const + { + return refWG + ( nevtOK() > 0 ? sumWGdiff / nevtOK() : 0 ); + } + // Variance matrix element + // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] + double varME() const { return ( sqsMEdiff - std::pow( sumMEdiff, 2 ) / nevtOK() ) / nevtOK(); } + // Variance sampling weight + // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] + double varWG() const { return ( sqsWGdiff - std::pow( sumWGdiff, 2 ) / nevtOK() ) / nevtOK(); } + // Standard deviation matrix element + double stdME() const { return std::sqrt( varME() ); } + // Standard deviation sampling weight + double stdWG() const { return std::sqrt( varWG() ); } + // Update reference matrix element + void updateRefME( const double newRef ) + { + const double deltaRef = refME - newRef; + sqsMEdiff += deltaRef * ( 2 * sumMEdiff + nevtOK() * deltaRef ); + sumMEdiff += deltaRef * nevtOK(); + refME = newRef; + } + // Update reference sampling weight + void updateRefWG( const double newRef ) + { + const double deltaRef = refWG - newRef; + sqsWGdiff += deltaRef * ( 2 * sumWGdiff + nevtOK() * deltaRef ); + sumWGdiff += deltaRef * nevtOK(); + refWG = newRef; + } + // Constructor + EventStatistics() + : nevtALL( 0 ) + , nevtABN( 0 ) + , nevtZERO( 0 ) + , minME( std::numeric_limits::max() ) + , maxME( std::numeric_limits::lowest() ) + , minWG( std::numeric_limits::max() ) + , maxWG( std::numeric_limits::lowest() ) + , refME( 0 ) + , refWG( 0 ) + , sumMEdiff( 0 ) + , sumWGdiff( 0 ) + , sqsMEdiff( 0 ) + , sqsWGdiff( 0 ) + , tag( "" ) {} + // Combine two EventStatistics +#ifdef __clang__ + // Disable optimizations for this function in HIP (work around FPE crash #1003: originally using #if __HIP_CLANG_ONLY__) + // Disable optimizations for this function in clang tout court (work around FPE crash #1005: now using #ifdef __clang__) + // See https://clang.llvm.org/docs/LanguageExtensions.html#extensions-for-selectively-disabling-optimization + __attribute__( ( optnone ) ) +#endif + EventStatistics& + operator+=( const EventStatistics& stats ) + { + EventStatistics s1 = *this; // temporary copy + EventStatistics s2 = stats; // temporary copy + EventStatistics& sum = *this; + sum.nevtALL = s1.nevtALL + s2.nevtALL; + sum.nevtABN = s1.nevtABN + s2.nevtABN; + sum.nevtZERO = s1.nevtZERO + s2.nevtZERO; + sum.minME = std::min( s1.minME, s2.minME ); + sum.maxME = std::max( s1.maxME, s2.maxME ); + sum.minWG = std::min( s1.minWG, s2.minWG ); + sum.maxWG = std::max( s1.maxWG, s2.maxWG ); + sum.refME = ( s1.meanME() * s1.nevtOK() + s2.meanME() * s2.nevtOK() ) / sum.nevtOK(); // new mean ME + s1.updateRefME( sum.refME ); + s2.updateRefME( sum.refME ); + sum.sumMEdiff = s1.sumMEdiff + s2.sumMEdiff; + sum.sqsMEdiff = s1.sqsMEdiff + s2.sqsMEdiff; + sum.refWG = ( s1.meanWG() * s1.nevtOK() + s2.meanWG() * s2.nevtOK() ) / sum.nevtOK(); // new mean WG + s1.updateRefWG( sum.refWG ); + s2.updateRefWG( sum.refWG ); + sum.sumWGdiff = s1.sumWGdiff + s2.sumWGdiff; + sum.sqsWGdiff = s1.sqsWGdiff + s2.sqsWGdiff; + return sum; + } + // Printout + void printout( std::ostream& out ) const + { + const EventStatistics& s = *this; + constexpr int meGeVexponent = -( 2 * ProcessData::npar - 8 ); + out << s.tag << "NumMatrixElems(notAbnormal) = " << s.nevtOK() << std::endl + << std::scientific // fixed format: affects all floats (default precision: 6) + << s.tag << "MeanMatrixElemValue = ( " << s.meanME() + << " +- " << s.stdME() / std::sqrt( s.nevtOK() ) << " ) GeV^" << meGeVexponent << std::endl // standard error + << s.tag << "[Min,Max]MatrixElemValue = [ " << s.minME + << " , " << s.maxME << " ] GeV^" << meGeVexponent << std::endl + << s.tag << "StdDevMatrixElemValue = ( " << s.stdME() + << std::string( 16, ' ' ) << " ) GeV^" << meGeVexponent << std::endl + << s.tag << "MeanWeight = ( " << s.meanWG() + << " +- " << s.stdWG() / std::sqrt( s.nevtOK() ) << std::endl // standard error + << s.tag << "[Min,Max]Weight = [ " << s.minWG + << " , " << s.maxWG << " ]" << std::endl + << s.tag << "StdDevWeight = ( " << s.stdWG() + << std::string( 16, ' ' ) << " )" << std::endl + << std::defaultfloat; // default format: affects all floats + } + }; + + //-------------------------------------------------------------------------- + + inline std::ostream& operator<<( std::ostream& out, const EventStatistics& s ) + { + s.printout( out ); + return out; + } + + //-------------------------------------------------------------------------- +} + +#endif // EventStatistics_H diff --git a/madgraph/iolibs/template_files/madmatrix/GpuAbstraction.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h similarity index 96% rename from madgraph/iolibs/template_files/madmatrix/GpuAbstraction.h rename to madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h index 6f709dfe9a..3dd375ecf7 100644 --- a/madgraph/iolibs/template_files/madmatrix/GpuAbstraction.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuAbstraction.h @@ -13,7 +13,7 @@ //-------------------------------------------------------------------------- -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ #ifndef MGONGPU_HAS_NO_BLAS #include "cublas_v2.h" @@ -33,6 +33,7 @@ #define gpuMemcpyDeviceToHost cudaMemcpyDeviceToHost #define gpuMemcpyDeviceToDevice cudaMemcpyDeviceToDevice #define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( cudaMemcpyToSymbol( type1, type2, size ) ) +#define gpuMemcpyFromSymbol( type1, type2, size ) checkGpu( cudaMemcpyFromSymbol( type1, type2, size ) ) #define gpuFree( ptr ) checkGpu( cudaFree( ptr ) ) #define gpuFreeHost( ptr ) checkGpu( cudaFreeHost( ptr ) ) @@ -101,6 +102,7 @@ #define gpuMemcpyDeviceToHost hipMemcpyDeviceToHost #define gpuMemcpyDeviceToDevice hipMemcpyDeviceToDevice #define gpuMemcpyToSymbol( type1, type2, size ) checkGpu( hipMemcpyToSymbol( type1, type2, size ) ) +#define gpuMemcpyFromSymbol( type1, type2, size ) checkGpu( hipMemcpyFromSymbol( type1, type2, size ) ) #define gpuFree( ptr ) checkGpu( hipFree( ptr ) ) #define gpuFreeHost( ptr ) checkGpu( hipHostFree( ptr ) ) diff --git a/madgraph/iolibs/template_files/madmatrix/GpuRuntime.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h similarity index 95% rename from madgraph/iolibs/template_files/madmatrix/GpuRuntime.h rename to madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h index 2c8ae1a804..6aa7df2374 100644 --- a/madgraph/iolibs/template_files/madmatrix/GpuRuntime.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/GpuRuntime.h @@ -17,7 +17,6 @@ //-------------------------------------------------------------------------- // See https://stackoverflow.com/a/14038590 -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ #define checkGpu( code ) { assertGpu( code, __FILE__, __LINE__ ); } inline void assertGpu( gpuError_t code, const char* file, int line, bool abort = true ) { @@ -27,11 +26,9 @@ inline void assertGpu( gpuError_t code, const char* file, int line, bool abort = if( abort ) assert( code == gpuSuccess ); } } -#endif /* clang-format on */ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ #ifndef MGONGPU_HAS_NO_BLAS #define checkGpuBlas( code ){ assertGpuBlas( code, __FILE__, __LINE__ ); } inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, bool abort = true ) @@ -43,12 +40,10 @@ inline void assertGpuBlas( gpuBlasStatus_t code, const char *file, int line, boo } } #endif -#endif /* clang-format on */ //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu +namespace madmatrix { // Instantiate a GpuRuntime at the beginnining of the application's main to // invoke gpuSetDevice(0) in the constructor and book a gpuDeviceReset() call in the destructor @@ -95,7 +90,6 @@ namespace mg5amcGpu } }; } -#endif //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc similarity index 65% rename from madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.cc rename to madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc index 872e4795e3..121aab5a60 100644 --- a/madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.cc @@ -6,7 +6,8 @@ #include "MatrixElementKernels.h" -#include "CPPProcess.h" +#include "ProcessData.h" +#include "SigmaKin.h" #include "GpuRuntime.h" // Includes the abstraction for Nvidia/AMD compilation #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" @@ -17,11 +18,7 @@ //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- @@ -50,7 +47,7 @@ namespace mg5amcCpu { //std::cout << "DEBUG: MatrixElementKernelBase ctor " << this << std::endl; #ifdef MGONGPU_CHANNELID_DEBUG - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) m_nevtProcessedByChannel[channelId] = 0; #endif } @@ -80,7 +77,7 @@ namespace mg5amcCpu { const size_t channelId = pHstChannelIds[ievt]; // Fortran indexing //assert( channelId > 0 ); - //assert( channelId < CPPProcess::ndiagrams ); + //assert( channelId < ProcessData::ndiagrams ); m_nevtProcessedByChannel[channelId]++; } } @@ -98,11 +95,11 @@ namespace mg5amcCpu void MatrixElementKernelBase::dumpNevtProcessedByChannel() { size_t nevtProcessed = 0; - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) nevtProcessed += m_nevtProcessedByChannel[channelId]; std::ostringstream sstr; sstr << " {"; - for( size_t channelId = 0; channelId < CPPProcess::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) { if( m_nevtProcessedByChannel[channelId] > 0 ) { @@ -117,7 +114,7 @@ namespace mg5amcCpu sstr << " }"; std::cout << "DEBUG: MEK " << this; if( m_tag != "" ) std::cout << " " << m_tag; - std::cout << " processed " << nevtProcessed << " events across " << CPPProcess::ndiagrams << " channels" << sstr.str() << std::endl; + std::cout << " processed " << nevtProcessed << " events across " << ProcessData::ndiagrams << " channels" << sstr.str() << std::endl; } #endif @@ -146,148 +143,10 @@ namespace mg5amcCpu //============================================================================ -#ifndef MGONGPUCPP_GPUIMPL -namespace mg5amcCpu -{ - - //-------------------------------------------------------------------------- - - MatrixElementKernelHost::MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt ) - : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) - , NumberOfEvents( nevt ) - , m_couplings( nevt ) - , m_numerators( nevt * CPPProcess::ndiagrams ) - , m_denominators( nevt ) - { - //std::cout << "DEBUG: MatrixElementKernelHost::ctor " << this << std::endl; - if( m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: momenta must be a host array" ); - if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: matrixElements must be a host array" ); - if( m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: channelIds must be a device array" ); - if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with momenta" ); - if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with matrixElements" ); - if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with channelIds" ); - if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with iflavorVec" ); - // Sanity checks for memory access (momenta buffer) - constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout - static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); - if( nevt % neppM != 0 ) - { - std::ostringstream sstr; - sstr << "MatrixElementKernelHost: nevt should be a multiple of neppM=" << neppM; - throw std::runtime_error( sstr.str() ); - } - // Fail gently and avoid "Illegal instruction (core dumped)" if the host does not support the SIMD used in the ME calculation - // Note: this prevents a crash on pmpe04 but not on some github CI nodes? - // [NB: SIMD vectorization in mg5amc C++ code is only used in the ME calculation below MatrixElementKernelHost!] - if( !MatrixElementKernelHost::hostSupportsSIMD() ) - throw std::runtime_error( "Host does not support the SIMD implementation of MatrixElementKernelsHost" ); - } - - //-------------------------------------------------------------------------- - - MatrixElementKernelHost::~MatrixElementKernelHost() - { - //std::cout << "DEBUG: MatrixElementKernelBase::dtor " << this << std::endl; - } - - //-------------------------------------------------------------------------- - - int MatrixElementKernelHost::computeGoodHelicities() - { - HostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); - // ... 0d1. Compute good helicity mask on the host - computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); - sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_numerators.data(), m_denominators.data(), hstIsGoodHel.data(), nevt() ); - // ... 0d2. Copy good helicity list to static memory on the host - // [FIXME! REMOVE THIS STATIC THAT BREAKS MULTITHREADING?] - return sigmaKin_setGoodHel( hstIsGoodHel.data() ); - } - - //-------------------------------------------------------------------------- - - void MatrixElementKernelHost::computeMatrixElements( const bool useChannelIds ) - { - computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); - const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); - sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_numerators.data(), m_denominators.data(), nullptr, true, nevt() ); -#ifdef MGONGPU_CHANNELID_DEBUG - //std::cout << "DEBUG: MatrixElementKernelHost::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; - MatrixElementKernelBase::updateNevtProcessedByChannel( pChannelIds, nevt() ); -#endif - } - - //-------------------------------------------------------------------------- - - // Does this host system support the SIMD used in the matrix element calculation? - bool MatrixElementKernelHost::hostSupportsSIMD( const bool verbose ) - { -#if defined __AVX512VL__ - bool known = true; - bool ok = __builtin_cpu_supports( "avx512vl" ); - const std::string tag = "skylake-avx512 (AVX512VL)"; -#elif defined __AVX2__ - bool known = true; - bool ok = __builtin_cpu_supports( "avx2" ); - const std::string tag = "haswell (AVX2)"; -#elif defined __SSE4_2__ -#ifdef __PPC__ - // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html - bool known = true; - bool ok = __builtin_cpu_supports( "vsx" ); - const std::string tag = "powerpc vsx (128bit as in SSE4.2)"; -#elif defined( __x86_64__ ) || defined( __i386__ ) - bool known = true; - bool ok = __builtin_cpu_supports( "sse4.2" ); - const std::string tag = "nehalem (SSE4.2)"; -#else // AV FIXME! Added by OM for Mac, should identify the correct __xxx__ flag that should be targeted - // DM now we have an explicit NEON target for ARM - bool known = false; // __builtin_cpu_supports is not supported - bool ok = true; // this is just an assumption! - const std::string tag = "simd arch not defined"; -#endif -#elif defined __ARM_NEON // consider using __BUILTIN_CPU_SUPPORTS__ - bool known = false; // __builtin_cpu_supports is not supported - // See https://stackoverflow.com/q/62783908 - // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu - bool ok = true; // this is just an assumption! - const std::string tag = "arm neon (128bit as in SSE4.2)"; -#else - bool known = true; - bool ok = true; - const std::string tag = "none"; -#endif - if( verbose ) - { - if( tag == "none" ) - std::cout << "INFO: The application does not require the host to support any AVX feature" << std::endl; - else if( ok && known ) - std::cout << "INFO: The application is built for " << tag << " and the host supports it" << std::endl; - else if( ok ) - std::cout << "WARNING: The application is built for " << tag << " but it is unknown if the host supports it" << std::endl; - else - std::cout << "ERROR! The application is built for " << tag << " but the host does not support it" << std::endl; - } - return ok; - } - - //-------------------------------------------------------------------------- - -} -#endif //============================================================================ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu +namespace madmatrix { //-------------------------------------------------------------------------- @@ -310,7 +169,7 @@ namespace mg5amcGpu , m_pHelJamps() , m_pHelNumerators() , m_pHelDenominators() - , m_colJamp2s( CPPProcess::ncolor * this->nevt() ) + , m_colJamp2s( ProcessData::ncolor * this->nevt() ) #ifdef MGONGPU_CHANNELID_DEBUG , m_hstChannelIds( this->nevt() ) #endif @@ -345,9 +204,9 @@ namespace mg5amcGpu throw std::runtime_error( sstr.str() ); } // Create the "one-helicity" jamp buffer that will be used for helicity filtering - m_pHelJamps.reset( new DeviceBufferSimple( CPPProcess::ncolor * mgOnGpu::nx2 * this->nevt() ) ); + m_pHelJamps.reset( new DeviceBufferSimple( ProcessData::ncolor * mgOnGpu::nx2 * this->nevt() ) ); // Create the "one-helicity" numerator and denominator buffers that will be used for helicity filtering - m_pHelNumerators.reset( new DeviceBufferSimple( this->nevt() * CPPProcess::ndiagrams ) ); + m_pHelNumerators.reset( new DeviceBufferSimple( this->nevt() * ProcessData::ndiagrams ) ); m_pHelDenominators.reset( new DeviceBufferSimple( this->nevt() ) ); // Decide at runtime whether to use BLAS for color sums // Decide at runtime whether TF32TENSOR math should be used in cuBLAS @@ -375,7 +234,7 @@ namespace mg5amcGpu #endif } #ifndef MGONGPU_HAS_NO_BLAS -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ // Analyse environment variable CUDACPP_RUNTIME_CUBLASTF32TENSOR const char* blasEnv2 = getenv( "CUDACPP_RUNTIME_CUBLASTF32TENSOR" ); if( blasEnv2 && std::string( blasEnv2 ) != "" ) @@ -412,7 +271,7 @@ namespace mg5amcGpu #ifndef MGONGPU_HAS_NO_BLAS if( m_blasHandle ) gpuBlasDestroy( m_blasHandle ); #endif - for( int ihel = 0; ihel < CPPProcess::ncomb; ihel++ ) + for( int ihel = 0; ihel < ProcessData::ncomb; ihel++ ) { if( m_helStreams[ihel] ) gpuStreamDestroy( m_helStreams[ihel] ); // do not destroy if nullptr } @@ -432,7 +291,7 @@ namespace mg5amcGpu int MatrixElementKernelDevice::computeGoodHelicities() { - PinnedHostBufferHelicityMask hstIsGoodHel( CPPProcess::ncomb ); + PinnedHostBufferHelicityMask hstIsGoodHel( ProcessData::ncomb ); // ... 0d1. Compute good helicity mask (a host variable) on the device gpuLaunchKernel( computeDependentCouplings, m_gpublocks, m_gputhreads, m_gs.data(), m_couplings.data() ); const int nevt = m_gpublocks * m_gputhreads; @@ -448,7 +307,7 @@ namespace mg5amcGpu if( m_blasColorSum ) { checkGpuBlas( gpuBlasCreate( &m_blasHandle ) ); -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) +#ifdef __CUDACC__ // this must be __CUDACC__ if( m_blasTf32Tensor ) checkGpuBlas( cublasSetMathMode( m_blasHandle, CUBLAS_TF32_TENSOR_OP_MATH ) ); // enable TF32 tensor cores #endif @@ -458,19 +317,19 @@ namespace mg5amcGpu m_pHelMEs.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); // ... Create the "many-helicity" super-buffer of nGoodHel ME buffers (dynamically allocated because nGoodHel is determined at runtime) // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) - m_pHelJamps.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); + m_pHelJamps.reset( new DeviceBufferSimple( nGoodHel * ProcessData::ncolor * mgOnGpu::nx2 * nevt ) ); // ... Create the "many-helicity" super-buffers of nGoodHel numerator and denominator buffers (dynamically allocated) // ... (calling reset here deletes the previously created "one-helicity" buffers used for helicity filtering) - m_pHelNumerators.reset( new DeviceBufferSimple( nGoodHel * CPPProcess::ndiagrams * nevt ) ); + m_pHelNumerators.reset( new DeviceBufferSimple( nGoodHel * ProcessData::ndiagrams * nevt ) ); m_pHelDenominators.reset( new DeviceBufferSimple( nGoodHel * nevt ) ); #ifndef MGONGPU_HAS_NO_BLAS // Create the "many-helicity" super-buffers of real/imag ncolor*nevt temporary buffers for cuBLAS/hipBLAS intermediate results in color_sum_blas #if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT // Mixed precision mode: need two fptype2[ncolor*2*nevt] buffers and one fptype2[nevt] buffer per good helicity - if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ( 2 * CPPProcess::ncolor * mgOnGpu::nx2 + 1 ) * nevt ) ); + if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ( 2 * ProcessData::ncolor * mgOnGpu::nx2 + 1 ) * nevt ) ); #else // Standard single/double precision mode: need one fptype2[ncolor*2*nevt] buffer per good helicity - if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * CPPProcess::ncolor * mgOnGpu::nx2 * nevt ) ); + if( m_blasColorSum ) m_pHelBlasTmp.reset( new DeviceBufferSimple2( nGoodHel * ProcessData::ncolor * mgOnGpu::nx2 * nevt ) ); #endif #endif // Return the number of good helicities @@ -504,6 +363,5 @@ namespace mg5amcGpu //-------------------------------------------------------------------------- } -#endif //============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h similarity index 74% rename from madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.h rename to madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h index 01041e43b7..aa406c60d1 100644 --- a/madgraph/iolibs/template_files/madmatrix/MatrixElementKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MatrixElementKernels.h @@ -9,18 +9,14 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "GpuAbstraction.h" #include "MemoryBuffers.h" #include #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- @@ -109,58 +105,9 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating matrix element calculations on a CPU host - class MatrixElementKernelHost final : public MatrixElementKernelBase, public NumberOfEvents - { - public: - - // Constructor from existing input and output buffers - MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta - const BufferGs& gs, // input: gs for alphaS - const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination - const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection - const BufferRndNumColor& rndcol, // input: random numbers for color selection - const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement - BufferMatrixElements& matrixElements, // output: matrix elements - BufferSelectedHelicity& selhel, // output: helicity selection - BufferSelectedColor& selcol, // output: color selection - const size_t nevt); - - // Destructor - virtual ~MatrixElementKernelHost(); - - // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) - int computeGoodHelicities() override final; - - // Compute matrix elements - void computeMatrixElements( const bool useChannelIds ) override final; - - // Is this a host or device kernel? - bool isOnDevice() const override final { return false; } - - private: - - // Does this host system support the SIMD used in the matrix element calculation? - // [NB: this is private, SIMD vectorization in mg5amc C++ code is currently only used in the ME calculations below MatrixElementKernelHost!] - static bool hostSupportsSIMD( const bool verbose = false ); // ZW: default verbose false - - private: - - // The buffer for the event-by-event couplings that depends on alphas QCD - HostBufferCouplings m_couplings; - - // The buffer for the event-by-event numerators of multichannel factors - HostBufferNumerators m_numerators; - - // The buffer for the event-by-event denominators of multichannel factors - HostBufferDenominators m_denominators; - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating matrix element calculations on a GPU device class MatrixElementKernelDevice : public MatrixElementKernelBase, public NumberOfEvents { @@ -235,7 +182,7 @@ namespace mg5amcCpu #endif // The array of GPU streams (one for each good helicity) - gpuStream_t m_helStreams[CPPProcess::ncomb]; // reserve ncomb streams (but only nGoodHel <= ncomb will be used) + gpuStream_t m_helStreams[ProcessData::ncomb]; // reserve ncomb streams (but only nGoodHel <= ncomb will be used) // The number of blocks in the GPU grid size_t m_gpublocks; @@ -243,7 +190,6 @@ namespace mg5amcCpu // The number of threads in the GPU grid size_t m_gputhreads; }; -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h new file mode 100644 index 0000000000..669ed8ead5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessAmplitudes.h @@ -0,0 +1,161 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessAmplitudes_H +#define MemoryAccessAmplitudes_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" + +#define MGONGPU_TRIVIAL_AMPLITUDES 1 + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + +#ifndef MGONGPU_TRIVIAL_AMPLITUDES + + // A class describing the internal layout of memory buffers for amplitudes + // This implementation uses an AOSOA[npagA][nx2][neppA] where nevt=npagA*neppA + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessAmplitudesBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the amplitude AOSOA memory buffer layout + static constexpr int neppA = 1; // AOS (just a test...) + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagA = ievt / neppA; // #event "A-page" + const int ieppA = ievt % neppA; // #event in the current event A-page + constexpr int ix2 = 0; + return &( buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA] ); // AOSOA[ipagA][ix2][ieppA] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ix2 ) + { + constexpr int ipagA = 0; + constexpr int ieppA = 0; + return buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA]; // AOSOA[ipagA][ix2][ieppA] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessAmplitudes : public MemoryAccessAmplitudesBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + +#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessAmplitudes + { + public: + +#ifndef MGONGPU_TRIVIAL_AMPLITUDES + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2 = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2Const = + KernelAccessHelper::template kernelAccessFieldConst; + +#else + + static __host__ __device__ inline cxtype_sv* + kernelAccess( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + static __host__ __device__ inline const cxtype_sv* + kernelAccessConst( const fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + +#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessAmplitudes HostAccessAmplitudes; + typedef KernelAccessAmplitudes DeviceAccessAmplitudes; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h new file mode 100644 index 0000000000..0b1c1162a3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessChannelIds.h @@ -0,0 +1,115 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: S. Roiser (Dec 2023, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessChannelIds_H +#define MemoryAccessChannelIds_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for channel ids + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessChannelIdsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] + static __host__ __device__ inline unsigned int* + ieventAccessRecord( unsigned int* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline unsigned int& + decodeRecord( unsigned int* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessChannelIds : public MemoryAccessChannelIdsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] + static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessChannelIds + { + public: + + // Expose selected functions from MemoryAccessChannelIds + static constexpr auto ieventAccessRecordConst = MemoryAccessChannelIds::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] + static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] + static __host__ __device__ inline const uint_sv& + kernelAccessConst( const unsigned int* buffer ) + { + const unsigned int& out = kernelAccessConst_s( buffer ); + return out; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessChannelIds HostAccessChannelIds; + typedef KernelAccessChannelIds DeviceAccessChannelIds; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h new file mode 100644 index 0000000000..491c6f8a5a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplings.h @@ -0,0 +1,240 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessCouplings_H +#define MemoryAccessCouplings_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM +#include "MemoryBuffers.h" // for HostBufferCouplings::isaligned + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for couplings + // This implementation uses an AOSOA[npagC][ndcoup][nx2][neppC] "super-buffer" where nevt=npagC*neppC + // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessCouplingsBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the coupling AOSOA memory buffer layout + static constexpr int neppC = MemoryAccessMomentaBase::neppM; // use the same AOSOA striding as for momenta + + // SANITY CHECK: check that neppC is a power of two + static_assert( ispoweroftwo( neppC ), "neppC is not a power of 2" ); + + //-------------------------------------------------------------------------- + // ** NB! A single super-buffer AOSOA[npagC][ndcoup][nx2][neppC] includes data for ndcoup different couplings ** + // ** NB! The ieventAccessRecord and kernelAccess functions refer to the buffer for one individual coupling ** + // ** NB! Use idcoupAccessBuffer to add a fixed offset and locate the buffer for one given individual coupling ** + //-------------------------------------------------------------------------- + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (non-const) ===> fptype* idcoupAccessBuffer( fptype* buffer, const int idcoup ) <===] + // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? + static __host__ __device__ inline fptype* + idcoupAccessBuffer( fptype* buffer, // input "super-buffer" + const int idcoup ) + { + constexpr int ipagC = 0; + constexpr int ieppC = 0; + constexpr int ix2 = 0; + // NB! this effectively adds an offset "idcoup * nx2 * neppC" + return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] + } + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (const) ===> const fptype* idcoupAccessBufferConst( const fptype* buffer, const int idcoup ) <===] + // NB: keep this in public even if exposed through KernelAccessCouplings: nvcc says it is inaccesible otherwise? + static __host__ __device__ inline const fptype* + idcoupAccessBufferConst( const fptype* buffer, // input "super-buffer" + const int idcoup ) + { + return idcoupAccessBuffer( const_cast( buffer ), idcoup ); + } + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of couplings that dependent on the running alphas QCD in this specific process + static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagC = ievt / neppC; // #event "C-page" + const int ieppC = ievt % neppC; // #event in the current event C-page + constexpr int idcoup = 0; + constexpr int ix2 = 0; + return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ix2 ) + { + constexpr int ipagC = 0; + constexpr int ieppC = 0; + // NB! the offset "idcoup * nx2 * neppC" has been added in idcoupAccessBuffer + constexpr int idcoup = 0; + return buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC]; // AOSOA[ipagC][idcoup][ix2][ieppC] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessCouplings : public MemoryAccessCouplingsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessCouplings + { + public: + + // Expose selected functions from MemoryAccessCouplingsBase + static constexpr auto idcoupAccessBuffer = MemoryAccessCouplingsBase::idcoupAccessBuffer; + static constexpr auto idcoupAccessBufferConst = MemoryAccessCouplingsBase::idcoupAccessBufferConst; + + // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. + // For dependent (event-by-event, running-alphas) couplings the value is an AOSOA record [nx2][neppC] + // (real and imaginary SIMD lanes), so consecutive flavor slots are nx2*neppC fptype's apart. + static constexpr int flv_stride = MemoryAccessCouplingsBase::neppC * mgOnGpu::nx2; + + // Expose selected functions from MemoryAccessCouplings + static constexpr auto ieventAccessRecordConst = MemoryAccessCouplings::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2_s = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2Const_s = + KernelAccessHelper::template kernelAccessFieldConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccessIx2( fptype* buffer, + const int ix2 ) + { + fptype& out = kernelAccessIx2_s( buffer, ix2 ); + return out; + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessIx2Const( const fptype* buffer, + const int ix2 ) + { + return kernelAccessIx2( const_cast( buffer ), ix2 ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non const, SCALAR OR VECTOR) ===> cxtype_sv_ref kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline cxtype_sv_ref + kernelAccess( fptype* buffer ) + { + /* + fptype_sv& real = kernelAccessIx2( buffer, 0 ); + fptype_sv& imag = kernelAccessIx2( buffer, 1 ); + printf( "C_ACCESS::kernelAccess: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); + return cxtype_sv_ref( real, imag ); + */ + return cxtype_sv_ref( kernelAccessIx2( buffer, 0 ), + kernelAccessIx2( buffer, 1 ) ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] + static __host__ __device__ inline cxtype_sv + kernelAccessConst( const fptype* buffer ) + { + /* + const fptype_sv& real = kernelAccessIx2Const( buffer, 0 ); + const fptype_sv& imag = kernelAccessIx2Const( buffer, 1 ); + printf( "C_ACCESS::kernelAccessConst: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); + return cxtype_sv( real, imag ); + */ + return cxtype_sv( kernelAccessIx2Const( buffer, 0 ), + kernelAccessIx2Const( buffer, 1 ) ); + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessCouplings HostAccessCouplings; + typedef KernelAccessCouplings DeviceAccessCouplings; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h new file mode 100644 index 0000000000..3eadf6f393 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessCouplingsFixed.h @@ -0,0 +1,84 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Apr 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessCouplingsFixed_H +#define MemoryAccessCouplingsFixed_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +//#include "MemoryAccessHelpers.h" + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for fixed couplings + // This implementation uses a STRUCT[ndcoup][nx2] "super-buffer" layout: in practice, the cIPC global array + // From the "super-buffer" for ndcoup different couplings, use idcoupAccessBuffer to access the buffer for one specific coupling + // [If many implementations are used, a suffix _Sv1 should be appended to the class name] + class MemoryAccessCouplingsFixedBase //_Sv1 + { + public: + + // Locate the buffer for a single coupling (output) in a memory super-buffer (input) from the given coupling index (input) + // [Signature (const) ===> const fptype* iicoupAccessBufferConst( const fptype* buffer, const int iicoup ) <===] + static __host__ __device__ inline const fptype* + iicoupAccessBufferConst( const fptype* buffer, // input "super-buffer": in practice, the cIPC global array + const int iicoup ) + { + constexpr int ix2 = 0; + // NB! this effectively adds an offset "iicoup * nx2" + return &( buffer[iicoup * nx2 + ix2] ); // STRUCT[idcoup][ix2] + } + + private: + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessCouplingsFixed + { + public: + + // Expose selected functions from MemoryAccessCouplingsFixedBase + static constexpr auto iicoupAccessBufferConst = MemoryAccessCouplingsFixedBase::iicoupAccessBufferConst; + + // Per-flavor stride (in fptype's) between two consecutive flavor slots of a flavored coupling value buffer. + // For fixed (independent) couplings the value is a single scalar complex (real,imag): nx2 fptype's, broadcast across the SIMD vector. + static constexpr int flv_stride = mgOnGpu::nx2; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> cxtype_sv kernelAccessConst( const fptype* buffer ) <===] + static __host__ __device__ inline const cxtype_sv + kernelAccessConst( const fptype* buffer ) + { + // TRIVIAL ACCESS to fixed-couplings buffers! + //return cxmake( fptype_sv{ buffer[0] }, fptype_sv{ buffer[1] } ); // NO! BUG #339! + const fptype_sv r_sv = fptype_sv{ 0 } + buffer[0]; + const fptype_sv i_sv = fptype_sv{ 0 } + buffer[1]; + return cxmake( r_sv, i_sv ); // ugly but effective + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessCouplingsFixed HostAccessCouplingsFixed; + typedef KernelAccessCouplingsFixed DeviceAccessCouplingsFixed; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h new file mode 100644 index 0000000000..9e5b87e674 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessDenominators.h @@ -0,0 +1,27 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessDenominators_H +#define MemoryAccessDenominators_H 1 + +#include "MemoryAccessGs.h" + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for denominators + // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs + + typedef KernelAccessGs HostAccessDenominators; + typedef KernelAccessGs DeviceAccessDenominators; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h new file mode 100644 index 0000000000..ce27a06de0 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessGs.h @@ -0,0 +1,153 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessGs_H +#define MemoryAccessGs_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for Gs + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessGsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessGs : public MemoryAccessGsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessGs + { + public: + + // Expose selected functions from MemoryAccessGs + static constexpr auto ieventAccessRecord = MemoryAccessGs::ieventAccessRecord; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccess( fptype* buffer ) <===] + static constexpr auto kernelAccess_s = + KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non-const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccess( fptype* buffer ) + { + fptype& out = kernelAccess_s( buffer ); + return out; + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (SCALAR OR VECTOR) ===> fptype_sv* kernelAccess( fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv* + kernelAccessP( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + static constexpr auto kernelAccessConst_s = + KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const fptype_sv& kernelAccess( const fptype* buffer ) <===] + static __host__ __device__ inline const fptype_sv& + kernelAccessConst( const fptype* buffer ) + { + const fptype& out = kernelAccessConst_s( buffer ); + return out; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessGs HostAccessGs; + typedef KernelAccessGs DeviceAccessGs; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessHelpers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h similarity index 98% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessHelpers.h rename to madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h index 8b5bd27f4d..b42bc349a2 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessHelpers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessHelpers.h @@ -106,13 +106,9 @@ class KernelAccessHelper : public MemoryAccessHelper } else { -#ifdef MGONGPUCPP_GPUIMPL const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid //printf( "kernelAccessRecord: ievt=%d threadId=%d\n", ievt, threadIdx.x ); return T::ieventAccessRecord( buffer, ievt ); // NB fptype and fptype_sv coincide for CUDA -#else - throw std::runtime_error( "kernelAccessRecord on device is only implemented in CUDA" ); -#endif } } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h new file mode 100644 index 0000000000..a6e5c356e4 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessIflavorVec.h @@ -0,0 +1,114 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: D. Massaro (Jan 2026, based on earlier work by A. Valassi) for the MG5aMC CUDACPP plugin. +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessIflavorVec_H +#define MemoryAccessIflavorVec_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for channel ids + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessIflavorVecBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> unsigned int* ieventAccessRecord( unsigned int* buffer, const int ievt ) <===] + static __host__ __device__ inline unsigned int* + ieventAccessRecord( unsigned int* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> unsigned int& decodeRecord( unsigned int* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline unsigned int& + decodeRecord( unsigned int* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessIflavorVec : public MemoryAccessIflavorVecBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const unsigned int* ieventAccessRecordConst( const unsigned int* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const unsigned int& decodeRecordConst( const unsigned int* buffer ) <===] + static constexpr auto decodeRecordConst = MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const unsigned int& ieventAccessConst( const unsigned int* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessIflavorVec + { + public: + + // Expose selected functions from MemoryAccessIflavorVec + static constexpr auto ieventAccessRecordConst = MemoryAccessIflavorVec::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const unsigned int& kernelAccessConst( const unsigned int* buffer ) <===] + static constexpr auto kernelAccessConst_s = KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const, SCALAR OR VECTOR) ===> const uint_sv& kernelAccess( const unsigned int* buffer ) <===] + static __host__ __device__ inline const uint_sv& + kernelAccessConst( const unsigned int* buffer ) + { + const unsigned int& out = kernelAccessConst_s( buffer ); + return out; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessIflavorVec HostAccessIflavorVec; + typedef KernelAccessIflavorVec DeviceAccessIflavorVec; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h new file mode 100644 index 0000000000..505312e236 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMatrixElements.h @@ -0,0 +1,136 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessMatrixElements_H +#define MemoryAccessMatrixElements_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" +#include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for matrix elements + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessMatrixElementsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessMatrixElements : public MemoryAccessMatrixElementsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessMatrixElements + { + public: + + // Expose selected functions from MemoryAccessMatrixElements + static constexpr auto ieventAccessRecord = MemoryAccessMatrixElements::ieventAccessRecord; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccess_s( fptype* buffer ) <===] + static constexpr auto kernelAccess_s = + KernelAccessHelper::template kernelAccessField<>; // requires cuda 11.4 + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non const, SCALAR OR VECTOR) ===> fptype_sv& kernelAccess( const fptype* buffer ) <===] + static __host__ __device__ inline fptype_sv& + kernelAccess( fptype* buffer ) + { + fptype& out = kernelAccess_s( buffer ); + return out; + } + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + static constexpr auto kernelAccessConst = + KernelAccessHelper::template kernelAccessFieldConst<>; // requires cuda 11.4 + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessMatrixElements HostAccessMatrixElements; + typedef KernelAccessMatrixElements DeviceAccessMatrixElements; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h new file mode 100644 index 0000000000..10fdc00c8e --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessMomenta.h @@ -0,0 +1,210 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessMomenta_H +#define MemoryAccessMomenta_H 1 + +#include "mgOnGpuConfig.h" + +#include "ProcessData.h" +#include "MemoryAccessHelpers.h" +#include "MemoryAccessVectors.h" + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for momenta + // This implementation uses an AOSOA[npagM][npar][np4][neppM] where nevt=npagM*neppM + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessMomentaBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the momenta AOSOA memory buffer layout + // (these are all best kept as a compile-time constants: see issue #23) + // ----------------------------------------------------------------------------------------------- + // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline + // --- This is relevant to ensure coalesced access to momenta in global memory + // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms + // ----------------------------------------------------------------------------------------------- + //static constexpr int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) + static constexpr int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) + //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 1.03E9 instead of 1.11E9 in eemumu) + + // SANITY CHECK: check that neppM is a power of two + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a 4-momentum + static constexpr int np4 = ProcessData::np4; + + // The number of particles in this physics process + static constexpr int npar = ProcessData::npar; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagM = ievt / neppM; // #event "M-page" + const int ieppM = ievt % neppM; // #event in the current event M-page + constexpr int ip4 = 0; + constexpr int ipar = 0; + return &( buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM] ); // AOSOA[ipagM][ipar][ip4][ieppM] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ip4, const int ipar" and rename "Field" as "Ip4Ipar"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ip4, + const int ipar ) + { + constexpr int ipagM = 0; + constexpr int ieppM = 0; + return buffer[ipagM * npar * np4 * neppM + ipar * np4 * neppM + ip4 * neppM + ieppM]; // AOSOA[ipagM][ipar][ip4][ieppM] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessMomenta : public MemoryAccessMomentaBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto decodeRecordIp4Ipar = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto decodeRecordIp4IparConst = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIp4Ipar( fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + static constexpr auto ieventAccessIp4Ipar = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + // DEFAULT VERSION + static constexpr auto ieventAccessIp4IparConst = + MemoryAccessHelper::template ieventAccessFieldConst; + + /* + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparConst( const fptype* buffer, const ievt, const int ipar, const int ipar ) <===] + // DEBUG VERSION WITH PRINTOUTS + static __host__ __device__ inline const fptype& + ieventAccessIp4IparConst( const fptype* buffer, + const int ievt, + const int ip4, + const int ipar ) + { + const fptype& out = MemoryAccessHelper::template ieventAccessFieldConst( buffer, ievt, ip4, ipar ); + printf( "ipar=%2d ip4=%2d ievt=%8d out=%8.3f\n", ipar, ip4, ievt, out ); + return out; + } + */ + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessMomenta + { + public: + + // Expose selected functions from MemoryAccessMomenta + static constexpr auto ieventAccessRecordConst = MemoryAccessMomenta::ieventAccessRecordConst; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const, SCALAR) ===> fptype& kernelAccessIp4Ipar( fptype* buffer, const int ipar, const int ipar ) <===] + static constexpr auto kernelAccessIp4Ipar = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // DEFAULT VERSION + static constexpr auto kernelAccessIp4IparConst_s = + KernelAccessHelper::template kernelAccessFieldConst; + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR) ===> const fptype& kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // DEBUG VERSION WITH PRINTOUTS + static __host__ __device__ inline const fptype& + kernelAccessIp4IparConst_s( const fptype* buffer, + const int ip4, + const int ipar ) + { + const fptype& out = KernelAccessHelper::template kernelAccessFieldConst( buffer, ip4, ipar ); + printf( "ipar=%2d ip4=%2d ievt='kernel' out=%8.3f\n", ipar, ip4, out ); + return out; + } + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const, SCALAR OR VECTOR) ===> fptype_sv kernelAccessIp4IparConst( const fptype* buffer, const int ipar, const int ipar ) <===] + // FIXME? Eventually return by const reference and support aligned arrays only? + // FIXME? Currently return by value to support also unaligned and arbitrary arrays + static __host__ __device__ inline fptype_sv + kernelAccessIp4IparConst( const fptype* buffer, + const int ip4, + const int ipar ) + { + const fptype& out = kernelAccessIp4IparConst_s( buffer, ip4, ipar ); + return out; + } + + // Is this a HostAccess or DeviceAccess class? + // [this is only needed for a warning printout in rambo.h for nparf==1 #358] + static __host__ __device__ inline constexpr bool + isOnDevice() + { + return onDevice; + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessMomenta HostAccessMomenta; + typedef KernelAccessMomenta DeviceAccessMomenta; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h new file mode 100644 index 0000000000..6df3b20cf4 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessNumerators.h @@ -0,0 +1,27 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessNumerators_H +#define MemoryAccessNumerators_H 1 + +#include "MemoryAccessGs.h" + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for numerators + // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs + + typedef KernelAccessGs HostAccessNumerators; + typedef KernelAccessGs DeviceAccessNumerators; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h new file mode 100644 index 0000000000..a20726428b --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessRandomNumbers.h @@ -0,0 +1,139 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessRandomNumbers_H +#define MemoryAccessRandomNumbers_H 1 + +#include "mgOnGpuConfig.h" + +#include "ProcessData.h" +#include "MemoryAccessHelpers.h" + +//---------------------------------------------------------------------------- + +// A class describing the internal layout of memory buffers for random numbers +// This implementation uses an AOSOA[npagR][nparf][np4][neppR] where nevt=npagR*neppR +// [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] +class MemoryAccessRandomNumbersBase //_AOSOAv1 +{ +public: /* clang-format off */ + + // Number of Events Per Page in the random number AOSOA memory buffer layout + // *** NB Different values of neppR lead to different physics results: the *** + // *** same 1d array is generated, but it is interpreted in different ways *** + static constexpr int neppR = 8; // HARDCODED TO GIVE ALWAYS THE SAME PHYSICS RESULTS! + //static constexpr int neppR = 1; // AOS (tests of sectors/requests) + +private: /* clang-format on */ + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a 4-momentum + static constexpr int np4 = ProcessData::np4; + + // The number of final state particles in this physics process + static constexpr int nparf = ProcessData::nparf; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagR = ievt / neppR; // #event "R-page" + const int ieppR = ievt % neppR; // #event in the current event R-page + constexpr int ip4 = 0; + constexpr int iparf = 0; + return &( buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR] ); // AOSOA[ipagR][iparf][ip4][ieppR] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ip4, const int iparf" and rename "Field" as "Ip4Iparf"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ip4, + const int iparf ) + { + constexpr int ipagR = 0; + constexpr int ieppR = 0; + return buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR]; // AOSOA[ipagR][iparf][ip4][ieppR] + } +}; + +//---------------------------------------------------------------------------- + +// A class providing access to memory buffers for a given event, based on explicit event numbers +// Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations +class MemoryAccessRandomNumbers : public MemoryAccessRandomNumbersBase +{ +public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto decodeRecordIp4Iparf = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto decodeRecordIp4IparfConst = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIp4Iparf( fptype* buffer, const ievt, const int ipar, const int iparf ) <===] + static constexpr auto ieventAccessIp4Iparf = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparfConst( const fptype* buffer, const ievt, const int ipar, const int iparf ) <===] + static constexpr auto ieventAccessIp4IparfConst = + MemoryAccessHelper::template ieventAccessFieldConst; +}; + +//---------------------------------------------------------------------------- + +// A class providing access to memory buffers for a given event, based on implicit kernel rules +// Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations +template +class KernelAccessRandomNumbers +{ +public: + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIp4Iparf( fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto kernelAccessIp4Iparf = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIp4IparfConst( const fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto kernelAccessIp4IparfConst = + KernelAccessHelper::template kernelAccessFieldConst; +}; + +//---------------------------------------------------------------------------- + +typedef KernelAccessRandomNumbers HostAccessRandomNumbers; +typedef KernelAccessRandomNumbers DeviceAccessRandomNumbers; + +//---------------------------------------------------------------------------- + +#endif // MemoryAccessRandomNumbers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h new file mode 100644 index 0000000000..f21f90987c --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessVectors.h @@ -0,0 +1,15 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessVectors_H +#define MemoryAccessVectors_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + + +#endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h new file mode 100644 index 0000000000..9406a9d75a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWavefunctions.h @@ -0,0 +1,166 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessWavefunctions_H +#define MemoryAccessWavefunctions_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" + +#define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + +#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + // A class describing the internal layout of memory buffers for wavefunctions + // This implementation uses an AOSOA[npagW][nw6][nx2][neppW] where nevt=npagW*neppW + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessWavefunctionsBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the wavefunction AOSOA memory buffer layout + static constexpr int neppW = 1; // AOS (just a test...) + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a (fermion or vector) wavefunction + static constexpr int nw6 = mgOnGpu::nw6; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagW = ievt / neppW; // #event "W-page" + const int ieppW = ievt % neppW; // #event in the current event W-page + constexpr int iw6 = 0; + constexpr int ix2 = 0; + return &( buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW] ); // AOSOA[ipagW][iw6][ix2][ieppW] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int iw6, const int ix2" and rename "Field" as "Iw6Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int iw6, + const int ix2 ) + { + constexpr int ipagW = 0; + constexpr int ieppW = 0; + return buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW]; // AOSOA[ipagW][iw6][ix2][ieppW] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessWavefunctions : public MemoryAccessWavefunctionsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto decodeRecordIw6Ix2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto decodeRecordIw6Ix2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIw6Ix2( fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] + static constexpr auto ieventAccessIw6Ix2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIw6Ix2Const( const fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] + static constexpr auto ieventAccessIw6Ix2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + +#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessWavefunctions + { + public: + +#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIw6Ix2( fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto kernelAccessIw6Ix2 = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIw6Ix2Const( const fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto kernelAccessIw6Ix2Const = + KernelAccessHelper::template kernelAccessFieldConst; + +#else + + static __host__ __device__ inline cxtype_sv* + kernelAccess( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + static __host__ __device__ inline const cxtype_sv* + kernelAccessConst( const fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + +#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessWavefunctions HostAccessWavefunctions; + typedef KernelAccessWavefunctions DeviceAccessWavefunctions; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h new file mode 100644 index 0000000000..eece092fa3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryAccessWeights.h @@ -0,0 +1,146 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessWeights_H +#define MemoryAccessWeights_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for weights + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessWeightsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessWeights : public MemoryAccessWeightsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessWeights + { + public: + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] + // FINAL IMPLEMENTATION FOR CUDA 11.4 + static constexpr auto kernelAccess = + KernelAccessHelper::template kernelAccessField<>; + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] + // TEMPORARY HACK FOR CUDA 11.1 + static __host__ __device__ inline fptype& + kernelAccess( fptype* buffer ) + { + return KernelAccessHelper::template kernelAccessField<>( buffer ); + } + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + // FINAL IMPLEMENTATION FOR CUDA 11.4 + static constexpr auto kernelAccessConst = + KernelAccessHelper::template kernelAccessFieldConst<>; + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + // TEMPORARY HACK FOR CUDA 11.1 + static __host__ __device__ inline const fptype& + kernelAccessConst( const fptype* buffer ) + { + return KernelAccessHelper::template kernelAccessFieldConst<>( buffer ); + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessWeights HostAccessWeights; + typedef KernelAccessWeights DeviceAccessWeights; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h similarity index 72% rename from madgraph/iolibs/template_files/madmatrix/MemoryBuffers.h rename to madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h index d259aa7456..33227eadfa 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryBuffers.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/MemoryBuffers.h @@ -11,32 +11,27 @@ #include "mgOnGpuCxtypes.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "GpuRuntime.h" #include "Parameters.h" -#include "processConfig.h" #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- namespace MemoryBuffers { // Process-independent compile-time constants - static constexpr size_t np4 = CPPProcess::np4; - static constexpr size_t nw6 = CPPProcess::nw6; + static constexpr size_t np4 = ProcessData::np4; + static constexpr size_t nw6 = ProcessData::nw6; static constexpr size_t nx2 = mgOnGpu::nx2; // Process-dependent compile-time constants - static constexpr size_t nparf = CPPProcess::nparf; - static constexpr size_t npar = CPPProcess::npar; + static constexpr size_t nparf = ProcessData::nparf; + static constexpr size_t npar = ProcessData::npar; static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; - static constexpr size_t ncolor = CPPProcess::ncolor; + static constexpr size_t ncolor = ProcessData::ncolor; } //-------------------------------------------------------------------------- @@ -90,39 +85,9 @@ namespace mg5amcCpu //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL - constexpr bool HostBufferALIGNED = false; // ismisaligned=false - constexpr bool HostBufferMISALIGNED = true; // ismisaligned=true - - // A class encapsulating a C++ host buffer - template - class HostBufferBase : public BufferBase - { - public: - HostBufferBase( const size_t size ) - : BufferBase( size, false ) - { - if constexpr( !ismisaligned ) - this->m_data = new( std::align_val_t( cppAlign ) ) T[size](); - else - this->m_data = new( std::align_val_t( cppAlign ) ) T[size + 1]() + 1; // TEST MISALIGNMENT! - } - virtual ~HostBufferBase() - { - if constexpr( !ismisaligned ) - ::operator delete[]( this->m_data, std::align_val_t( cppAlign ) ); - else - ::operator delete[]( ( this->m_data ) - 1, std::align_val_t( cppAlign ) ); // TEST MISALIGNMENT! - } - static constexpr bool isaligned() { return !ismisaligned; } - public: - static constexpr size_t cppAlign = mgOnGpu::cppAlign; - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating a CUDA pinned host buffer template class PinnedHostBufferBase : public BufferBase @@ -138,11 +103,9 @@ namespace mg5amcCpu gpuFreeHost( this->m_data ); } }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating a CUDA device buffer template class DeviceBufferBase : public BufferBase @@ -158,33 +121,12 @@ namespace mg5amcCpu gpuFree( this->m_data ); } }; -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for a given number of events - template - class HostBuffer : public HostBufferBase, virtual private NumberOfEvents - { - public: - HostBuffer( const size_t nevt ) - : NumberOfEvents( nevt ) - , HostBufferBase( sizePerEvent * nevt ) - { - //std::cout << "HostBuffer::ctor " << this << " " << nevt << std::endl; - } - virtual ~HostBuffer() - { - //std::cout << "HostBuffer::dtor " << this << std::endl; - } - virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } - }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating a CUDA pinned host buffer for a given number of events template class PinnedHostBuffer : public PinnedHostBufferBase, virtual private NumberOfEvents @@ -196,11 +138,9 @@ namespace mg5amcCpu virtual ~PinnedHostBuffer() {} virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating a CUDA device buffer for a given number of events template class DeviceBuffer : public DeviceBufferBase, virtual protected NumberOfEvents @@ -218,15 +158,12 @@ namespace mg5amcCpu } virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // A class encapsulating a simple CUDA device buffer managed on an ad-hoc basis typedef DeviceBuffer DeviceBufferSimple; typedef DeviceBuffer DeviceBufferSimple2; -#endif //-------------------------------------------------------------------------- @@ -236,41 +173,13 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for momenta random numbers constexpr size_t sizePerEventRndNumMomenta = MemoryBuffers::np4 * MemoryBuffers::nparf; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for momenta random numbers - typedef HostBuffer HostBufferRndNumMomenta; -#else // A class encapsulating a CUDA pinned host buffer for momenta random numbers typedef PinnedHostBuffer PinnedHostBufferRndNumMomenta; // A class encapsulating a CUDA device buffer for momenta random numbers typedef DeviceBuffer DeviceBufferRndNumMomenta; -#endif //-------------------------------------------------------------------------- - /* - // A base class encapsulating a memory buffer with ONE fptype per event - typedef BufferBase BufferOneFp; - - // The size (number of elements) per event in a memory buffer with ONE fptype per event - constexpr size_t sizePerEventOneFp = 1; - -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer with ONE fptype per event - typedef HostBuffer HostBufferOneFp; -#else - // A class encapsulating a CUDA pinned host buffer for gs - typedef PinnedHostBuffer PinnedHostBufferOneFp; - // A class encapsulating a CUDA device buffer for gs - typedef DeviceBuffer DeviceBufferOneFp; -#endif - - // Memory buffers for Gs (related to the event-by-event strength of running coupling constant alphas QCD) - typedef BufferOneFp BufferGs; - typedef HostBufferOneFp HostBufferGs; - typedef PinnedHostBufferOneFp PinnedHostBufferGs; - typedef DeviceBufferOneFp DeviceBufferGs; - */ //-------------------------------------------------------------------------- @@ -280,15 +189,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for Gs constexpr size_t sizePerEventGs = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for gs - typedef HostBuffer HostBufferGs; -#else // A class encapsulating a CUDA pinned host buffer for gs typedef PinnedHostBuffer PinnedHostBufferGs; // A class encapsulating a CUDA device buffer for gs typedef DeviceBuffer DeviceBufferGs; -#endif //-------------------------------------------------------------------------- @@ -297,17 +201,12 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for numerators // (should be equal to the number of diagrams in the process) - constexpr size_t sizePerEventNumerators = processConfig::ndiagrams; + constexpr size_t sizePerEventNumerators = ProcessData::ndiagrams; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for numerators - typedef HostBuffer HostBufferNumerators; -#else // A class encapsulating a CUDA pinned host buffer for numerators typedef PinnedHostBuffer PinnedHostBufferNumerators; // A class encapsulating a CUDA device buffer for numerators typedef DeviceBuffer DeviceBufferNumerators; -#endif //-------------------------------------------------------------------------- @@ -317,15 +216,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for denominators constexpr size_t sizePerEventDenominators = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for denominators - typedef HostBuffer HostBufferDenominators; -#else // A class encapsulating a CUDA pinned host buffer for denominators typedef PinnedHostBuffer PinnedHostBufferDenominators; // A class encapsulating a CUDA device buffer for denominators typedef DeviceBuffer DeviceBufferDenominators; -#endif //-------------------------------------------------------------------------- @@ -335,15 +229,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for random numbers constexpr size_t sizePerEventCouplings = MemoryBuffers::ndcoup * MemoryBuffers::nx2; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for couplings - typedef HostBuffer HostBufferCouplings; -#else // A class encapsulating a CUDA pinned host buffer for couplings typedef PinnedHostBuffer PinnedHostBufferCouplings; // A class encapsulating a CUDA device buffer for couplings typedef DeviceBuffer DeviceBufferCouplings; -#endif //-------------------------------------------------------------------------- @@ -353,16 +242,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for momenta constexpr size_t sizePerEventMomenta = MemoryBuffers::np4 * MemoryBuffers::npar; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for momenta - typedef HostBuffer HostBufferMomenta; - //typedef HostBuffer HostBufferMomenta; // TEST MISALIGNMENT! -#else // A class encapsulating a CUDA pinned host buffer for momenta typedef PinnedHostBuffer PinnedHostBufferMomenta; // A class encapsulating a CUDA device buffer for momenta typedef DeviceBuffer DeviceBufferMomenta; -#endif //-------------------------------------------------------------------------- @@ -372,15 +255,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for sampling weights constexpr size_t sizePerEventWeights = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for sampling weights - typedef HostBuffer HostBufferWeights; -#else // A class encapsulating a CUDA pinned host buffer for sampling weights typedef PinnedHostBuffer PinnedHostBufferWeights; // A class encapsulating a CUDA device buffer for sampling weights typedef DeviceBuffer DeviceBufferWeights; -#endif //-------------------------------------------------------------------------- @@ -390,30 +268,20 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for matrix elements constexpr size_t sizePerEventMatrixElements = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for matrix elements - typedef HostBuffer HostBufferMatrixElements; -#else // A class encapsulating a CUDA pinned host buffer for matrix elements typedef PinnedHostBuffer PinnedHostBufferMatrixElements; // A class encapsulating a CUDA device buffer for matrix elements typedef DeviceBuffer DeviceBufferMatrixElements; -#endif //-------------------------------------------------------------------------- // A base class encapsulating a memory buffer for the helicity mask typedef BufferBase BufferHelicityMask; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for the helicity mask - typedef HostBufferBase HostBufferHelicityMask; -#else // A class encapsulating a CUDA pinned host buffer for the helicity mask typedef PinnedHostBufferBase PinnedHostBufferHelicityMask; // A class encapsulating a CUDA device buffer for the helicity mask typedef DeviceBufferBase DeviceBufferHelicityMask; -#endif //-------------------------------------------------------------------------- @@ -423,15 +291,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for wavefunctions constexpr size_t sizePerEventWavefunctions = MemoryBuffers::nw6 * MemoryBuffers::nx2; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for wavefunctions - typedef HostBuffer HostBufferWavefunctions; -#else // A class encapsulating a CUDA pinned host buffer for wavefunctions typedef PinnedHostBuffer PinnedHostBufferWavefunctions; // A class encapsulating a CUDA device buffer for wavefunctions typedef DeviceBuffer DeviceBufferWavefunctions; -#endif //-------------------------------------------------------------------------- @@ -441,15 +304,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for helicity random numbers constexpr size_t sizePerEventRndNumHelicity = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for helicity random numbers - typedef HostBuffer HostBufferRndNumHelicity; -#else // A class encapsulating a CUDA pinned host buffer for helicity random numbers typedef PinnedHostBuffer PinnedHostBufferRndNumHelicity; // A class encapsulating a CUDA device buffer for helicity random numbers typedef DeviceBuffer DeviceBufferRndNumHelicity; -#endif //-------------------------------------------------------------------------- @@ -459,15 +317,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for color random numbers constexpr size_t sizePerEventRndNumColor = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for color random numbers - typedef HostBuffer HostBufferRndNumColor; -#else // A class encapsulating a CUDA pinned host buffer for color random numbers typedef PinnedHostBuffer PinnedHostBufferRndNumColor; // A class encapsulating a CUDA device buffer for color random numbers typedef DeviceBuffer DeviceBufferRndNumColor; -#endif //-------------------------------------------------------------------------- @@ -477,15 +330,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for channel ids constexpr size_t sizePerEventChannelId = 1; -#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) - // A class encapsulating a C++ host buffer for channel ids - typedef HostBuffer HostBufferChannelIds; -#else // A class encapsulating a CUDA pinned host buffer for channel ids typedef PinnedHostBuffer PinnedHostBufferChannelIds; // A class encapsulating a CUDA device buffer for channel ids typedef DeviceBuffer DeviceBufferChannelIds; -#endif //-------------------------------------------------------------------------- @@ -495,15 +343,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for channel ids constexpr size_t sizePerEventIflavorVec = 1; -#ifndef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) - // A class encapsulating a C++ host buffer for channel ids - typedef HostBuffer HostBufferIflavorVec; -#else // A class encapsulating a CUDA pinned host buffer for channel ids typedef PinnedHostBuffer PinnedHostBufferIflavorVec; // A class encapsulating a CUDA device buffer for channel ids typedef DeviceBuffer DeviceBufferIflavorVec; -#endif //-------------------------------------------------------------------------- @@ -513,15 +356,10 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for helicity selection constexpr size_t sizePerEventSelectedHelicity = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for helicity selection - typedef HostBuffer HostBufferSelectedHelicity; -#else // A class encapsulating a CUDA pinned host buffer for helicity selection typedef PinnedHostBuffer PinnedHostBufferSelectedHelicity; // A class encapsulating a CUDA device buffer for helicity selection typedef DeviceBuffer DeviceBufferSelectedHelicity; -#endif //-------------------------------------------------------------------------- @@ -531,29 +369,21 @@ namespace mg5amcCpu // The size (number of elements) per event in a memory buffer for color selection constexpr size_t sizePerEventSelectedColor = 1; -#ifndef MGONGPUCPP_GPUIMPL - // A class encapsulating a C++ host buffer for color selection - typedef HostBuffer HostBufferSelectedColor; -#else // A class encapsulating a CUDA pinned host buffer for color selection typedef PinnedHostBuffer PinnedHostBufferSelectedColor; // A class encapsulating a CUDA device buffer for color selection typedef DeviceBuffer DeviceBufferSelectedColor; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL // The size (number of elements) per event in a memory buffer for jamps constexpr size_t sizePerEventJamps = MemoryBuffers::ncolor * MemoryBuffers::nx2; // A class encapsulating a CUDA device buffer for color selection typedef DeviceBuffer DeviceBufferJamps; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL template void copyDeviceFromHost( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy { @@ -572,11 +402,9 @@ namespace mg5amcCpu // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyHostToDevice ); } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL template void copyHostFromDevice( Tdst& dst, const Tsrc& src ) // keep the same order of arguments as in memcpy { @@ -595,7 +423,6 @@ namespace mg5amcCpu // NB (PR #45): cudaMemcpy involves an intermediate memcpy to pinned memory if host array is a not a pinned host array gpuMemcpy( dst.data(), src.data(), src.bytes(), gpuMemcpyDeviceToHost ); } -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc new file mode 100644 index 0000000000..280b298441 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.cc @@ -0,0 +1,620 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin, calculate_jamps, good-helicity filtering. +// The diagram/vertex-call sequence (helas_calls) is process-specific and +// lives in the P1-generated EvaluateDiagrams.inc, #include'd below. + +#include "SigmaKin.h" + +#include "CPPProcess.h" // ProcessData.h, Parameters.h, HelAmps_.h transitively +#include "ProcessTables.h" + +#include "GpuRuntime.h" +#include "MemoryAccessAmplitudes.h" +#include "MemoryAccessChannelIds.h" +#include "MemoryAccessCouplings.h" +#include "MemoryAccessCouplingsFixed.h" +#include "MemoryAccessDenominators.h" +#include "MemoryAccessGs.h" +#include "MemoryAccessIflavorVec.h" +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessNumerators.h" +#include "MemoryAccessWavefunctions.h" +#include "color_sum.h" +#include "ColorData.h" + +namespace madmatrix +{ + using namespace ProcessData; + using namespace ProcessTables; + using Parameters_dependentCouplings::ndcoup; // #couplings that vary event by event (depend on running alphas QCD) + using Parameters_independentCouplings::nicoup; // #couplings that are fixed for all events (do not depend on running alphas QCD) + constexpr int nParity = 1; // CUDA/HIP process one event per thread, no host-side SIMD paging + + // Helicity/flavor tables and SM parameter/coupling storage, populated once + // by CPPProcess's constructor/initProc via the setters below. + __device__ __constant__ short cHel[ncomb][npar]; + __device__ __constant__ int dcNGoodHel; + __device__ __constant__ int dcGoodHel[ncomb]; + __device__ __constant__ short cFlavors[nmaxflavor][npar]; + static int cNGoodHel; + static int cGoodHel[ncomb]; + __device__ __constant__ fptype cIPD[nIPD > 0 ? nIPD : 1]; + __device__ __constant__ fptype cIPC[nIPC > 0 ? nIPC * 2 : 1]; + __device__ __constant__ int cIPF_partner1[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + __device__ __constant__ int cIPF_partner2[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + __device__ __constant__ fptype cIPF_value[ProcessTables::nMF * nIPF * 2 > 0 ? ProcessTables::nMF * nIPF * 2 : 1]; + __device__ __constant__ double bsmIndepParam[Parameters::nBsmIndepParam > 0 ? Parameters::nBsmIndepParam : 1]; + + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ) + { + gpuMemcpyToSymbol( cHel, tHel, ncomb * npar * sizeof( short ) ); + gpuMemcpyToSymbol( cFlavors, tFlavors, nmaxflavor * npar * sizeof( short ) ); + } + + void setIndependentParams( const fptype* tIPD ) + { + if( nIPD > 0 ) gpuMemcpyToSymbol( cIPD, tIPD, nIPD * sizeof( fptype ) ); + } + + void setIndependentCouplings( const cxtype* tIPC ) + { + if( nIPC > 0 ) gpuMemcpyToSymbol( cIPC, tIPC, nIPC * sizeof( cxtype ) ); + } + + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ) + { + if( nIPF == 0 ) return; + gpuMemcpyToSymbol( cIPF_partner1, tIPF_partner1, nMF * nIPF * sizeof( int ) ); + gpuMemcpyToSymbol( cIPF_partner2, tIPF_partner2, nMF * nIPF * sizeof( int ) ); + gpuMemcpyToSymbol( cIPF_value, tIPF_value, nMF * nIPF * sizeof( cxtype ) ); + } + + void setBsmIndepParam( const double* values, int n ) + { + if( n > 0 ) gpuMemcpyToSymbol( bsmIndepParam, values, n * sizeof( double ) ); + } + + //-------------------------------------------------------------------------- + + class DeviceAccessJamp2 + { + public: + static __device__ inline fptype& + kernelAccessIcol( fptype* buffer, const int icol ) + { + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + return buffer[icol * nevt + ievt]; + } + static __device__ inline const fptype& + kernelAccessIcolConst( const fptype* buffer, const int icol ) + { + const int nevt = gridDim.x * blockDim.x; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; + return buffer[icol * nevt + ievt]; + } + }; + + //-------------------------------------------------------------------------- + + __device__ INLINE unsigned int + gpu_channelId( const unsigned int* allChannelIds ) + { + unsigned int channelId = 0; // disable multichannel single-diagram enhancement unless allChannelIds != nullptr + using CID_ACCESS = DeviceAccessChannelIds; // non-trivial access: buffer includes all events + // SCALAR channelId for the current event (CUDA) + if( allChannelIds != nullptr ) + { + const unsigned int* channelIds = allChannelIds; // fix #899 (distinguish channelIds and allChannelIds) + const uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) + // NB: channelIds_sv is a scalar in CUDA + channelId = channelIds_sv; + assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) + } + return channelId; + } + + //-------------------------------------------------------------------------- + + __global__ void /* clang-format off */ + computeDependentCouplings( const fptype* allgs, // input: Gs[nevt] + fptype* allcouplings // output: couplings[nevt*ndcoup*2] + ) /* clang-format on */ + { + using G_ACCESS = DeviceAccessGs; + using C_ACCESS = DeviceAccessCouplings; + G2COUP( allgs, allcouplings, bsmIndepParam ); + } + + //-------------------------------------------------------------------------- + + // Evaluate QCD partial amplitudes jamps for this given helicity from Feynman diagrams. + // This function processes a single event (one CUDA thread). + __global__ void /* clang-format off */ + calculate_jamps( int ihel, + const fptype* allmomenta, // input: momenta[nevt*npar*4] + const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] + const unsigned int* iflavorVec, // input: indices of the flavor combinations + fptype* allJamps, // output: jamp[2*ncolor*nevt] buffer for one helicity _within a super-buffer for dcNGoodHel helicities_ + bool storeChannelWeights, + fptype* allNumerators, // input/output: multichannel numerators[nevt], add helicity ihel + fptype* allDenominators, // input/output: multichannel denominators[nevt], add helicity ihel + fptype* colAllJamp2s, // output: allJamp2s[ncolor][nevt] super-buffer, sum over col/hel (nullptr to disable) + const int nevt, // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + const bool processAllHelicities // input: if true, use blockIdx.y to index helicities + ) /* clang-format on */ + { + using M_ACCESS = DeviceAccessMomenta; // non-trivial access: buffer includes all events + using W_ACCESS = DeviceAccessWavefunctions; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event + using A_ACCESS = DeviceAccessAmplitudes; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event + using CD_ACCESS = DeviceAccessCouplings; // non-trivial access (dependent couplings): buffer includes all events + using CI_ACCESS = DeviceAccessCouplingsFixed; // TRIVIAL access (independent couplings): buffer for one event + using F_ACCESS = DeviceAccessIflavorVec; // non-trivial access: buffer includes all events + using NUM_ACCESS = DeviceAccessNumerators; // non-trivial access: buffer includes all events + using DEN_ACCESS = DeviceAccessDenominators; // non-trivial access: buffer includes all events + mgDebug( 0, __FUNCTION__ ); + if( processAllHelicities ) + { + int ighel = blockIdx.y; + ihel = dcGoodHel[ighel]; + allJamps = allJamps + ighel * nevt; + allNumerators = allNumerators + ighel * nevt * ndiagrams; + allDenominators = allDenominators + ighel * nevt; + } + + fptype_sv pvec_sv[nwf][np4]; + cxtype_sv w_sv[nwf][nw6]; // particle wavefunctions within Feynman diagrams + cxtype_sv amp_sv[1]; // invariant amplitude for one given Feynman diagram + ALOHAOBJ aloha_obj[nwf]; + for( int iwf = 0; iwf < nwf; iwf++ ) aloha_obj[iwf] = ALOHAOBJ{ pvec_sv[iwf], w_sv[iwf] }; + fptype* amp_fp = reinterpret_cast( amp_sv ); + + // jamp: sum (for one event) of the invariant amplitudes for all Feynman diagrams in a given color combination + cxtype_sv jamp_sv[ncolor] = {}; // all zeros + + for( int iParity = 0; iParity < nParity; ++iParity ) + { + constexpr size_t nxcoup = ndcoup + nIPC; // both dependent and independent couplings + const fptype* allCOUPs[nxcoup]; +#ifdef __CUDACC__ // this must be __CUDACC__ +#pragma nv_diagnostic push +#pragma nv_diag_suppress 186 // e.g. <> +#endif + for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) + allCOUPs[idcoup] = CD_ACCESS::idcoupAccessBufferConst( allcouplings, idcoup ); // dependent couplings, vary event-by-event + for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) + allCOUPs[ndcoup + iicoup] = CI_ACCESS::iicoupAccessBufferConst( cIPC, iicoup ); // independent couplings, fixed for all events +#ifdef __CUDACC__ // this must be __CUDACC__ +#pragma nv_diagnostic pop +#endif + // CUDA kernels take input/output buffers with momenta/MEs for all events + const fptype* momenta = allmomenta; + const fptype* COUPs[nxcoup]; + for( size_t ixcoup = 0; ixcoup < nxcoup; ixcoup++ ) COUPs[ixcoup] = allCOUPs[ixcoup]; + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid + fptype* numerators = &allNumerators[ievt * ndiagrams]; + fptype* denominators = allDenominators; + // Create an array of views over the Flavor Couplings + FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; + + // Dependent (event-by-event, running-alphas) flavor couplings (Step 3): the per-flavor + // values are NOT baked in (they run per event). Gather the current values of the + // underlying dependent couplings for this event page into an AOSOA buffer dpf_value + // (one nx2*neppC SIMD record per (coupling,flavor) slot, matching CD_ACCESS), then build + // an ordinary value-based view over it. The flavor index is constant across a SIMD lane + // (guaranteed by the phase-space integrator), so each lane gets its own running value + // while sharing the same flavor selection. This is the direct analogue of Fortran's + // FLV_xx%VAL(k)%P => GC_yyy(J). The vertex routines are instantiated with CD_ACCESS so + // get_coupling_def reads dpf_value with the right per-flavor stride (CD_ACCESS::flv_stride). + constexpr int ndpfbuf = ( nDPF > 0 ? nDPF * nMF * CD_ACCESS::flv_stride : 1 ); + fptype dpf_value[ndpfbuf]{}; + for( int idpf = 0; idpf < nDPF; idpf++ ) + for( int imf = 0; imf < nMF; imf++ ) + { + const int idc = cDPF_idcoup[idpf * nMF + imf]; + if( idc >= 0 ) + CD_ACCESS::kernelAccess( dpf_value + ( idpf * nMF + imf ) * CD_ACCESS::flv_stride ) = + CD_ACCESS::kernelAccessConst( COUPs[idc] ); + } + FLV_COUPLING_ARRAY flvCOUPs_dep{ cDPF_partner1, cDPF_partner2, dpf_value }; + + // Reset color flows (reset jamp_sv) at the beginning of a new event or event page + for( int i = 0; i < ncolor; i++ ) { jamp_sv[i] = cxzero_sv(); } + + // Numerators and denominators for the current event (CUDA) + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + // Scalar iflavor for the current event + const unsigned int iflavor = F_ACCESS::kernelAccessConst( iflavorVec ); +#include "EvaluateDiagrams.inc" + + // *** COLOR CHOICE BELOW *** + // Store the leading color flows for choice of color + assert( iParity == 0 ); // sanity check for J2_ACCESS + using J2_ACCESS = DeviceAccessJamp2; + if( colAllJamp2s ) // disable color choice if nullptr + { + for( int icol = 0; icol < ncolor; icol++ ) + // NB: atomicAdd is needed after moving to cuda streams with one helicity per stream! + atomicAdd( &J2_ACCESS::kernelAccessIcol( colAllJamp2s, icol ), cxabs2( jamp_sv[icol] ) ); + } + + // *** PREPARE OUTPUT JAMPS *** + // In CUDA, copy the local jamp to the output global-memory jamp + constexpr int ihel0 = 0; // the allJamps buffer already points to a specific helicity _within a super-buffer for dcNGoodHel helicities_ + using J_ACCESS = DeviceAccessJamp; + for( int icol = 0; icol < ncolor; icol++ ) + J_ACCESS::kernelAccessIcolIhelNhel( allJamps, icol, ihel0, dcNGoodHel ) = jamp_sv[icol]; + } + // END LOOP ON IPARITY + + mgDebug( 1, __FUNCTION__ ); + return; + } + + //-------------------------------------------------------------------------- + + void /* clang-format off */ + sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta[nevt*npar*4] + const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] + const unsigned int* iflavorVec, // input: index of the flavor combination + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + fptype* allNumerators, // output: multichannel numerators[nevt], running_sum_over_helicities + fptype* allDenominators, // output: multichannel denominators[nevt], running_sum_over_helicities + fptype_sv* allJamps, // tmp: jamp[ncolor*2*nevt] _for one helicity_ (reused in the getGoodHel helicity loop) + bool* isGoodHel, // output: isGoodHel[ncomb] - host array + const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + { /* clang-format on */ + const int maxtry0 = 16; + fptype hstMEs[maxtry0]; + const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt 1 ) + { + for( int i = broken_sym_component_starts[icomp] - 1; i < broken_sym_component_ends[icomp]; i++ ) + { + if( pid_work[i] == 0 ) + continue; + int n_tot = 1; + for( int j = i + 1; j < broken_sym_component_ends[icomp]; j++ ) + { + if( pid_work[i] != pid_work[j] ) + continue; + bool same_block = ( broken_sym_block_lengths[i] == broken_sym_block_lengths[j] ); + for( int k = 0; same_block && k < broken_sym_block_lengths[i]; k++ ) + { + if( cFlavors[iflavor][broken_sym_block_starts[i] - 1 + k] != cFlavors[iflavor][broken_sym_block_starts[j] - 1 + k] ) + same_block = false; + } + if( same_block ) + { + pid_work[j] = 0; + n_tot = n_tot + 1; + old_factor = old_factor / n_tot; + } + } + } + } + total_factor = total_factor * old_factor; + } + return total_factor; + } + + //-------------------------------------------------------------------------- + + __global__ void + normalise_output( fptype* allMEs, // output: allMEs[nevt], |M|^2 running_sum_over_helicities + const unsigned int* iflavorVec, + fptype* ghelAllNumerators, // input/tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + fptype* ghelAllDenominators, // input/tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable SDE enhancement (fix #899/#911) + bool storeChannelWeights, // if true, compute final multichannel weights + bool mulChannelWeight, // if true, multiply matrix element by channel weight + const fptype globaldenom) + { + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) + allMEs[ievt] = allMEs[ievt] * broken_symmetry_factor(iflavorVec[ievt]) / globaldenom; + const int nevt = gridDim.x * blockDim.x; + if( storeChannelWeights ) // fix segfault #892 (not 'channelIds[0] != 0') + { + fptype* totAllNumerators = ghelAllNumerators; // reuse "helicity #0" buffer to compute the total over all helicities + fptype* totAllDenominators = ghelAllDenominators; // reuse "helicity #0" buffer to compute the total over all helicities + for( int ighel = 1; ighel < dcNGoodHel; ighel++ ) // NB: the loop starts at ighel=1 + { + fptype* hAllDenominators = ghelAllDenominators + ighel * nevt; + totAllDenominators[ievt] += hAllDenominators[ievt]; + fptype* hAllNumerators = ghelAllNumerators + ( ievt + ighel * nevt ) * ndiagrams; + fptype* firstNumerator = ghelAllNumerators + ievt * ndiagrams; + for( int idiag = 0; idiag < ndiagrams; ++idiag ) + { + firstNumerator[idiag] += hAllNumerators[idiag]; + } + } + if( mulChannelWeight ) + { + unsigned int channelId = allChannelIds[ievt]; + allMEs[ievt] *= totAllNumerators[channelId - 1 + ievt * ndiagrams] / totAllDenominators[ievt]; + } + } + return; + } + + //-------------------------------------------------------------------------- + + __global__ void + add_and_select_hel( int* allselhel, // output: helicity selection[nevt] + const fptype* allrndhel, // input: random numbers[nevt] for helicity selection + fptype* ghelAllMEs, // input/tmp: allMEs for nGoodHel <= ncomb individual/runningsum helicities (index is ighel) + fptype* allMEs, // output: allMEs[nevt], final sum over helicities + const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + { + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) + // Compute the sum of MEs over all good helicities (defer this after the helicity loop to avoid breaking streams parallelism) + for( int ighel = 0; ighel < dcNGoodHel; ighel++ ) + { + allMEs[ievt] += ghelAllMEs[ighel * nevt + ievt]; + ghelAllMEs[ighel * nevt + ievt] = allMEs[ievt]; // reuse the buffer to store the running sum for helicity selection + } + // Event-by-event random choice of helicity #403 + for( int ighel = 0; ighel < dcNGoodHel; ighel++ ) + { + if( allrndhel[ievt] < ( ghelAllMEs[ighel * nevt + ievt] / allMEs[ievt] ) ) + { + const int ihelF = dcGoodHel[ighel] + 1; // NB Fortran [1,ncomb], cudacpp [0,ncomb-1] + allselhel[ievt] = ihelF; + break; + } + } + return; + } + + //-------------------------------------------------------------------------- + + __global__ void + select_col_and_diag( int* allselcol, // output: color selection[nevt] + unsigned int* allDiagramIdsOut, // output: sampled diagram ids + const fptype* allrndcol, // input: random numbers[nevt] for color selection + const fptype* allrnddiagram, // input: random numbers[nevt] for diagram selection + const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable SDE enhancement (fix #899/#911) + const fptype_sv* allJamp2s, // input: jamp2[ncolor][nevt] for color choice (nullptr if disabled) + const fptype* allNumerators, // input: all numerators + const fptype* allDenominators, // input: all denominators + const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) + { + const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) + // SCALAR channelId for the current event (CUDA) + unsigned int channelId = gpu_channelId( allChannelIds ); + // Event-by-event random choice of channel + if( allrnddiagram != nullptr ) + { + fptype numerator_sum = 0., normalization = 0.; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + normalization += allNumerators[ievt * ndiagrams + ichan]; + } + channelId = mgOnGpu::nchannels; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + numerator_sum += allNumerators[ievt * ndiagrams + ichan]; + if( allrnddiagram[ievt] < numerator_sum / normalization ) + { + channelId = ichan + 1; + break; + } + } + allDiagramIdsOut[ievt] = channelId; + } + + if( channelId != 0 ) // no event-by-event choice of color if channelId == 0 (fix FPE #783) + { + if( channelId > mgOnGpu::nchannels ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which is greater than nchannels=%d\n", channelId, mgOnGpu::nchannels ); + assert( channelId <= mgOnGpu::nchannels ); // SANITY CHECK #919 #910 + } + // Determine the jamp2 for this event + fptype_sv jamp2_sv[ncolor] = { 0 }; + assert( allJamp2s != nullptr ); // sanity check + using J2_ACCESS = DeviceAccessJamp2; + for( int icolC = 0; icolC < ncolor; icolC++ ) + jamp2_sv[icolC] = J2_ACCESS::kernelAccessIcolConst( allJamp2s, icolC ); + const int iconfig = mgOnGpu::channel2iconfig[channelId - 1]; // map N_diagrams to N_config <= N_diagrams configs (fix LHE color mismatch #856: see also #826, #852, #853) + if( iconfig <= 0 ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which has no associated SDE iconfig\n", channelId ); + assert( iconfig > 0 ); // SANITY CHECK #917 + } + else if( iconfig > (int)mgOnGpu::nconfigSDE ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d (invalid SDE iconfig=%d\n > nconfig=%d)", channelId, iconfig, mgOnGpu::nconfigSDE ); + assert( iconfig <= (int)mgOnGpu::nconfigSDE ); // SANITY CHECK #917 + } + fptype targetamp[ncolor] = { 0 }; + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( icolC == 0 ) + targetamp[icolC] = 0; + else + targetamp[icolC] = targetamp[icolC - 1]; + if( mgOnGpu::icolamp[iconfig - 1][icolC] ) targetamp[icolC] += jamp2_sv[icolC]; + } + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( allrndcol[ievt] < ( targetamp[icolC] / targetamp[ncolor - 1] ) ) + { + allselcol[ievt] = icolC + 1; // NB Fortran [1,ncolor], cudacpp [0,ncolor-1] + break; + } + } + } + else + { + allselcol[ievt] = 0; // no color selected in Fortran range [1,ncolor] if channelId == 0 (see #931) + } + return; + } + + //-------------------------------------------------------------------------- + // Evaluate |M|^2, part independent of incoming flavour + + void /* clang-format off */ + sigmaKin( const fptype* allmomenta, // input: momenta[nevt*npar*4] + const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] + const unsigned int* iflavorVec, // input: indices of the flavor combinations + const fptype* allrndhel, // input: random numbers[nevt] for helicity selection + const fptype* allrndcol, // input: random numbers[nevt] for color selection + const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable single-diagram enhancement (fix #899/#911) + const fptype* allrnddiagram, // input: random numbers[nevt] for channel sampling + fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities + int* allselhel, // output: helicity selection[nevt] + int* allselcol, // output: helicity selection[nevt] + fptype* colAllJamp2s, // tmp: allJamp2s super-buffer for ncolor individual colors, running sum over colors and helicities + fptype* ghelAllNumerators, // tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + fptype* ghelAllDenominators, // tmp: allDenominators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + unsigned int* allDiagramIdsOut, // output: multichannel channelIds[nevt] (1 to #diagrams) + bool mulChannelWeight, // if true, multiply channel weight to ME output + fptype* ghelAllMEs, // tmp: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) + fptype* ghelAllJamps, // tmp: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities + fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities + gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle + gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) + const bool async, // input: if true, run everything asynchronously in first stream in ghelStreams + const int gpublocks, // input: cuda gpublocks + const int gputhreads // input: cuda gputhreads + ) /* clang-format on */ + { + mgDebugInitialise(); + + // SANITY CHECKS for cudacpp code generation (see issues #272 and #343 and PRs #619, #626, #360, #396 and #754) + { + // nprocesses == 2 may happen for "mirror processes" such as P0_uux_ttx within pp_tt012j (see PR #754) + static_assert( nproc == 1 || nproc == 2, "Assume nprocesses == 1 or 2" ); + static_assert( proc_id == 1, "Assume process_id == 1" ); + } + + const int nevt = gpublocks * gputhreads; + gpuMemset( allMEs, 0, nevt * sizeof( fptype ) ); + gpuMemset( ghelAllJamps, 0, cNGoodHel * ncolor * mgOnGpu::nx2 * nevt * sizeof( fptype ) ); + gpuMemset( colAllJamp2s, 0, ncolor * nevt * sizeof( fptype ) ); + gpuMemset( ghelAllNumerators, 0, cNGoodHel * ndiagrams * nevt * sizeof( fptype ) ); + gpuMemset( ghelAllDenominators, 0, cNGoodHel * nevt * sizeof( fptype ) ); + gpuMemset( ghelAllMEs, 0, cNGoodHel * nevt * sizeof( fptype ) ); + + // *** HELICITY LOOP: CALCULATE WAVEFUNCTIONS (one event per GPU thread) *** + // Use CUDA/HIP streams to process different helicities in parallel (one good helicity per stream) + // (1) First, within each helicity stream, compute the QCD partial amplitudes jamp's for each helicity + // In multichannel mode, also compute the running sums over helicities of numerators, denominators and squared jamp2s + bool storeChannelWeights = allChannelIds != nullptr || allrnddiagram != nullptr; + if( async ) + { + gpuLaunchKernel2D( calculate_jamps, gpublocks, cNGoodHel, gputhreads, ghelStreams[0], 0, allmomenta, allcouplings, iflavorVec, ghelAllJamps, storeChannelWeights, ghelAllNumerators, ghelAllDenominators, colAllJamp2s, nevt, true ); + color_sum_gpu( ghelAllMEs, ghelAllJamps, ghelAllBlasTmp, pBlasHandle, ghelStreams, cNGoodHel, gpublocks, gputhreads, true ); + } + else + { + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + const int ihel = cGoodHel[ighel]; + fptype* hAllJamps = ghelAllJamps + ighel * nevt; // HACK: bypass DeviceAccessJamp (consistent with layout defined there) + fptype* hAllNumerators = ghelAllNumerators + ighel * nevt * ndiagrams; + fptype* hAllDenominators = ghelAllDenominators + ighel * nevt; + gpuLaunchKernelStream( calculate_jamps, gpublocks, gputhreads, ghelStreams[ighel], ihel, allmomenta, allcouplings, iflavorVec, hAllJamps, storeChannelWeights, hAllNumerators, hAllDenominators, colAllJamp2s, nevt, false ); + } + // (2) Then compute the ME for that helicity from the color sum of QCD partial amplitudes jamps + color_sum_gpu( ghelAllMEs, ghelAllJamps, ghelAllBlasTmp, pBlasHandle, ghelStreams, cNGoodHel, gpublocks, gputhreads, false ); + checkGpu( gpuDeviceSynchronize() ); // do not start helicity/color selection until the loop over helicities has completed + // (3) Wait for all helicity streams to complete, then finally compute the ME sum over all helicities and choose one helicity and one color + } + // Event-by-event random choice of helicity #403 and ME sum over helicities (defer this after the helicity loop to avoid breaking streams parallelism) + gpuLaunchKernel( add_and_select_hel, gpublocks, gputhreads, allselhel, allrndhel, ghelAllMEs, allMEs, gpublocks * gputhreads ); + + gpuLaunchKernel( normalise_output, gpublocks, gputhreads, allMEs, iflavorVec, ghelAllNumerators, ghelAllDenominators, allChannelIds, storeChannelWeights, mulChannelWeight, helcolDenominators[0] ); + + // Event-by-event random choice of color and diagram #402 + gpuLaunchKernel( select_col_and_diag, gpublocks, gputhreads, allselcol, allDiagramIdsOut, allrndcol, allrnddiagram, allChannelIds, colAllJamp2s, ghelAllNumerators, ghelAllDenominators, gpublocks * gputhreads ); + + mgDebugFinalise(); + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h new file mode 100644 index 0000000000..4461cdef50 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/SigmaKin.h @@ -0,0 +1,73 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin and everything it calls (calculate_jamps, +// good-helicity filtering, color/channel selection). Declared here so +// CPPProcess.cc's constructor/initProc (P1-generated) can call the setters +// that populate this file's storage, and so umami.cc/MatrixElementKernels.cc +// can call sigmaKin/computeDependentCouplings. + +#ifndef SIGMAKIN_H +#define SIGMAKIN_H 1 + +#include "mgOnGpuConfig.h" +#include "mgOnGpuCxtypes.h" // for cxtype, fptype_sv +#include "GpuAbstraction.h" // for gpuBlasHandle_t, gpuStream_t + +namespace madmatrix +{ + __global__ void + computeDependentCouplings( const fptype* allgs, + fptype* allcouplings ); + + void + sigmaKin_getGoodHel( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + fptype* allMEs, + fptype* allNumerators, + fptype* allDenominators, + fptype_sv* allJamps, + bool* isGoodHel, + const int nevt ); + + int + sigmaKin_setGoodHel( const bool* isGoodHel ); + + void + sigmaKin( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + const fptype* allrndhel, + const fptype* allrndcol, + const unsigned int* allChannelIds, + const fptype* allrnddiagram, + fptype* allMEs, + int* allselhel, + int* allselcol, + fptype* colAllJamp2s, + fptype* ghelAllNumerators, + fptype* ghelAllDenominators, + unsigned int* allDiagramIdsOut, + bool mulChannelWeight, + fptype* ghelAllMEs, + fptype* ghelAllJamps, + fptype2* ghelAllBlasTmp, + gpuBlasHandle_t* pBlasHandle, + gpuStream_t* ghelStreams, + const bool async, + const int gpublocks, + const int gputhreads ); + + // Setters: called once by CPPProcess (P1-generated) to populate this file's + // otherwise-internal storage, since it can no longer be written directly + // from a different translation unit. + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ); + void setIndependentParams( const fptype* tIPD ); + void setIndependentCouplings( const cxtype* tIPC ); + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ); + void setBsmIndepParam( const double* values, int n ); +} + +#endif // SIGMAKIN_H diff --git a/madgraph/iolibs/template_files/madmatrix/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc similarity index 78% rename from madgraph/iolibs/template_files/madmatrix/color_sum.cc rename to madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc index 30c6799932..3a35a0a72d 100644 --- a/madgraph/iolibs/template_files/madmatrix/color_sum.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.cc @@ -8,22 +8,15 @@ #include "mgOnGpuConfig.h" +#include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps #include "MemoryAccessMatrixElements.h" -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { - constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors + using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor //-------------------------------------------------------------------------- - // *** COLOR MATRIX BELOW *** -%(color_matrix_lines)s - -#ifdef MGONGPUCPP_GPUIMPL // The normalized color matrix (divide each column by denom) template struct NormalizedColorMatrix @@ -39,11 +32,9 @@ namespace mg5amcCpu }; // The fptype2 version is the default used by kernels (supporting mixed floating point mode also in blas) static __device__ fptype2 s_pNormalizedColorMatrix2[ncolor * ncolor]; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL void createNormalizedColorMatrix() { static bool first = true; @@ -54,107 +45,12 @@ namespace mg5amcCpu gpuMemcpyToSymbol( s_pNormalizedColorMatrix2, normalizedColorMatrix2.value, ncolor * ncolor * sizeof( fptype2 ) ); } } -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL - void - color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity - const int ievt0 ) // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) - { - // Pre-compute a constexpr triangular color matrix properly normalized #475 - struct TriangularNormalizedColorMatrix - { - // See https://stackoverflow.com/a/34465458 - __host__ __device__ constexpr TriangularNormalizedColorMatrix() - : value() - { - for( int icol = 0; icol < ncolor; icol++ ) - { - // Diagonal terms - value[icol][icol] = colorMatrix[icol][icol] / colorDenom[icol]; - // Off-diagonal terms - for( int jcol = icol + 1; jcol < ncolor; jcol++ ) - value[icol][jcol] = 2 * colorMatrix[icol][jcol] / colorDenom[icol]; - } - } - fptype2 value[ncolor][ncolor]; - }; - static constexpr auto cf2 = TriangularNormalizedColorMatrix(); - // Use the property that M is a real matrix (see #475): - // we can rewrite the quadratic form (A-iB)(M)(A+iB) as AMA - iBMA + iBMA + BMB = AMA + BMB - // In addition, on C++ use the property that M is symmetric (see #475), - // and also use constexpr to compute "2*" and "/colorDenom[icol]" once and for all at compile time: - // we gain (not a factor 2...) in speed here as we only loop over the up diagonal part of the matrix. - // Strangely, CUDA is slower instead, so keep the old implementation for the moment. - fptype_sv deltaMEs = { 0 }; -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype_sv deltaMEs_next = { 0 }; - // Mixed mode: merge two neppV vectors into one neppV2 vector - fptype2_sv jampR_sv[ncolor]; - fptype2_sv jampI_sv[ncolor]; - for( int icol = 0; icol < ncolor; icol++ ) - { - jampR_sv[icol] = fpvmerge( cxreal( allJamp_sv[icol] ), cxreal( allJamp_sv[ncolor + icol] ) ); - jampI_sv[icol] = fpvmerge( cximag( allJamp_sv[icol] ), cximag( allJamp_sv[ncolor + icol] ) ); - } -#else - const cxtype_sv* jamp_sv = allJamp_sv; -#endif - // Loop over icol - for( int icol = 0; icol < ncolor; icol++ ) - { - // Diagonal terms -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype2_sv& jampRi_sv = jampR_sv[icol]; - fptype2_sv& jampIi_sv = jampI_sv[icol]; -#else - fptype2_sv jampRi_sv = (fptype2_sv)( cxreal( jamp_sv[icol] ) ); - fptype2_sv jampIi_sv = (fptype2_sv)( cximag( jamp_sv[icol] ) ); -#endif - fptype2_sv ztempR_sv = cf2.value[icol][icol] * jampRi_sv; - fptype2_sv ztempI_sv = cf2.value[icol][icol] * jampIi_sv; - // Loop over jcol - for( int jcol = icol + 1; jcol < ncolor; jcol++ ) - { - // Off-diagonal terms -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype2_sv& jampRj_sv = jampR_sv[jcol]; - fptype2_sv& jampIj_sv = jampI_sv[jcol]; -#else - fptype2_sv jampRj_sv = (fptype2_sv)( cxreal( jamp_sv[jcol] ) ); - fptype2_sv jampIj_sv = (fptype2_sv)( cximag( jamp_sv[jcol] ) ); -#endif - ztempR_sv += cf2.value[icol][jcol] * jampRj_sv; - ztempI_sv += cf2.value[icol][jcol] * jampIj_sv; - } - fptype2_sv deltaMEs2 = ( jampRi_sv * ztempR_sv + jampIi_sv * ztempI_sv ); // may underflow #831 -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - deltaMEs += fpvsplit0( deltaMEs2 ); - deltaMEs_next += fpvsplit1( deltaMEs2 ); -#else - deltaMEs += deltaMEs2; -#endif - } - // *** STORE THE RESULTS *** - using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events - fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); - // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) - fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); - MEs_sv += deltaMEs; // fix #435 -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - fptype* MEs_next = E_ACCESS::ieventAccessRecord( allMEs, ievt0 + neppV ); - fptype_sv& MEs_sv_next = E_ACCESS::kernelAccess( MEs_next ); - MEs_sv_next += deltaMEs_next; -#endif - } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL __global__ void color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity @@ -210,11 +106,9 @@ namespace mg5amcCpu // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) E_ACCESS::kernelAccess( allMEs ) += deltaMEs; // fix #435 } -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL #ifndef MGONGPU_HAS_NO_BLAS #if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT __global__ void @@ -233,12 +127,10 @@ namespace mg5amcCpu allJamps[ix2 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt]; } #endif -#endif #endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL #ifndef MGONGPU_HAS_NO_BLAS #if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT __global__ void @@ -249,12 +141,10 @@ namespace mg5amcCpu allMEs[ievt] = allMEsFpt2[ievt]; } #endif -#endif #endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ #ifndef MGONGPU_HAS_NO_BLAS void color_sum_blas( fptype* ghelAllMEs, // output: allMEs super-buffer[nhel][nevt], add |M|^2 separately for each helicity @@ -371,11 +261,9 @@ namespace mg5amcCpu #endif } #endif /* clang-format on */ -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL void color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities @@ -427,7 +315,6 @@ namespace mg5amcCpu #endif } } -#endif //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h similarity index 85% rename from madgraph/iolibs/template_files/madmatrix/color_sum.h rename to madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h index 347184c4e1..df434100d2 100644 --- a/madgraph/iolibs/template_files/madmatrix/color_sum.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/color_sum.h @@ -9,27 +9,22 @@ #include "mgOnGpuConfig.h" -#include "mgOnGpuVectors.h" +#include "mgOnGpuCxtypes.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "GpuAbstraction.h" -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL class DeviceAccessJamp { public: static __device__ inline cxtype_ref kernelAccessIcolIhelNhel( fptype* buffer, const int icol, const int ihel, const int nhel ) { - const int ncolor = CPPProcess::ncolor; // the number of leading colors + const int ncolor = ProcessData::ncolor; // the number of leading colors const int nevt = gridDim.x * blockDim.x; const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) @@ -44,7 +39,7 @@ namespace mg5amcCpu static __device__ inline const cxtype kernelAccessIcolIhelNhelConst( const fptype* buffer, const int icol, const int ihel, const int nhel ) { - const int ncolor = CPPProcess::ncolor; // the number of leading colors + const int ncolor = ProcessData::ncolor; // the number of leading colors const int nevt = gridDim.x * blockDim.x; const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // (ONE HELICITY) Original "old" striding for CUDA kernels: ncolor separate 2*nevt matrices for each color (ievt last) @@ -57,26 +52,16 @@ namespace mg5amcCpu buffer[1 * ncolor * nhel * nevt + icol * nhel * nevt + ihel * nevt + ievt] ); } }; -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL void createNormalizedColorMatrix(); -#endif //-------------------------------------------------------------------------- -#ifndef MGONGPUCPP_GPUIMPL - void - color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity - const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity - const int ievt0 ); // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL void color_sum_gpu( fptype* ghelAllMEs, // output: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) const fptype* ghelAllJamps, // input: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities @@ -87,17 +72,14 @@ namespace mg5amcCpu const int gpublocks, // input: cuda gpublocks const int gputhreads, // input: cuda gputhreads const bool processAllHelicities); // input: if true, use blockIdx.y to index helicities -#endif //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL __global__ void color_sum_kernel( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity const fptype* allJamps, // input: jamp[ncolor*2*nevt] for one specific helicity const int nGoodHel, // input: number of good helicities const int nevtIfAllHelicities); // input: zero in single-helicity mode, number of events in multi-helicity mode -#endif //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h new file mode 100644 index 0000000000..30eea25afa --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/constexpr_math.h @@ -0,0 +1,354 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Feb 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef constexpr_math_h +#define constexpr_math_h 1 + +#include "mgOnGpuConfig.h" + +#include +#include // for feenableexcept, fegetexcept and FE_XXX +#include +#include +#include +#include + +// SWITCH ON/OFF DEBUGGING +#undef CONSTEXPR_MATH_DEBUG // no-debug +//#define CONSTEXPR_MATH_DEBUG 1 // debug + +// FOR DEBUGGING +#ifdef CONSTEXPR_MATH_DEBUG +#define CONSTEXPRMATHFUN inline +#define CONSTEXPRMATHVAR const +#else +#define CONSTEXPRMATHFUN constexpr +#define CONSTEXPRMATHVAR constexpr +#endif + +//One namespace. Split ber backend. +namespace madmatrix +{ + // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) + constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) + { + return curr == prev ? curr : sqrtNewtonRaphson( xx, 0.5 * ( curr + xx / curr ), curr ); + } + constexpr long double constexpr_sqrt( const long double xx ) + { + return xx >= 0 // && x < std::numeric_limits::infinity() // avoid -Wtautological-constant-compare warning in fast math + ? sqrtNewtonRaphson( xx, xx, 0 ) + : std::numeric_limits::quiet_NaN(); + } + + // SQRT constants + constexpr long double constexpr_sqrt2 = constexpr_sqrt( 2 ); + + // Constexpr implementation of floor (see https://stackoverflow.com/a/66146159) + constexpr int constexpr_floor( const long double xx ) + { + const int i = static_cast( xx ); + return xx < i ? i - 1 : i; + } + + // Constexpr implementation of pow + constexpr long double constexpr_pow( const long double base, const long double exp, const bool requireExpGe0 = false ) + { + // NB(1): this iterative integer implementation of constexpr_pow requires exponent >= 0 + if( requireExpGe0 ) assert( exp >= 0 ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + if( exp < 0 ) return 1. / constexpr_pow( base, -exp, true ); + // NB(2): this iterative integer implementation of constexpr_pow requires an integer exponent, excexpt for special cases (1/2, 1/4) + if( exp == 0.5 ) return constexpr_sqrt( base ); + if( exp == 0.25 ) return constexpr_sqrt( constexpr_sqrt( base ) ); + const int iexp = constexpr_floor( exp ); + assert( static_cast( iexp ) == exp ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + // Iterative implementation of pow if exp is a non negative integer + return iexp == 0 ? 1 : base * constexpr_pow( base, iexp - 1 ); + } + + // PI constants + // NB1: M_PIl from from cmath is not defined on Mac + // NB2: std::numbers::pi needs c++20 but we are still using c++17 + // NB3: I could use my constexpr_atan(1)*4... but a literal is better? + //constexpr long double constexpr_pi = M_PIl; // pi + //constexpr long double constexpr_pi_by_2 = M_PI_2l; // pi/2 + //constexpr long double constexpr_pi_by_4 = M_PI_4l; // pi/4 + constexpr long double constexpr_pi = 3.141592653589793238462643383279502884L; // same as M_PIl in gcc + constexpr long double constexpr_pi_by_2 = 1.570796326794896619231321691639751442L; // same as M_PI_2l in gcc + constexpr long double constexpr_pi_by_4 = 0.785398163397448309615660845819875721L; // same as M_PI_4l in gcc + static_assert( constexpr_pi_by_4 * 4 == constexpr_pi ); + static_assert( constexpr_pi_by_4 * 2 == constexpr_pi_by_2 ); + static_assert( constexpr_pi_by_2 * 2 == constexpr_pi ); + + // Constexpr implementation of sin for 0= constexpr_pi_by_4 ) std::cout << "sinTaylor xx=" << xx << std::endl; +#endif + assert( xx >= 0 && "The argument of sinTaylor is lower than the expected range [0,pi/4)" ); + assert( xx < constexpr_pi_by_4 && "The argument of sinTaylor is higher than the expected range [0,pi/4)" ); + long double sinx = 0; + int ipow = 1; + long double delta = xx; + while( true ) + { + long double sinxlast = sinx; + sinx += delta; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + if( sinx == sinxlast ) break; + // Next iteration + ipow += 2; + delta *= -xx * xx / ( ipow - 1 ) / ipow; + } +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + return sinx; + } + +#ifdef CONSTEXPR_MATH_DEBUG + // Debug printouts for trig functions + inline void debug_constexpr_trig( const long double xx, size_t call ) + { + CONSTEXPRMATHVAR long double xxminuspi = xx - constexpr_pi; + CONSTEXPRMATHVAR long double twopiminusxx = 2 * constexpr_pi - xx; + std::cout << std::setprecision( 40 ); + std::cout << "constexpr_sin_quad call=" << call << std::endl + << " xx=" << xx << std::endl + << " (xx-pi)=" << xxminuspi << std::endl + << " (2pi-xx)=" << twopiminusxx << std::endl; + std::cout << std::setprecision( 6 ); + if( xx < 0 ) // (-inf, 0) + std::cout << " -- case 1 (xx < 0)" << std::endl; + else if( xx == 0 ) // [0] *** NEW + std::cout << " -- case 2 (xx == 0)" << std::endl; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + std::cout << " -- case 3 (xx < pi/4)" << std::endl; + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + std::cout << " -- case 4 (xx == pi/4)" << std::endl; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + std::cout << " -- case 5 (xx < pi/2)" << std::endl; + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + std::cout << " -- case 6 (xx == pi/2)" << std::endl; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + std::cout << " -- case 7 (xx < 3*pi/4)" << std::endl; + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + std::cout << " -- case 8 (xx == 3*pi/4)" << std::endl; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + std::cout << " -- case 9 (xx < pi)" << std::endl; + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + std::cout << " -- case 10 (xx == pi)" << std::endl; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + std::cout << " -- case 11 (xx < 2*pi)" << std::endl; + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + std::cout << " -- case 12 (xx == 2*pi)" << std::endl; + else // (2*pi, +inf) + std::cout << " -- case 13 (xx > 2*pi)" << std::endl; + } +#endif + + // Mapping to [0,2*pi) range (long double signature) + constexpr long double mapIn0to2Pi( const long double xx ) + { + return xx - constexpr_floor( xx / 2 / constexpr_pi ) * 2 * constexpr_pi; + } + + // Constexpr implementation of cos (long double signature) + CONSTEXPRMATHFUN long double constexpr_cos_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 1; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx ), 2 ) ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return sinTaylor( constexpr_pi_by_2 - xx ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 0; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return -sinTaylor( xx - constexpr_pi_by_2 ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return -1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return -constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi - xx ), 2 ) ); + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + return -1; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return constexpr_cos_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 1; + else // (2*pi, +inf) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of cos (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_cos( const double x ) + { + return constexpr_cos_quad( x ); + } + + // Constexpr implementation of sin (long double signature) + CONSTEXPRMATHFUN long double constexpr_sin_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 0; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return sinTaylor( xx ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi_by_2 - xx ), 2 ) ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 1; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx - constexpr_pi_by_2 ), 2 ) ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return sinTaylor( constexpr_pi - xx ); + else if( xx == constexpr_pi ) // [pi] *** NEW (1st fix #903 infinite recursion) + return 0; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return -constexpr_sin_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 0; + else // (2*pi, +inf) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of sin (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_sin( const double x ) + { + return constexpr_sin_quad( x ); + } + + // Constexpr implementation of tan (long double signature) + CONSTEXPRMATHFUN long double constexpr_tan_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } + if( xx < 0 ) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + else if( xx < 2 * constexpr_pi ) // [0, 2*pi) + return constexpr_sin_quad( xx, assume0to2Pi ) / constexpr_cos_quad( xx, assume0to2Pi ); + else // [8/4*pi, +inf) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of tan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_tan( const double x ) + { + return constexpr_tan_quad( x ); + } + + // Constexpr implementation of atan for -1= -1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + assert( xx < 1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + long double atanx = 0; + int ipow = 1; + long double xpow = xx; + while( true ) + { + long double atanxlast = atanx; + atanx += xpow / ipow; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", xpow=" << xpow << ", atanx=" << atanx << std::endl; // for debugging (not constexpr) +#endif + if( atanx == atanxlast ) break; + // Next iteration + ipow += 2; + xpow *= -xx * xx; + } + return atanx; + } + + // Constexpr implementation of atan (long double signature) + CONSTEXPRMATHFUN long double constexpr_atan_quad( const long double xx ) + { + if( xx > 1 ) + return constexpr_pi_by_2 - atanTaylor( 1 / xx ); + else if( xx == 1 ) + return constexpr_pi_by_4; + else if( xx > -1 ) + return atanTaylor( xx ); + else if( xx == -1 ) + return -constexpr_pi_by_4; + else // if( xx < -1 ) + return -constexpr_pi_by_2 - atanTaylor( 1 / xx ); + } + + // Constexpr implementation of atan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_atan( const double x ) + { + return constexpr_atan_quad( x ); + } + + // Enable FPE traps (see #701, #733, #831 - except on MacOS where feenableexcept is not defined #730) + // [NB1: Fortran default is -ffpe-trap=none, i.e. FPE traps are not enabled, https://gcc.gnu.org/onlinedocs/gfortran/Debugging-Options.html] + // [NB2: Fortran default is -ffpe-summary=invalid,zero,overflow,underflow,denormal, i.e. warn at the end on STOP] + inline void + fpeEnable() + { + static bool first = true; // FIXME: quick and dirty hack to do this only once (can be removed when separate C++/CUDA builds are implemented) + if( !first ) return; + first = false; +#ifndef __APPLE__ // on MacOS feenableexcept is not defined #730 + constexpr bool enableFPE = true; // this is hardcoded and no longer controlled by getenv( "CUDACPP_RUNTIME_ENABLEFPE" ) + if( enableFPE ) + { + std::cout << "INFO: The following Floating Point Exceptions will cause SIGFPE program aborts: FE_DIVBYZERO, FE_INVALID, FE_OVERFLOW" << std::endl; + feenableexcept( FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW ); // new strategy #831 (do not enable FE_UNDERFLOW) + } +#else + //std::cout << "INFO: Keep default SIGFPE settings because feenableexcept is not available on MacOS" << std::endl; +#endif + } +} + +#endif // constexpr_math_h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h new file mode 100644 index 0000000000..9cecf1e8fb --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuConfig.h @@ -0,0 +1,316 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jul 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCONFIG_H +#define MGONGPUCONFIG_H 1 + +#include +#include + +// Is this a GPU (CUDA, HIP) or CPU implementation? +#ifdef __CUDACC__ // this must be __CUDACC__ +#define MGONGPUCPP_GPUIMPL cuda +#elif defined __HIPCC__ +#define MGONGPUCPP_GPUIMPL hip +#include "hip/hip_runtime.h" // needed for blockDim, blockIdx, threadIdx: better in mgOnGpuConfig.h than in GpuAbstraction.h +#else +#undef MGONGPUCPP_GPUIMPL +#endif + +// Make sure that __HIP_PLATFORM_NVIDIA__ is undefined +// (__HIP_PLATFORM_AMD__ is defined by hipcc or in HiprandRandomNumberKernel.cc) +#undef __HIP_PLATFORM_NVIDIA__ // disable hiprand for NVidia (curand) + +// ** NB1 Throughputs (e.g. 6.8E8) are events/sec for "./gcheck.exe -p 65536 128 12" +// ** NB2 Baseline on b7g47n0004 fluctuates (probably depends on load on other VMs) + +// Choose if curand is supported for generating random numbers +// For HIP, by default, do not allow curand to be used (hiprand or common random numbers will be used instead) +// For both CUDA and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_CURAND +// (there exist CUDA installations, e.g. using the HPC package, which do not include curand - see PR #784 and #785) +#if defined __HIPCC__ +#define MGONGPU_HAS_NO_CURAND 1 +#else +//#ifdef __CUDACC__ // this must be __CUDACC__ +//#undef MGONGPU_HAS_NO_CURAND // default +////#define MGONGPU_HAS_NO_CURAND 1 +//#else +//#undef MGONGPU_HAS_NO_CURAND // default +////#define MGONGPU_HAS_NO_CURAND 1 +//#endif +#endif + +// Choose if hiprand is supported for generating random numbers +// For CUDA, by default, do not allow hiprand to be used (curand or common random numbers will be used instead) +// For both HIP and C++, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_HIPRAND +// (there may exist HIP installations which do not include hiprand?) +#if defined __CUDACC__ // this must be __CUDACC__ +#define MGONGPU_HAS_NO_HIPRAND 1 +#else +//#ifdef __HIPCC__ +//#undef MGONGPU_HAS_NO_HIPRAND // default +////#define MGONGPU_HAS_NO_HIPRAND 1 +//#else +//#undef MGONGPU_HAS_NO_HIPRAND // default +////#define MGONGPU_HAS_NO_HIPRAND 1 +//#endif +#endif + +// Choose floating point precision (for everything but color algebra #537) +// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE_FLOAT, nothing happens (issue #167) +#if not defined MGONGPU_FPTYPE_DOUBLE and not defined MGONGPU_FPTYPE_FLOAT +// Floating point precision (CHOOSE ONLY ONE) +#define MGONGPU_FPTYPE_DOUBLE 1 // default +//#define MGONGPU_FPTYPE_FLOAT 1 // 2x faster +#endif + +// Choose floating point precision (for color algebra alone #537) +// If one of these macros has been set from outside with e.g. -DMGONGPU_FPTYPE2_FLOAT, nothing happens (issue #167) +#if not defined MGONGPU_FPTYPE2_DOUBLE and not defined MGONGPU_FPTYPE2_FLOAT +// Floating point precision (CHOOSE ONLY ONE) +#define MGONGPU_FPTYPE2_DOUBLE 1 // default +//#define MGONGPU_FPTYPE2_FLOAT 1 // 2x faster +#endif + +// Choose whether to inline all HelAmps functions +// This optimization can gain almost a factor 4 in C++, similar to -flto (issue #229) +// By default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_INLINE_HELAMPS +//#undef MGONGPU_INLINE_HELAMPS // default +////#define MGONGPU_INLINE_HELAMPS 1 + +// Choose whether to hardcode the cIPD physics parameters rather than reading them from user cards +// This optimization can gain 20% in CUDA in eemumu (issue #39) +// By default, do not hardcode, but allow this macro to be set from outside with e.g. -DMGONGPU_HARDCODE_PARAM +// ** NB: The option to use hardcoded cIPD physics parameters is supported again even now when alphas is running (#373) +// ** NB: Note however that it now only refers to cIPD parameters (cIPC parameters are always accessed through global memory) +//#undef MGONGPU_HARDCODE_PARAM // default +////#define MGONGPU_HARDCODE_PARAM 1 + +/* clang-format off */ +// Complex type in CUDA: thrust or cucomplex or cxsmpl (CHOOSE ONLY ONE) +// (NB THIS IS MGONGPU_*CU*CXTYPE_xxx) +#ifdef __CUDACC__ // this must be __CUDACC__ +#define MGONGPU_CUCXTYPE_THRUST 1 // default (~1.15E9/double, ~3.2E9/float) +//#define MGONGPU_CUCXTYPE_CUCOMPLEX 1 // ~10 percent slower (1.03E9/double, ~2.8E9/float) +//#define MGONGPU_CUCXTYPE_CXSMPL 1 // ~10 percent slower (1.00E9/double, ~2.9E9/float) + +// Complex type in HIP: cxsmpl (ONLY ONE OPTION POSSIBLE? #810) +// (NB THIS IS MGONGPU_*HIP*CXTYPE_xxx) +#elif defined __HIPCC__ +#define MGONGPU_HIPCXTYPE_CXSMPL 1 // default for HIP + +// Complex type in C++: std::complex or cxsmpl (CHOOSE ONLY ONE) +// (NB THIS IS MGONGPU_*CPP*CXTYPE_xxx) +#else +//#define MGONGPU_CPPCXTYPE_STDCOMPLEX 1 // ~8 percent slower on float, same on double (5.1E6/double, 9.4E6/float) +#define MGONGPU_CPPCXTYPE_CXSMPL 1 // new default (5.1E6/double, 10.2E6/float) +#endif + +// Choose if cuBLAS and hipBLAS are supported for generating random numbers +// For both CUDA and HIP, by default, do not inline, but allow this macro to be set from outside with e.g. -DMGONGPU_HAS_NO_BLAS +// (there may exist CUDA/HIP installations, e.g. using the HPC package, which do not include cuBLAS/hipBLAS?) +#ifdef __CUDACC__ // this must be __CUDACC__ +//#undef MGONGPU_HAS_NO_BLAS // default +////#define MGONGPU_HAS_NO_BLAS 1 +#elif defined __HIPCC__ +//#undef MGONGPU_HAS_NO_BLAS // default +////#define MGONGPU_HAS_NO_BLAS 1 +#else +#define MGONGPU_HAS_NO_BLAS 1 +#endif + +// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation +#ifdef __CUDACC__ // this must be __CUDACC__ +#undef MGONGPU_NSIGHT_DEBUG // default in CUDA +//#define MGONGPU_NSIGHT_DEBUG 1 // CURRENTLY NO LONGER SUPPORTED! +#else +#undef MGONGPU_NSIGHT_DEBUG // only option in HIP or C++ +#endif /* clang-format on */ + +// SANITY CHECKS (floating point precision for everything but color algebra #537) +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE_DOUBLE or defined MGONGPU_FPTYPE_FLOAT +#endif + +// SANITY CHECKS (floating point precision for color algebra alone #537) +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE2_DOUBLE or defined MGONGPU_FPTYPE2_FLOAT +#endif +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You cannot use double precision for color algebra and single precision elsewhere +#endif + +// SANITY CHECKS (CUDA complex number implementation) +#ifdef __CUDACC__ // this must be __CUDACC__ +#if defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CUCOMPLEX +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX for CUDA +#elif defined MGONGPU_CUCXTYPE_THRUST and defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CXSMPL for CUDA +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX and defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE OF MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL for CUDA +#endif +#endif + +// SANITY CHECKS (C++ complex number implementation) + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu +{ + + // --- Type definitions + + // Floating point type (for everything but color algebra #537): fptype +#if defined MGONGPU_FPTYPE_DOUBLE + typedef double fptype; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE_FLOAT + typedef float fptype; // single precision (4 bytes, fp32) +#endif + + // Floating point type (for color algebra alone #537): fptype2 +#if defined MGONGPU_FPTYPE2_DOUBLE + typedef double fptype2; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE2_FLOAT + typedef float fptype2; // single precision (4 bytes, fp32) +#endif + + // --- Platform-specific software implementation details + + // Maximum number of blocks per grid + // ** NB Some arrays of pointers will be allocated statically to fit all these blocks + // ** (the actual memory for each block will then be allocated dynamically only for existing blocks) + //const int nbpgMAX = 2048; + + // Maximum number of threads per block + //const int ntpbMAX = 256; // AV Apr2021: why had I set this to 256? + const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" + + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + // (using reinterpret_cast with non aligned memory may lead to segmentation faults!) + // Only needed for C++ code but can be enforced also in NVCC builds of C++ code using CUDA>=11.2 and C++17 (#318, #319, #333) + + // Retrieve the compiler that was used to build this module + inline std::string + getCompiler() + { + std::stringstream out; + // HIP version (HIPCC) + // [Use __HIPCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'hipcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'hipcc -dM -E -x hip -I ../../src CPPProcess.cc | grep HIP'] +#ifdef __HIPCC__ +#if defined HIP_VERSION_MAJOR && defined HIP_VERSION_MINOR && defined HIP_VERSION_PATCH + out << "hipcc " << HIP_VERSION_MAJOR << "." << HIP_VERSION_MINOR << "." << HIP_VERSION_PATCH; +#else + out << "hipcc UNKNOWN"; +#endif + out << " ("; +#endif + // CUDA version (NVCC) + // [Use __NVCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'nvcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'nvcc --compiler-options -dM -E dummy.c | grep CUDA': see https://stackoverflow.com/a/53713712] +#ifdef __NVCC__ +#if defined __CUDACC_VER_MAJOR__ && defined __CUDACC_VER_MINOR__ && defined __CUDACC_VER_BUILD__ + out << "nvcc " << __CUDACC_VER_MAJOR__ << "." << __CUDACC_VER_MINOR__ << "." << __CUDACC_VER_BUILD__; +#else + out << "nvcc UNKNOWN"; +#endif + out << " ("; +#endif + // ICX version (either as CXX or as host compiler inside NVCC) +#if defined __INTEL_COMPILER +#error "icc is no longer supported: please use icx" +#elif defined __INTEL_LLVM_COMPILER // alternative: __INTEL_CLANG_COMPILER + out << "icx " << __INTEL_LLVM_COMPILER; +#ifdef __NVCC__ + out << ", "; +#else + out << " ("; +#endif +#endif + // CLANG version (either as CXX or as host compiler inside NVCC or inside ICX) +#if defined __clang__ +#if defined __clang_major__ && defined __clang_minor__ && defined __clang_patchlevel__ +#ifdef __APPLE__ + out << "Apple clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#else + out << "clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#endif +#else + out << "clang UNKNOWKN"; +#endif +#else + // GCC version (either as CXX or as host compiler inside NVCC) +#if defined __GNUC__ && defined __GNUC_MINOR__ && defined __GNUC_PATCHLEVEL__ + out << "gcc " << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__; +#else + out << "gcc UNKNOWKN"; +#endif +#endif +#if defined __HIPCC__ or defined __NVCC__ or defined __INTEL_LLVM_COMPILER + out << ")"; +#endif + return out.str(); + } +} + +// Expose typedefs and operators outside the namespace +using mgOnGpu::fptype; +using mgOnGpu::fptype2; + +// Undefine ARM_NEON (hack for cppnone on Apple silicon ARM) +#ifdef MGONGPU_NOARMNEON +#undef __ARM_NEON +#endif + +// C++ SIMD vectorization width (this will be used to set neppV) +#undef MGONGPU_CPPSIMD + +/* clang-format off */ +// CUDA nsight compute (ncu) debug: add dummy lines to ease SASS program flow navigation [NB: CURRENTLY NO LONGER SUPPORTED!] +// Arguments (not used so far): text is __FUNCTION__, code is 0 (start) or 1 (end) +//#if defined __CUDACC__ && defined MGONGPU_NSIGHT_DEBUG // this must be __CUDACC__ +//#define mgDebugDeclare() __shared__ float mgDebugCounter[mgOnGpu::ntpbMAX]; +//#define mgDebugInitialise() { mgDebugCounter[threadIdx.x] = 0; } +//#define mgDebug( code, text ) { mgDebugCounter[threadIdx.x] += 1; } +//#define mgDebugFinalise() { if ( blockIdx.x == 0 && threadIdx.x == 0 ) printf( "MGDEBUG: counter=%f\n", mgDebugCounter[threadIdx.x] ); } +//#else +#define mgDebugDeclare() /*noop*/ +#define mgDebugInitialise() /*noop*/ +#define mgDebug( code, text ) /*noop*/ +#define mgDebugFinalise() /*noop*/ +//#endif /* clang-format on */ + +// Define empty CUDA/HIP declaration specifiers for C++ + +// For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two (https://stackoverflow.com/a/108360) +inline constexpr bool +ispoweroftwo( int n ) +{ + return ( n > 0 ) && !( n & ( n - 1 ) ); +} + +// Compiler version support (#96): require nvcc from CUDA >= 11.2, e.g. to use C++17 (see #333) +#ifdef __NVCC__ +#if( __CUDACC_VER_MAJOR__ < 11 ) || ( __CUDACC_VER_MAJOR__ == 11 && __CUDACC_VER_MINOR__ < 2 ) +#error Unsupported CUDA version: please use CUDA >= 11.2 +#endif +#endif + +// Compiler version support (#96): require clang >= 11 +#if defined __clang__ +#if( __clang_major__ < 11 ) +#error Unsupported clang version: please use clang >= 11 +#endif +// Compiler version support (#96): require gcc >= 9.3, e.g. for some OMP issues (see #269) +// [NB skip this check for the gcc toolchain below clang or icx (TEMPORARY? #355)] +#elif defined __GNUC__ +#if( __GNUC__ < 9 ) || ( __GNUC__ == 9 && __GNUC_MINOR__ < 3 ) +#error Unsupported gcc version: please gcc >= 9.3 +#endif +#endif + +#endif // MGONGPUCONFIG_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h new file mode 100644 index 0000000000..f819da5783 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuCxtypes.h @@ -0,0 +1,782 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022, based on earlier work by D. Smith) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCXTYPES_H +#define MGONGPUCXTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +#include + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) HEADERS +//========================================================================== + +#include + +// Complex type in cuda: thrust or cucomplex or cxsmpl +#ifdef __CUDACC__ // this must be __CUDACC__ +#if defined MGONGPU_CUCXTYPE_THRUST +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) +#include +#pragma clang diagnostic pop +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX +#include +#elif not defined MGONGPU_CUCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CUCXTYPE_THRUST or MGONGPU_CUCXTYPE_CUCOMPLEX or MGONGPU_CUCXTYPE_CXSMPL +#endif +// Complex type in HIP: cxsmpl +#elif defined __HIPCC__ +#if not defined MGONGPU_HIPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_HIPCXTYPE_CXSMPL +#endif +#else +// Complex type in c++ or HIP: std::complex or cxsmpl +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX +#include +#elif not defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL +#endif +#endif + +//========================================================================== +// COMPLEX TYPES: INSTRUMENTED CUCOMPLEX CLASS (cucomplex) +//========================================================================== + +#ifdef __CUDACC__ // this must be __CUDACC__ +#if defined MGONGPU_CUCXTYPE_CUCOMPLEX +namespace madmatrix +{ +#if defined MGONGPU_FPTYPE_DOUBLE + class cucomplex + { + public: + __host__ __device__ cucomplex( const double& r = 0, const double& i = 0 ) + : m_ri( make_cuDoubleComplex( r, i ) ) {} + __host__ __device__ constexpr cucomplex( const cuDoubleComplex& ri ) + : m_ri( ri ) {} + //__host__ __device__ operator cuDoubleComplex&() { return m_ri; } + __host__ __device__ constexpr operator cuDoubleComplex() const { return m_ri; } + __host__ __device__ double real() const { return cuCreal( m_ri ); } + __host__ __device__ double imag() const { return cuCimag( m_ri ); } + inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) + { + m_ri = cuCadd( m_ri, c ); + return *this; + } + inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) + { + m_ri = cuCsub( m_ri, c ); + return *this; + } + private: + cuDoubleComplex m_ri; + }; +#elif defined MGONGPU_FPTYPE_FLOAT + class cucomplex + { + public: + __host__ __device__ cucomplex( const float& r = 0, const float& i = 0 ) + : m_ri( make_cuFloatComplex( r, i ) ) {} + __host__ __device__ constexpr cucomplex( const cuFloatComplex& ri ) + : m_ri( ri ) {} + //__host__ __device__ operator cuFloatComplex&() { return m_ri; } + __host__ __device__ constexpr operator cuFloatComplex() const { return m_ri; } + __host__ __device__ float real() const { return cuCrealf( m_ri ); } + __host__ __device__ float imag() const { return cuCimagf( m_ri ); } + inline __host__ __device__ cucomplex& operator+=( const cucomplex& c ) + { + m_ri = cuCaddf( m_ri, c ); + return *this; + } + inline __host__ __device__ cucomplex& operator-=( const cucomplex& c ) + { + m_ri = cuCsubf( m_ri, c ); + return *this; + } + private: + cuFloatComplex m_ri; + }; +#endif +} +#endif +#endif + +//========================================================================== +// COMPLEX TYPES: SIMPLE COMPLEX CLASS (cxsmpl) +//========================================================================== + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu /* clang-format off */ +{ + // The number of floating point types in a complex type (real, imaginary) + constexpr int nx2 = 2; + + // --- Type definition (simple complex type derived from cxtype_v) + template + class cxsmpl + { + public: + __host__ __device__ constexpr cxsmpl() : m_real( 0 ), m_imag( 0 ) {} + cxsmpl( const cxsmpl& ) = default; + cxsmpl( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl( const FP& r, const FP& i = 0 ) : m_real( r ), m_imag( i ) {} + __host__ __device__ constexpr cxsmpl( const std::complex& c ) : m_real( c.real() ), m_imag( c.imag() ) {} + cxsmpl& operator=( const cxsmpl& ) = default; + cxsmpl& operator=( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl& operator+=( const cxsmpl& c ) { m_real += c.real(); m_imag += c.imag(); return *this; } + __host__ __device__ constexpr cxsmpl& operator-=( const cxsmpl& c ) { m_real -= c.real(); m_imag -= c.imag(); return *this; } + __host__ __device__ constexpr const FP& real() const { return m_real; } + __host__ __device__ constexpr const FP& imag() const { return m_imag; } + template __host__ __device__ constexpr operator cxsmpl() const { return cxsmpl( m_real, m_imag ); } +#ifdef __CUDACC__ // this must be __CUDACC__ +#ifdef MGONGPU_CUCXTYPE_THRUST + template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX + __host__ __device__ constexpr operator madmatrix::cucomplex() const { return madmatrix::cucomplex( m_real, m_imag ); } +#endif +#else +#ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX + template __host__ __device__ constexpr operator std::complex() const { return std::complex( m_real, m_imag ); } +#endif +#endif + private: + FP m_real, m_imag; // RI + }; + + template + constexpr // (NB: now valid code? in the past this failed as "a constexpr function cannot have a nonliteral return type mgOnGpu::cxsmpl") + inline __host__ __device__ cxsmpl + conj( const cxsmpl& c ) + { + return cxsmpl( c.real(), -c.imag() ); + } +} /* clang-format on */ + +// Expose the cxsmpl class outside the namespace +using mgOnGpu::cxsmpl; + +// Printout to stream for user defined types +namespace madmatrix +{ + template + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxsmpl& c ) + { + //out << std::complex( c.real(), c.imag() ); + out << "(" << c.real() << ", " << c.imag() << ")"; // add a space after the comma + return out; + } + + // Operators for cxsmpl + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl a ) + { + return a; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a ) + { + return cxsmpl( -a.real(), -a.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() + b.real(), a.imag() + b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) + b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() - b.real(), a.imag() - b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) - b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const double& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const double& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const cxsmpl& b ) + { + FP bnorm = b.real() * b.real() + b.imag() * b.imag(); + return cxsmpl( ( a.real() * b.real() + a.imag() * b.imag() ) / bnorm, + ( a.imag() * b.real() - a.real() * b.imag() ) / bnorm ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) / b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const FP& b ) + { + return a + cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const FP& b ) + { + return a - cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const FP& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const FP& b ) + { + return a / cxsmpl( b, 0 ); + } +} + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS +//========================================================================== + +//One namespace. Split ber backend. +namespace madmatrix +{ + // --- Type definitions (complex type: cxtype) +#ifdef __CUDACC__ // this must be __CUDACC__ +#if defined MGONGPU_CUCXTYPE_THRUST + typedef thrust::complex cxtype; +#elif defined MGONGPU_CUCXTYPE_CUCOMPLEX + typedef cucomplex cxtype; +#else + typedef cxsmpl cxtype; +#endif +#else // c++ +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX + typedef std::complex cxtype; +#else + typedef cxsmpl cxtype; +#endif +#endif + + // SANITY CHECK: memory access may be based on casts of fptype[2] to cxtype (e.g. for wavefunctions) + static_assert( sizeof( cxtype ) == mgOnGpu::nx2 * sizeof( fptype ), "sizeof(cxtype) is not 2*sizeof(fptype)" ); +} + +// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) +// DO NOT expose typedefs and operators outside the namespace +//using mgOnGpu::cxtype; + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS +//========================================================================== + +//One namespace. Split ber backend. +namespace madmatrix +{ +#if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL + + //------------------------------ + // CUDA or C++ - using cxsmpl + //------------------------------ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // cxsmpl constructor + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return c.real(); // cxsmpl::real() + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return c.imag(); // cxsmpl::imag() + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( cxsmpl ) + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +#endif // #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL + + //========================================================================== + +#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST // cuda + thrust (this must be __CUDACC__) + + //------------------------------ + // CUDA - using thrust::complex + //------------------------------ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // thrust::complex constructor + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return c.real(); // thrust::complex::real() + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return c.imag(); // thrust::complex::imag() + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( thrust::complex ) + } + + inline __host__ __device__ const cxtype& + cxmake( const cxtype& c ) + { + return c; + } + +#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_THRUST + + //========================================================================== + +#if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX // cuda + cucomplex (this must be __CUDACC__) + + //------------------------------ + // CUDA - using cuComplex + //------------------------------ + +#if defined MGONGPU_FPTYPE_DOUBLE // cuda + cucomplex + double + + //+++++++++++++++++++++++++ + // cuDoubleComplex ONLY + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return make_cuDoubleComplex( r, i ); + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return cuCreal( c ); // returns by value + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return cuCimag( c ); // returns by value + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const cxtype& b ) + { + return cuCadd( a, b ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const cxtype& b ) + { + return cuCsub( a, b ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const cxtype& b ) + { + return cuCmul( a, b ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const cxtype& b ) + { + return cuCdiv( a, b ); + } + + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + //out << std::complex( cxreal( c ), cximag( c ) ); + out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma + return out; + } + +#elif defined MGONGPU_FPTYPE_FLOAT // cuda + cucomplex + float + + //+++++++++++++++++++++++++ + // cuFloatComplex ONLY + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return make_cuFloatComplex( r, i ); + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return cuCrealf( c ); // returns by value + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return cuCimagf( c ); // returns by value + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const cxtype& b ) + { + return cuCaddf( a, b ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const cxtype& b ) + { + return cuCsubf( a, b ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const cxtype& b ) + { + return cuCmulf( a, b ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const cxtype& b ) + { + return cuCdivf( a, b ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cucomplex (cast double-to-float) + { + return cxmake( (fptype)c.real(), (fptype)c.imag() ); + } + + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + //out << std::complex( cxreal( c ), cximag( c ) ); + out << "(" << cxreal( c ) << ", " << cximag( c ) << ")"; // add a space after the comma + return out; + } + +#endif + + //+++++++++++++++++++++++++ + // cuDoubleComplex OR + // cuFloatComplex + //+++++++++++++++++++++++++ + + inline __host__ __device__ cxtype + operator+( const cxtype a ) + { + return a; + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a ) + { + return cxmake( -cxreal( a ), -cximag( a ) ); + } + + inline __host__ __device__ cxtype + operator+( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) + b; + } + + inline __host__ __device__ cxtype + operator-( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) - b; + } + + inline __host__ __device__ cxtype + operator*( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) * b; + } + + inline __host__ __device__ cxtype + operator/( const fptype& a, const cxtype& b ) + { + return cxmake( a, 0 ) / b; + } + + inline __host__ __device__ cxtype + operator+( const cxtype& a, const fptype& b ) + { + return a + cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator-( const cxtype& a, const fptype& b ) + { + return a - cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator*( const cxtype& a, const fptype& b ) + { + return a * cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + operator/( const cxtype& a, const fptype& b ) + { + return a / cxmake( b, 0 ); + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return cxmake( cxreal( c ), -cximag( c ) ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cucomplex (float-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +#endif // #if defined __CUDACC__ and defined MGONGPU_CUCXTYPE_CUCOMPLEX + + //========================================================================== + +#if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX // c++/hip + stdcomplex (this must be __CUDACC__) + + //------------------------------ + // C++ - using std::complex + //------------------------------ + + inline cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // std::complex constructor + } + + inline fptype + cxreal( const cxtype& c ) + { + return c.real(); // std::complex::real() + } + + inline fptype + cximag( const cxtype& c ) + { + return c.imag(); // std::complex::imag() + } + + inline cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( std::complex ) + } + + inline const cxtype& + cxmake( const cxtype& c ) // std::complex to std::complex (float-to-float or double-to-double) + { + return c; + } + +#if defined MGONGPU_FPTYPE_FLOAT + inline cxtype + cxmake( const std::complex& c ) // std::complex to std::complex (cast double-to-float) + { + return cxmake( (fptype)c.real(), (fptype)c.imag() ); + } +#endif + +#endif // #if not defined __CUDACC__ and defined MGONGPU_CPPCXTYPE_STDCOMPLEX + + //========================================================================== + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +} // end namespace madmatrix + +//========================================================================== +// COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) +//========================================================================== + +//One namespace. Split ber backend. +namespace madmatrix +{ + // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] + // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined + // It is now always included in the code because it is needed also to access an fptype wavefunction buffer as a cxtype + class cxtype_ref + { + public: + cxtype_ref() = delete; + cxtype_ref( const cxtype_ref& ) = delete; + cxtype_ref( cxtype_ref&& ) = default; // copy const refs + __host__ __device__ cxtype_ref( fptype& r, fptype& i ) + : m_preal( &r ), m_pimag( &i ) {} // copy (create from) const refs + cxtype_ref& operator=( const cxtype_ref& ) = delete; + //__host__ __device__ cxtype_ref& operator=( cxtype_ref&& c ) {...} // REMOVED! Should copy refs or copy values? No longer needed in cxternary + __host__ __device__ cxtype_ref& operator=( const cxtype& c ) + { + *m_preal = cxreal( c ); + *m_pimag = cximag( c ); + return *this; + } // copy (assign) non-const values + __host__ __device__ operator cxtype() const { return cxmake( *m_preal, *m_pimag ); } + private: + fptype* const m_preal; // const pointer to non-const fptype R + fptype* const m_pimag; // const pointer to non-const fptype I + }; + + // Printout to stream for user defined types + inline __host__ __device__ std::ostream& + operator<<( std::ostream& out, const cxtype_ref& c ) + { + out << (cxtype)c; + return out; + } + + //-------------------------------------------------------------------------- + + // all needed from mgOnGpuVectors.h for gpu + const int neppV = 1; + + // Printout to std::cout for user defined types + inline __host__ __device__ void + print( const fptype& f ) + { + printf( "%f\n", f ); + } + inline __host__ __device__ void + print( const cxtype& c ) + { + printf( "[%f, %f]\n", cxreal( c ), cximag( c ) ); + } + + inline __host__ __device__ fptype + fpternary( const bool& mask, const fptype& a, const fptype& b ) + { + return ( mask ? a : b ); + } + + inline __host__ __device__ cxtype + cxternary( const bool& mask, const cxtype& a, const cxtype& b ) + { + return ( mask ? a : b ); + } + + inline __host__ __device__ bool + maskand( const bool& mask ) + { + return mask; + } + + //vector is scalar on gpu + typedef bool bool_sv; + typedef fptype fptype_sv; + typedef fptype2 fptype2_sv; + typedef unsigned int uint_sv; + typedef cxtype cxtype_sv; + typedef cxtype_ref cxtype_sv_ref; + + //vector is scalar on gpu + inline __host__ __device__ cxtype cxzero_sv() { return cxtype( 0, 0 ); } + + // Functions and operators for cxtype_sv + inline __host__ __device__ fptype_sv + cxabs2( const cxtype_sv& c ) + { + return cxreal( c ) * cxreal( c ) + cximag( c ) * cximag( c ); + } + +} // end namespace madmatrix + +//========================================================================== + +#endif // MGONGPUCXTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h new file mode 100644 index 0000000000..3a101094dd --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/mgOnGpuFptypes.h @@ -0,0 +1,71 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUFPTYPES_H +#define MGONGPUFPTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include +#include + +//One namespace. Split ber backend. +namespace madmatrix +{ + //========================================================================== + + + //------------------------------ + // Floating point types - Cuda + //------------------------------ + + /* + inline __host__ __device__ fptype + fpmax( const fptype& a, const fptype& b ) + { + return max( a, b ); + } + + inline __host__ __device__ fptype + fpmin( const fptype& a, const fptype& b ) + { + return min( a, b ); + } + */ + + inline __host__ __device__ const fptype& + fpmax( const fptype& a, const fptype& b ) + { + return ( ( b < a ) ? a : b ); + } + + inline __host__ __device__ const fptype& + fpmin( const fptype& a, const fptype& b ) + { + return ( ( a < b ) ? a : b ); + } + + inline __host__ __device__ fptype + fpsqrt( const fptype& f ) + { +#if defined MGONGPU_FPTYPE_FLOAT + // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__SINGLE.html + return sqrtf( f ); +#else + // See https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH__DOUBLE.html + return sqrt( f ); +#endif + } + + + //========================================================================== + + + //========================================================================== + +} // end namespace madmatrix + +#endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/umami.cc b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc similarity index 61% rename from madgraph/iolibs/template_files/madmatrix/umami.cc rename to madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc index d19c93bb9b..de9470c1f9 100644 --- a/madgraph/iolibs/template_files/madmatrix/umami.cc +++ b/madgraph/iolibs/template_files/madmatrix/backend/gpu/umami.cc @@ -6,7 +6,9 @@ #include "umami.h" -#include "CPPProcess.h" +#include "ProcessData.h" +#include "CPPProcess.h" // needed to construct/initProc the process object (umami_initialize) +#include "SigmaKin.h" // sigmaKin_getGoodHel/setGoodHel #include "GpuRuntime.h" #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" @@ -16,11 +18,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else -using namespace mg5amcCpu; -#endif +using namespace madmatrix; namespace { @@ -30,19 +28,15 @@ namespace const fptype* couplings, const unsigned int* flavor_indices, fptype* matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL fptype* color_jamps, -#endif fptype* numerators, fptype* denominators, std::size_t count ) { - bool is_good_hel[CPPProcess::ncomb]; + bool is_good_hel[ProcessData::ncomb]; sigmaKin_getGoodHel( momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, -#ifdef MGONGPUCPP_GPUIMPL color_jamps, -#endif is_good_hel, count ); sigmaKin_setGoodHel( is_good_hel ); @@ -54,26 +48,20 @@ namespace const fptype* couplings, const unsigned int* flavor_indices, fptype* matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL fptype* color_jamps, -#endif fptype* numerators, fptype* denominators, std::size_t count ) { // static local initialization is called exactly once in a thread-safe way static void* dummy = initialize_impl( momenta, couplings, flavor_indices, matrix_elements, -#ifdef MGONGPUCPP_GPUIMPL color_jamps, -#endif numerators, denominators, count ); } -#ifdef MGONGPUCPP_GPUIMPL __device__ -#endif void transpose_momenta( const double* momenta_in, fptype* momenta_out, std::size_t i_event_in, std::size_t i_event_out, std::size_t stride ) { @@ -81,17 +69,16 @@ namespace std::size_t i_page = i_event_out / page_size; std::size_t i_vector = i_event_out % page_size; - for( std::size_t i_part = 0; i_part < CPPProcess::npar; ++i_part ) + for( std::size_t i_part = 0; i_part < ProcessData::npar; ++i_part ) { for( std::size_t i_mom = 0; i_mom < 4; ++i_mom ) { - momenta_out[i_page * CPPProcess::npar * 4 * page_size + - i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( CPPProcess::npar * i_mom + i_part ) + i_event_in]; + momenta_out[i_page * ProcessData::npar * 4 * page_size + + i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( ProcessData::npar * i_mom + i_part ) + i_event_in]; } } } -#ifdef MGONGPUCPP_GPUIMPL __global__ void copy_inputs( const double* momenta_in, @@ -144,9 +131,9 @@ namespace if( amp2_out ) { double denominator = denominators[i_event]; - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) + for( std::size_t i_diag = 0; i_diag < ProcessData::ndiagrams; ++i_diag ) { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_event * CPPProcess::ndiagrams + i_diag] / denominator; + amp2_out[stride * i_diag + i_event + offset] = numerators[i_event * ProcessData::ndiagrams + i_diag] / denominator; } } if( diagram_out ) diagram_out[i_event + offset] = diagram_index[i_event] - 1; @@ -154,7 +141,6 @@ namespace if( helicity_out ) helicity_out[i_event + offset] = helicity_index[i_event] - 1; } -#endif // MGONGPUCPP_GPUIMPL struct InterfaceInstance { @@ -174,33 +160,29 @@ extern "C" case UMAMI_META_DEVICE: { UmamiDevice& device = *static_cast( result ); -#ifdef MGONGPUCPP_GPUIMPL #ifdef __CUDACC__ device = UMAMI_DEVICE_CUDA; #elif defined( __HIPCC__ ) device = UMAMI_DEVICE_HIP; -#endif -#else - device = UMAMI_DEVICE_CPU; #endif break; } case UMAMI_META_PARTICLE_COUNT: - *static_cast( result ) = CPPProcess::npar; + *static_cast( result ) = ProcessData::npar; break; case UMAMI_META_DIAGRAM_COUNT: - *static_cast( result ) = CPPProcess::ndiagrams; + *static_cast( result ) = ProcessData::ndiagrams; break; case UMAMI_META_HELICITY_COUNT: - *static_cast( result ) = CPPProcess::ncomb; + *static_cast( result ) = ProcessData::ncomb; break; case UMAMI_META_COLOR_COUNT: return UMAMI_ERROR_UNSUPPORTED_META; case UMAMI_META_MASSES: { - if( g_externalMasses.size() != (size_t)CPPProcess::npar ) return UMAMI_ERROR_UNINITIALIZED_META; + if( g_externalMasses.size() != (size_t)ProcessData::npar ) return UMAMI_ERROR_UNINITIALIZED_META; - for( int ipar = 0; ipar < CPPProcess::npar; ++ipar ) + for( int ipar = 0; ipar < ProcessData::npar; ++ipar ) static_cast( result )[ipar] = g_externalMasses[ipar]; break; } @@ -231,11 +213,7 @@ extern "C" UmamiStatus umami_supported_outputs( bool const** supported, int* count ) { // MATRIX_ELEMENT, DIAGRAM_AMP2, COLOR_INDEX, HELICITY_INDEX, DIAGRAM_INDEX, GPU_STREAM -#ifdef MGONGPUCPP_GPUIMPL static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true, true }; -#else - static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true }; -#endif *supported = data; *count = UMAMI_OUTPUT_KEY_COUNT; return UMAMI_SUCCESS; @@ -326,9 +304,7 @@ extern "C" } if( !momenta_in ) return UMAMI_ERROR_MISSING_INPUT; -#ifdef MGONGPUCPP_GPUIMPL gpuStream_t gpu_stream = nullptr; -#endif double* m2_out = nullptr; double* amp2_out = nullptr; int* diagram_out = nullptr; @@ -354,17 +330,14 @@ extern "C" case UMAMI_OUT_DIAGRAM_INDEX: diagram_out = static_cast( output ); break; -#ifdef MGONGPUCPP_GPUIMPL case UMAMI_OUT_GPU_STREAM: gpu_stream = static_cast( output ); break; -#endif default: return UMAMI_ERROR_UNSUPPORTED_OUTPUT; } } -#ifdef MGONGPUCPP_GPUIMPL std::size_t n_threads = 256; std::size_t n_blocks = ( count + n_threads - 1 ) / n_threads; std::size_t rounded_count = n_blocks * n_threads; @@ -374,9 +347,9 @@ extern "C" int *helicity_index, *color_index; unsigned int *flavor_indices, *diagram_index; - std::size_t n_coup = mg5amcGpu::Parameters_dependentCouplings::ndcoup; + std::size_t n_coup = madmatrix::Parameters_dependentCouplings::ndcoup; std::array, 16> ptrs_and_sizes = {{ - {reinterpret_cast(&momenta), rounded_count * CPPProcess::npar * 4 * sizeof( fptype )}, + {reinterpret_cast(&momenta), rounded_count * ProcessData::npar * 4 * sizeof( fptype )}, {reinterpret_cast(&couplings), rounded_count * n_coup * 2 * sizeof( fptype )}, {reinterpret_cast(&g_s), rounded_count * sizeof( fptype )}, {reinterpret_cast(&flavor_indices), rounded_count * sizeof( unsigned int )}, @@ -385,13 +358,13 @@ extern "C" {reinterpret_cast(&diagram_random), rounded_count * sizeof( fptype )}, {reinterpret_cast(&matrix_elements), rounded_count * sizeof( fptype )}, {reinterpret_cast(&diagram_index), rounded_count * sizeof( unsigned int )}, - {reinterpret_cast(&color_jamps), rounded_count * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, - {reinterpret_cast(&numerators), rounded_count * CPPProcess::ndiagrams * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&denominators), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, + {reinterpret_cast(&color_jamps), rounded_count * ProcessData::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, + {reinterpret_cast(&numerators), rounded_count * ProcessData::ndiagrams * ProcessData::ncomb * sizeof( fptype )}, + {reinterpret_cast(&denominators), rounded_count * ProcessData::ncomb * sizeof( fptype )}, {reinterpret_cast(&helicity_index), rounded_count * sizeof( int )}, {reinterpret_cast(&color_index), rounded_count * sizeof( int )}, - {reinterpret_cast(&ghel_matrix_elements), rounded_count * CPPProcess::ncomb * sizeof( fptype )}, - {reinterpret_cast(&ghel_jamps), rounded_count * CPPProcess::ncomb * CPPProcess::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, + {reinterpret_cast(&ghel_matrix_elements), rounded_count * ProcessData::ncomb * sizeof( fptype )}, + {reinterpret_cast(&ghel_jamps), rounded_count * ProcessData::ncomb * ProcessData::ncolor * mgOnGpu::nx2 * sizeof( fptype )}, }}; std::size_t total_size = 0; constexpr std::size_t MAX_SIZE = std::max(sizeof(fptype), sizeof(int)); @@ -479,182 +452,6 @@ extern "C" checkGpu( gpuPeekAtLastError() ); gpuFreeAsync( buffer, gpu_stream ); -#else // MGONGPUCPP_GPUIMPL - constexpr std::size_t vector_size = MemoryAccessMomentaBase::neppM; - // need to round to round to double page size for some reason - constexpr std::size_t page_size2 = 2 * vector_size; - std::vector permutation; - std::size_t rounded_count; - - constexpr std::size_t flavor_count = CPPProcess::nmaxflavor; - HostBufferBase flavor_indices( ((count + page_size2 - 1) / page_size2 + flavor_count) * page_size2 ); - bool sort_flavors = vector_size > 1 && flavor_count > 1 && flavor_indices_in; - if ( sort_flavors ) - { - permutation.resize(count); - std::size_t voffset = 0; - std::size_t vector_indices[flavor_count] = {}; - std::size_t vector_counts[flavor_count] = {}; - // determine permutation of inputs such that all entries in a SIMD vector - // have the same flavor index - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - unsigned int flav = flavor_indices_in[i_event + offset]; - auto& vcount = vector_counts[flav]; - auto& vindex = vector_indices[flav]; - if ( vcount == 0 ) - { - vindex = voffset * page_size2; - for ( std::size_t i = 0; i < page_size2; ++i) { - flavor_indices[voffset * page_size2 + i] = flav; - } - voffset += 1; - } - permutation[i_event] = vindex + vcount; - vcount = (vcount + 1) % page_size2; - } - rounded_count = voffset * page_size2; - } else { - rounded_count = ( count + page_size2 - 1 ) / page_size2 * page_size2; - } - - HostBufferBase momenta( rounded_count * CPPProcess::npar * 4 ); - HostBufferBase couplings( rounded_count * mg5amcCpu::Parameters_dependentCouplings::ndcoup * 2 ); - HostBufferBase g_s( rounded_count ); - HostBufferBase helicity_random( rounded_count ); - HostBufferBase color_random( rounded_count ); - HostBufferBase diagram_random( rounded_count ); - HostBufferBase matrix_elements( rounded_count ); - HostBufferBase diagram_index( rounded_count ); - HostBufferBase numerators( rounded_count * CPPProcess::ndiagrams ); - HostBufferBase denominators( rounded_count ); - HostBufferBase helicity_index( rounded_count ); - HostBufferBase color_index( rounded_count ); - if ( sort_flavors ) { - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - std::size_t i_sorted = permutation[i_event]; - transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_sorted, stride ); - helicity_random[i_sorted] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; - color_random[i_sorted] = random_color_in ? random_color_in[i_event + offset] : 0.5; - diagram_random[i_sorted] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; - g_s[i_sorted] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; - } - } else { - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_event, stride ); - helicity_random[i_event] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; - color_random[i_event] = random_color_in ? random_color_in[i_event + offset] : 0.5; - diagram_random[i_event] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; - g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; - flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; - } - for ( std::size_t i_event = count; i_event < rounded_count; ++i_event ) { - flavor_indices[i_event] = 0; - } - } - computeDependentCouplings( g_s.data(), couplings.data(), rounded_count ); - - InterfaceInstance* instance = static_cast( handle ); - if( !instance->initialized ) - { - initialize( - momenta.data(), - couplings.data(), - flavor_indices.data(), - matrix_elements.data(), - numerators.data(), - denominators.data(), - rounded_count ); - instance->initialized = true; - } - - sigmaKin( - momenta.data(), - couplings.data(), - flavor_indices.data(), - helicity_random.data(), - color_random.data(), - nullptr, - diagram_random.data(), - matrix_elements.data(), - helicity_index.data(), - color_index.data(), - numerators.data(), - denominators.data(), - diagram_index.data(), - false, - rounded_count ); - - if ( sort_flavors ) - { - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - std::size_t i_sorted = permutation[i_event]; - std::size_t page_size = MemoryAccessMomentaBase::neppM; - std::size_t i_page = i_sorted / page_size; - std::size_t i_vector = i_sorted % page_size; // vector lane - - double denominator = denominators[i_sorted]; - if( m2_out != nullptr ) - { - m2_out[i_event + offset] = matrix_elements[i_sorted]; - } - if( amp2_out != nullptr ) - { - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) - { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; - } - } - if( diagram_out != nullptr ) - { - diagram_out[i_event + offset] = diagram_index[i_sorted] - 1; - } - if( color_out != nullptr ) - { - color_out[i_event + offset] = color_index[i_sorted] - 1; - } - if( helicity_out != nullptr ) - { - helicity_out[i_event + offset] = helicity_index[i_sorted] - 1; - } - } - } else { - std::size_t page_size = MemoryAccessMomentaBase::neppM; - for( std::size_t i_event = 0; i_event < count; ++i_event ) - { - std::size_t i_page = i_event / page_size; - std::size_t i_vector = i_event % page_size; - - double denominator = denominators[i_event]; - if( m2_out != nullptr ) - { - m2_out[i_event + offset] = matrix_elements[i_event]; - } - if( amp2_out != nullptr ) - { - for( std::size_t i_diag = 0; i_diag < CPPProcess::ndiagrams; ++i_diag ) - { - amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * CPPProcess::ndiagrams + i_diag * page_size + i_vector] / denominator; - } - } - if( diagram_out != nullptr ) - { - diagram_out[i_event + offset] = diagram_index[i_event] - 1; - } - if( color_out != nullptr ) - { - color_out[i_event + offset] = color_index[i_event] - 1; - } - if( helicity_out != nullptr ) - { - helicity_out[i_event + offset] = helicity_index[i_event] - 1; - } - } - } -#endif // MGONGPUCPP_GPUIMPL return UMAMI_SUCCESS; } diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc new file mode 100644 index 0000000000..29efce440d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.cc @@ -0,0 +1,187 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "CrossSectionKernels.h" + +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessWeights.h" +#include "MemoryBuffers.h" + +#include +#include + +// ****************************************************************************************** +// *** NB: Disabling fast math is essential here, otherwise results are undefined *** +// *** NB: This file CrossSectionKernels.cc IS BUILT WITH -fno-fast-math in the Makefile! *** +// *** NB: Attempts with __attribute__((optimize("-fno-fast-math"))) were unsatisfactory *** +// ****************************************************************************************** + +inline bool +fp_is_nan( const fptype& fp ) +{ + //#pragma clang diagnostic push + //#pragma clang diagnostic ignored "-Wtautological-compare" // for icpx2021/clang13 (https://stackoverflow.com/a/15864661) + return std::isnan( fp ); // always false for clang in fast math mode (tautological compare)? + //#pragma clang diagnostic pop +} + +inline bool +fp_is_abnormal( const fptype& fp ) +{ + if( fp_is_nan( fp ) ) return true; + if( fp != fp ) return true; + return false; +} + +inline bool +fp_is_zero( const fptype& fp ) +{ + if( fp == 0 ) return true; + return false; +} + +// See https://en.cppreference.com/w/cpp/numeric/math/FP_categories +inline const char* +fp_show_class( const fptype& fp ) +{ + switch( std::fpclassify( fp ) ) + { + case FP_INFINITE: return "Inf"; + case FP_NAN: return "NaN"; + case FP_NORMAL: return "normal"; + case FP_SUBNORMAL: return "subnormal"; + case FP_ZERO: return "zero"; + default: return "unknown"; + } +} + +inline void +debug_me_is_abnormal( const fptype& me, size_t ievtALL ) +{ + std::cout << "DEBUG[" << ievtALL << "]" + << " ME=" << me + << " fpisabnormal=" << fp_is_abnormal( me ) + << " fpclass=" << fp_show_class( me ) + << " (me==me)=" << ( me == me ) + << " (me==me+1)=" << ( me == me + 1 ) + << " isnan=" << fp_is_nan( me ) + << " isfinite=" << std::isfinite( me ) + << " isnormal=" << std::isnormal( me ) + << " is0=" << ( me == 0 ) + << " is1=" << ( me == 1 ) + << " abs(ME)=" << std::abs( me ) + << " isnan=" << fp_is_nan( std::abs( me ) ) + << std::endl; +} + +//============================================================================ + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ) + { + for( unsigned int ievt = 0; ievt < nevt; ievt++ ) + { + if( fp_is_abnormal( hstMEs[ievt] ) ) + { + std::cout << "WARNING! flagging abnormal ME for ievt=" << ievt << std::endl; + hstMEs[ievt] = std::sqrt( -1. ); + } + } + } + + //-------------------------------------------------------------------------- + + CrossSectionKernelHost::CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t nevt ) + : CrossSectionKernelBase( samplingWeights, matrixElements, stats ) + , NumberOfEvents( nevt ) + { + if( m_samplingWeights.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: samplingWeights must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "CrossSectionKernelHost: matrixElements must be a host array" ); + if( this->nevt() != m_samplingWeights.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with samplingWeights" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "CrossSectionKernelHost: nevt mismatch with matrixElements" ); + } + + //-------------------------------------------------------------------------- + + void CrossSectionKernelHost::updateEventStatistics( const bool debug ) + { + EventStatistics stats; // new statistics for the new nevt events + // FIRST PASS: COUNT ALL/ABN/ZERO EVENTS, COMPUTE MIN/MAX, COMPUTE REFS AS MEANS OF SIMPLE SUMS + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + const size_t ievtALL = m_iter * nevt() + ievt; + // The following events are abnormal in a run with "-p 2048 256 12 -d" + // - check.exe/commonrand: ME[310744,451171,3007871,3163868,4471038,5473927] with fast math + // - check.exe/curand: ME[578162,1725762,2163579,5407629,5435532,6014690] with fast math + // - gcheck.exe/curand: ME[596016,1446938] with fast math + // Debug NaN/abnormal issues + //if ( ievtALL == 310744 ) // this ME is abnormal both with and without fast math + // debug_me_is_abnormal( me, ievtALL ); + //if ( ievtALL == 5473927 ) // this ME is abnormal only with fast math + // debug_me_is_abnormal( me, ievtALL ); + stats.nevtALL++; + if( fp_is_abnormal( me ) ) + { + if( debug ) // only printed out with "-p -d" (matrixelementALL is not filled without -p) + std::cout << "WARNING! ME[" << ievtALL << "] is NaN/abnormal" << std::endl; + stats.nevtABN++; + continue; + } + if( fp_is_zero( me ) ) stats.nevtZERO++; + stats.minME = std::min( stats.minME, (double)me ); + stats.maxME = std::max( stats.maxME, (double)me ); + stats.minWG = std::min( stats.minWG, (double)wg ); + stats.maxWG = std::max( stats.maxWG, (double)wg ); + stats.sumMEdiff += me; // NB stats.refME is 0 here + stats.sumWGdiff += wg; // NB stats.refWG is 0 here + } + stats.refME = stats.meanME(); // draft ref + stats.refWG = stats.meanWG(); // draft ref + stats.sumMEdiff = 0; + stats.sumWGdiff = 0; + // SECOND PASS: IMPROVE MEANS FROM SUMS OF DIFFS TO PREVIOUS REF, UPDATE REF + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + if( fp_is_abnormal( me ) ) continue; + stats.sumMEdiff += ( me - stats.refME ); + stats.sumWGdiff += ( wg - stats.refWG ); + } + stats.refME = stats.meanME(); // final ref + stats.refWG = stats.meanWG(); // final ref + stats.sumMEdiff = 0; + stats.sumWGdiff = 0; + // THIRD PASS: COMPUTE STDDEV FROM SQUARED SUMS OF DIFFS TO REF + for( size_t ievt = 0; ievt < nevt(); ++ievt ) // Loop over all events in this iteration + { + const fptype& me = MemoryAccessMatrixElements::ieventAccessConst( m_matrixElements.data(), ievt ); + const fptype& wg = MemoryAccessWeights::ieventAccessConst( m_samplingWeights.data(), ievt ); + if( fp_is_abnormal( me ) ) continue; + stats.sqsMEdiff += std::pow( me - stats.refME, 2 ); + stats.sqsWGdiff += std::pow( wg - stats.refWG, 2 ); + } + // FOURTH PASS: UPDATE THE OVERALL STATS BY ADDING THE NEW STATS + m_stats += stats; + // Increment the iterations counter + m_iter++; + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h new file mode 100644 index 0000000000..59a4c098ec --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/CrossSectionKernels.h @@ -0,0 +1,98 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef CROSSSECTIONKERNELS_H +#define CROSSSECTIONKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "EventStatistics.h" +#include "MemoryBuffers.h" + +//============================================================================ + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + // Helper function for Bridge.h: must be compiled without fast math + // Iterate through all output MEs and replace any NaN/abnormal ones by sqrt(-1) + void flagAbnormalMEs( fptype* hstMEs, unsigned int nevt ); + + //-------------------------------------------------------------------------- + + // A base class encapsulating the calculation of event statistics on a CPU host or on a GPU device + class CrossSectionKernelBase //: virtual public ICrossSectionKernel + { + protected: + + // Constructor from existing input and output buffers + CrossSectionKernelBase( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats ) // output: event statistics + : m_samplingWeights( samplingWeights ) + , m_matrixElements( matrixElements ) + , m_stats( stats ) + , m_iter( 0 ) + { + // NB: do not initialise EventStatistics (you may be asked to update an existing result) + } + + public: + + // Destructor + virtual ~CrossSectionKernelBase() {} + + // Update event statistics + virtual void updateEventStatistics( const bool debug = false ) = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + protected: + + // The buffer for the sampling weights + const BufferWeights& m_samplingWeights; + + // The buffer for the output matrix elements + const BufferMatrixElements& m_matrixElements; + + // The event statistics + EventStatistics& m_stats; + + // The number of iterations processed so far + size_t m_iter; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating the calculation of event statistics on a CPU host + class CrossSectionKernelHost final : public CrossSectionKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + CrossSectionKernelHost( const BufferWeights& samplingWeights, // input: sampling weights + const BufferMatrixElements& matrixElements, // input: matrix elements + EventStatistics& stats, // output: event statistics + const size_t nevt ); + + // Destructor + virtual ~CrossSectionKernelHost() {} + + // Update event statistics + void updateEventStatistics( const bool debug = false ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + }; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- +} +#endif // CROSSSECTIONKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h new file mode 100644 index 0000000000..df62a74918 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/EventStatistics.h @@ -0,0 +1,171 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef EventStatistics_H +#define EventStatistics_H 1 + +#include "mgOnGpuConfig.h" + +#include "ProcessData.h" // for npar (meGeVexponent) + +#include +#include +#include +#include +#include + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + // The EventStatistics struct is used to accumulate running aggregates of event statistics. + // This will eventually include the process cross section and the process maximum weight: + // one important case of EventStatistics will then be the "gridpack" result set, which is + // the output of the "integration" step and the input to "unweighted event generation" step. + // The current implementation only includes statistics for matrix elements (ME) and sampling weights (WG); + // in first approximation, the process cross section and maximum weight are just the mean ME and maximum ME, + // but eventually the sampling weights WG (e.g. from Rambo) must also be taken into account in the calculation. + // The implementation uses differences to reference values to improve numerical precision. + struct EventStatistics + { + public: + size_t nevtALL; // total number of events used + size_t nevtABN; // number of events used, where ME is abnormal (nevtABN <= nevtALL) + size_t nevtZERO; // number of not-abnormal events used, where ME is zero (nevtZERO <= nevtOK) + double minME; // minimum matrix element + double maxME; // maximum matrix element + double minWG; // minimum sampling weight + double maxWG; // maximum sampling weight + double refME; // "reference" matrix element (normally the current mean) + double refWG; // "reference" sampling weight (normally the current mean) + double sumMEdiff; // sum of diff to ref for matrix element + double sumWGdiff; // sum of diff to ref for sampling weight + double sqsMEdiff; // squared sum of diff to ref for matrix element + double sqsWGdiff; // squared sum of diff to ref for sampling weight + std::string tag; // a text tag for printouts + // Number of events used, where ME is not abnormal + size_t nevtOK() const { return nevtALL - nevtABN; } + // Mean matrix element + // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] + double meanME() const + { + return refME + ( nevtOK() > 0 ? sumMEdiff / nevtOK() : 0 ); + } + // Mean sampling weight + // [x = ref+d => mean(x) = sum(x)/n = ref+sum(d)/n] + double meanWG() const + { + return refWG + ( nevtOK() > 0 ? sumWGdiff / nevtOK() : 0 ); + } + // Variance matrix element + // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] + double varME() const { return ( sqsMEdiff - std::pow( sumMEdiff, 2 ) / nevtOK() ) / nevtOK(); } + // Variance sampling weight + // [x = ref+d => n*var(x) = sum((x-mean(x))^2) = sum((ref+d-ref-sum(d)/n)^2) = sum((d-sum(d)/n)^2)/n = sum(d^2)-(sum(d))^2/n] + double varWG() const { return ( sqsWGdiff - std::pow( sumWGdiff, 2 ) / nevtOK() ) / nevtOK(); } + // Standard deviation matrix element + double stdME() const { return std::sqrt( varME() ); } + // Standard deviation sampling weight + double stdWG() const { return std::sqrt( varWG() ); } + // Update reference matrix element + void updateRefME( const double newRef ) + { + const double deltaRef = refME - newRef; + sqsMEdiff += deltaRef * ( 2 * sumMEdiff + nevtOK() * deltaRef ); + sumMEdiff += deltaRef * nevtOK(); + refME = newRef; + } + // Update reference sampling weight + void updateRefWG( const double newRef ) + { + const double deltaRef = refWG - newRef; + sqsWGdiff += deltaRef * ( 2 * sumWGdiff + nevtOK() * deltaRef ); + sumWGdiff += deltaRef * nevtOK(); + refWG = newRef; + } + // Constructor + EventStatistics() + : nevtALL( 0 ) + , nevtABN( 0 ) + , nevtZERO( 0 ) + , minME( std::numeric_limits::max() ) + , maxME( std::numeric_limits::lowest() ) + , minWG( std::numeric_limits::max() ) + , maxWG( std::numeric_limits::lowest() ) + , refME( 0 ) + , refWG( 0 ) + , sumMEdiff( 0 ) + , sumWGdiff( 0 ) + , sqsMEdiff( 0 ) + , sqsWGdiff( 0 ) + , tag( "" ) {} + // Combine two EventStatistics +#ifdef __clang__ + // Disable optimizations for this function in HIP (work around FPE crash #1003: originally using #if __HIP_CLANG_ONLY__) + // Disable optimizations for this function in clang tout court (work around FPE crash #1005: now using #ifdef __clang__) + // See https://clang.llvm.org/docs/LanguageExtensions.html#extensions-for-selectively-disabling-optimization + __attribute__( ( optnone ) ) +#endif + EventStatistics& + operator+=( const EventStatistics& stats ) + { + EventStatistics s1 = *this; // temporary copy + EventStatistics s2 = stats; // temporary copy + EventStatistics& sum = *this; + sum.nevtALL = s1.nevtALL + s2.nevtALL; + sum.nevtABN = s1.nevtABN + s2.nevtABN; + sum.nevtZERO = s1.nevtZERO + s2.nevtZERO; + sum.minME = std::min( s1.minME, s2.minME ); + sum.maxME = std::max( s1.maxME, s2.maxME ); + sum.minWG = std::min( s1.minWG, s2.minWG ); + sum.maxWG = std::max( s1.maxWG, s2.maxWG ); + sum.refME = ( s1.meanME() * s1.nevtOK() + s2.meanME() * s2.nevtOK() ) / sum.nevtOK(); // new mean ME + s1.updateRefME( sum.refME ); + s2.updateRefME( sum.refME ); + sum.sumMEdiff = s1.sumMEdiff + s2.sumMEdiff; + sum.sqsMEdiff = s1.sqsMEdiff + s2.sqsMEdiff; + sum.refWG = ( s1.meanWG() * s1.nevtOK() + s2.meanWG() * s2.nevtOK() ) / sum.nevtOK(); // new mean WG + s1.updateRefWG( sum.refWG ); + s2.updateRefWG( sum.refWG ); + sum.sumWGdiff = s1.sumWGdiff + s2.sumWGdiff; + sum.sqsWGdiff = s1.sqsWGdiff + s2.sqsWGdiff; + return sum; + } + // Printout + void printout( std::ostream& out ) const + { + const EventStatistics& s = *this; + constexpr int meGeVexponent = -( 2 * ProcessData::npar - 8 ); + out << s.tag << "NumMatrixElems(notAbnormal) = " << s.nevtOK() << std::endl + << std::scientific // fixed format: affects all floats (default precision: 6) + << s.tag << "MeanMatrixElemValue = ( " << s.meanME() + << " +- " << s.stdME() / std::sqrt( s.nevtOK() ) << " ) GeV^" << meGeVexponent << std::endl // standard error + << s.tag << "[Min,Max]MatrixElemValue = [ " << s.minME + << " , " << s.maxME << " ] GeV^" << meGeVexponent << std::endl + << s.tag << "StdDevMatrixElemValue = ( " << s.stdME() + << std::string( 16, ' ' ) << " ) GeV^" << meGeVexponent << std::endl + << s.tag << "MeanWeight = ( " << s.meanWG() + << " +- " << s.stdWG() / std::sqrt( s.nevtOK() ) << std::endl // standard error + << s.tag << "[Min,Max]Weight = [ " << s.minWG + << " , " << s.maxWG << " ]" << std::endl + << s.tag << "StdDevWeight = ( " << s.stdWG() + << std::string( 16, ' ' ) << " )" << std::endl + << std::defaultfloat; // default format: affects all floats + } + }; + + //-------------------------------------------------------------------------- + + inline std::ostream& operator<<( std::ostream& out, const EventStatistics& s ) + { + s.printout( out ); + return out; + } + + //-------------------------------------------------------------------------- +} + +#endif // EventStatistics_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc new file mode 100644 index 0000000000..3aeeadd9e8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.cc @@ -0,0 +1,285 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "MatrixElementKernels.h" + +#include "ProcessData.h" +#include "SigmaKin.h" +#include "MemoryAccessMomenta.h" +#include "MemoryBuffers.h" + +#include +#include // for fetestexcept +#include +#include + +//============================================================================ + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + MatrixElementKernelBase::MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol ) // output: color selection + : m_momenta( momenta ) + , m_gs( gs ) + , m_iflavorVec( iflavorVec ) + , m_rndhel( rndhel ) + , m_rndcol( rndcol ) + , m_channelIds( channelIds ) + , m_matrixElements( matrixElements ) + , m_selhel( selhel ) + , m_selcol( selcol ) +#ifdef MGONGPU_CHANNELID_DEBUG + , m_nevtProcessedByChannel() + , m_tag() +#endif + { + //std::cout << "DEBUG: MatrixElementKernelBase ctor " << this << std::endl; +#ifdef MGONGPU_CHANNELID_DEBUG + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + m_nevtProcessedByChannel[channelId] = 0; +#endif + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelBase::~MatrixElementKernelBase() + { + //std::cout << "DEBUG: MatrixElementKernelBase dtor " << this << std::endl; +#ifdef MGONGPU_CHANNELID_DEBUG + MatrixElementKernelBase::dumpNevtProcessedByChannel(); +#endif +#ifdef MGONGPUCPP_VERBOSE + MatrixElementKernelBase::dumpSignallingFPEs(); +#endif + } + + //-------------------------------------------------------------------------- + +#ifdef MGONGPU_CHANNELID_DEBUG + void MatrixElementKernelBase::updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ) + { + if( pHstChannelIds != nullptr ) + { + //std::cout << "DEBUG " << this << ": not nullptr " << nevt << std::endl; + for( unsigned int ievt = 0; ievt < nevt; ievt++ ) + { + const size_t channelId = pHstChannelIds[ievt]; // Fortran indexing + //assert( channelId > 0 ); + //assert( channelId < ProcessData::ndiagrams ); + m_nevtProcessedByChannel[channelId]++; + } + } + else + { + //std::cout << "DEBUG " << this << ": nullptr " << std::endl; + m_nevtProcessedByChannel[0] += nevt; + } + } +#endif + + //-------------------------------------------------------------------------- + +#ifdef MGONGPU_CHANNELID_DEBUG + void MatrixElementKernelBase::dumpNevtProcessedByChannel() + { + size_t nevtProcessed = 0; + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + nevtProcessed += m_nevtProcessedByChannel[channelId]; + std::ostringstream sstr; + sstr << " {"; + for( size_t channelId = 0; channelId < ProcessData::ndiagrams + 1; channelId++ ) // [0...ndiagrams] (TEMPORARY: 0=multichannel) + { + if( m_nevtProcessedByChannel[channelId] > 0 ) + { + if( sstr.str() != " {" ) sstr << ","; + if( channelId == 0 ) + sstr << " no-multichannel"; + else + sstr << " " << channelId; + sstr << " : " << m_nevtProcessedByChannel[channelId]; + } + } + sstr << " }"; + std::cout << "DEBUG: MEK " << this; + if( m_tag != "" ) std::cout << " " << m_tag; + std::cout << " processed " << nevtProcessed << " events across " << ProcessData::ndiagrams << " channels" << sstr.str() << std::endl; + } +#endif + + //-------------------------------------------------------------------------- + + void MatrixElementKernelBase::dumpSignallingFPEs() + { + // New strategy for issue #831: add a final report of FPEs + // Note: normally only underflow will be reported here (inexact is switched off because it would almost always signal; + // divbyzero, invalid and overflow are configured by feenablexcept to send a SIGFPE signal, and are normally fixed in the code) + // Note: this is now called in the individual destructors of MEK classes rather than in that of MatrixElementKernelBase(#837) + std::string fpes; + if( std::fetestexcept( FE_DIVBYZERO ) ) fpes += " FE_DIVBYZERO"; + if( std::fetestexcept( FE_INVALID ) ) fpes += " FE_INVALID"; + if( std::fetestexcept( FE_OVERFLOW ) ) fpes += " FE_OVERFLOW"; + if( std::fetestexcept( FE_UNDERFLOW ) ) fpes += " FE_UNDERFLOW"; + //if( std::fetestexcept( FE_INEXACT ) ) fpes += " FE_INEXACT"; // do not print this out: this would almost always signal! + if( fpes == "" ) + std::cout << "INFO: No Floating Point Exceptions have been reported" << std::endl; + else + std::cerr << "INFO: The following Floating Point Exceptions have been reported:" << fpes << std::endl; + } + + //-------------------------------------------------------------------------- +} + +//============================================================================ + +namespace madmatrix +{ + + //-------------------------------------------------------------------------- + + MatrixElementKernelHost::MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt ) + : MatrixElementKernelBase( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, matrixElements, selhel, selcol ) + , NumberOfEvents( nevt ) + , m_couplings( nevt ) + , m_numerators( nevt * ProcessData::ndiagrams ) + , m_denominators( nevt ) + { + //std::cout << "DEBUG: MatrixElementKernelHost::ctor " << this << std::endl; + if( m_momenta.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: momenta must be a host array" ); + if( m_matrixElements.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: matrixElements must be a host array" ); + if( m_channelIds.isOnDevice() ) throw std::runtime_error( "MatrixElementKernelHost: channelIds must be a device array" ); + if( this->nevt() != m_momenta.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with momenta" ); + if( this->nevt() != m_matrixElements.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with matrixElements" ); + if( this->nevt() != m_channelIds.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with channelIds" ); + if( this->nevt() != m_iflavorVec.nevt() ) throw std::runtime_error( "MatrixElementKernelHost: nevt mismatch with iflavorVec" ); + // Sanity checks for memory access (momenta buffer) + constexpr int neppM = MemoryAccessMomenta::neppM; // AOSOA layout + static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); + if( nevt % neppM != 0 ) + { + std::ostringstream sstr; + sstr << "MatrixElementKernelHost: nevt should be a multiple of neppM=" << neppM; + throw std::runtime_error( sstr.str() ); + } + // Fail gently and avoid "Illegal instruction (core dumped)" if the host does not support the SIMD used in the ME calculation + // Note: this prevents a crash on pmpe04 but not on some github CI nodes? + // [NB: SIMD vectorization in mg5amc C++ code is only used in the ME calculation below MatrixElementKernelHost!] + if( !MatrixElementKernelHost::hostSupportsSIMD() ) + throw std::runtime_error( "Host does not support the SIMD implementation of MatrixElementKernelsHost" ); + } + + //-------------------------------------------------------------------------- + + MatrixElementKernelHost::~MatrixElementKernelHost() + { + //std::cout << "DEBUG: MatrixElementKernelBase::dtor " << this << std::endl; + } + + //-------------------------------------------------------------------------- + + int MatrixElementKernelHost::computeGoodHelicities() + { + HostBufferHelicityMask hstIsGoodHel( ProcessData::ncomb ); + // ... 0d1. Compute good helicity mask on the host + computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); + sigmaKin_getGoodHel( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_matrixElements.data(), m_numerators.data(), m_denominators.data(), hstIsGoodHel.data(), nevt() ); + // ... 0d2. Copy good helicity list to static memory on the host + // [FIXME! REMOVE THIS STATIC THAT BREAKS MULTITHREADING?] + return sigmaKin_setGoodHel( hstIsGoodHel.data() ); + } + + //-------------------------------------------------------------------------- + + void MatrixElementKernelHost::computeMatrixElements( const bool useChannelIds ) + { + computeDependentCouplings( m_gs.data(), m_couplings.data(), m_gs.size() ); + const unsigned int* pChannelIds = ( useChannelIds ? m_channelIds.data() : nullptr ); + sigmaKin( m_momenta.data(), m_couplings.data(), m_iflavorVec.data(), m_rndhel.data(), m_rndcol.data(), pChannelIds, nullptr, m_matrixElements.data(), m_selhel.data(), m_selcol.data(), m_numerators.data(), m_denominators.data(), nullptr, true, nevt() ); +#ifdef MGONGPU_CHANNELID_DEBUG + //std::cout << "DEBUG: MatrixElementKernelHost::computeMatrixElements " << this << " " << ( useChannelIds ? "T" : "F" ) << " " << nevt() << std::endl; + MatrixElementKernelBase::updateNevtProcessedByChannel( pChannelIds, nevt() ); +#endif + } + + //-------------------------------------------------------------------------- + + // Does this host system support the SIMD used in the matrix element calculation? + bool MatrixElementKernelHost::hostSupportsSIMD( const bool verbose ) + { +#if defined __AVX512VL__ + bool known = true; + bool ok = __builtin_cpu_supports( "avx512vl" ); + const std::string tag = "skylake-avx512 (AVX512VL)"; +#elif defined __AVX2__ + bool known = true; + bool ok = __builtin_cpu_supports( "avx2" ); + const std::string tag = "haswell (AVX2)"; +#elif defined __SSE4_2__ +#ifdef __PPC__ + // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html + bool known = true; + bool ok = __builtin_cpu_supports( "vsx" ); + const std::string tag = "powerpc vsx (128bit as in SSE4.2)"; +#elif defined( __x86_64__ ) || defined( __i386__ ) + bool known = true; + bool ok = __builtin_cpu_supports( "sse4.2" ); + const std::string tag = "nehalem (SSE4.2)"; +#else // AV FIXME! Added by OM for Mac, should identify the correct __xxx__ flag that should be targeted + // DM now we have an explicit NEON target for ARM + bool known = false; // __builtin_cpu_supports is not supported + bool ok = true; // this is just an assumption! + const std::string tag = "simd arch not defined"; +#endif +#elif defined __ARM_NEON // consider using __BUILTIN_CPU_SUPPORTS__ + bool known = false; // __builtin_cpu_supports is not supported + // See https://stackoverflow.com/q/62783908 + // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu + bool ok = true; // this is just an assumption! + const std::string tag = "arm neon (128bit as in SSE4.2)"; +#else + bool known = true; + bool ok = true; + const std::string tag = "none"; +#endif + if( verbose ) + { + if( tag == "none" ) + std::cout << "INFO: The application does not require the host to support any AVX feature" << std::endl; + else if( ok && known ) + std::cout << "INFO: The application is built for " << tag << " and the host supports it" << std::endl; + else if( ok ) + std::cout << "WARNING: The application is built for " << tag << " but it is unknown if the host supports it" << std::endl; + else + std::cout << "ERROR! The application is built for " << tag << " but the host does not support it" << std::endl; + } + return ok; + } + + //-------------------------------------------------------------------------- + +} + +//============================================================================ + + +//============================================================================ diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h new file mode 100644 index 0000000000..e7cf857b47 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MatrixElementKernels.h @@ -0,0 +1,158 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: D. Massaro, J. Teig, A. Thete, A. Valassi, Z. Wettersten (2022-2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MATRIXELEMENTKERNELS_H +#define MATRIXELEMENTKERNELS_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryBuffers.h" + +#include +#include + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + // A base class encapsulating matrix element calculations on a CPU host or on a GPU device + class MatrixElementKernelBase //: virtual public IMatrixElementKernel + { + protected: + + // Constructor from existing input and output buffers + MatrixElementKernelBase( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol); // output: color selection + + public: + + // Destructor + virtual ~MatrixElementKernelBase(); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + virtual int computeGoodHelicities() = 0; + + // Compute matrix elements + virtual void computeMatrixElements( const bool useChannelIds ) = 0; + + // Is this a host or device kernel? + virtual bool isOnDevice() const = 0; + + // Dump signalling FPEs (#831 and #837) + static void dumpSignallingFPEs(); + +#ifdef MGONGPU_CHANNELID_DEBUG + // Add a MEK identifier for the channelId debug printout + void setTagForNevtProcessedByChannel( const std::string& tag ) { m_tag = tag; } + + protected: + // Update number of events processed by channel + void updateNevtProcessedByChannel( const unsigned int* pHstChannelIds, const size_t nevt ); + + // Dump number of events processed by channel + void dumpNevtProcessedByChannel(); +#endif + + protected: + + // The buffer for the input momenta + const BufferMomenta& m_momenta; + + // The buffer for the gs to calculate the alphaS values + const BufferGs& m_gs; + + // The buffer for the flavor indices for the flavor combination + const BufferIflavorVec& m_iflavorVec; + + // The buffer for the random numbers for helicity selection + const BufferRndNumHelicity& m_rndhel; + + // The buffer for the random numbers for color selection + const BufferRndNumColor& m_rndcol; + + // The buffer for the channel ids for single-diagram enhancement + const BufferChannelIds& m_channelIds; + + // The buffer for the output matrix elements + BufferMatrixElements& m_matrixElements; + + // The buffer for the output helicity selection + BufferSelectedHelicity& m_selhel; + + // The buffer for the output color selection + BufferSelectedColor& m_selcol; + +#ifdef MGONGPU_CHANNELID_DEBUG + // The events-per-channel counter for debugging + std::map m_nevtProcessedByChannel; + + // The tag for events-per-channel debugging + std::string m_tag; +#endif + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating matrix element calculations on a CPU host + class MatrixElementKernelHost final : public MatrixElementKernelBase, public NumberOfEvents + { + public: + + // Constructor from existing input and output buffers + MatrixElementKernelHost( const BufferMomenta& momenta, // input: momenta + const BufferGs& gs, // input: gs for alphaS + const BufferIflavorVec& iflavorVec, // input: flavor indices for the flavor combination + const BufferRndNumHelicity& rndhel, // input: random numbers for helicity selection + const BufferRndNumColor& rndcol, // input: random numbers for color selection + const BufferChannelIds& channelIds, // input: channel ids for single-diagram enhancement + BufferMatrixElements& matrixElements, // output: matrix elements + BufferSelectedHelicity& selhel, // output: helicity selection + BufferSelectedColor& selcol, // output: color selection + const size_t nevt); + + // Destructor + virtual ~MatrixElementKernelHost(); + + // Compute good helicities (returns nGoodHel, the number of good helicity combinations out of ncomb) + int computeGoodHelicities() override final; + + // Compute matrix elements + void computeMatrixElements( const bool useChannelIds ) override final; + + // Is this a host or device kernel? + bool isOnDevice() const override final { return false; } + + private: + + // Does this host system support the SIMD used in the matrix element calculation? + // [NB: this is private, SIMD vectorization in mg5amc C++ code is currently only used in the ME calculations below MatrixElementKernelHost!] + static bool hostSupportsSIMD( const bool verbose = false ); // ZW: default verbose false + + private: + + // The buffer for the event-by-event couplings that depends on alphas QCD + HostBufferCouplings m_couplings; + + // The buffer for the event-by-event numerators of multichannel factors + HostBufferNumerators m_numerators; + + // The buffer for the event-by-event denominators of multichannel factors + HostBufferDenominators m_denominators; + }; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- +} +#endif // MATRIXELEMENTKERNELS_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h new file mode 100644 index 0000000000..669ed8ead5 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessAmplitudes.h @@ -0,0 +1,161 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessAmplitudes_H +#define MemoryAccessAmplitudes_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" + +#define MGONGPU_TRIVIAL_AMPLITUDES 1 + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + +#ifndef MGONGPU_TRIVIAL_AMPLITUDES + + // A class describing the internal layout of memory buffers for amplitudes + // This implementation uses an AOSOA[npagA][nx2][neppA] where nevt=npagA*neppA + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessAmplitudesBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the amplitude AOSOA memory buffer layout + static constexpr int neppA = 1; // AOS (just a test...) + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagA = ievt / neppA; // #event "A-page" + const int ieppA = ievt % neppA; // #event in the current event A-page + constexpr int ix2 = 0; + return &( buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA] ); // AOSOA[ipagA][ix2][ieppA] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ix2" and rename "Field" as "Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ix2 ) + { + constexpr int ipagA = 0; + constexpr int ieppA = 0; + return buffer[ipagA * nx2 * neppA + ix2 * neppA + ieppA]; // AOSOA[ipagA][ix2][ieppA] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessAmplitudes : public MemoryAccessAmplitudesBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ix2 ) <===] + static constexpr auto decodeRecordIx2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIx2( fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIx2Const( const fptype* buffer, const ievt, const int ix2 ) <===] + static constexpr auto ieventAccessIx2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + +#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessAmplitudes + { + public: + +#ifndef MGONGPU_TRIVIAL_AMPLITUDES + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIx2( fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2 = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIx2Const( const fptype* buffer, const int ix2 ) <===] + static constexpr auto kernelAccessIx2Const = + KernelAccessHelper::template kernelAccessFieldConst; + +#else + + static __host__ __device__ inline cxtype_sv* + kernelAccess( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + static __host__ __device__ inline const cxtype_sv* + kernelAccessConst( const fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + +#endif // #ifndef MGONGPU_TRIVIAL_AMPLITUDES + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessAmplitudes HostAccessAmplitudes; + typedef KernelAccessAmplitudes DeviceAccessAmplitudes; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessAmplitudes_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessChannelIds.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h similarity index 92% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessChannelIds.h rename to madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h index 331e7f12c3..f2939a8ca7 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessChannelIds.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessChannelIds.h @@ -13,12 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -103,14 +99,10 @@ namespace mg5amcCpu kernelAccessConst( const unsigned int* buffer ) { const unsigned int& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferChannelIds::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif + return madmatrix::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } }; @@ -121,6 +113,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessChannelIds_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplings.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h similarity index 91% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplings.h rename to madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h index fd3fb80c65..347570a22e 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplings.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplings.h @@ -15,12 +15,8 @@ #include "MemoryAccessMomenta.h" // for MemoryAccessMomentaBase::neppM #include "MemoryBuffers.h" // for HostBufferCouplings::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -92,7 +88,7 @@ namespace mg5amcCpu const int ievt ) { const int ipagC = ievt / neppC; // #event "C-page" - const int ieppC = ievt %% neppC; // #event in the current event C-page + const int ieppC = ievt % neppC; // #event in the current event C-page constexpr int idcoup = 0; constexpr int ix2 = 0; return &( buffer[ipagC * ndcoup * nx2 * neppC + idcoup * nx2 * neppC + ix2 * neppC + ieppC] ); // AOSOA[ipagC][idcoup][ix2][ieppC] @@ -189,17 +185,13 @@ namespace mg5amcCpu const int ix2 ) { fptype& out = kernelAccessIx2_s( buffer, ix2 ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif + static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( madmatrix::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) @@ -225,10 +217,10 @@ namespace mg5amcCpu // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays constexpr int neppC = MemoryAccessCouplingsBase::neppC; static_assert( neppC >= neppV ); // ASSUME CONTIGUOUS ARRAYS - static_assert( neppC %% neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS - static_assert( mg5amcCpu::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - //assert( (size_t)( buffer ) %% mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + static_assert( neppC % neppV == 0 ); // ASSUME CONTIGUOUS ARRAYS + static_assert( madmatrix::HostBufferCouplings::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast #endif } */ @@ -241,7 +233,7 @@ namespace mg5amcCpu /* fptype_sv& real = kernelAccessIx2( buffer, 0 ); fptype_sv& imag = kernelAccessIx2( buffer, 1 ); - printf( "C_ACCESS::kernelAccess: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + printf( "C_ACCESS::kernelAccess: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); return cxtype_sv_ref( real, imag ); */ return cxtype_sv_ref( kernelAccessIx2( buffer, 0 ), @@ -256,7 +248,7 @@ namespace mg5amcCpu /* const fptype_sv& real = kernelAccessIx2Const( buffer, 0 ); const fptype_sv& imag = kernelAccessIx2Const( buffer, 1 ); - printf( "C_ACCESS::kernelAccessConst: pbuffer=%%p pr=%%p pi=%%p\n", buffer, &real, &imag ); + printf( "C_ACCESS::kernelAccessConst: pbuffer=%p pr=%p pi=%p\n", buffer, &real, &imag ); return cxtype_sv( real, imag ); */ return cxtype_sv( kernelAccessIx2Const( buffer, 0 ), @@ -271,6 +263,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessCouplings_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplingsFixed.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h similarity index 93% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplingsFixed.h rename to madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h index 757de7b6f2..df36536afe 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessCouplingsFixed.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessCouplingsFixed.h @@ -14,12 +14,8 @@ //#include "MemoryAccessHelpers.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -84,6 +80,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessCouplingsFixed_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h new file mode 100644 index 0000000000..9e5b87e674 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessDenominators.h @@ -0,0 +1,27 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessDenominators_H +#define MemoryAccessDenominators_H 1 + +#include "MemoryAccessGs.h" + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for denominators + // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs + + typedef KernelAccessGs HostAccessDenominators; + typedef KernelAccessGs DeviceAccessDenominators; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessDenominators_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessGs.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h similarity index 92% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessGs.h rename to madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h index bfcb3e3222..9dc2aba11e 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessGs.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessGs.h @@ -13,12 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -119,14 +115,10 @@ namespace mg5amcCpu kernelAccess( fptype* buffer ) { fptype& out = kernelAccess_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) @@ -148,14 +140,10 @@ namespace mg5amcCpu kernelAccessConst( const fptype* buffer ) { const fptype& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferGs::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } }; @@ -166,6 +154,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessGs_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h new file mode 100644 index 0000000000..525196d770 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessHelpers.h @@ -0,0 +1,152 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessHelpers_H +#define MemoryAccessHelpers_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +//---------------------------------------------------------------------------- + +// A templated helper class that includes the boilerplate code for MemoryAccess classes +template +class MemoryAccessHelper +{ +public: + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = T::ieventAccessRecord; + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline const FT* + ieventAccessRecordConst( const FT* buffer, + const int ievt ) + { + return ieventAccessRecord( const_cast( buffer ), ievt ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + static constexpr auto decodeRecord = T::decodeRecord; + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + decodeRecordConst( const FT* buffer, + Ts... args ) // variadic template + { + return T::decodeRecord( const_cast( buffer ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessField( fptype* buffer, const ievt, Ts... args ) <===] + template + static __host__ __device__ inline FT& + ieventAccessField( FT* buffer, + const int ievt, + Ts... args ) // variadic template + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + return T::decodeRecord( T::ieventAccessRecord( buffer, ievt ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessFieldConst( const fptype* buffer, const ievt, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + ieventAccessFieldConst( const FT* buffer, + const int ievt, + Ts... args ) // variadic template + { + return ieventAccessField( const_cast( buffer ), ievt, args... ); + } +}; + +//---------------------------------------------------------------------------- + +// A templated helper class that includes the boilerplate code for KernelAccess classes +template +class KernelAccessHelper : public MemoryAccessHelper +{ +public: + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (non-const) ===> fptype* kernelAccessRecord( fptype* buffer ) <===] + static __host__ __device__ inline FT* + kernelAccessRecord( FT* buffer ) + { + if constexpr( !onDevice ) // requires c++17 also in CUDA (#333) + { + // FIXME #436: clarify that buffer includes all events on device, and only the record for an event subset on host! + // FIXME #436: am I not assuming that the following line is always identical to buffer for all access classes T? + return T::ieventAccessRecord( buffer, 0 ); + } + else + { + throw std::runtime_error( "kernelAccessRecord on device is only implemented in CUDA" ); + } + } + + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) + // [Signature (const) ===> const fptype* kernelAccessRecordConst( const fptype* buffer ) <===] + static __host__ __device__ inline const FT* + kernelAccessRecordConst( const FT* buffer ) + { + return kernelAccessRecord( const_cast( buffer ) ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessField( fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline FT& + kernelAccessField( FT* buffer, + Ts... args ) // variadic template + { + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + return T::decodeRecord( kernelAccessRecord( buffer ), args... ); + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessFieldConst( const fptype* buffer, Ts... args ) <===] + template + static __host__ __device__ inline const FT& + kernelAccessFieldConst( const FT* buffer, + Ts... args ) // variadic template + { + return kernelAccessField( const_cast( buffer ), args... ); + } + + //-------------------------------------------------------------------------- +}; + +#endif // MemoryAccessHelpers_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessIflavorVec.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h similarity index 92% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessIflavorVec.h rename to madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h index ffc74b0f0c..c64ce4b307 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessIflavorVec.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessIflavorVec.h @@ -12,12 +12,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL // fix #893 (not __CUDACC__) -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -102,14 +98,10 @@ namespace mg5amcCpu kernelAccessConst( const unsigned int* buffer ) { const unsigned int& out = kernelAccessConst_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferIflavorVec::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif + return madmatrix::uintvFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } }; @@ -120,6 +112,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessIflavorVec_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessMatrixElements.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h similarity index 94% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessMatrixElements.h rename to madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h index 4236e20602..7dd8626eb7 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessMatrixElements.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMatrixElements.h @@ -13,12 +13,8 @@ #include "MemoryAccessVectors.h" #include "MemoryBuffers.h" // for HostBufferMatrixElements::isaligned -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -119,14 +115,10 @@ namespace mg5amcCpu kernelAccess( fptype* buffer ) { fptype& out = kernelAccess_s( buffer ); -#ifndef MGONGPU_CPPSIMD - return out; -#else // NB: derived from MemoryAccessMomenta, restricting the implementation to contiguous aligned arrays (#435) - static_assert( mg5amcCpu::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) + static_assert( madmatrix::HostBufferMatrixElements::isaligned() ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) //assert( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ); // ASSUME ALIGNED ARRAYS (reinterpret_cast will segfault otherwise!) - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast -#endif + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) @@ -142,6 +134,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessMatrixElements_H diff --git a/madgraph/iolibs/template_files/madmatrix/MemoryAccessMomenta.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h similarity index 88% rename from madgraph/iolibs/template_files/madmatrix/MemoryAccessMomenta.h rename to madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h index 3ef4d76fbd..c1ed427a36 100644 --- a/madgraph/iolibs/template_files/madmatrix/MemoryAccessMomenta.h +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessMomenta.h @@ -9,16 +9,12 @@ #include "mgOnGpuConfig.h" -#include "CPPProcess.h" +#include "ProcessData.h" #include "MemoryAccessHelpers.h" #include "MemoryAccessVectors.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { //---------------------------------------------------------------------------- @@ -31,32 +27,17 @@ namespace mg5amcCpu // Number of Events Per Page in the momenta AOSOA memory buffer layout // (these are all best kept as a compile-time constants: see issue #23) -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - // ----------------------------------------------------------------------------------------------- - // --- GPUs: neppM is best set to a power of 2 times the number of fptype's in a 32-byte cacheline - // --- This is relevant to ensure coalesced access to momenta in global memory - // --- Note that neppR is hardcoded and may differ from neppM and neppV on some platforms - // ----------------------------------------------------------------------------------------------- - //static constexpr int neppM = 64/sizeof(fptype); // 2x 32-byte GPU cache lines (512 bits): 8 (DOUBLE) or 16 (FLOAT) - static constexpr int neppM = 32/sizeof(fptype); // (DEFAULT) 32-byte GPU cache line (256 bits): 4 (DOUBLE) or 8 (FLOAT) - //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 1.03E9 instead of 1.11E9 in eemumu) -#else // ----------------------------------------------------------------------------------------------- // --- CPUs: neppM is best set equal to the number of fptype's (neppV) in a vector register // --- This is relevant to ensure faster access to momenta from C++ memory cache lines // --- However, neppM is now decoupled from neppV (issue #176) and can be separately hardcoded // --- In practice, neppR, neppM and neppV could now (in principle) all be different // ----------------------------------------------------------------------------------------------- -#ifdef MGONGPU_CPPSIMD static constexpr int neppM = MGONGPU_CPPSIMD; // (DEFAULT) neppM=neppV for optimal performance //static constexpr int neppM = 64/sizeof(fptype); // maximum CPU vector width (512 bits): 8 (DOUBLE) or 16 (FLOAT) //static constexpr int neppM = 32/sizeof(fptype); // lower CPU vector width (256 bits): 4 (DOUBLE) or 8 (FLOAT) //static constexpr int neppM = 1; // *** NB: this is equivalent to AOS *** (slower: 4.66E6 instead of 5.09E9 in eemumu) //static constexpr int neppM = MGONGPU_CPPSIMD*2; // FOR TESTS -#else - static constexpr int neppM = 1; // (DEFAULT) neppM=neppV for optimal performance (NB: this is equivalent to AOS) -#endif -#endif /* clang-format on */ // SANITY CHECK: check that neppM is a power of two static_assert( ispoweroftwo( neppM ), "neppM is not a power of 2" ); @@ -68,10 +49,10 @@ namespace mg5amcCpu friend class KernelAccessHelper; // The number of components of a 4-momentum - static constexpr int np4 = CPPProcess::np4; + static constexpr int np4 = ProcessData::np4; // The number of particles in this physics process - static constexpr int npar = CPPProcess::npar; + static constexpr int npar = ProcessData::npar; //-------------------------------------------------------------------------- // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" @@ -208,9 +189,6 @@ namespace mg5amcCpu const int ipar ) { const fptype& out = kernelAccessIp4IparConst_s( buffer, ip4, ipar ); -#ifndef MGONGPU_CPPSIMD - return out; -#else constexpr int neppM = MemoryAccessMomentaBase::neppM; constexpr bool useContiguousEventsIfPossible = true; // DEFAULT //constexpr bool useContiguousEventsIfPossible = false; // FOR PERFORMANCE TESTS (treat as arbitrary array even if it is an AOSOA) @@ -224,21 +202,21 @@ namespace mg5amcCpu //static bool first=true; if( first ){ std::cout << "WARNING! assume aligned AOSOA, skip check" << std::endl; first=false; } // SLOWER (5.06E6) // FASTEST? (5.09E6 in eemumu 512y) // This assumes alignment for momenta1d without checking - causes segmentation fault in reinterpret_cast if not aligned! - return mg5amcCpu::fptypevFromAlignedArray( out ); // use reinterpret_cast + return madmatrix::fptypevFromAlignedArray( out ); // use reinterpret_cast } else if( (size_t)( buffer ) % mgOnGpu::cppAlign == 0 ) { //static bool first=true; if( first ){ std::cout << "WARNING! aligned AOSOA, reinterpret cast" << std::endl; first=false; } // SLOWER (5.00E6) // DEFAULT! A tiny bit (<1%) slower because of the alignment check (5.07E6 in eemumu 512y) // This explicitly checks buffer alignment to avoid segmentation faults in reinterpret_cast - return mg5amcCpu::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast + return madmatrix::fptypevFromAlignedArray( out ); // SIMD bulk load of neppV, use reinterpret_cast } else { //static bool first=true; if( first ){ std::cout << "WARNING! AOSOA but no reinterpret cast" << std::endl; first=false; } // SLOWER (4.93E6) // A bit (1%) slower (5.05E6 in eemumu 512y) // This does not require buffer alignment, but it requires AOSOA with neppM>=neppV and neppM%neppV==0 - return mg5amcCpu::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) + return madmatrix::fptypevFromUnalignedArray( out ); // SIMD bulk load of neppV, do not use reinterpret_cast (fewer SIMD operations) } } else @@ -250,9 +228,8 @@ namespace mg5amcCpu auto decoderIeppv = [buffer, ip4, ipar]( int ieppV ) -> const fptype& { return MemoryAccessMomenta::ieventAccessIp4IparConst( buffer, ievt0 + ieppV, ip4, ipar ); }; - return mg5amcCpu::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) + return madmatrix::fptypevFromArbitraryArray( decoderIeppv ); // iterate over ieppV in neppV (no SIMD) } -#endif } // Is this a HostAccess or DeviceAccess class? @@ -271,6 +248,6 @@ namespace mg5amcCpu //---------------------------------------------------------------------------- -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MemoryAccessMomenta_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h new file mode 100644 index 0000000000..6df3b20cf4 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessNumerators.h @@ -0,0 +1,27 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (May 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessNumerators_H +#define MemoryAccessNumerators_H 1 + +#include "MemoryAccessGs.h" + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for numerators + // This implementation reuses the plain ARRAY[nevt] implementation of MemoryAccessGs + + typedef KernelAccessGs HostAccessNumerators; + typedef KernelAccessGs DeviceAccessNumerators; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessNumerators_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h new file mode 100644 index 0000000000..a20726428b --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessRandomNumbers.h @@ -0,0 +1,139 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessRandomNumbers_H +#define MemoryAccessRandomNumbers_H 1 + +#include "mgOnGpuConfig.h" + +#include "ProcessData.h" +#include "MemoryAccessHelpers.h" + +//---------------------------------------------------------------------------- + +// A class describing the internal layout of memory buffers for random numbers +// This implementation uses an AOSOA[npagR][nparf][np4][neppR] where nevt=npagR*neppR +// [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] +class MemoryAccessRandomNumbersBase //_AOSOAv1 +{ +public: /* clang-format off */ + + // Number of Events Per Page in the random number AOSOA memory buffer layout + // *** NB Different values of neppR lead to different physics results: the *** + // *** same 1d array is generated, but it is interpreted in different ways *** + static constexpr int neppR = 8; // HARDCODED TO GIVE ALWAYS THE SAME PHYSICS RESULTS! + //static constexpr int neppR = 1; // AOS (tests of sectors/requests) + +private: /* clang-format on */ + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a 4-momentum + static constexpr int np4 = ProcessData::np4; + + // The number of final state particles in this physics process + static constexpr int nparf = ProcessData::nparf; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagR = ievt / neppR; // #event "R-page" + const int ieppR = ievt % neppR; // #event in the current event R-page + constexpr int ip4 = 0; + constexpr int iparf = 0; + return &( buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR] ); // AOSOA[ipagR][iparf][ip4][ieppR] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int ip4, const int iparf" and rename "Field" as "Ip4Iparf"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int ip4, + const int iparf ) + { + constexpr int ipagR = 0; + constexpr int ieppR = 0; + return buffer[ipagR * nparf * np4 * neppR + iparf * np4 * neppR + ip4 * neppR + ieppR]; // AOSOA[ipagR][iparf][ip4][ieppR] + } +}; + +//---------------------------------------------------------------------------- + +// A class providing access to memory buffers for a given event, based on explicit event numbers +// Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations +class MemoryAccessRandomNumbers : public MemoryAccessRandomNumbersBase +{ +public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto decodeRecordIp4Iparf = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto decodeRecordIp4IparfConst = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIp4Iparf( fptype* buffer, const ievt, const int ipar, const int iparf ) <===] + static constexpr auto ieventAccessIp4Iparf = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIp4IparfConst( const fptype* buffer, const ievt, const int ipar, const int iparf ) <===] + static constexpr auto ieventAccessIp4IparfConst = + MemoryAccessHelper::template ieventAccessFieldConst; +}; + +//---------------------------------------------------------------------------- + +// A class providing access to memory buffers for a given event, based on implicit kernel rules +// Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations +template +class KernelAccessRandomNumbers +{ +public: + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIp4Iparf( fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto kernelAccessIp4Iparf = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIp4IparfConst( const fptype* buffer, const int ipar, const int iparf ) <===] + static constexpr auto kernelAccessIp4IparfConst = + KernelAccessHelper::template kernelAccessFieldConst; +}; + +//---------------------------------------------------------------------------- + +typedef KernelAccessRandomNumbers HostAccessRandomNumbers; +typedef KernelAccessRandomNumbers DeviceAccessRandomNumbers; + +//---------------------------------------------------------------------------- + +#endif // MemoryAccessRandomNumbers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h new file mode 100644 index 0000000000..fb96f9ab26 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessVectors.h @@ -0,0 +1,64 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessVectors_H +#define MemoryAccessVectors_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +namespace madmatrix // this is only needed for CPU SIMD vectorization +{ + + //-------------------------------------------------------------------------- + + // Cast one non-const fptype_v reference (one vector of neppV fptype values) from one non-const fptype reference (#435), + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned + inline fptype_v& fptypevFromAlignedArray( fptype& ref ) + { + return *reinterpret_cast( &ref ); + } + + inline uint_v& uintvFromAlignedArray( unsigned int& ref ) + { + return *reinterpret_cast( &ref ); + } + + // Cast one const fptype_v reference (one vector of neppV fptype values) from one const fptype reference, + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", and that the arrays are aligned + inline const fptype_v& fptypevFromAlignedArray( const fptype& ref ) + { + return *reinterpret_cast( &ref ); + } + + inline const uint_v& uintvFromAlignedArray( const unsigned int& ref ) + { + return *reinterpret_cast( &ref ); + } + + // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, + // assuming that "pointer(evt#0)+1" indicates "pointer(evt#1)", but that the arrays are not aligned + inline fptype_v fptypevFromUnalignedArray( const fptype& ref ) + { + return fptype_v{ *( &ref ), // explicit initialization of all array elements (2) + *( &ref + 1 ) }; + } + + // Build one fptype_v (one vector of neppV fptype values) from one fptype reference, + // with no a priori assumption on how the input fptype array should be decoded + template + inline fptype_v fptypevFromArbitraryArray( Functor decoderIeppv ) + { + return fptype_v{ decoderIeppv( 0 ), // explicit initialization of all array elements (2) + decoderIeppv( 1 ) }; + } + + //-------------------------------------------------------------------------- + +} // end namespace + +#endif // MemoryAccessVectors_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h new file mode 100644 index 0000000000..9406a9d75a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWavefunctions.h @@ -0,0 +1,166 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessWavefunctions_H +#define MemoryAccessWavefunctions_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "MemoryAccessHelpers.h" + +#define MGONGPU_TRIVIAL_WAVEFUNCTIONS 1 + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + +#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + // A class describing the internal layout of memory buffers for wavefunctions + // This implementation uses an AOSOA[npagW][nw6][nx2][neppW] where nevt=npagW*neppW + // [If many implementations are used, a suffix _AOSOAv1 should be appended to the class name] + class MemoryAccessWavefunctionsBase //_AOSOAv1 + { + public: + + // Number of Events Per Page in the wavefunction AOSOA memory buffer layout + static constexpr int neppW = 1; // AOS (just a test...) + + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + // The number of components of a (fermion or vector) wavefunction + static constexpr int nw6 = mgOnGpu::nw6; + + // The number of floating point components of a complex number + static constexpr int nx2 = mgOnGpu::nx2; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + const int ipagW = ievt / neppW; // #event "W-page" + const int ieppW = ievt % neppW; // #event in the current event W-page + constexpr int iw6 = 0; + constexpr int ix2 = 0; + return &( buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW] ); // AOSOA[ipagW][iw6][ix2][ieppW] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to "const int iw6, const int ix2" and rename "Field" as "Iw6Ix2"] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer, + const int iw6, + const int ix2 ) + { + constexpr int ipagW = 0; + constexpr int ieppW = 0; + return buffer[ipagW * nw6 * nx2 * neppW + iw6 * nx2 * neppW + ix2 * neppW + ieppW]; // AOSOA[ipagW][iw6][ix2][ieppW] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessWavefunctions : public MemoryAccessWavefunctionsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto decodeRecordIw6Ix2 = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto decodeRecordIw6Ix2Const = + MemoryAccessHelper::template decodeRecordConst; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccessIw6Ix2( fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] + static constexpr auto ieventAccessIw6Ix2 = + MemoryAccessHelper::template ieventAccessField; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessIw6Ix2Const( const fptype* buffer, const ievt, const int iw6, const int ix2 ) <===] + static constexpr auto ieventAccessIw6Ix2Const = + MemoryAccessHelper::template ieventAccessFieldConst; + }; + +#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessWavefunctions + { + public: + +#ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccessIw6Ix2( fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto kernelAccessIw6Ix2 = + KernelAccessHelper::template kernelAccessField; + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessIw6Ix2Const( const fptype* buffer, const int iw6, const int ix2 ) <===] + static constexpr auto kernelAccessIw6Ix2Const = + KernelAccessHelper::template kernelAccessFieldConst; + +#else + + static __host__ __device__ inline cxtype_sv* + kernelAccess( fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + + static __host__ __device__ inline const cxtype_sv* + kernelAccessConst( const fptype* buffer ) + { + return reinterpret_cast( buffer ); + } + +#endif // #ifndef MGONGPU_TRIVIAL_WAVEFUNCTIONS + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessWavefunctions HostAccessWavefunctions; + typedef KernelAccessWavefunctions DeviceAccessWavefunctions; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessWavefunctions_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h new file mode 100644 index 0000000000..eece092fa3 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryAccessWeights.h @@ -0,0 +1,146 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryAccessWeights_H +#define MemoryAccessWeights_H 1 + +#include "mgOnGpuConfig.h" + +#include "MemoryAccessHelpers.h" + +//One namespace. Split ber backend. +namespace madmatrix +{ + //---------------------------------------------------------------------------- + + // A class describing the internal layout of memory buffers for weights + // This implementation uses a plain ARRAY[nevt] + // [If many implementations are used, a suffix _ARRAYv1 should be appended to the class name] + class MemoryAccessWeightsBase //_ARRAYv1 + { + private: + + friend class MemoryAccessHelper; + friend class KernelAccessHelper; + friend class KernelAccessHelper; + + //-------------------------------------------------------------------------- + // NB all KernelLaunchers assume that memory access can be decomposed as "accessField = decodeRecord( accessRecord )" + // (in other words: first locate the event record for a given event, then locate an element in that record) + //-------------------------------------------------------------------------- + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static __host__ __device__ inline fptype* + ieventAccessRecord( fptype* buffer, + const int ievt ) + { + return &( buffer[ievt] ); // ARRAY[nevt] + } + + //-------------------------------------------------------------------------- + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer, Ts... args ) <===] + // [NB: expand variadic template "Ts... args" to empty and rename "Field" as empty] + static __host__ __device__ inline fptype& + decodeRecord( fptype* buffer ) + { + constexpr int ievt = 0; + return buffer[ievt]; // ARRAY[nevt] + } + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on explicit event numbers + // Its methods use the MemoryAccessHelper templates - note the use of the template keyword in template function instantiations + class MemoryAccessWeights : public MemoryAccessWeightsBase + { + public: + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (non-const) ===> fptype* ieventAccessRecord( fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecord = MemoryAccessHelper::ieventAccessRecord; + + // Locate an event record (output) in a memory buffer (input) from the given event number (input) + // [Signature (const) ===> const fptype* ieventAccessRecordConst( const fptype* buffer, const int ievt ) <===] + static constexpr auto ieventAccessRecordConst = MemoryAccessHelper::ieventAccessRecordConst; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (non-const) ===> fptype& decodeRecord( fptype* buffer ) <===] + static constexpr auto decodeRecord = MemoryAccessHelper::decodeRecord; + + // Locate a field (output) of an event record (input) from the given field indexes (input) + // [Signature (const) ===> const fptype& decodeRecordConst( const fptype* buffer ) <===] + static constexpr auto decodeRecordConst = + MemoryAccessHelper::template decodeRecordConst<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (non-const) ===> fptype& ieventAccess( fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccess = + MemoryAccessHelper::template ieventAccessField<>; + + // Locate a field (output) in a memory buffer (input) from the given event number (input) and the given field indexes (input) + // [Signature (const) ===> const fptype& ieventAccessConst( const fptype* buffer, const ievt ) <===] + static constexpr auto ieventAccessConst = + MemoryAccessHelper::template ieventAccessFieldConst<>; + }; + + //---------------------------------------------------------------------------- + + // A class providing access to memory buffers for a given event, based on implicit kernel rules + // Its methods use the KernelAccessHelper template - note the use of the template keyword in template function instantiations + template + class KernelAccessWeights + { + public: + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] + // FINAL IMPLEMENTATION FOR CUDA 11.4 + static constexpr auto kernelAccess = + KernelAccessHelper::template kernelAccessField<>; + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (non-const) ===> fptype& kernelAccess( fptype* buffer ) <===] + // TEMPORARY HACK FOR CUDA 11.1 + static __host__ __device__ inline fptype& + kernelAccess( fptype* buffer ) + { + return KernelAccessHelper::template kernelAccessField<>( buffer ); + } + + /* + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + // FINAL IMPLEMENTATION FOR CUDA 11.4 + static constexpr auto kernelAccessConst = + KernelAccessHelper::template kernelAccessFieldConst<>; + */ + + // Locate a field (output) in a memory buffer (input) from a kernel event-indexing mechanism (internal) and the given field indexes (input) + // [Signature (const) ===> const fptype& kernelAccessConst( const fptype* buffer ) <===] + // TEMPORARY HACK FOR CUDA 11.1 + static __host__ __device__ inline const fptype& + kernelAccessConst( const fptype* buffer ) + { + return KernelAccessHelper::template kernelAccessFieldConst<>( buffer ); + } + }; + + //---------------------------------------------------------------------------- + + typedef KernelAccessWeights HostAccessWeights; + typedef KernelAccessWeights DeviceAccessWeights; + + //---------------------------------------------------------------------------- + +} // end namespace madmatrix + +#endif // MemoryAccessWeights_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h new file mode 100644 index 0000000000..88e90aeafb --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/MemoryBuffers.h @@ -0,0 +1,340 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Dec 2021, based on earlier work by S. Hageboeck) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, J. Teig, A. Valassi (2021-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MemoryBuffers_H +#define MemoryBuffers_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuCxtypes.h" + +#include "ProcessData.h" +#include "Parameters.h" + +#include + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + namespace MemoryBuffers + { + // Process-independent compile-time constants + static constexpr size_t np4 = ProcessData::np4; + static constexpr size_t nw6 = ProcessData::nw6; + static constexpr size_t nx2 = mgOnGpu::nx2; + // Process-dependent compile-time constants + static constexpr size_t nparf = ProcessData::nparf; + static constexpr size_t npar = ProcessData::npar; + static constexpr size_t ndcoup = Parameters_dependentCouplings::ndcoup; + static constexpr size_t ncolor = ProcessData::ncolor; + } + + //-------------------------------------------------------------------------- + + // An abstract interface encapsulating a given number of events + class INumberOfEvents + { + public: + virtual ~INumberOfEvents() {} + virtual size_t nevt() const = 0; + }; + + //-------------------------------------------------------------------------- + + // A class encapsulating a given number of events + class NumberOfEvents : virtual public INumberOfEvents + { + public: + NumberOfEvents( const size_t nevt ) + : m_nevt( nevt ) {} + virtual ~NumberOfEvents() {} + virtual size_t nevt() const override { return m_nevt; } + private: + const size_t m_nevt; + }; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer (not necessarily an event buffer) + template + class BufferBase : virtual public INumberOfEvents + { + protected: + BufferBase( const size_t size, const bool onDevice ) + : m_size( size ), m_data( nullptr ), m_isOnDevice( onDevice ) {} + public: + virtual ~BufferBase() {} + T* data() { return m_data; } + const T* data() const { return m_data; } + T& operator[]( const size_t index ) { return m_data[index]; } + const T& operator[]( const size_t index ) const { return m_data[index]; } + size_t size() const { return m_size; } + size_t bytes() const { return m_size * sizeof( T ); } + bool isOnDevice() const { return m_isOnDevice; } + virtual size_t nevt() const override { throw std::runtime_error( "This BufferBase is not an event buffer" ); } + protected: + const size_t m_size; + T* m_data; + const bool m_isOnDevice; + }; + + //-------------------------------------------------------------------------- + + constexpr bool HostBufferALIGNED = false; // ismisaligned=false + constexpr bool HostBufferMISALIGNED = true; // ismisaligned=true + + // A class encapsulating a C++ host buffer + template + class HostBufferBase : public BufferBase + { + public: + HostBufferBase( const size_t size ) + : BufferBase( size, false ) + { + if constexpr( !ismisaligned ) + this->m_data = new( std::align_val_t( cppAlign ) ) T[size](); + else + this->m_data = new( std::align_val_t( cppAlign ) ) T[size + 1]() + 1; // TEST MISALIGNMENT! + } + virtual ~HostBufferBase() + { + if constexpr( !ismisaligned ) + ::operator delete[]( this->m_data, std::align_val_t( cppAlign ) ); + else + ::operator delete[]( ( this->m_data ) - 1, std::align_val_t( cppAlign ) ); // TEST MISALIGNMENT! + } + static constexpr bool isaligned() { return !ismisaligned; } + public: + static constexpr size_t cppAlign = mgOnGpu::cppAlign; + }; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + // A class encapsulating a C++ host buffer for a given number of events + template + class HostBuffer : public HostBufferBase, virtual private NumberOfEvents + { + public: + HostBuffer( const size_t nevt ) + : NumberOfEvents( nevt ) + , HostBufferBase( sizePerEvent * nevt ) + { + //std::cout << "HostBuffer::ctor " << this << " " << nevt << std::endl; + } + virtual ~HostBuffer() + { + //std::cout << "HostBuffer::dtor " << this << std::endl; + } + virtual size_t nevt() const override final { return NumberOfEvents::nevt(); } + }; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for momenta random numbers + typedef BufferBase BufferRndNumMomenta; + + // The size (number of elements) per event in a memory buffer for momenta random numbers + constexpr size_t sizePerEventRndNumMomenta = MemoryBuffers::np4 * MemoryBuffers::nparf; + + // A class encapsulating a C++ host buffer for momenta random numbers + typedef HostBuffer HostBufferRndNumMomenta; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for Gs (related to the event-by-event strength of running coupling constant alphas QCD) + typedef BufferBase BufferGs; + + // The size (number of elements) per event in a memory buffer for Gs + constexpr size_t sizePerEventGs = 1; + + // A class encapsulating a C++ host buffer for gs + typedef HostBuffer HostBufferGs; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for numerators (of the multichannel single-diagram enhancement factors) + typedef BufferBase BufferNumerators; + + // The size (number of elements) per event in a memory buffer for numerators + // (should be equal to the number of diagrams in the process) + constexpr size_t sizePerEventNumerators = ProcessData::ndiagrams; + + // A class encapsulating a C++ host buffer for numerators + typedef HostBuffer HostBufferNumerators; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for denominators (of the multichannel single-diagram enhancement factors) + typedef BufferBase BufferDenominators; + + // The size (number of elements) per event in a memory buffer for denominators + constexpr size_t sizePerEventDenominators = 1; + + // A class encapsulating a C++ host buffer for denominators + typedef HostBuffer HostBufferDenominators; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for couplings that depend on the event-by-event running coupling constant alphas QCD + typedef BufferBase BufferCouplings; + + // The size (number of elements) per event in a memory buffer for random numbers + constexpr size_t sizePerEventCouplings = MemoryBuffers::ndcoup * MemoryBuffers::nx2; + + // A class encapsulating a C++ host buffer for couplings + typedef HostBuffer HostBufferCouplings; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for momenta + typedef BufferBase BufferMomenta; + + // The size (number of elements) per event in a memory buffer for momenta + constexpr size_t sizePerEventMomenta = MemoryBuffers::np4 * MemoryBuffers::npar; + + // A class encapsulating a C++ host buffer for momenta + typedef HostBuffer HostBufferMomenta; + //typedef HostBuffer HostBufferMomenta; // TEST MISALIGNMENT! + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for sampling weights + typedef BufferBase BufferWeights; + + // The size (number of elements) per event in a memory buffer for sampling weights + constexpr size_t sizePerEventWeights = 1; + + // A class encapsulating a C++ host buffer for sampling weights + typedef HostBuffer HostBufferWeights; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for matrix elements + typedef BufferBase BufferMatrixElements; + + // The size (number of elements) per event in a memory buffer for matrix elements + constexpr size_t sizePerEventMatrixElements = 1; + + // A class encapsulating a C++ host buffer for matrix elements + typedef HostBuffer HostBufferMatrixElements; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for the helicity mask + typedef BufferBase BufferHelicityMask; + + // A class encapsulating a C++ host buffer for the helicity mask + typedef HostBufferBase HostBufferHelicityMask; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for wavefunctions + typedef BufferBase BufferWavefunctions; + + // The size (number of elements) per event in a memory buffer for wavefunctions + constexpr size_t sizePerEventWavefunctions = MemoryBuffers::nw6 * MemoryBuffers::nx2; + + // A class encapsulating a C++ host buffer for wavefunctions + typedef HostBuffer HostBufferWavefunctions; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for helicity random numbers + typedef BufferBase BufferRndNumHelicity; + + // The size (number of elements) per event in a memory buffer for helicity random numbers + constexpr size_t sizePerEventRndNumHelicity = 1; + + // A class encapsulating a C++ host buffer for helicity random numbers + typedef HostBuffer HostBufferRndNumHelicity; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for color random numbers + typedef BufferBase BufferRndNumColor; + + // The size (number of elements) per event in a memory buffer for color random numbers + constexpr size_t sizePerEventRndNumColor = 1; + + // A class encapsulating a C++ host buffer for color random numbers + typedef HostBuffer HostBufferRndNumColor; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for channel ids + typedef BufferBase BufferChannelIds; + + // The size (number of elements) per event in a memory buffer for channel ids + constexpr size_t sizePerEventChannelId = 1; + + // A class encapsulating a C++ host buffer for channel ids + typedef HostBuffer HostBufferChannelIds; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for channel ids + typedef BufferBase BufferIflavorVec; + + // The size (number of elements) per event in a memory buffer for channel ids + constexpr size_t sizePerEventIflavorVec = 1; + + // A class encapsulating a C++ host buffer for channel ids + typedef HostBuffer HostBufferIflavorVec; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for helicity selection + typedef BufferBase BufferSelectedHelicity; + + // The size (number of elements) per event in a memory buffer for helicity selection + constexpr size_t sizePerEventSelectedHelicity = 1; + + // A class encapsulating a C++ host buffer for helicity selection + typedef HostBuffer HostBufferSelectedHelicity; + + //-------------------------------------------------------------------------- + + // A base class encapsulating a memory buffer for color selection + typedef BufferBase BufferSelectedColor; + + // The size (number of elements) per event in a memory buffer for color selection + constexpr size_t sizePerEventSelectedColor = 1; + + // A class encapsulating a C++ host buffer for color selection + typedef HostBuffer HostBufferSelectedColor; + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- +} + +#endif // MemoryBuffers_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc new file mode 100644 index 0000000000..84679743c8 --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.cc @@ -0,0 +1,651 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin, calculate_jamps, good-helicity filtering. +// The diagram/vertex-call sequence (helas_calls) is process-specific and +// lives in the P1-generated EvaluateDiagrams.inc, #include'd below. + +#include "SigmaKin.h" + +#include "CPPProcess.h" // ProcessData.h, Parameters.h, HelAmps_.h transitively +#include "ProcessTables.h" + +#include "MemoryAccessAmplitudes.h" +#include "MemoryAccessChannelIds.h" +#include "MemoryAccessCouplings.h" +#include "MemoryAccessCouplingsFixed.h" +#include "MemoryAccessDenominators.h" +#include "MemoryAccessGs.h" +#include "MemoryAccessIflavorVec.h" +#include "MemoryAccessMatrixElements.h" +#include "MemoryAccessMomenta.h" +#include "MemoryAccessNumerators.h" +#include "MemoryAccessWavefunctions.h" +#include "color_sum.h" +#include "ColorData.h" + +#include +#include + +namespace madmatrix +{ + using namespace ProcessData; + using namespace ProcessTables; + using Parameters_dependentCouplings::ndcoup; // #couplings that vary event by event (depend on running alphas QCD) + using Parameters_independentCouplings::nicoup; // #couplings that are fixed for all events (do not depend on running alphas QCD) + + // The number of SIMD vectors of events processed by calculate_jamps +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + constexpr int nParity = 2; +#else + constexpr int nParity = 1; +#endif + + // Helicity/flavor tables and SM parameter/coupling storage, populated once + // by CPPProcess's constructor/initProc via the setters below. + static short cHel[ncomb][npar]; + static short cFlavors[nmaxflavor][npar]; + static int cNGoodHel; + static int cGoodHel[ncomb]; + static fptype cIPD[nIPD > 0 ? nIPD : 1]; + static fptype cIPC[nIPC > 0 ? nIPC * 2 : 1]; + static int cIPF_partner1[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + static int cIPF_partner2[ProcessTables::nMF * nIPF > 0 ? ProcessTables::nMF * nIPF : 1]; + static fptype cIPF_value[ProcessTables::nMF * nIPF * 2 > 0 ? ProcessTables::nMF * nIPF * 2 : 1]; + static double bsmIndepParam[Parameters::nBsmIndepParam > 0 ? Parameters::nBsmIndepParam : 1]; + + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ) + { + memcpy( cHel, tHel, ncomb * npar * sizeof( short ) ); + memcpy( cFlavors, tFlavors, nmaxflavor * npar * sizeof( short ) ); + } + + void setIndependentParams( const fptype* tIPD ) + { + if( nIPD > 0 ) memcpy( cIPD, tIPD, nIPD * sizeof( fptype ) ); + } + + void setIndependentCouplings( const cxtype* tIPC ) + { + if( nIPC > 0 ) memcpy( cIPC, tIPC, nIPC * sizeof( cxtype ) ); + } + + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ) + { + if( nIPF == 0 ) return; + memcpy( cIPF_partner1, tIPF_partner1, nMF * nIPF * sizeof( int ) ); + memcpy( cIPF_partner2, tIPF_partner2, nMF * nIPF * sizeof( int ) ); + memcpy( cIPF_value, tIPF_value, nMF * nIPF * sizeof( cxtype ) ); + } + + void setBsmIndepParam( const double* values, int n ) + { + if( n > 0 ) memcpy( bsmIndepParam, values, n * sizeof( double ) ); + } + + //-------------------------------------------------------------------------- + + __device__ INLINE unsigned int + getChannelId( const unsigned int* allChannelIds, const int ievt00, bool sanityCheckMixedPrecision = true ) + { + unsigned int channelId = 0; // disable multichannel single-diagram enhancement unless allChannelIds != nullptr + using CID_ACCESS = HostAccessChannelIds; // non-trivial access: buffer includes all events + // SCALAR channelId for the whole SIMD neppV2 event page (C++), i.e. one or two neppV event page(s) + // The cudacpp implementation ASSUMES (and checks! #898) that all channelIds are the same in a neppV2 SIMD event page + // **NB! in "mixed" precision, using SIMD, calculate_wavefunctions computes MEs for TWO neppV pages with a single channelId! #924 + if( allChannelIds != nullptr ) + { + // First - and/or only - neppV page of channels (iParity=0 => ievt0 = ievt00 + 0 * neppV) + const unsigned int* channelIds = CID_ACCESS::ieventAccessRecordConst( allChannelIds, ievt00 ); // fix bug #899/#911 + uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) + // NB: channelIds_sv is a vector in SIMD C++ + channelId = channelIds_sv[0]; // element[0] + for( int i = 1; i < neppV; ++i ) // elements[1...neppV-1] + { + assert( channelId == channelIds_sv[i] ); // SANITY CHECK #898: check that all events in a SIMD vector have the same channelId + } + assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) + if( sanityCheckMixedPrecision ) + { +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Second neppV page of channels (iParity=1 => ievt0 = ievt00 + 1 * neppV) + const unsigned int* channelIds2 = CID_ACCESS::ieventAccessRecordConst( allChannelIds, ievt00 + neppV ); // fix bug #899/#911 + uint_v channelIds2_v = CID_ACCESS::kernelAccessConst( channelIds2 ); // fix #895 (compute this only once for all diagrams) + // **NB! in "mixed" precision, using SIMD, calculate_wavefunctions computes MEs for TWO neppV pages with a single channelId! #924 + for( int i = 0; i < neppV; ++i ) + { + assert( channelId == channelIds2_v[i] ); // SANITY CHECKS #898 #924: all events in the 2nd SIMD vector have the same channelId as that of the 1st SIMD vector + } +#endif + } + } + return channelId; + } + + //-------------------------------------------------------------------------- + + __global__ void + computeDependentCouplings( const fptype* allgs, fptype* allcouplings, const int nevt ) + { + using G_ACCESS = HostAccessGs; + using C_ACCESS = HostAccessCouplings; + for( int ipagV = 0; ipagV < nevt / neppV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + const fptype* gs = MemoryAccessGs::ieventAccessRecordConst( allgs, ievt0 ); + fptype* couplings = MemoryAccessCouplings::ieventAccessRecord( allcouplings, ievt0 ); + G2COUP( gs, couplings, bsmIndepParam ); + } + } + + //-------------------------------------------------------------------------- + + // Evaluate QCD partial amplitudes jamps for this given helicity from Feynman diagrams. + // This function processes a single event "page" or SIMD vector (or for two in "mixed" + // precision mode, nParity=2). Accepts a SCALAR channelId because it is GUARANTEED that + // all events in a SIMD vector have the same channelId #898. + void + calculate_jamps( int ihel, + const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + cxtype_sv* allJamp_sv, + bool storeChannelWeights, + fptype* allNumerators, + fptype* allDenominators, + fptype_sv* jamp2_sv, + const int ievt00 ) + { + using M_ACCESS = HostAccessMomenta; + using W_ACCESS = HostAccessWavefunctions; + using A_ACCESS = HostAccessAmplitudes; + using CD_ACCESS = HostAccessCouplings; + using CI_ACCESS = HostAccessCouplingsFixed; + using F_ACCESS = HostAccessIflavorVec; + using NUM_ACCESS = HostAccessNumerators; + using DEN_ACCESS = HostAccessDenominators; + mgDebug( 0, __FUNCTION__ ); + + fptype_sv pvec_sv[nwf][np4]; + cxtype_sv w_sv[nwf][nw6]; // particle wavefunctions within Feynman diagrams + cxtype_sv amp_sv[1]; // invariant amplitude for one given Feynman diagram + ALOHAOBJ aloha_obj[nwf]; + for( int iwf = 0; iwf < nwf; iwf++ ) aloha_obj[iwf] = ALOHAOBJ{ pvec_sv[iwf], w_sv[iwf] }; + fptype* amp_fp = reinterpret_cast( amp_sv ); + + // jamp: sum (for one event or event page) of the invariant amplitudes for + // all Feynman diagrams in a given color combination + cxtype_sv jamp_sv[ncolor] = {}; // all zeros + + for( int iParity = 0; iParity < nParity; ++iParity ) + { + const int ievt0 = ievt00 + iParity * neppV; + + constexpr size_t nxcoup = ndcoup + nIPC; // both dependent and independent couplings + const fptype* allCOUPs[nxcoup]; + for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) + allCOUPs[idcoup] = CD_ACCESS::idcoupAccessBufferConst( allcouplings, idcoup ); // dependent couplings, vary event-by-event + for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) + allCOUPs[ndcoup + iicoup] = CI_ACCESS::iicoupAccessBufferConst( cIPC, iicoup ); // independent couplings, fixed for all events + // C++ kernels take input/output buffers with momenta/MEs for one specific event (the first in the current event page) + const fptype* momenta = M_ACCESS::ieventAccessRecordConst( allmomenta, ievt0 ); + const fptype* COUPs[nxcoup]; + for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) + COUPs[idcoup] = CD_ACCESS::ieventAccessRecordConst( allCOUPs[idcoup], ievt0 ); // dependent couplings, vary event-by-event + for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) + COUPs[ndcoup + iicoup] = allCOUPs[ndcoup + iicoup]; // independent couplings, fixed for all events + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + // Create an array of views over the Flavor Couplings + FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; + + // Dependent (event-by-event, running-alphas) flavor couplings (Step 3): the per-flavor + // values are NOT baked in (they run per event). Gather the current values of the + // underlying dependent couplings for this event page into an AOSOA buffer dpf_value + // (one nx2*neppC SIMD record per (coupling,flavor) slot, matching CD_ACCESS), then build + // an ordinary value-based view over it. The flavor index is constant across a SIMD lane + // (guaranteed by the phase-space integrator), so each lane gets its own running value + // while sharing the same flavor selection. This is the direct analogue of Fortran's + // FLV_xx%VAL(k)%P => GC_yyy(J). The vertex routines are instantiated with CD_ACCESS so + // get_coupling_def reads dpf_value with the right per-flavor stride (CD_ACCESS::flv_stride). + constexpr int ndpfbuf = ( nDPF > 0 ? nDPF * nMF * CD_ACCESS::flv_stride : 1 ); + alignas( mgOnGpu::cppAlign ) fptype dpf_value[ndpfbuf]{}; + for( int idpf = 0; idpf < nDPF; idpf++ ) + for( int imf = 0; imf < nMF; imf++ ) + { + const int idc = cDPF_idcoup[idpf * nMF + imf]; + if( idc >= 0 ) + CD_ACCESS::kernelAccess( dpf_value + ( idpf * nMF + imf ) * CD_ACCESS::flv_stride ) = + CD_ACCESS::kernelAccessConst( COUPs[idc] ); + } + FLV_COUPLING_ARRAY flvCOUPs_dep{ cDPF_partner1, cDPF_partner2, dpf_value }; + + // Reset color flows (reset jamp_sv) at the beginning of a new event or event page + for( int i = 0; i < ncolor; i++ ) { jamp_sv[i] = cxzero_sv(); } + + // Numerators and denominators for the current event (CUDA) or SIMD event page (C++) + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + // Scalar iflavor for the current event (constant across the SIMD vector) + const unsigned int* iflavor_rec = F_ACCESS::ieventAccessRecordConst( iflavorVec, ievt0 ); + const uint_sv iflavor_sv = F_ACCESS::kernelAccessConst( iflavor_rec ); + const unsigned int iflavor = reinterpret_cast(&iflavor_sv)[0]; +#include "EvaluateDiagrams.inc" + + // *** COLOR CHOICE BELOW *** + // Store the leading color flows for choice of color + if( jamp2_sv ) // disable color choice if nullptr + { + for( int icol = 0; icol < ncolor; icol++ ) + jamp2_sv[ncolor * iParity + icol] += cxabs2( jamp_sv[icol] ); // may underflow #831 + } + + // *** PREPARE OUTPUT JAMPS *** + // In C++, copy the local jamp to the output array passed as function argument + for( int icol = 0; icol < ncolor; icol++ ) + allJamp_sv[iParity * ncolor + icol] = jamp_sv[icol]; + } + // END LOOP ON IPARITY + + mgDebug( 1, __FUNCTION__ ); + return; + } + + //-------------------------------------------------------------------------- + + void + sigmaKin_getGoodHel( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + fptype* allMEs, + fptype* allNumerators, + fptype* allDenominators, + bool* isGoodHel, + const int nevt ) + { + // Allocate arrays at build time to contain at least 16 events (or at least neppV events if neppV>16, e.g. in future VPUs) + constexpr int maxtry0 = std::max( 16, neppV ); // 16, but at least neppV (otherwise the npagV loop does not even start) + assert( nevt >= neppV ); + const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt 1 ) + { + for( int i = broken_sym_component_starts[icomp] - 1; i < broken_sym_component_ends[icomp]; i++ ) + { + if( pid_work[i] == 0 ) + continue; + int n_tot = 1; + for( int j = i + 1; j < broken_sym_component_ends[icomp]; j++ ) + { + if( pid_work[i] != pid_work[j] ) + continue; + bool same_block = ( broken_sym_block_lengths[i] == broken_sym_block_lengths[j] ); + for( int k = 0; same_block && k < broken_sym_block_lengths[i]; k++ ) + { + if( cFlavors[iflavor][broken_sym_block_starts[i] - 1 + k] != cFlavors[iflavor][broken_sym_block_starts[j] - 1 + k] ) + same_block = false; + } + if( same_block ) + { + pid_work[j] = 0; + n_tot = n_tot + 1; + old_factor = old_factor / n_tot; + } + } + } + } + total_factor = total_factor * old_factor; + } + return total_factor; + } + + //-------------------------------------------------------------------------- + // Evaluate |M|^2, part independent of incoming flavour + + void + sigmaKin( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + const fptype* allrndhel, + const fptype* allrndcol, + const unsigned int* allChannelIds, + const fptype* allrnddiagram, + fptype* allMEs, + int* allselhel, + int* allselcol, + fptype* allNumerators, + fptype* allDenominators, + unsigned int* allDiagramIdsOut, + bool mulChannelWeight, + const int nevt ) + { + mgDebugInitialise(); + + // SANITY CHECKS for cudacpp code generation (see issues #272 and #343 and PRs #619, #626, #360, #396 and #754) + { + // nprocesses == 2 may happen for "mirror processes" such as P0_uux_ttx within pp_tt012j (see PR #754) + static_assert( nproc == 1 || nproc == 2, "Assume nprocesses == 1 or 2" ); + static_assert( proc_id == 1, "Assume process_id == 1" ); + } + + using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events + using NUM_ACCESS = HostAccessNumerators; // non-trivial access: buffer includes all events + using DEN_ACCESS = HostAccessDenominators; // non-trivial access: buffer includes all events + + // Reset the "matrix elements" - running sums of |M|^2 over helicities for the given event + const int npagV = nevt / neppV; + for( int ipagV = 0; ipagV < npagV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv = fptype_sv{ 0 }; + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + for( int i = 0; i < ndiagrams; ++i ) + { + numerators_sv[i] = fptype_sv{ 0 }; + } + denominators_sv = fptype_sv{ 0 }; + } + + // HELICITY LOOP: CALCULATE WAVEFUNCTIONS (using precomputed good helicities) +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + // Mixed fptypes #537: float for color algebra and double elsewhere + // Delay color algebra and ME updates (only on even pages) + assert( npagV % 2 == 0 ); // SANITY CHECK for mixed fptypes: two neppV-pages are merged to one 2*neppV-page + const int npagV2 = npagV / 2; // loop on two SIMD pages (neppV events) at a time +#else + const int npagV2 = npagV; // loop on one SIMD page (neppV events) at a time +#endif +#ifdef _OPENMP + // OMP multithreading #575 (NB: tested only with gcc11 so far) +#define _OMPLIST0 allcouplings, allMEs, allmomenta, allrndcol, allrndhel, allselcol, allselhel, cGoodHel, cNGoodHel, npagV2 +#define _OMPLIST1 , allDenominators, allNumerators, allChannelIds, mgOnGpu::icolamp, mgOnGpu::channel2iconfig +#pragma omp parallel for default( none ) shared( _OMPLIST0 _OMPLIST1 ) +#undef _OMPLIST0 +#undef _OMPLIST1 +#endif // _OPENMP + for( int ipagV2 = 0; ipagV2 < npagV2; ++ipagV2 ) + { +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int ievt00 = ipagV2 * neppV * 2; // loop on two SIMD pages (neppV events) at a time +#else + const int ievt00 = ipagV2 * neppV; // loop on one SIMD page (neppV events) at a time +#endif + // Running sum of partial amplitudes squared for event by event color selection (#402) + fptype_sv jamp2_sv[nParity * ncolor] = {}; + fptype_sv MEs_ighel[ncomb] = {}; // sum of MEs for all good helicities up to ighel (for the first - and/or only - neppV page) +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype_sv MEs_ighel2[ncomb] = {}; // sum of MEs for all good helicities up to ighel (for the second neppV page) +#endif + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + const int ihel = cGoodHel[ighel]; + cxtype_sv jamp_sv[nParity * ncolor] = {}; // fixed nasty bug (omitting 'nParity' caused memory corruptions after calling calculate_jamps) + bool storeChannelWeights = allChannelIds != nullptr || allrnddiagram != nullptr; + calculate_jamps( ihel, allmomenta, allcouplings, iflavorVec, jamp_sv, storeChannelWeights, allNumerators, allDenominators, jamp2_sv, ievt00 ); + color_sum_cpu( allMEs, jamp_sv, ievt00 ); + MEs_ighel[ighel] = E_ACCESS::kernelAccess( E_ACCESS::ieventAccessRecord( allMEs, ievt00 ) ); +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + MEs_ighel2[ighel] = E_ACCESS::kernelAccess( E_ACCESS::ieventAccessRecord( allMEs, ievt00 + neppV ) ); +#endif + } + // Event-by-event random choice of helicity #403 + for( int ieppV = 0; ieppV < neppV; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + const bool okhel = allrndhel[ievt] < ( MEs_ighel[ighel][ieppV] / MEs_ighel[cNGoodHel - 1][ieppV] ); + if( okhel ) + { + const int ihelF = cGoodHel[ighel] + 1; // NB Fortran [1,ncomb], cudacpp [0,ncomb-1] + allselhel[ievt] = ihelF; + break; + } + } +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int ievt2 = ievt00 + ieppV + neppV; + for( int ighel = 0; ighel < cNGoodHel; ighel++ ) + { + if( allrndhel[ievt2] < ( MEs_ighel2[ighel][ieppV] / MEs_ighel2[cNGoodHel - 1][ieppV] ) ) + { + const int ihelF = cGoodHel[ighel] + 1; // NB Fortran [1,ncomb], cudacpp [0,ncomb-1] + allselhel[ievt2] = ihelF; + break; + } + } +#endif + } +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int vecsize = 2 * neppV; +#else + const int vecsize = neppV; +#endif + unsigned int channelIdVec[vecsize]; + if( allChannelIds != nullptr ) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + channelIdVec[ieppV] = allChannelIds[ievt]; + } + } + + // Event-by-event random choice of channel + if( allrnddiagram != nullptr ) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + fptype numerator_sum = 0., normalization = 0.; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + normalization += allNumerators[ievt / neppV * neppV * ndiagrams + + ichan * neppV + ieppV % neppV]; + } + channelIdVec[ieppV] = mgOnGpu::nchannels; + for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) + { + if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; + numerator_sum += allNumerators[ievt / neppV * neppV * ndiagrams + + ichan * neppV + ieppV % neppV]; + if( allrnddiagram[ievt] < numerator_sum / normalization ) + { + channelIdVec[ieppV] = ichan + 1; + break; + } + } + allDiagramIdsOut[ievt] = channelIdVec[ieppV]; + } + } + + // Event-by-event random choice of color #402 + if( allChannelIds != nullptr || allrnddiagram != nullptr ) // no event-by-event choice of color if channelId == 0 (fix FPE #783) + { + for( int ieppV = 0; ieppV < vecsize; ++ieppV ) + { + unsigned int channelId = channelIdVec[ieppV]; + if( channelId > mgOnGpu::nchannels ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which is greater than nchannels=%d\n", channelId, mgOnGpu::nchannels ); + assert( channelId <= mgOnGpu::nchannels ); // SANITY CHECK #919 #910 + } + const int iconfig = mgOnGpu::channel2iconfig[channelId - 1]; // map N_diagrams to N_config <= N_diagrams configs (fix LHE color mismatch #856: see also #826, #852, #853) + if( iconfig <= 0 ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d which has no associated SDE iconfig\n", channelId ); + assert( iconfig > 0 ); // SANITY CHECK #917 + } + else if( iconfig > (int)mgOnGpu::nconfigSDE ) + { + printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%d (invalid SDE iconfig=%d\n > nconfig=%d)", channelId, iconfig, mgOnGpu::nconfigSDE ); + assert( iconfig <= (int)mgOnGpu::nconfigSDE ); // SANITY CHECK #917 + } + fptype targetamp[ncolor] = { 0 }; + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( icolC == 0 ) + targetamp[icolC] = 0; + else + targetamp[icolC] = targetamp[icolC - 1]; + if( mgOnGpu::icolamp[iconfig - 1][icolC] ) targetamp[icolC] += + jamp2_sv[icolC + ncolor * ( ieppV / neppV )][ieppV % neppV]; + } + const int ievt = ievt00 + ieppV; + for( int icolC = 0; icolC < ncolor; icolC++ ) + { + if( allrndcol[ievt] < ( targetamp[icolC] / targetamp[ncolor - 1] ) ) + { + allselcol[ievt] = icolC + 1; // NB Fortran [1,ncolor], cudacpp [0,ncolor-1] + break; + } + } + } + } + else + { + for( int ieppV = 0; ieppV < neppV; ++ieppV ) + { + const int ievt = ievt00 + ieppV; + allselcol[ievt] = 0; // no color selected in Fortran range [1,ncolor] if channelId == 0 (see #931) +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int ievt2 = ievt00 + ieppV + neppV; + allselcol[ievt2] = 0; // no color selected in Fortran range [1,ncolor] if channelId == 0 (see #931) +#endif + } + } + } + // *** END OF PART 1b - C++ (loop on event pages) + + // PART 2 - FINALISATION (after calculate_jamps) + // Get the final |M|^2 as an average over helicities/colors of the running sum of |M|^2 over helicities for the given event + for( int ipagV = 0; ipagV < npagV; ++ipagV ) + { + const int ievt0 = ipagV * neppV; + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv = MEs_sv * broken_symmetry_factor( iflavorVec[ievt0] ) / helcolDenominators[0]; + if( mulChannelWeight && allChannelIds != nullptr ) // fix segfault #892 (not 'channelIds[0] != 0') + { + const unsigned int channelId = getChannelId( allChannelIds, ievt0, false ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ndiagrams ); + fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); + fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); + fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); + MEs_sv *= numerators_sv[channelId - 1] / denominators_sv; + } + } + mgDebugFinalise(); + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h new file mode 100644 index 0000000000..81956d459d --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/SigmaKin.h @@ -0,0 +1,64 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Integrated with the MadGraph7 project in Feb 2026. +// +// Backend-owned driver: sigmaKin and everything it calls (calculate_jamps, +// good-helicity filtering, color/channel selection). Declared here so +// CPPProcess.cc's constructor/initProc (P1-generated) can call the setters +// that populate this file's storage, and so umami.cc/MatrixElementKernels.cc +// can call sigmaKin/computeDependentCouplings. + +#ifndef SIGMAKIN_H +#define SIGMAKIN_H 1 + +#include "mgOnGpuConfig.h" +#include "mgOnGpuCxtypes.h" // for cxtype + +namespace madmatrix +{ + __global__ void + computeDependentCouplings( const fptype* allgs, + fptype* allcouplings, + const int nevt ); + + void + sigmaKin_getGoodHel( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + fptype* allMEs, + fptype* allNumerators, + fptype* allDenominators, + bool* isGoodHel, + const int nevt ); + + int + sigmaKin_setGoodHel( const bool* isGoodHel ); + + void + sigmaKin( const fptype* allmomenta, + const fptype* allcouplings, + const unsigned int* iflavorVec, + const fptype* allrndhel, + const fptype* allrndcol, + const unsigned int* allChannelIds, + const fptype* allrnddiagram, + fptype* allMEs, + int* allselhel, + int* allselcol, + fptype* allNumerators, + fptype* allDenominators, + unsigned int* allDiagramIdsOut, + bool mulChannelWeight, + const int nevt ); + + // Setters: called once by CPPProcess (P1-generated) to populate this file's + // otherwise-internal storage, since it can no longer be written directly + // from a different translation unit. + void setHelicitiesAndFlavors( const short* tHel, const short* tFlavors ); + void setIndependentParams( const fptype* tIPD ); + void setIndependentCouplings( const cxtype* tIPC ); + void setFlavorCouplings( const int* tIPF_partner1, const int* tIPF_partner2, const cxtype* tIPF_value ); + void setBsmIndepParam( const double* values, int n ); +} + +#endif // SIGMAKIN_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc new file mode 100644 index 0000000000..15b28a305a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.cc @@ -0,0 +1,115 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#include "color_sum.h" + +#include "mgOnGpuConfig.h" + +#include "ColorData.h" // P1-generated: colorMatrix/colorDenom, channel/config maps +#include "MemoryAccessMatrixElements.h" + +namespace madmatrix +{ + using namespace ColorMatrixData; // colorMatrix, colorDenom, ncolor + + + //-------------------------------------------------------------------------- + + void + color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity + const int ievt0 ) // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) + { + // Pre-compute a constexpr triangular color matrix properly normalized #475 + struct TriangularNormalizedColorMatrix + { + // See https://stackoverflow.com/a/34465458 + __host__ __device__ constexpr TriangularNormalizedColorMatrix() + : value() + { + for( int icol = 0; icol < ncolor; icol++ ) + { + // Diagonal terms + value[icol][icol] = colorMatrix[icol][icol] / colorDenom[icol]; + // Off-diagonal terms + for( int jcol = icol + 1; jcol < ncolor; jcol++ ) + value[icol][jcol] = 2 * colorMatrix[icol][jcol] / colorDenom[icol]; + } + } + fptype2 value[ncolor][ncolor]; + }; + static constexpr auto cf2 = TriangularNormalizedColorMatrix(); + // Use the property that M is a real matrix (see #475): + // we can rewrite the quadratic form (A-iB)(M)(A+iB) as AMA - iBMA + iBMA + BMB = AMA + BMB + // In addition, on C++ use the property that M is symmetric (see #475), + // and also use constexpr to compute "2*" and "/colorDenom[icol]" once and for all at compile time: + // we gain (not a factor 2...) in speed here as we only loop over the up diagonal part of the matrix. + // Strangely, CUDA is slower instead, so keep the old implementation for the moment. + fptype_sv deltaMEs = { 0 }; +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype_sv deltaMEs_next = { 0 }; + // Mixed mode: merge two neppV vectors into one neppV2 vector + fptype2_sv jampR_sv[ncolor]; + fptype2_sv jampI_sv[ncolor]; + for( int icol = 0; icol < ncolor; icol++ ) + { + jampR_sv[icol] = fpvmerge( cxreal( allJamp_sv[icol] ), cxreal( allJamp_sv[ncolor + icol] ) ); + jampI_sv[icol] = fpvmerge( cximag( allJamp_sv[icol] ), cximag( allJamp_sv[ncolor + icol] ) ); + } +#else + const cxtype_sv* jamp_sv = allJamp_sv; +#endif + // Loop over icol + for( int icol = 0; icol < ncolor; icol++ ) + { + // Diagonal terms +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype2_sv& jampRi_sv = jampR_sv[icol]; + fptype2_sv& jampIi_sv = jampI_sv[icol]; +#else + fptype2_sv jampRi_sv = (fptype2_sv)( cxreal( jamp_sv[icol] ) ); + fptype2_sv jampIi_sv = (fptype2_sv)( cximag( jamp_sv[icol] ) ); +#endif + fptype2_sv ztempR_sv = cf2.value[icol][icol] * jampRi_sv; + fptype2_sv ztempI_sv = cf2.value[icol][icol] * jampIi_sv; + // Loop over jcol + for( int jcol = icol + 1; jcol < ncolor; jcol++ ) + { + // Off-diagonal terms +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype2_sv& jampRj_sv = jampR_sv[jcol]; + fptype2_sv& jampIj_sv = jampI_sv[jcol]; +#else + fptype2_sv jampRj_sv = (fptype2_sv)( cxreal( jamp_sv[jcol] ) ); + fptype2_sv jampIj_sv = (fptype2_sv)( cximag( jamp_sv[jcol] ) ); +#endif + ztempR_sv += cf2.value[icol][jcol] * jampRj_sv; + ztempI_sv += cf2.value[icol][jcol] * jampIj_sv; + } + fptype2_sv deltaMEs2 = ( jampRi_sv * ztempR_sv + jampIi_sv * ztempI_sv ); // may underflow #831 +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + deltaMEs += fpvsplit0( deltaMEs2 ); + deltaMEs_next += fpvsplit1( deltaMEs2 ); +#else + deltaMEs += deltaMEs2; +#endif + } + // *** STORE THE RESULTS *** + using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events + fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); + // NB: color_sum ADDS |M|^2 for one helicity to the running sum of |M|^2 over helicities for the given event(s) + fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); + MEs_sv += deltaMEs; // fix #435 +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + fptype* MEs_next = E_ACCESS::ieventAccessRecord( allMEs, ievt0 + neppV ); + fptype_sv& MEs_sv_next = E_ACCESS::kernelAccess( MEs_next ); + MEs_sv_next += deltaMEs_next; +#endif + } + + //-------------------------------------------------------------------------- + +} // end namespace diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h new file mode 100644 index 0000000000..f86a9a6c8a --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/color_sum.h @@ -0,0 +1,43 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Sep 2025) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2025). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef COLOR_SUM_H +#define COLOR_SUM_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuVectors.h" + +#include "ProcessData.h" + +namespace madmatrix +{ + //-------------------------------------------------------------------------- + + // No-op on cpu/simd: the normalized color matrix is already a compile-time + // constexpr there (see color_sum.cc); only gpu needs a runtime push to device memory. + inline void createNormalizedColorMatrix() {} + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + void + color_sum_cpu( fptype* allMEs, // output: allMEs[nevt], add |M|^2 for one specific helicity + const cxtype_sv* allJamp_sv, // input: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for one specific helicity + const int ievt0 ); // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- + + + //-------------------------------------------------------------------------- +} + +#endif // COLOR_SUM_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h new file mode 100644 index 0000000000..30eea25afa --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/constexpr_math.h @@ -0,0 +1,354 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Feb 2024) for the MG5aMC CUDACPP plugin. +// Further modified by: A. Valassi (2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef constexpr_math_h +#define constexpr_math_h 1 + +#include "mgOnGpuConfig.h" + +#include +#include // for feenableexcept, fegetexcept and FE_XXX +#include +#include +#include +#include + +// SWITCH ON/OFF DEBUGGING +#undef CONSTEXPR_MATH_DEBUG // no-debug +//#define CONSTEXPR_MATH_DEBUG 1 // debug + +// FOR DEBUGGING +#ifdef CONSTEXPR_MATH_DEBUG +#define CONSTEXPRMATHFUN inline +#define CONSTEXPRMATHVAR const +#else +#define CONSTEXPRMATHFUN constexpr +#define CONSTEXPRMATHVAR constexpr +#endif + +//One namespace. Split ber backend. +namespace madmatrix +{ + // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) + constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) + { + return curr == prev ? curr : sqrtNewtonRaphson( xx, 0.5 * ( curr + xx / curr ), curr ); + } + constexpr long double constexpr_sqrt( const long double xx ) + { + return xx >= 0 // && x < std::numeric_limits::infinity() // avoid -Wtautological-constant-compare warning in fast math + ? sqrtNewtonRaphson( xx, xx, 0 ) + : std::numeric_limits::quiet_NaN(); + } + + // SQRT constants + constexpr long double constexpr_sqrt2 = constexpr_sqrt( 2 ); + + // Constexpr implementation of floor (see https://stackoverflow.com/a/66146159) + constexpr int constexpr_floor( const long double xx ) + { + const int i = static_cast( xx ); + return xx < i ? i - 1 : i; + } + + // Constexpr implementation of pow + constexpr long double constexpr_pow( const long double base, const long double exp, const bool requireExpGe0 = false ) + { + // NB(1): this iterative integer implementation of constexpr_pow requires exponent >= 0 + if( requireExpGe0 ) assert( exp >= 0 ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + if( exp < 0 ) return 1. / constexpr_pow( base, -exp, true ); + // NB(2): this iterative integer implementation of constexpr_pow requires an integer exponent, excexpt for special cases (1/2, 1/4) + if( exp == 0.5 ) return constexpr_sqrt( base ); + if( exp == 0.25 ) return constexpr_sqrt( constexpr_sqrt( base ) ); + const int iexp = constexpr_floor( exp ); + assert( static_cast( iexp ) == exp ); // NB would fail at compile time with "error: call to non-‘constexpr’ function ‘void __assert_fail'" + // Iterative implementation of pow if exp is a non negative integer + return iexp == 0 ? 1 : base * constexpr_pow( base, iexp - 1 ); + } + + // PI constants + // NB1: M_PIl from from cmath is not defined on Mac + // NB2: std::numbers::pi needs c++20 but we are still using c++17 + // NB3: I could use my constexpr_atan(1)*4... but a literal is better? + //constexpr long double constexpr_pi = M_PIl; // pi + //constexpr long double constexpr_pi_by_2 = M_PI_2l; // pi/2 + //constexpr long double constexpr_pi_by_4 = M_PI_4l; // pi/4 + constexpr long double constexpr_pi = 3.141592653589793238462643383279502884L; // same as M_PIl in gcc + constexpr long double constexpr_pi_by_2 = 1.570796326794896619231321691639751442L; // same as M_PI_2l in gcc + constexpr long double constexpr_pi_by_4 = 0.785398163397448309615660845819875721L; // same as M_PI_4l in gcc + static_assert( constexpr_pi_by_4 * 4 == constexpr_pi ); + static_assert( constexpr_pi_by_4 * 2 == constexpr_pi_by_2 ); + static_assert( constexpr_pi_by_2 * 2 == constexpr_pi ); + + // Constexpr implementation of sin for 0= constexpr_pi_by_4 ) std::cout << "sinTaylor xx=" << xx << std::endl; +#endif + assert( xx >= 0 && "The argument of sinTaylor is lower than the expected range [0,pi/4)" ); + assert( xx < constexpr_pi_by_4 && "The argument of sinTaylor is higher than the expected range [0,pi/4)" ); + long double sinx = 0; + int ipow = 1; + long double delta = xx; + while( true ) + { + long double sinxlast = sinx; + sinx += delta; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + if( sinx == sinxlast ) break; + // Next iteration + ipow += 2; + delta *= -xx * xx / ( ipow - 1 ) / ipow; + } +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", delta=" << delta << ", sinx=" << sinx << std::endl; // for debugging (not constexpr) +#endif + return sinx; + } + +#ifdef CONSTEXPR_MATH_DEBUG + // Debug printouts for trig functions + inline void debug_constexpr_trig( const long double xx, size_t call ) + { + CONSTEXPRMATHVAR long double xxminuspi = xx - constexpr_pi; + CONSTEXPRMATHVAR long double twopiminusxx = 2 * constexpr_pi - xx; + std::cout << std::setprecision( 40 ); + std::cout << "constexpr_sin_quad call=" << call << std::endl + << " xx=" << xx << std::endl + << " (xx-pi)=" << xxminuspi << std::endl + << " (2pi-xx)=" << twopiminusxx << std::endl; + std::cout << std::setprecision( 6 ); + if( xx < 0 ) // (-inf, 0) + std::cout << " -- case 1 (xx < 0)" << std::endl; + else if( xx == 0 ) // [0] *** NEW + std::cout << " -- case 2 (xx == 0)" << std::endl; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + std::cout << " -- case 3 (xx < pi/4)" << std::endl; + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + std::cout << " -- case 4 (xx == pi/4)" << std::endl; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + std::cout << " -- case 5 (xx < pi/2)" << std::endl; + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + std::cout << " -- case 6 (xx == pi/2)" << std::endl; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + std::cout << " -- case 7 (xx < 3*pi/4)" << std::endl; + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + std::cout << " -- case 8 (xx == 3*pi/4)" << std::endl; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + std::cout << " -- case 9 (xx < pi)" << std::endl; + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + std::cout << " -- case 10 (xx == pi)" << std::endl; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + std::cout << " -- case 11 (xx < 2*pi)" << std::endl; + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + std::cout << " -- case 12 (xx == 2*pi)" << std::endl; + else // (2*pi, +inf) + std::cout << " -- case 13 (xx > 2*pi)" << std::endl; + } +#endif + + // Mapping to [0,2*pi) range (long double signature) + constexpr long double mapIn0to2Pi( const long double xx ) + { + return xx - constexpr_floor( xx / 2 / constexpr_pi ) * 2 * constexpr_pi; + } + + // Constexpr implementation of cos (long double signature) + CONSTEXPRMATHFUN long double constexpr_cos_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_cos_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 1; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx ), 2 ) ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return sinTaylor( constexpr_pi_by_2 - xx ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 0; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return -sinTaylor( xx - constexpr_pi_by_2 ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return -1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return -constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi - xx ), 2 ) ); + else if( xx == constexpr_pi ) // [pi] *** NEW (2nd fix #903 infinite recursion) + return -1; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return constexpr_cos_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 1; + else // (2*pi, +inf) + return constexpr_cos_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of cos (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_cos( const double x ) + { + return constexpr_cos_quad( x ); + } + + // Constexpr implementation of sin (long double signature) + CONSTEXPRMATHFUN long double constexpr_sin_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } +#ifdef CONSTEXPR_MATH_DEBUG + static size_t call = 0; + if( !assume0to2Pi ) + call = 0; + else + call++; + if( call > 10 ) debug_constexpr_trig( xx, call ); + assert( call < 20 ); +#endif + if( xx < 0 ) // (-inf, 0) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + else if( xx == 0 ) // [0] *** NEW + return 0; + else if( xx < constexpr_pi_by_4 ) // (0, 1/4*pi) + return sinTaylor( xx ); + else if( xx == constexpr_pi_by_4 ) // [1/4*pi] *** NEW (3rd fix #903 assert fails) + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi_by_2 ) // (1/4*pi, 1/2*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( constexpr_pi_by_2 - xx ), 2 ) ); + else if( xx == constexpr_pi_by_2 ) // [1/2*pi] ** NEW + return 1; + else if( xx < 3 * constexpr_pi_by_4 ) // (1/2*pi, 3/4*pi) + return constexpr_sqrt( 1 - constexpr_pow( sinTaylor( xx - constexpr_pi_by_2 ), 2 ) ); + else if( xx == 3 * constexpr_pi_by_4 ) // [3/4*pi] ** NEW + return 1 / constexpr_sqrt2; + else if( xx < constexpr_pi ) // (3/4*pi, pi) + return sinTaylor( constexpr_pi - xx ); + else if( xx == constexpr_pi ) // [pi] *** NEW (1st fix #903 infinite recursion) + return 0; + else if( xx < 2 * constexpr_pi ) // (pi, 2*pi) + return -constexpr_sin_quad( 2 * constexpr_pi - xx, true ); + else if( xx == 2 * constexpr_pi ) // [2*pi] ** NEW + return 0; + else // (2*pi, +inf) + return constexpr_sin_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of sin (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_sin( const double x ) + { + return constexpr_sin_quad( x ); + } + + // Constexpr implementation of tan (long double signature) + CONSTEXPRMATHFUN long double constexpr_tan_quad( const long double xx, const bool assume0to2Pi = false ) + { + if( assume0to2Pi ) + { + assert( xx >= 0 && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + assert( xx < 2 * constexpr_pi && "The argument of constexpr_sin_quad is assumed to be in [0,2*pi)" ); + } + if( xx < 0 ) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + else if( xx < 2 * constexpr_pi ) // [0, 2*pi) + return constexpr_sin_quad( xx, assume0to2Pi ) / constexpr_cos_quad( xx, assume0to2Pi ); + else // [8/4*pi, +inf) + return constexpr_tan_quad( mapIn0to2Pi( xx ), true ); + } + + // Constexpr implementation of tan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_tan( const double x ) + { + return constexpr_tan_quad( x ); + } + + // Constexpr implementation of atan for -1= -1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + assert( xx < 1 && "The argument of atanTaylor is assumed to be in (-1,+1)" ); + long double atanx = 0; + int ipow = 1; + long double xpow = xx; + while( true ) + { + long double atanxlast = atanx; + atanx += xpow / ipow; +#ifdef CONSTEXPR_MATH_DEBUG + //std::cout << "ipow=" << ipow << ", xpow=" << xpow << ", atanx=" << atanx << std::endl; // for debugging (not constexpr) +#endif + if( atanx == atanxlast ) break; + // Next iteration + ipow += 2; + xpow *= -xx * xx; + } + return atanx; + } + + // Constexpr implementation of atan (long double signature) + CONSTEXPRMATHFUN long double constexpr_atan_quad( const long double xx ) + { + if( xx > 1 ) + return constexpr_pi_by_2 - atanTaylor( 1 / xx ); + else if( xx == 1 ) + return constexpr_pi_by_4; + else if( xx > -1 ) + return atanTaylor( xx ); + else if( xx == -1 ) + return -constexpr_pi_by_4; + else // if( xx < -1 ) + return -constexpr_pi_by_2 - atanTaylor( 1 / xx ); + } + + // Constexpr implementation of atan (double signature, internally implemented as long double) + CONSTEXPRMATHFUN double constexpr_atan( const double x ) + { + return constexpr_atan_quad( x ); + } + + // Enable FPE traps (see #701, #733, #831 - except on MacOS where feenableexcept is not defined #730) + // [NB1: Fortran default is -ffpe-trap=none, i.e. FPE traps are not enabled, https://gcc.gnu.org/onlinedocs/gfortran/Debugging-Options.html] + // [NB2: Fortran default is -ffpe-summary=invalid,zero,overflow,underflow,denormal, i.e. warn at the end on STOP] + inline void + fpeEnable() + { + static bool first = true; // FIXME: quick and dirty hack to do this only once (can be removed when separate C++/CUDA builds are implemented) + if( !first ) return; + first = false; +#ifndef __APPLE__ // on MacOS feenableexcept is not defined #730 + constexpr bool enableFPE = true; // this is hardcoded and no longer controlled by getenv( "CUDACPP_RUNTIME_ENABLEFPE" ) + if( enableFPE ) + { + std::cout << "INFO: The following Floating Point Exceptions will cause SIGFPE program aborts: FE_DIVBYZERO, FE_INVALID, FE_OVERFLOW" << std::endl; + feenableexcept( FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW ); // new strategy #831 (do not enable FE_UNDERFLOW) + } +#else + //std::cout << "INFO: Keep default SIGFPE settings because feenableexcept is not available on MacOS" << std::endl; +#endif + } +} + +#endif // constexpr_math_h diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h new file mode 100644 index 0000000000..966f10492e --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuConfig.h @@ -0,0 +1,224 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jul 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Hageboeck, O. Mattelaer, S. Roiser, J. Teig, A. Valassi (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCONFIG_H +#define MGONGPUCONFIG_H 1 + +#include +#include + +// simd backend: always built with a plain host compiler, never nvcc/hipcc, so the +// GPU-backend selection macro (see gpu/mgOnGpuConfig.h) is deliberately never +// defined here - single-file, all-backend headers rely on that to pick branches. + +// Choose floating point precision (for everything but color algebra #537) +// If set from outside with e.g. -DMGONGPU_FPTYPE_FLOAT, nothing happens (#167) +#if not defined MGONGPU_FPTYPE_DOUBLE and not defined MGONGPU_FPTYPE_FLOAT +#define MGONGPU_FPTYPE_DOUBLE 1 // default +//#define MGONGPU_FPTYPE_FLOAT 1 // 2x faster +#endif + +// Choose floating point precision (for color algebra alone #537) +#if not defined MGONGPU_FPTYPE2_DOUBLE and not defined MGONGPU_FPTYPE2_FLOAT +#define MGONGPU_FPTYPE2_DOUBLE 1 // default +//#define MGONGPU_FPTYPE2_FLOAT 1 // 2x faster +#endif + +// Choose whether to inline all HelAmps functions (can gain ~4x, issue #229) +// By default off; set from outside with -DMGONGPU_INLINE_HELAMPS +//#define MGONGPU_INLINE_HELAMPS 1 + +// Choose whether to hardcode cIPD physics parameters instead of reading user cards +// By default off; set from outside with -DMGONGPU_HARDCODE_PARAM +//#define MGONGPU_HARDCODE_PARAM 1 + +// Complex type in C++: cxsmpl by default, or std::complex (CHOOSE ONLY ONE) +//#define MGONGPU_CPPCXTYPE_STDCOMPLEX 1 // ~8% slower on float, same on double +#define MGONGPU_CPPCXTYPE_CXSMPL 1 // default + +// No BLAS on the simd backend (cuBLAS/hipBLAS are GPU-only) +#define MGONGPU_HAS_NO_BLAS 1 + +// nsight compute (ncu) debugging is CUDA-only; always off here +#undef MGONGPU_NSIGHT_DEBUG + +// SANITY CHECKS +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE_DOUBLE or MGONGPU_FPTYPE_FLOAT +#endif +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_FPTYPE2_DOUBLE or MGONGPU_FPTYPE2_FLOAT +#endif +#if defined MGONGPU_FPTYPE2_DOUBLE and defined MGONGPU_FPTYPE_FLOAT +#error You cannot use double precision for color algebra and single precision elsewhere +#endif +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX and defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL for C++ +#endif + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu +{ + // Floating point type (for everything but color algebra #537): fptype +#if defined MGONGPU_FPTYPE_DOUBLE + typedef double fptype; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE_FLOAT + typedef float fptype; // single precision (4 bytes, fp32) +#endif + + // Floating point type (for color algebra alone #537): fptype2 +#if defined MGONGPU_FPTYPE2_DOUBLE + typedef double fptype2; // double precision (8 bytes, fp64) +#elif defined MGONGPU_FPTYPE2_FLOAT + typedef float fptype2; // single precision (4 bytes, fp32) +#endif + + // Maximum number of threads per block + const int ntpbMAX = 1024; // NB: 512 is ok, but 1024 does fail with "too many resources requested for launch" + + // Alignment requirement for using reinterpret_cast with SIMD vectorized code + constexpr int cppAlign = 64; // 64-byte i.e. 512-bit + + // Retrieve the compiler that was used to build this module + inline std::string + getCompiler() + { + std::stringstream out; + // HIP version (HIPCC) + // [Use __HIPCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'hipcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'hipcc -dM -E -x hip -I ../../src CPPProcess.cc | grep HIP'] +#ifdef __HIPCC__ +#if defined HIP_VERSION_MAJOR && defined HIP_VERSION_MINOR && defined HIP_VERSION_PATCH + out << "hipcc " << HIP_VERSION_MAJOR << "." << HIP_VERSION_MINOR << "." << HIP_VERSION_PATCH; +#else + out << "hipcc UNKNOWN"; +#endif + out << " ("; +#endif + // CUDA version (NVCC) + // [Use __NVCC__ instead of MGONGPUCPP_GPUIMPL here!] + // [This tests if 'nvcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] + // [Check 'nvcc --compiler-options -dM -E dummy.c | grep CUDA': see https://stackoverflow.com/a/53713712] +#ifdef __NVCC__ +#if defined __CUDACC_VER_MAJOR__ && defined __CUDACC_VER_MINOR__ && defined __CUDACC_VER_BUILD__ + out << "nvcc " << __CUDACC_VER_MAJOR__ << "." << __CUDACC_VER_MINOR__ << "." << __CUDACC_VER_BUILD__; +#else + out << "nvcc UNKNOWN"; +#endif + out << " ("; +#endif + // ICX version (either as CXX or as host compiler inside NVCC) +#if defined __INTEL_COMPILER +#error "icc is no longer supported: please use icx" +#elif defined __INTEL_LLVM_COMPILER // alternative: __INTEL_CLANG_COMPILER + out << "icx " << __INTEL_LLVM_COMPILER; +#ifdef __NVCC__ + out << ", "; +#else + out << " ("; +#endif +#endif + // CLANG version (either as CXX or as host compiler inside NVCC or inside ICX) +#if defined __clang__ +#if defined __clang_major__ && defined __clang_minor__ && defined __clang_patchlevel__ +#ifdef __APPLE__ + out << "Apple clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#else + out << "clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; +#endif +#else + out << "clang UNKNOWKN"; +#endif +#else + // GCC version (either as CXX or as host compiler inside NVCC) +#if defined __GNUC__ && defined __GNUC_MINOR__ && defined __GNUC_PATCHLEVEL__ + out << "gcc " << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__; +#else + out << "gcc UNKNOWKN"; +#endif +#endif +#if defined __HIPCC__ or defined __NVCC__ or defined __INTEL_LLVM_COMPILER + out << ")"; +#endif + return out.str(); + } +} + +// Expose typedefs outside the namespace +using mgOnGpu::fptype; +using mgOnGpu::fptype2; + +// Undefine ARM_NEON (hack for cppnone on Apple silicon ARM) +#ifdef MGONGPU_NOARMNEON +#undef __ARM_NEON +#endif + +// C++ SIMD vectorization width (this will be used to set neppV) +#if defined __AVX512VL__ && defined MGONGPU_PVW512 // "512z" AVX512 512-bit: 8 (DOUBLE) or 16 (FLOAT) +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 8 +#else +#define MGONGPU_CPPSIMD 16 +#endif +#elif defined __AVX512VL__ // "512y" AVX512 256-bit: 4 (DOUBLE) or 8 (FLOAT) [gcc default] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif +#elif defined __AVX2__ // "avx2" 256-bit: 4 (DOUBLE) or 8 (FLOAT) [clang default] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 4 +#else +#define MGONGPU_CPPSIMD 8 +#endif +#elif defined __SSE4_2__ // "sse4" SSE4.2 128-bit: 2 (DOUBLE) or 4 (FLOAT) [Power9 default] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 +#else +#define MGONGPU_CPPSIMD 4 +#endif +#elif defined __ARM_NEON // ARM NEON 128-bit: 2 (DOUBLE) or 4 (FLOAT) [ARM default] +#ifdef MGONGPU_FPTYPE_DOUBLE +#define MGONGPU_CPPSIMD 2 +#else +#define MGONGPU_CPPSIMD 4 +#endif +#else // "none" i.e. no SIMD +#undef MGONGPU_CPPSIMD +#endif + +// No-op debug macros (nsight-based debugging is CUDA-only, unused here) +#define mgDebugDeclare() /*noop*/ +#define mgDebugInitialise() /*noop*/ +#define mgDebug( code, text ) /*noop*/ +#define mgDebugFinalise() /*noop*/ + +// Define empty CUDA/HIP declaration specifiers for C++ +#define __global__ +#define __host__ +#define __device__ + +// For SANITY CHECKS: check that neppR, neppM, neppV... are powers of two +inline constexpr bool +ispoweroftwo( int n ) +{ + return ( n > 0 ) && !( n & ( n - 1 ) ); +} + +// Compiler version support (#96) +#if defined __clang__ +#if( __clang_major__ < 11 ) +#error Unsupported clang version: please use clang >= 11 +#endif +#elif defined __GNUC__ +#if( __GNUC__ < 9 ) || ( __GNUC__ == 9 && __GNUC_MINOR__ < 3 ) +#error Unsupported gcc version: please gcc >= 9.3 +#endif +#endif + +#endif // MGONGPUCONFIG_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h new file mode 100644 index 0000000000..5a7f09d0cc --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuCxtypes.h @@ -0,0 +1,380 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022, based on earlier work by D. Smith) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUCXTYPES_H +#define MGONGPUCXTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include "mgOnGpuFptypes.h" + +#include + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) HEADERS +//========================================================================== + +#include + +// Complex type in c++: std::complex or cxsmpl +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX +#include +#elif not defined MGONGPU_CPPCXTYPE_CXSMPL +#error You must CHOOSE (ONE AND) ONLY ONE of MGONGPU_CPPCXTYPE_STDCOMPLEX or MGONGPU_CPPCXTYPE_CXSMPL +#endif + +//========================================================================== +// COMPLEX TYPES: SIMPLE COMPLEX CLASS (cxsmpl) +//========================================================================== + +// NB: namespace mgOnGpu includes types which are defined in exactly the same way for CPU and GPU builds (see #318 and #725) +namespace mgOnGpu /* clang-format off */ +{ + // The number of floating point types in a complex type (real, imaginary) + constexpr int nx2 = 2; + + // --- Type definition (simple complex type derived from cxtype_v) + template + class cxsmpl + { + public: + __host__ __device__ constexpr cxsmpl() : m_real( 0 ), m_imag( 0 ) {} + cxsmpl( const cxsmpl& ) = default; + cxsmpl( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl( const FP& r, const FP& i = 0 ) : m_real( r ), m_imag( i ) {} + __host__ __device__ constexpr cxsmpl( const std::complex& c ) : m_real( c.real() ), m_imag( c.imag() ) {} + cxsmpl& operator=( const cxsmpl& ) = default; + cxsmpl& operator=( cxsmpl&& ) = default; + __host__ __device__ constexpr cxsmpl& operator+=( const cxsmpl& c ) { m_real += c.real(); m_imag += c.imag(); return *this; } + __host__ __device__ constexpr cxsmpl& operator-=( const cxsmpl& c ) { m_real -= c.real(); m_imag -= c.imag(); return *this; } + __host__ __device__ constexpr const FP& real() const { return m_real; } + __host__ __device__ constexpr const FP& imag() const { return m_imag; } + template __host__ __device__ constexpr operator cxsmpl() const { return cxsmpl( m_real, m_imag ); } +#ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX + template __host__ __device__ constexpr operator std::complex() const { return std::complex( m_real, m_imag ); } +#endif + private: + FP m_real, m_imag; // RI + }; + + template + constexpr // (NB: now valid code? in the past this failed as "a constexpr function cannot have a nonliteral return type mgOnGpu::cxsmpl") + inline __host__ __device__ cxsmpl + conj( const cxsmpl& c ) + { + return cxsmpl( c.real(), -c.imag() ); + } +} /* clang-format on */ + +// Expose the cxsmpl class outside the namespace +using mgOnGpu::cxsmpl; + +// Printout to stream for user defined types +namespace madmatrix +{ + template + inline __host__ std::ostream& + operator<<( std::ostream& out, const cxsmpl& c ) + { + //out << std::complex( c.real(), c.imag() ); + out << "(" << c.real() << ", " << c.imag() << ")"; // add a space after the comma + return out; + } + + // Operators for cxsmpl + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl a ) + { + return a; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a ) + { + return cxsmpl( -a.real(), -a.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() + b.real(), a.imag() + b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) + b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() - b.real(), a.imag() - b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) - b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const cxsmpl& b ) + { + return cxsmpl( a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag() ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const double& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) * b; + } + + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const double& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const cxsmpl& b ) + { + FP bnorm = b.real() * b.real() + b.imag() * b.imag(); + return cxsmpl( ( a.real() * b.real() + a.imag() * b.imag() ) / bnorm, + ( a.imag() * b.real() - a.real() * b.imag() ) / bnorm ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const FP& a, const cxsmpl& b ) + { + return cxsmpl( a, 0 ) / b; + } + + template + inline __host__ __device__ constexpr cxsmpl + operator+( const cxsmpl& a, const FP& b ) + { + return a + cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator-( const cxsmpl& a, const FP& b ) + { + return a - cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator*( const cxsmpl& a, const FP& b ) + { + return a * cxsmpl( b, 0 ); + } + + template + inline __host__ __device__ constexpr cxsmpl + operator/( const cxsmpl& a, const FP& b ) + { + return a / cxsmpl( b, 0 ); + } +} + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS +//========================================================================== + +namespace madmatrix +{ + // --- Type definitions (complex type: cxtype) +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX + typedef std::complex cxtype; +#else + typedef cxsmpl cxtype; +#endif + + // SANITY CHECK: memory access may be based on casts of fptype[2] to cxtype (e.g. for wavefunctions) + static_assert( sizeof( cxtype ) == mgOnGpu::nx2 * sizeof( fptype ), "sizeof(cxtype) is not 2*sizeof(fptype)" ); +} + +// DANGEROUS! this was mixing different cxtype definitions for CPU and GPU builds (see #318 and #725) +// DO NOT expose typedefs and operators outside the namespace +//using mgOnGpu::cxtype; + +//========================================================================== +// COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS +//========================================================================== + +namespace madmatrix +{ +#if defined MGONGPU_CPPCXTYPE_CXSMPL + + //------------------------------ + // C++ - using cxsmpl + //------------------------------ + + inline __host__ __device__ cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // cxsmpl constructor + } + + inline __host__ __device__ fptype + cxreal( const cxtype& c ) + { + return c.real(); // cxsmpl::real() + } + + inline __host__ __device__ fptype + cximag( const cxtype& c ) + { + return c.imag(); // cxsmpl::imag() + } + + inline __host__ __device__ cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( cxsmpl ) + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ cxtype // NOT __device__ + cxmake( const std::complex& c ) // std::complex to cxsmpl (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +#endif // #if defined MGONGPU_CPPCXTYPE_CXSMPL + + //========================================================================== + +#if defined MGONGPU_CPPCXTYPE_STDCOMPLEX + + //------------------------------ + // C++ - using std::complex + //------------------------------ + + inline cxtype + cxmake( const fptype& r, const fptype& i ) + { + return cxtype( r, i ); // std::complex constructor + } + + inline fptype + cxreal( const cxtype& c ) + { + return c.real(); // std::complex::real() + } + + inline fptype + cximag( const cxtype& c ) + { + return c.imag(); // std::complex::imag() + } + + inline cxtype + cxconj( const cxtype& c ) + { + return conj( c ); // conj( std::complex ) + } + + inline const cxtype& + cxmake( const cxtype& c ) // std::complex to std::complex (float-to-float or double-to-double) + { + return c; + } + +#if defined MGONGPU_FPTYPE_FLOAT + inline cxtype + cxmake( const std::complex& c ) // std::complex to std::complex (cast double-to-float) + { + return cxmake( (fptype)c.real(), (fptype)c.imag() ); + } +#endif + +#endif // #if defined MGONGPU_CPPCXTYPE_STDCOMPLEX + + //========================================================================== + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (float-to-float or float-to-double) + { + return cxmake( c.real(), c.imag() ); + } + + inline __host__ __device__ const cxtype + cxmake( const cxsmpl& c ) // cxsmpl to cxtype (double-to-float or double-to-double) + { + return cxmake( c.real(), c.imag() ); + } + +} // end namespace madmatrix + +//========================================================================== +// COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) +//========================================================================== + +//One namespace. Split ber backend. +namespace madmatrix +{ + // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] + // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined + // It is now always included in the code because it is needed also to access an fptype wavefunction buffer as a cxtype + class cxtype_ref + { + public: + cxtype_ref() = delete; + cxtype_ref( const cxtype_ref& ) = delete; + cxtype_ref( cxtype_ref&& ) = default; // copy const refs + __host__ __device__ cxtype_ref( fptype& r, fptype& i ) + : m_preal( &r ), m_pimag( &i ) {} // copy (create from) const refs + cxtype_ref& operator=( const cxtype_ref& ) = delete; + //__host__ __device__ cxtype_ref& operator=( cxtype_ref&& c ) {...} // REMOVED! Should copy refs or copy values? No longer needed in cxternary + __host__ __device__ cxtype_ref& operator=( const cxtype& c ) + { + *m_preal = cxreal( c ); + *m_pimag = cximag( c ); + return *this; + } // copy (assign) non-const values + __host__ __device__ operator cxtype() const { return cxmake( *m_preal, *m_pimag ); } + private: + fptype* const m_preal; // const pointer to non-const fptype R + fptype* const m_pimag; // const pointer to non-const fptype I + }; + + // Printout to stream for user defined types + inline __host__ __device__ std::ostream& + operator<<( std::ostream& out, const cxtype_ref& c ) + { + out << (cxtype)c; + return out; + } + +} // end namespace madmatrix + +//========================================================================== + +#endif // MGONGPUCXTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h new file mode 100644 index 0000000000..d6fe740b8f --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuFptypes.h @@ -0,0 +1,51 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Jan 2022) for the MG5aMC CUDACPP plugin. +// Further modified by: J. Teig, A. Valassi (2022-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUFPTYPES_H +#define MGONGPUFPTYPES_H 1 + +#include "mgOnGpuConfig.h" + +#include +#include + +//One namespace. Split ber backend. +namespace madmatrix +{ + //========================================================================== + + + //========================================================================== + + + //------------------------------ + // Floating point types - C++ + //------------------------------ + + inline const fptype& + fpmax( const fptype& a, const fptype& b ) + { + return std::max( a, b ); + } + + inline const fptype& + fpmin( const fptype& a, const fptype& b ) + { + return std::min( a, b ); + } + + inline fptype + fpsqrt( const fptype& f ) + { + return std::sqrt( f ); + } + + + //========================================================================== + +} // end namespace madmatrix + +#endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h new file mode 100644 index 0000000000..14bf7ea0bc --- /dev/null +++ b/madgraph/iolibs/template_files/madmatrix/backend/simd/mgOnGpuVectors.h @@ -0,0 +1,755 @@ +// Copyright (C) 2020-2026 CERN and UCLouvain. +// Licensed under the GNU Lesser General Public License (version 3 or later). +// Created originally by: A. Valassi (Nov 2020) for the MG5aMC CUDACPP plugin. +// Further modified by: S. Roiser, A. Valassi, Z. Wettersten (2020-2024). +// Integrated with the MadGraph7 project in Feb 2026. + +#ifndef MGONGPUVECTORS_H +#define MGONGPUVECTORS_H 1 + +#include "mgOnGpuCxtypes.h" +#include "mgOnGpuFptypes.h" + +#include + +//========================================================================== + +//------------------------------ +// Vector types - C++ +//------------------------------ + +#ifdef __clang__ +// If set: return a pair of (fptype&, fptype&) by non-const reference in cxtype_v::operator[] +// This is forbidden in clang ("non-const reference cannot bind to vector element") +// See also https://stackoverflow.com/questions/26554829 +//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // clang test (compilation fails also on clang 12.0, issue #182) +#undef MGONGPU_HAS_CPPCXTYPEV_BRK // clang default +#elif defined __INTEL_COMPILER +//#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // icc default? +#undef MGONGPU_HAS_CPPCXTYPEV_BRK // icc test +#else +#define MGONGPU_HAS_CPPCXTYPEV_BRK 1 // gcc default +//#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) +#endif + +//One namespace. Split ber backend. +namespace madmatrix +{ + + const int neppV = MGONGPU_CPPSIMD; + + // SANITY CHECK: cppAlign must be a multiple of neppV * sizeof(fptype) + static_assert( mgOnGpu::cppAlign % ( neppV * sizeof( fptype ) ) == 0 ); + + // SANITY CHECK: check that neppV is a power of two + static_assert( ispoweroftwo( neppV ), "neppV is not a power of 2" ); + + // --- Type definition (using vector compiler extensions: need -march=...) + // For gcc: https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html + // For clang: https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors +#ifdef __clang__ + typedef fptype fptype_v __attribute__( ( ext_vector_type( neppV ) ) ); // RRRR +#else + typedef fptype fptype_v __attribute__( ( vector_size( neppV * sizeof(fptype) ), aligned( neppV * sizeof(fptype) ) ) ); // RRRR +#endif + + // Mixed fptypes #537: float for color algebra and double elsewhere +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + const int neppV2 = MGONGPU_CPPSIMD * 2; + static_assert( mgOnGpu::cppAlign % ( neppV2 * sizeof( fptype2 ) ) == 0 ); + static_assert( ispoweroftwo( neppV2 ), "neppV2 is not a power of 2" ); +#ifdef __clang__ + typedef fptype2 fptype2_v __attribute__( ( ext_vector_type( neppV2 ) ) ); // RRRRRRRR +#else + typedef fptype2 fptype2_v __attribute__( ( vector_size( neppV2 * sizeof( fptype2 ) ), aligned( neppV2 * sizeof( fptype2 ) ) ) ); // RRRRRRRR +#endif +#else + typedef fptype_v fptype2_v; +#endif + + // --- Type definition (using vector compiler extensions: need -march=...) + class cxtype_v // no need for "class alignas(2*sizeof(fptype_v)) cxtype_v" + { + public: + // Array initialization: zero-out as "{0}" (C and C++) or as "{}" (C++ only) + // See https://en.cppreference.com/w/c/language/array_initialization#Notes + cxtype_v() + : m_real{ 0 }, m_imag{ 0 } {} // RRRR=0000 IIII=0000 + cxtype_v( const cxtype_v& ) = default; + cxtype_v( cxtype_v&& ) = default; + cxtype_v( const fptype_v& r, const fptype_v& i ) + : m_real( r ), m_imag( i ) {} + cxtype_v( const fptype_v& r ) + : m_real( r ), m_imag{ 0 } {} // IIII=0000 + cxtype_v( const fptype& r ) + : m_real( fptype_v{} + r ), m_imag{ 0 } {} // IIII=0000 + cxtype_v& operator=( const cxtype_v& ) = default; + cxtype_v& operator=( cxtype_v&& ) = default; + cxtype_v& operator+=( const cxtype_v& c ) + { + m_real += c.real(); + m_imag += c.imag(); + return *this; + } + cxtype_v& operator-=( const cxtype_v& c ) + { + m_real -= c.real(); + m_imag -= c.imag(); + return *this; + } +#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK + // NB: THIS IS THE FUNDAMENTAL DIFFERENCE BETWEEN MGONGPU_HAS_CPPCXTYPEV_BRK DEFINED AND NOT DEFINED + // NB: the alternative "clang" implementation is simpler: it simply does not have any bracket operator[] + //cxtype_ref operator[]( size_t i ) const { return cxtype_ref( m_real[i], m_imag[i] ); } // gcc14.2 build fails #1004 + cxtype_ref operator[]( size_t i ) { return cxtype_ref( m_real[i], m_imag[i] ); } + cxtype operator[]( size_t i ) const { return cxtype( m_real[i], m_imag[i] ); } +#endif + const fptype_v& real() const + { + return m_real; + } + const fptype_v& imag() const { return m_imag; } + private: + fptype_v m_real, m_imag; // RRRRIIII + }; + + // --- Type definition (using vector compiler extensions: need -march=...) +#ifdef __clang__ // https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors + typedef unsigned int uint_v __attribute__( ( ext_vector_type( neppV ) ) ); +#if defined MGONGPU_FPTYPE_DOUBLE + typedef long int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb +#elif defined MGONGPU_FPTYPE_FLOAT + typedef int bool_v __attribute__( ( ext_vector_type( neppV ) ) ); // bbbb +#endif +#else // gcc + typedef unsigned int uint_v __attribute__( ( vector_size( neppV * sizeof( unsigned int ) ), aligned( neppV * sizeof( unsigned int ) ) ) ); +#if defined MGONGPU_FPTYPE_DOUBLE + typedef long int bool_v __attribute__( ( vector_size( neppV * sizeof( long int ) ), aligned( neppV * sizeof( long int ) ) ) ); // bbbb +#elif defined MGONGPU_FPTYPE_FLOAT + typedef int bool_v __attribute__( ( vector_size( neppV * sizeof( int ) ), aligned( neppV * sizeof( int ) ) ) ); // bbbb +#endif +#endif + +} + +//-------------------------------------------------------------------------- + +//========================================================================== + +//One namespace. Split ber backend. +namespace madmatrix +{ + + // Printout to stream for user defined types + +#ifndef MGONGPU_CPPCXTYPE_CXSMPL // operator<< for cxsmpl has already been defined! + inline std::ostream& + operator<<( std::ostream& out, const cxtype& c ) + { + out << "[" << cxreal( c ) << "," << cximag( c ) << "]"; + //out << cxreal(c) << "+i" << cximag(c); + return out; + } +#endif + + inline std::ostream& + operator<<( std::ostream& out, const fptype_v& v ) + { + out << "{ " << v[0]; + for( int i = 1; i < neppV; i++ ) out << ", " << v[i]; + out << " }"; + return out; + } + +#if defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT + inline std::ostream& + operator<<( std::ostream& out, const fptype2_v& v ) + { + out << "{ " << v[0]; + for( int i = 1; i < neppV2; i++ ) out << ", " << v[i]; + out << " }"; + return out; + } +#endif + + inline std::ostream& + operator<<( std::ostream& out, const cxtype_v& v ) + { +#ifdef MGONGPU_HAS_CPPCXTYPEV_BRK + out << "{ " << v[0]; + for( int i = 1; i < neppV; i++ ) out << ", " << v[i]; +#else + out << "{ " << cxmake( v.real()[0], v.imag()[0] ); + for( int i = 1; i < neppV; i++ ) out << ", " << cxmake( v.real()[i], v.imag()[i] ); +#endif + out << " }"; + return out; + } + + inline std::ostream& + operator<<( std::ostream& out, const uint_v& v ) + { + out << "{ " << v[0]; + for( int i = 1; i < neppV; i++ ) out << ", " << v[i]; + out << " }"; + return out; + } + + //-------------------------------------------------------------------------- + + // Functions and operators for fptype_v + + inline fptype_v + fpsqrt( const volatile fptype_v& v ) // volatile fixes #736 + { + // See https://stackoverflow.com/questions/18921049/gcc-vector-extensions-sqrt + fptype_v out = {}; // avoid warning 'out' may be used uninitialized: see #594 + for( int i = 0; i < neppV; i++ ) + { + volatile fptype outi = 0; // volatile fixes #736 + if( v[i] > 0 ) outi = fpsqrt( (fptype)v[i] ); + out[i] = outi; + } + return out; + } + + inline fptype_v + fpsqrt( const fptype_v& v ) + { + // See https://stackoverflow.com/questions/18921049/gcc-vector-extensions-sqrt + fptype_v out = {}; // avoid warning 'out' may be used uninitialized: see #594 + for( int i = 0; i < neppV; i++ ) out[i] = fpsqrt( v[i] ); + return out; + } + + //-------------------------------------------------------------------------- + + // Functions and operators for cxtype_v + + + /* + inline cxtype_v + cxvmake( const cxtype c ) + { + cxtype_v out; + for ( int i=0; i +#include +#include +#include + +using namespace madmatrix; + +namespace +{ + + void* initialize_impl( + const fptype* momenta, + const fptype* couplings, + const unsigned int* flavor_indices, + fptype* matrix_elements, + fptype* numerators, + fptype* denominators, + std::size_t count ) + { + bool is_good_hel[ProcessData::ncomb]; + sigmaKin_getGoodHel( + momenta, couplings, flavor_indices, matrix_elements, numerators, denominators, + is_good_hel, + count ); + sigmaKin_setGoodHel( is_good_hel ); + return nullptr; + } + + void initialize( + const fptype* momenta, + const fptype* couplings, + const unsigned int* flavor_indices, + fptype* matrix_elements, + fptype* numerators, + fptype* denominators, + std::size_t count ) + { + // static local initialization is called exactly once in a thread-safe way + static void* dummy = initialize_impl( momenta, couplings, flavor_indices, matrix_elements, + numerators, + denominators, + count ); + } + + void + transpose_momenta( const double* momenta_in, fptype* momenta_out, std::size_t i_event_in, std::size_t i_event_out, std::size_t stride ) + { + std::size_t page_size = MemoryAccessMomentaBase::neppM; + std::size_t i_page = i_event_out / page_size; + std::size_t i_vector = i_event_out % page_size; + + for( std::size_t i_part = 0; i_part < ProcessData::npar; ++i_part ) + { + for( std::size_t i_mom = 0; i_mom < 4; ++i_mom ) + { + momenta_out[i_page * ProcessData::npar * 4 * page_size + + i_part * 4 * page_size + i_mom * page_size + i_vector] = momenta_in[stride * ( ProcessData::npar * i_mom + i_part ) + i_event_in]; + } + } + } + + + struct InterfaceInstance + { + bool initialized = false; + }; + + std::vector g_externalMasses; + +} + +extern "C" +{ + UmamiStatus umami_get_meta( UmamiMetaKey meta_key, void* result ) + { + switch( meta_key ) + { + case UMAMI_META_DEVICE: + { + UmamiDevice& device = *static_cast( result ); + device = UMAMI_DEVICE_CPU; + break; + } + case UMAMI_META_PARTICLE_COUNT: + *static_cast( result ) = ProcessData::npar; + break; + case UMAMI_META_DIAGRAM_COUNT: + *static_cast( result ) = ProcessData::ndiagrams; + break; + case UMAMI_META_HELICITY_COUNT: + *static_cast( result ) = ProcessData::ncomb; + break; + case UMAMI_META_COLOR_COUNT: + return UMAMI_ERROR_UNSUPPORTED_META; + case UMAMI_META_MASSES: + { + if( g_externalMasses.size() != (size_t)ProcessData::npar ) return UMAMI_ERROR_UNINITIALIZED_META; + + for( int ipar = 0; ipar < ProcessData::npar; ++ipar ) + static_cast( result )[ipar] = g_externalMasses[ipar]; + break; + } + default: + return UMAMI_ERROR_UNSUPPORTED_META; + } + return UMAMI_SUCCESS; + } + + UmamiStatus umami_supported_inputs( bool const** supported, int* count ) + { + // MOMENTA, ALPHA_S, FLAVOR_INDEX, RANDOM_COLOR, RANDOM_HELICITY, RANDOM_DIAGRAM, + // HELICITY_INDEX=false, DIAGRAM_INDEX=true, CHANNEL_INDEX=false + static const bool data[UMAMI_INPUT_KEY_COUNT] = { true, true, true, true, true, true, false, true }; + *supported = data; + *count = UMAMI_INPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_required_inputs( bool const** required, int* count ) + { + static const bool data[UMAMI_INPUT_KEY_COUNT] = { true }; // MOMENTA only + *required = data; + *count = UMAMI_INPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_supported_outputs( bool const** supported, int* count ) + { + // MATRIX_ELEMENT, DIAGRAM_AMP2, COLOR_INDEX, HELICITY_INDEX, DIAGRAM_INDEX, GPU_STREAM + static const bool data[UMAMI_OUTPUT_KEY_COUNT] = { true, true, true, true, true }; + *supported = data; + *count = UMAMI_OUTPUT_KEY_COUNT; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_initialize( UmamiHandle* handle, char const* param_card_path ) + { + CPPProcess process; + process.initProc( param_card_path ); + + const std::vector& masses = process.getMasses(); + g_externalMasses.assign( masses.begin(), masses.end() ); + + auto instance = new InterfaceInstance(); + *handle = instance; + return UMAMI_SUCCESS; + } + + UmamiStatus umami_set_parameter( + [[maybe_unused]] UmamiHandle handle, + [[maybe_unused]] char const* name, + [[maybe_unused]] double parameter_real, + [[maybe_unused]] double parameter_imag ) + { + return UMAMI_ERROR_NOT_IMPLEMENTED; + } + + UmamiStatus umami_get_parameter( + [[maybe_unused]] UmamiHandle handle, + [[maybe_unused]] char const* name, + [[maybe_unused]] double* parameter_real, + [[maybe_unused]] double* parameter_imag ) + { + return UMAMI_ERROR_NOT_IMPLEMENTED; + } + + UmamiStatus umami_matrix_element( + UmamiHandle handle, + size_t count, + size_t stride, + size_t offset, + size_t input_count, + UmamiInputKey const* input_keys, + void const* const* inputs, + size_t output_count, + UmamiOutputKey const* output_keys, + void* const* outputs ) + { + const double* momenta_in = nullptr; + const double* alpha_s_in = nullptr; + const unsigned int* flavor_indices_in = nullptr; + const double* random_color_in = nullptr; + const double* random_helicity_in = nullptr; + const double* random_diagram_in = nullptr; + [[maybe_unused]] const int* diagram_in = nullptr; // TODO: unused + + for( std::size_t i = 0; i < input_count; ++i ) + { + const void* input = inputs[i]; + switch( input_keys[i] ) + { + case UMAMI_IN_MOMENTA: + momenta_in = static_cast( input ); + break; + case UMAMI_IN_ALPHA_S: + alpha_s_in = static_cast( input ); + break; + case UMAMI_IN_FLAVOR_INDEX: + flavor_indices_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_COLOR: + random_color_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_HELICITY: + random_helicity_in = static_cast( input ); + break; + case UMAMI_IN_RANDOM_DIAGRAM: + random_diagram_in = static_cast( input ); + break; + case UMAMI_IN_HELICITY_INDEX: + return UMAMI_ERROR_UNSUPPORTED_INPUT; + case UMAMI_IN_DIAGRAM_INDEX: + diagram_in = static_cast( input ); + break; + default: + return UMAMI_ERROR_UNSUPPORTED_INPUT; + } + } + if( !momenta_in ) return UMAMI_ERROR_MISSING_INPUT; + + double* m2_out = nullptr; + double* amp2_out = nullptr; + int* diagram_out = nullptr; + int* color_out = nullptr; + int* helicity_out = nullptr; + for( std::size_t i = 0; i < output_count; ++i ) + { + void* output = outputs[i]; + switch( output_keys[i] ) + { + case UMAMI_OUT_MATRIX_ELEMENT: + m2_out = static_cast( output ); + break; + case UMAMI_OUT_DIAGRAM_AMP2: + amp2_out = static_cast( output ); + break; + case UMAMI_OUT_COLOR_INDEX: + color_out = static_cast( output ); + break; + case UMAMI_OUT_HELICITY_INDEX: + helicity_out = static_cast( output ); + break; + case UMAMI_OUT_DIAGRAM_INDEX: + diagram_out = static_cast( output ); + break; + default: + return UMAMI_ERROR_UNSUPPORTED_OUTPUT; + } + } + + constexpr std::size_t vector_size = MemoryAccessMomentaBase::neppM; + // need to round to round to double page size for some reason + constexpr std::size_t page_size2 = 2 * vector_size; + std::vector permutation; + std::size_t rounded_count; + + constexpr std::size_t flavor_count = ProcessData::nmaxflavor; + HostBufferBase flavor_indices( ((count + page_size2 - 1) / page_size2 + flavor_count) * page_size2 ); + bool sort_flavors = vector_size > 1 && flavor_count > 1 && flavor_indices_in; + if ( sort_flavors ) + { + permutation.resize(count); + std::size_t voffset = 0; + std::size_t vector_indices[flavor_count] = {}; + std::size_t vector_counts[flavor_count] = {}; + // determine permutation of inputs such that all entries in a SIMD vector + // have the same flavor index + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + unsigned int flav = flavor_indices_in[i_event + offset]; + auto& vcount = vector_counts[flav]; + auto& vindex = vector_indices[flav]; + if ( vcount == 0 ) + { + vindex = voffset * page_size2; + for ( std::size_t i = 0; i < page_size2; ++i) { + flavor_indices[voffset * page_size2 + i] = flav; + } + voffset += 1; + } + permutation[i_event] = vindex + vcount; + vcount = (vcount + 1) % page_size2; + } + rounded_count = voffset * page_size2; + } else { + rounded_count = ( count + page_size2 - 1 ) / page_size2 * page_size2; + } + + HostBufferBase momenta( rounded_count * ProcessData::npar * 4 ); + HostBufferBase couplings( rounded_count * madmatrix::Parameters_dependentCouplings::ndcoup * 2 ); + HostBufferBase g_s( rounded_count ); + HostBufferBase helicity_random( rounded_count ); + HostBufferBase color_random( rounded_count ); + HostBufferBase diagram_random( rounded_count ); + HostBufferBase matrix_elements( rounded_count ); + HostBufferBase diagram_index( rounded_count ); + HostBufferBase numerators( rounded_count * ProcessData::ndiagrams ); + HostBufferBase denominators( rounded_count ); + HostBufferBase helicity_index( rounded_count ); + HostBufferBase color_index( rounded_count ); + if ( sort_flavors ) { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_sorted = permutation[i_event]; + transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_sorted, stride ); + helicity_random[i_sorted] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; + color_random[i_sorted] = random_color_in ? random_color_in[i_event + offset] : 0.5; + diagram_random[i_sorted] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; + g_s[i_sorted] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + } + } else { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + transpose_momenta( &momenta_in[offset], momenta.data(), i_event, i_event, stride ); + helicity_random[i_event] = random_helicity_in ? random_helicity_in[i_event + offset] : 0.5; + color_random[i_event] = random_color_in ? random_color_in[i_event + offset] : 0.5; + diagram_random[i_event] = random_diagram_in ? random_diagram_in[i_event + offset] : 0.5; + g_s[i_event] = alpha_s_in ? sqrt( 4 * M_PI * alpha_s_in[i_event + offset] ) : 1.2177157847767195; + flavor_indices[i_event] = flavor_indices_in ? flavor_indices_in[i_event + offset] : 0; + } + for ( std::size_t i_event = count; i_event < rounded_count; ++i_event ) { + flavor_indices[i_event] = 0; + } + } + computeDependentCouplings( g_s.data(), couplings.data(), rounded_count ); + + InterfaceInstance* instance = static_cast( handle ); + if( !instance->initialized ) + { + initialize( + momenta.data(), + couplings.data(), + flavor_indices.data(), + matrix_elements.data(), + numerators.data(), + denominators.data(), + rounded_count ); + instance->initialized = true; + } + + sigmaKin( + momenta.data(), + couplings.data(), + flavor_indices.data(), + helicity_random.data(), + color_random.data(), + nullptr, + diagram_random.data(), + matrix_elements.data(), + helicity_index.data(), + color_index.data(), + numerators.data(), + denominators.data(), + diagram_index.data(), + false, + rounded_count ); + + if ( sort_flavors ) + { + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_sorted = permutation[i_event]; + std::size_t page_size = MemoryAccessMomentaBase::neppM; + std::size_t i_page = i_sorted / page_size; + std::size_t i_vector = i_sorted % page_size; // vector lane + + double denominator = denominators[i_sorted]; + if( m2_out != nullptr ) + { + m2_out[i_event + offset] = matrix_elements[i_sorted]; + } + if( amp2_out != nullptr ) + { + for( std::size_t i_diag = 0; i_diag < ProcessData::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * ProcessData::ndiagrams + i_diag * page_size + i_vector] / denominator; + } + } + if( diagram_out != nullptr ) + { + diagram_out[i_event + offset] = diagram_index[i_sorted] - 1; + } + if( color_out != nullptr ) + { + color_out[i_event + offset] = color_index[i_sorted] - 1; + } + if( helicity_out != nullptr ) + { + helicity_out[i_event + offset] = helicity_index[i_sorted] - 1; + } + } + } else { + std::size_t page_size = MemoryAccessMomentaBase::neppM; + for( std::size_t i_event = 0; i_event < count; ++i_event ) + { + std::size_t i_page = i_event / page_size; + std::size_t i_vector = i_event % page_size; + + double denominator = denominators[i_event]; + if( m2_out != nullptr ) + { + m2_out[i_event + offset] = matrix_elements[i_event]; + } + if( amp2_out != nullptr ) + { + for( std::size_t i_diag = 0; i_diag < ProcessData::ndiagrams; ++i_diag ) + { + amp2_out[stride * i_diag + i_event + offset] = numerators[i_page * page_size * ProcessData::ndiagrams + i_diag * page_size + i_vector] / denominator; + } + } + if( diagram_out != nullptr ) + { + diagram_out[i_event + offset] = diagram_index[i_event] - 1; + } + if( color_out != nullptr ) + { + color_out[i_event + offset] = color_index[i_event] - 1; + } + if( helicity_out != nullptr ) + { + helicity_out[i_event + offset] = helicity_index[i_event] - 1; + } + } + } + return UMAMI_SUCCESS; + } + + UmamiStatus umami_free( UmamiHandle handle ) + { + InterfaceInstance* instance = static_cast( handle ); + delete instance; + return UMAMI_SUCCESS; + } +} diff --git a/madgraph/iolibs/template_files/madmatrix/check_sa.cc b/madgraph/iolibs/template_files/madmatrix/check_sa.cc index 68e93edb50..7b41f4a30e 100644 --- a/madgraph/iolibs/template_files/madmatrix/check_sa.cc +++ b/madgraph/iolibs/template_files/madmatrix/check_sa.cc @@ -26,13 +26,15 @@ #include "mgOnGpuConfig.h" #include "CPPProcess.h" +#ifdef MGONGPUCPP_GPUIMPL #include "GpuAbstraction.h" #include "GpuRuntime.h" +#endif #include "MemoryAccessMomenta.h" #include "MemoryBuffers.h" #include "RamboSamplingKernels.h" #include "RandomNumberKernels.h" -#include "epoch_process_id.h" +#include "ProcessData.h" #include "read_slha.h" #include "timermap.h" #include "umami.h" @@ -56,11 +58,7 @@ namespace { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif + using namespace madmatrix; // Fixed physics inputs fptype kEnergy = 1500.; // Ecms = 1.5 TeV and changed for the matrix mode to 1TeV diff --git a/madgraph/iolibs/template_files/madmatrix/coloramps.h b/madgraph/iolibs/template_files/madmatrix/coloramps.h deleted file mode 100644 index 027f1aa44e..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/coloramps.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Dec 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: O. Mattelaer, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef COLORAMPS_H -#define COLORAMPS_H 1 - -#include "CPPProcess.h" - - -namespace mgOnGpu -{ - // Summary of numbering and indexing conventions for the relevant concepts (see issue #826 and PR #852) - // - Diagram number (no variable) in [0, N_diagrams-1]: all values are allowed (N_diagrams distinct values) - // It follows the same C-style indexing of MadSpace - // => this number is displayed for information before each block of code in CPPProcess.cc - // - Channel number ("channelId" in C, CHANNEL_ID in F) in [1, N_channels]: not all values are allowed (N_config <= N_channels <= N_diagrams distinct values) - // *** NB channelId is a diagram number: but ALL diagrams > N_channels, and also some < N_channels, do not have an associated SDE config number (#919) *** - // => this number (with F indexing as in ps/pdf output) is passed around as an API argument between cudacpp functions - // Note: the old API passes around a single CHANNEL_ID (and uses CHANNEL_ID=0 to indicate no-multichannel mode, but this is not used in coloramps.h), - // while the new API passes around an array of CHANNEL_ID's (and uses a NULL array pointer to indicate no-multichannel mode) - // - Channel number in C indexing: "channelID - 1" - // => this number (with C indexing) is used as the index of the channel2iconfig array below - // - Config number ("iconfig" in C, ICONFIG in F) in [1, N_config]: all values are allowed (N_config <= N_channels <= N_diagrams distinct values) - // - Config number in C indexing: "iconfig - 1" - // => this number (with C indexing) is used as the index of the icolamp array below - - // The number of channels in the channel2iconfig array below - // *** NB this is not guaranteed to be equal to ndiagrams, it can be lower as the remaining diagrams all have no associated SDE iconfig (#919) *** - constexpr unsigned int nchannels = %(nb_diag)i; -#ifdef MGONGPUCPP_GPUIMPL - static_assert( nchannels <= mg5amcGpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 -#else - static_assert( nchannels <= mg5amcCpu::CPPProcess::ndiagrams, "nchannels should be <= ndiagrams" ); // sanity check #910 and #919 -#endif - - // Map channel to iconfig (e.g. "iconfig = channel2iconfig[channelId - 1]": input index uses C indexing, output index uses F indexing) - // Note: iconfig=-1 indicates channels/diagrams with no associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (presence of 4-point interaction?) - // This array has N_diagrams elements, but only N_config <= N_diagrams valid values (iconfig>0) - // (NB: this array is created on the host in C++ code and on the device in GPU code, but a host copy is also needed in runTest #917) - __device__ constexpr int channel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed -%(channelc2iconfig_lines)s - }; - - // Host copy of the channel2iconfig array (this is needed in runTest #917) -#ifndef MGONGPUCPP_GPUIMPL - constexpr const int* hostChannel2iconfig = channel2iconfig; -#else - constexpr int hostChannel2iconfig[%(nb_diag)i] = { // note: a trailing comma in the initializer list is allowed -%(channelc2iconfig_lines)s - }; -#endif - - // The number N_config of channels/diagrams with an associated iconfig for single-diagram enhancement in the MadEvent sampling algorithm (#917) - constexpr unsigned int nconfigSDE = %(nb_channel)s; - - // Map iconfig to the mask of allowed colors (e.g. "colormask = icolamp[iconfig - 1]": input index uses C indexing) - // This array has N_config <= N_diagrams elements - // (NB: this array is created on the host in C++ code and on the device in GPU code) - __device__ constexpr bool icolamp[%(nb_channel)s][%(nb_color)s] = { // note: a trailing comma in the initializer list is allowed -%(is_LC)s - }; - -} - -#endif // COLORAMPS_H diff --git a/madgraph/iolibs/template_files/madmatrix/constexpr_math.h b/madgraph/iolibs/template_files/madmatrix/constexpr_math.h index 32c1bbb963..03e9ffaf5c 100644 --- a/madgraph/iolibs/template_files/madmatrix/constexpr_math.h +++ b/madgraph/iolibs/template_files/madmatrix/constexpr_math.h @@ -27,12 +27,8 @@ #define CONSTEXPRMATHVAR constexpr #endif -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { // Constexpr implementation of sqrt (see https://stackoverflow.com/a/34134071) constexpr long double sqrtNewtonRaphson( const long double xx, const long double curr, const long double prev ) diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc b/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc index 8dde0fff4b..144f2384ed 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_hel_amps_h.inc @@ -17,7 +17,11 @@ #include "mgOnGpuConfig.h" -#include "mgOnGpuVectors.h" +#if __has_include("mgOnGpuVectors.h") // simd only +#include "mgOnGpuVectors.h" +#else +#include "mgOnGpuCxtypes.h" +#endif #include "Parameters.h" @@ -27,11 +31,7 @@ //#include //#include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { // ALOHA-style object for easy flavor consolidation and non-template API diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc index 4650fc76c8..c1cebdbd02 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_cc.inc @@ -16,11 +16,7 @@ #include #include -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; -#else -using namespace mg5amcCpu; -#endif +using namespace madmatrix; #ifndef MGONGPU_HARDCODE_PARAM diff --git a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc index c13f8ebe2f..e6db352680 100644 --- a/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/cpp_model_parameters_h.inc @@ -17,7 +17,9 @@ #include "mgOnGpuConfig.h" #include "mgOnGpuCxtypes.h" -#include "mgOnGpuVectors.h" +#if __has_include("mgOnGpuVectors.h") // simd only +#include "mgOnGpuVectors.h" +#endif #include "constexpr_math.h" @@ -32,12 +34,8 @@ #include "read_slha.h" -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { // Define FLV_COUPLING struct struct FLV_COUPLING { @@ -112,19 +110,29 @@ namespace mg5amcCpu static Parameters* instance; }; -} // end namespace mg5amcGpu/mg5amcCpu + // Generic readers for initProc: dereference an array of pointer-to-Parameters-member + // (a per-process compile-time table) into a matching array of tIPD/tIPC values. + template + inline void gatherFptype( const Parameters* pars, double Parameters::* const ( &members )[N], fptype ( &out )[N] ) + { + for( int i = 0; i < N; ++i ) out[i] = (fptype)( pars->*members[i] ); + } + + template + inline void gatherCxtype( const Parameters* pars, cxtype Parameters::* const ( &members )[N], cxtype ( &out )[N] ) + { + for( int i = 0; i < N; ++i ) out[i] = cxmake( pars->*members[i] ); + } + +} // end namespace madmatrix #else%(eftwarn1)s #include #include -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { // Hardcoded constexpr physics parameters namespace Parameters // keep the same name rather than HardcodedParameters for simplicity @@ -156,18 +164,14 @@ namespace mg5amcCpu %(hasbsmip)s__device__ constexpr double mdl_bsmIndepParam[nBsmIndepParam] = { %(bsmip)s }; } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { namespace Parameters_dependentCouplings { @@ -257,7 +261,7 @@ namespace mg5amcCpu } #pragma GCC diagnostic pop -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/epoch_process_id.h b/madgraph/iolibs/template_files/madmatrix/epoch_process_id.h deleted file mode 100644 index bdf579affd..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/epoch_process_id.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Oct 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef EPOCH_PROCESS_ID_H -#define EPOCH_PROCESS_ID_H 1 - -// No need to indicate EPOCHX_ any longer for auto-generated code -// However, keep the name of the file as it may be useful again for new manual developments -#define MG_EPOCH_PROCESS_ID %(processid_uppercase)s - -// For simplicity, define here the name of the process-dependent reference file for tests -#define MG_EPOCH_REFERENCE_FILE_NAME "../../test/ref/dump_CPUTest.%(processid)s.txt" - -#endif // EPOCH_PROCESS_ID_H diff --git a/madgraph/iolibs/template_files/madmatrix/fbridge.cc b/madgraph/iolibs/template_files/madmatrix/fbridge.cc deleted file mode 100644 index 05c3b3e0c4..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/fbridge.cc +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (Oct 2021) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, S. Roiser, J. Teig, A. Thete, A. Valassi (2021-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "Bridge.h" -#include "CPPProcess.h" -#include "GpuRuntime.h" - -extern "C" -{ - /** - * The namespace where the Bridge class is taken from. - * - * In the current implementation, two separate shared libraries are created for the GPU/CUDA and CPU/C++ implementations. - * Actually, two shared libraries for GPU and CPU are created for each of the five SIMD implementations on CPUs (none, sse4, avx2, 512y, 512z). - * A single fcreatebridge_ symbol is created in each library with the same name, connected to the appropriate Bridge on CPU or GPU. - * The Fortran MadEvent code is always the same: the choice whether to use a CPU or GPU implementation is done by linking the appropriate library. - * As the names of the two CPU/GPU libraries are the same in the five SIMD implementations, the choice of SIMD is done by setting LD_LIBRARY_PATH. - * - * In a future implementation, a single heterogeneous shared library may be created, with the same interface. - * Using the same Fortran MadEvent code, linking to the hetrerogeneous library would allow access to both CPU and GPU implementations. - * The specific heterogeneous configuration (how many GPUs, how many threads on each CPU, etc) could be loaded in CUDA/C++ from a data file. - */ -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif - - /** - * The floating point precision used in Fortran arrays. - * This is presently hardcoded to double precision (REAL*8). - */ - using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays - //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays - - /** - * Create a Bridge and return its pointer. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param nevtF the pointer to the number of events in the Fortran arrays - * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - */ - void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ) - { -#ifdef MGONGPUCPP_GPUIMPL - GpuRuntime::setUp(); -#endif - // (NB: CPPProcess::initProc no longer needs to be executed here because it is called in the Bridge constructor) - // FIXME: disable OMP in Bridge when called from Fortran - *ppbridge = new Bridge( *pnevtF, *pnparF, *pnp4F ); - } - - /** - * Delete a Bridge. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - */ - void fbridgedelete_( CppObjectInFortran** ppbridge ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgedelete_: invalid Bridge address" ); - delete pbridge; -#ifdef MGONGPUCPP_GPUIMPL - GpuRuntime::tearDown(); -#endif - } - - /** - * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant alphas) - * @param rndhel the pointer to the input random numbers for helicity selection - * @param rndcol the pointer to the input random numbers for color selection - * @param channelIds the pointer to the input channels i.e. Feynman diagrams to enhance (1 to n: 0 is an invalid value!) - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void fbridgesequence_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - //printf("fbridgesequence_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgesequence_: invalid Bridge address" ); -#ifdef MGONGPUCPP_GPUIMPL - // Use the device/GPU implementation in the CUDA library - // (there is also a host implementation in this library) - pbridge->gpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); -#else - // Use the host/CPU implementation in the C++ library - // (there is no device implementation in this library) - pbridge->cpu_sequence( momenta, gs, iflavorVec, rndhel, rndcol, channelIds, mes, selhel, selcol, *pgoodHelOnly ); -#endif - } - - /** - * Execute the matrix-element calculation "sequence" via a Bridge on GPU/CUDA or CUDA/C++, without multi-channel mode. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param gs the pointer to the input Gs (running QCD coupling constant alphas) - * @param rndhel the pointer to the input random numbers for helicity selection - * @param rndcol the pointer to the input random numbers for color selection - * @param mes the pointer to the output matrix elements - * @param selhel the pointer to the output selected helicities - * @param selcol the pointer to the output selected colors - * @param goodHelOnly quit after computing good helicities? - */ - void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ) - { - //printf("fbridgesequence_nomultichannel_ goodHelOnly=%d\n", ( *pgoodHelOnly ? 1 : 0 ) ); - fbridgesequence_( ppbridge, momenta, gs, iflavorVec, rndhel, rndcol, nullptr, mes, selhel, selcol, pgoodHelOnly ); - } - - /** - * Retrieve the number of good helicities for helicity filtering in the Bridge. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppbridge the pointer to the Bridge pointer (the Bridge pointer is handled in Fortran as an INTEGER*8 variable) - * @param pngoodhel the pointer to the output number of good helicities - * @param pntothel the pointer to the output total number of helicities - */ - void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, - unsigned int* pngoodhel, - unsigned int* pntothel ) - { - Bridge* pbridge = dynamic_cast*>( *ppbridge ); - if( pbridge == 0 ) throw std::runtime_error( "fbridgegetngoodhel_: invalid Bridge address" ); - *pngoodhel = pbridge->nGoodHel(); - *pntothel = pbridge->nTotHel(); - } -} diff --git a/madgraph/iolibs/template_files/madmatrix/fbridge.h b/madgraph/iolibs/template_files/madmatrix/fbridge.h deleted file mode 100644 index e676df0a33..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/fbridge.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: Z. Wettersten (Oct 2024) for the MG5aMC CUDACPP plugin. -// Further modified by: D. Massaro, A. Thete, A. Valassi (2025). - -#include "Bridge.h" -#include "CPPProcess.h" -#include "GpuRuntime.h" - -#ifndef _FBRIDGE_H_ -#define _FBRIDGE_H_ - -extern "C" -{ -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif - - using FORTRANFPTYPE = double; - - void fbridgecreate_( CppObjectInFortran** ppbridge, const int* pnevtF, const int* pnparF, const int* pnp4F ); - - void fbridgedelete_( CppObjectInFortran** ppbridge ); - - void fbridgesequence_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - const unsigned int* channelIds, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ); - - void fbridgesequence_nomultichannel_( CppObjectInFortran** ppbridge, - const FORTRANFPTYPE* momenta, - const FORTRANFPTYPE* gs, - const unsigned int* iflavorVec, - const FORTRANFPTYPE* rndhel, - const FORTRANFPTYPE* rndcol, - FORTRANFPTYPE* mes, - int* selhel, - int* selcol, - const bool* pgoodHelOnly ); - - void fbridgegetngoodhel_( CppObjectInFortran** ppbridge, unsigned int* pngoodhel, unsigned int* pntothel ); -} -#endif // _FBRIDGE_H_ diff --git a/madgraph/iolibs/template_files/madmatrix/fsampler.cc b/madgraph/iolibs/template_files/madmatrix/fsampler.cc deleted file mode 100644 index 8a4468fb3c..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/fsampler.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: A. Valassi (Feb 2022) for the MG5aMC CUDACPP plugin. -// Further modified by: J. Teig, A. Valassi (2022-2024). -// Integrated with the MadGraph7 project in Feb 2026. - -#include "mgOnGpuConfig.h" - -#include "Bridge.h" -#include "CPPProcess.h" -#include "MemoryBuffers.h" -#include "RamboSamplingKernels.h" -#include "RandomNumberKernels.h" - -//-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif -{ - template - class Sampler final : public CppObjectInFortran - { - public: - // Constructor - // @param nevtF (VECSIZE_USED, vector.inc) number of events in Fortran arrays - // @param nparF (NEXTERNAL, nexternal.inc) number of external particles in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) - // @param np4F number of momenta components, usually 4, in Fortran arrays (KEPT FOR SANITY CHECKS ONLY: remove it?) - Sampler( int nevtF, int nparF, int np4F ); - // Destructor - virtual ~Sampler() {} - // Delete copy/move constructors and assignment operators - Sampler( const Sampler& ) = delete; - Sampler( Sampler&& ) = delete; - Sampler& operator=( const Sampler& ) = delete; - Sampler& operator=( Sampler&& ) = delete; - // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta - void samplerHostSequence( FORTRANFPTYPE* fortranMomenta ); - private: - const int m_nevt; // The number of events in each iteration - int m_iiter; // The iteration counter (for random number seeding) -#ifndef MGONGPUCPP_GPUIMPL - HostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers - HostBufferMomenta m_hstMomenta; // Memory buffers for momenta - HostBufferWeights m_hstWeights; // Memory buffers for sampling weights -#else - PinnedHostBufferRndNumMomenta m_hstRndmom; // Memory buffers for random numbers - PinnedHostBufferMomenta m_hstMomenta; // Memory buffers for momenta - PinnedHostBufferWeights m_hstWeights; // Memory buffers for sampling weights -#endif - std::unique_ptr m_prnk; // The appropriate RandomNumberKernel - std::unique_ptr m_prsk; // The appropriate SamplingKernel - // HARDCODED DEFAULTS - static constexpr fptype energy = 1500; // historical default, Ecms = 1500 GeV = 1.5 TeV (above the Z peak) - }; - - template - Sampler::Sampler( int nevtF, int nparF, int np4F ) - : m_nevt( nevtF ) - , m_iiter( 0 ) - , m_hstRndmom( nevtF ) - , m_hstMomenta( nevtF ) - , m_hstWeights( nevtF ) - , m_prnk( new CommonRandomNumberKernel( m_hstRndmom ) ) - , m_prsk( new RamboSamplingKernelHost( energy, m_hstRndmom, m_hstMomenta, m_hstWeights, nevtF ) ) - { - if( nparF != CPPProcess::npar ) throw std::runtime_error( "Sampler constructor: npar mismatch" ); - if( np4F != CPPProcess::np4 ) throw std::runtime_error( "Sampler constructor: np4 mismatch" ); - std::cout << "WARNING! Instantiate host Sampler (nevt=" << m_nevt << ")" << std::endl; - } - - // Draw random numbers and convert them to momenta in C++, then transpose them to Fortran momenta - template - void Sampler::samplerHostSequence( FORTRANFPTYPE* fortranMomenta ) - { - std::cout << "Iteration #" << m_iiter + 1 << std::endl; - // === STEP 1 OF 3 - // --- 1a. Seed rnd generator (to get same results on host and device in curand) - // [NB This should not be necessary using the host API: "Generation functions - // can be called multiple times on the same generator to generate successive - // blocks of results. For pseudorandom generators, multiple calls to generation - // functions will yield the same result as a single call with a large size."] - // *** NB! REMEMBER THAT THE FORTRAN SAMPLER ALWAYS USES COMMON RANDOM NUMBERS! *** - constexpr unsigned long long seed = 20200805; - m_prnk->seedGenerator( seed + m_iiter ); - m_iiter++; - // --- 1b. Generate all relevant numbers to build nevt events (i.e. nevt phase space points) on the host - m_prnk->generateRnarray(); - //std::cout << "Got random numbers" << std::endl; - // === STEP 2 OF 3 - // --- 2a. Fill in momenta of initial state particles on the device - m_prsk->getMomentaInitial(); - //std::cout << "Got initial momenta" << std::endl; - // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device - // (i.e. map random numbers to final-state particle momenta for each of nevt events) - m_prsk->getMomentaFinal(); - //std::cout << "Got final momenta" << std::endl; - // --- 2c. TransposeC2F - hst_transposeMomentaC2F( m_hstMomenta.data(), fortranMomenta, m_nevt ); - } -} - -//-------------------------------------------------------------------------- - -extern "C" -{ -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif - - /** - * The floating point precision used in Fortran arrays. - * This is presently hardcoded to double precision (REAL*8). - */ - using FORTRANFPTYPE = double; // for Fortran double precision (REAL*8) arrays - //using FORTRANFPTYPE = float; // for Fortran single precision (REAL*4) arrays - - /** - * Create a Sampler and return its pointer. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - * @param nevtF the pointer to the number of events in the Fortran arrays - * @param nparF the pointer to the number of external particles in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - * @param np4F the pointer to the number of momenta components, usually 4, in the Fortran arrays (KEPT FOR SANITY CHECKS ONLY) - */ - void fsamplercreate_( CppObjectInFortran** ppsampler, const int* pnevtF, const int* pnparF, const int* pnp4F ) - { - *ppsampler = new Sampler( *pnevtF, *pnparF, *pnp4F ); - } - - /** - * Delete a Sampler. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - */ - void fsamplerdelete_( CppObjectInFortran** ppsampler ) - { - Sampler* psampler = dynamic_cast*>( *ppsampler ); - if( psampler == 0 ) throw std::runtime_error( "fsamplerdelete_: invalid Sampler address" ); - delete psampler; - } - - /** - * Execute the matrix-element calculation "sequence" via a Sampler on GPU/CUDA or CUDA/C++. - * This is a C symbol that should be called from the Fortran code (in auto_dsig1.f). - * - * @param ppsampler the pointer to the Sampler pointer (the Sampler pointer is handled in Fortran as an INTEGER*8 variable) - * @param momenta the pointer to the input 4-momenta - * @param mes the pointer to the output matrix elements - */ - void fsamplersequence_( CppObjectInFortran** ppsampler, FORTRANFPTYPE* momenta ) - { - Sampler* psampler = dynamic_cast*>( *ppsampler ); - if( psampler == 0 ) throw std::runtime_error( "fsamplersequence_: invalid Sampler address" ); - // Use the host/CPU implementation (there is no device implementation) - psampler->samplerHostSequence( momenta ); - } -} - -//-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk index c82234690e..c54be1436a 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix.mk @@ -31,6 +31,11 @@ ifeq ($(HRDCOD),) override HRDCOD = 0 endif +# default USEBUILDDIR = 1 +ifeq ($(USEBUILDDIR),) + override USEBUILDDIR = 1 +endif + # Check that the user-defined choices of BACKEND, FPTYPE, HELINL, HRDCOD are supported # (NB: use 'filter' and 'words' instead of 'findstring' because they properly handle whitespace-separated words) override SUPPORTED_BACKENDS = cuda hip cppnone cppsse4 cppavx2 cpp512y cpp512z cppauto @@ -72,7 +77,7 @@ endif # Build directory "full" tag (used for build lockfiles to prevent mixing builds with different options) override DIRTAG := $(patsubst cpp%%,%%,$(BACKEND))_$(FPTYPE)_inl$(HELINL)_hrd$(HRDCOD) -# Build directory: current directory by default, or build. if USEBUILDDIR==1 +# Build directory: build. by default (USEBUILDDIR=1), or current directory if USEBUILDDIR=0 # NB: using '=' (not ':=') ensures BACKEND is evaluated lazily after potential cppauto resolution ifeq ($(USEBUILDDIR),1) override MADMATRIX_BUILDDIR = build.$(BACKEND) @@ -321,7 +326,13 @@ ifeq ($(BACKEND),cuda) else ifeq ($(BACKEND),hip) # example architecture values MI200:gfx90a, MI350X:gfx942 - MADGRAPH_HIP_ARCHITECTURE ?= gfx942 + # auto detect arch if not set; if fail gfx942 default + ifeq ($(origin MADGRAPH_HIP_ARCHITECTURE),undefined) + MADGRAPH_HIP_ARCHITECTURE := $(shell $(HIP_HOME)/bin/rocm_agent_enumerator 2>/dev/null | grep -v gfx000 | sort -u | head -1) + endif + ifeq ($(MADGRAPH_HIP_ARCHITECTURE),) + MADGRAPH_HIP_ARCHITECTURE := gfx942 + endif # Set GPUCC as $(HIP_HOME)/bin/hipcc (it was already checked above that this exists) GPUCC = $(HIP_HOME)/bin/hipcc XCOMPILERFLAG = @@ -378,6 +389,16 @@ export GPUSUFFIX # Export BACKEND (resolved from cppauto above if needed; used e.g. to name the common library) export BACKEND +# Map BACKEND to its backend/ source subdirectory +ifneq ($(GPUCC),) + override BACKENDDIR = gpu +else ifeq ($(BACKEND),cppnone) + override BACKENDDIR = cpu +else + override BACKENDDIR = simd +endif +export BACKENDDIR + #------------------------------------------------------------------------------- #=== Configure ccache for C++ and CUDA/HIP builds @@ -399,7 +420,7 @@ endif #=== Configure common compiler flags for C++ and CUDA/HIP -INCFLAGS = -I. +INCFLAGS = -I. -I../../backend/$(BACKENDDIR) OPTFLAGS = -O3 # HIP requires -O2 to avoid "Memory access fault" in gq_ttq (#806) @@ -421,7 +442,7 @@ endif # The common library name carries the full BACKEND suffix so each vectorisation/GPU variant is distinct. MADMATRIX_COMMONLIB = madmatrix_common_$(BACKEND) LIBFLAGS = -L$(LIBDIR) -l$(MADMATRIX_COMMONLIB) -INCFLAGS += -I$(SRC) +INCFLAGS += -I$(SRC) -I$(SRC)/rambo #------------------------------------------------------------------------------- @@ -671,7 +692,7 @@ override TAG = $(patsubst cpp%%,%%,$(BACKEND))_$(FPTYPE)_inl$(HELINL)_hrd$(HRDCO # Export TAG (so that there is no need to check/define it again in src/Makefile) export TAG -# Build directory for object files: current directory by default, or build. if USEBUILDDIR==1 +# Build directory for object files: build. by default, or current directory if USEBUILDDIR=0 override BUILDDIR = $(MADMATRIX_BUILDDIR) ###override INCDIR = ../../include @@ -704,7 +725,13 @@ processid_short=$(shell basename $(CURDIR)) ###$(info processid_short=$(processid_short)) MADMATRIX_LIB = madmatrix_$(processid_short)_$(BACKEND) -objects_lib=$(BUILDDIR)/CPPProcess.o $(BUILDDIR)/color_sum.o $(BUILDDIR)/MatrixElementKernels.o $(BUILDDIR)/CrossSectionKernels.o $(BUILDDIR)/umami.o +objects_lib=$(BUILDDIR)/CPPProcess.o $(BUILDDIR)/color_sum.o $(BUILDDIR)/MatrixElementKernels.o $(BUILDDIR)/CrossSectionKernels.o $(BUILDDIR)/umami.o $(BUILDDIR)/SigmaKin.o + +# Backend-owned sources +vpath %%.cc ../../backend/$(BACKENDDIR) + +# Rambo/random-number sources +vpath %%.cc ../../src/rambo # Explicitly define the default goal (this is not necessary as it is the first target, which is implicitly the default goal) .DEFAULT_GOAL := all.$(TAG) @@ -751,11 +778,11 @@ endif # incompatible backends (different BACKEND, FPTYPE, etc.) in the same directory. # Use USEBUILDDIR=1 to build for multiple backends simultaneously without cleaning. ifeq ($(GPUCC),) -$(BUILDDIR)/%%.o : %%.cc *.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) +$(BUILDDIR)/%%.o : %%.cc *.h ../../backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(SRC)/rambo/*.h $(BUILDDIR)/.build.$(TAG) @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi $(CXX) $(CPPFLAGS) $(INCFLAGS) $(CXXFLAGS) -c $< -o $@ else -$(BUILDDIR)/%%.o : %%.cc *.h $(SRC)/*.h $(BUILDDIR)/.build.$(TAG) +$(BUILDDIR)/%%.o : %%.cc *.h ../../backend/$(BACKENDDIR)/*.h $(SRC)/*.h $(SRC)/rambo/*.h $(BUILDDIR)/.build.$(TAG) @if [ ! -d $(BUILDDIR) ]; then echo "mkdir -p $(BUILDDIR)"; mkdir -p $(BUILDDIR); fi $(GPUCC) $(CPPFLAGS) $(INCFLAGS) $(GPUFLAGS) -c -x $(GPULANGUAGE) $< -o $@ endif diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix_src.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix_src.mk index 8b13f7f4e1..ba88f6b923 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix_src.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix_src.mk @@ -12,7 +12,8 @@ SHELL := /bin/bash #------------------------------------------------------------------------------- #=== Configure common compiler flags for CUDA and C++ -INCFLAGS = -I. +# BACKENDDIR has been exported from the subprocess Makefile +INCFLAGS = -I. -I../backend/$(BACKENDDIR) #------------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk b/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk index 618cb9e42a..7fd3af109f 100644 --- a/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk +++ b/madgraph/iolibs/template_files/madmatrix/madmatrix_standalone.mk @@ -19,11 +19,22 @@ include madmatrix.mk #=== Standalone driver (check_sa.exe) configuration # Standalone-only object files (compiled via the generic %%.o pattern rule from -# madmatrix.mk). +# madmatrix.mk; RamboSamplingKernels.cc/CommonRandomNumberKernel.cc are found +# in ../../src/rambo/ via the vpath in madmatrix.mk override standalone_objects = $(BUILDDIR)/RamboSamplingKernels.o \ $(BUILDDIR)/CommonRandomNumberKernel.o \ $(BUILDDIR)/check_sa.o +# force relink when changing to before compiled backend +CHECK_SA_BACKEND_MARKER = .check_sa_backend +check_sa.exe: $(CHECK_SA_BACKEND_MARKER) +$(CHECK_SA_BACKEND_MARKER): FORCE + @if [ ! -f $(CHECK_SA_BACKEND_MARKER) ] || [ "$$(cat $(CHECK_SA_BACKEND_MARKER) 2>/dev/null)" != "$(BACKEND)" ]; then \ + echo $(BACKEND) > $(CHECK_SA_BACKEND_MARKER); \ + fi +.PHONY: FORCE +FORCE: + # Top-level standalone goal: process lib + standalone driver. .PHONY: standalone_all standalone_all: all.$(TAG) check_sa.exe @@ -49,7 +60,7 @@ clean: clean_standalone .PHONY: clean_standalone clean_standalone: rm -f $(BUILDDIR)/RamboSamplingKernels.o $(BUILDDIR)/CommonRandomNumberKernel.o $(BUILDDIR)/check_sa.o - rm -f check_sa.exe + rm -f check_sa.exe $(CHECK_SA_BACKEND_MARKER) # 'cleanall' from madmatrix.mk also wipes build.* directories, which already # covers our standalone objects when USEBUILDDIR=1. We only need to take care diff --git a/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h b/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h index d6e1d02e05..c966c17c40 100644 --- a/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h +++ b/madgraph/iolibs/template_files/madmatrix/mgOnGpuCxtypes.h @@ -51,7 +51,7 @@ #ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) #if defined MGONGPU_CUCXTYPE_CUCOMPLEX -namespace mg5amcGpu +namespace madmatrix { #if defined MGONGPU_FPTYPE_DOUBLE class cucomplex @@ -139,7 +139,7 @@ namespace mgOnGpu /* clang-format off */ #ifdef MGONGPU_CUCXTYPE_THRUST template __host__ __device__ constexpr operator thrust::complex() const { return thrust::complex( m_real, m_imag ); } #elif defined MGONGPU_CUCXTYPE_CUCOMPLEX - __host__ __device__ constexpr operator mg5amcGpu::cucomplex() const { return mg5amcGpu::cucomplex( m_real, m_imag ); } + __host__ __device__ constexpr operator madmatrix::cucomplex() const { return madmatrix::cucomplex( m_real, m_imag ); } #endif #else #ifdef MGONGPU_CPPCXTYPE_STDCOMPLEX @@ -163,11 +163,7 @@ namespace mgOnGpu /* clang-format off */ using mgOnGpu::cxsmpl; // Printout to stream for user defined types -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { template inline __host__ std::ostream& @@ -296,12 +292,8 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) TYPEDEFS //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { // --- Type definitions (complex type: cxtype) #ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) @@ -332,12 +324,8 @@ namespace mg5amcCpu // COMPLEX TYPES: (PLATFORM-SPECIFIC) FUNCTIONS AND OPERATORS //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { #if defined MGONGPU_CUCXTYPE_CXSMPL or defined MGONGPU_HIPCXTYPE_CXSMPL or defined MGONGPU_CPPCXTYPE_CXSMPL @@ -692,18 +680,14 @@ namespace mg5amcCpu return cxmake( c.real(), c.imag() ); } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix //========================================================================== // COMPLEX TYPES: WRAPPER OVER RI FLOATING POINT PAIR (cxtype_ref) //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { // The cxtype_ref class (a const reference to two non-const fp variables) was originally designed for cxtype_v::operator[] // It used to be included in the code only when MGONGPU_HAS_CPPCXTYPEV_BRK (originally MGONGPU_HAS_CPPCXTYPE_REF) is defined @@ -738,7 +722,7 @@ namespace mg5amcCpu return out; } -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h b/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h index a2489d05d5..0bc805cf72 100644 --- a/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h +++ b/madgraph/iolibs/template_files/madmatrix/mgOnGpuFptypes.h @@ -12,11 +12,11 @@ #include #include -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) +//One namespace. Split ber backend. #ifdef MGONGPUCPP_GPUIMPL // cuda -namespace mg5amcGpu +namespace madmatrix #else -namespace mg5amcCpu +namespace madmatrix #endif { //========================================================================== @@ -97,6 +97,6 @@ namespace mg5amcCpu //========================================================================== -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MGONGPUFPTYPES_H diff --git a/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h b/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h index 74d93f05a3..8a13174491 100644 --- a/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h +++ b/madgraph/iolibs/template_files/madmatrix/mgOnGpuVectors.h @@ -32,12 +32,8 @@ //#undef MGONGPU_HAS_CPPCXTYPEV_BRK // gcc test (very slightly slower? issue #172) #endif -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { #ifdef MGONGPU_CPPSIMD @@ -156,12 +152,8 @@ namespace mg5amcCpu //========================================================================== -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { #ifndef MGONGPUCPP_GPUIMPL @@ -927,6 +919,6 @@ namespace mg5amcCpu //========================================================================== -} // end namespace mg5amcGpu/mg5amcCpu +} // end namespace madmatrix #endif // MGONGPUVECTORS_H diff --git a/madgraph/iolibs/template_files/madmatrix/processConfig.h b/madgraph/iolibs/template_files/madmatrix/processConfig.h deleted file mode 100644 index bfaf20065b..0000000000 --- a/madgraph/iolibs/template_files/madmatrix/processConfig.h +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (C) 2020-2026 CERN and UCLouvain. -// Licensed under the GNU Lesser General Public License (version 3 or later). -// Created originally by: S. Roiser (May 2025) for the MG5aMC CUDACPP plugin. -// Integrated with the MadGraph7 project in Feb 2026. - -#ifndef MG5_CONFIG_%(processid_uppercase)s_H -#define MG5_CONFIG_%(processid_uppercase)s_H 1 - -namespace processConfig { - - constexpr int ndiagrams = %(ndiagrams)d; - -} - -#endif // MG5_CONFIG_%(processid_uppercase)s_H diff --git a/madgraph/iolibs/template_files/madmatrix/process_cc.inc b/madgraph/iolibs/template_files/madmatrix/process_cc.inc index 4dff7e869d..bf5f79e7c8 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_cc.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_cc.inc @@ -15,7 +15,6 @@ #include "mgOnGpuConfig.h" -#include "GpuRuntime.h" %(hel_amps_h)s #include "MemoryAccessAmplitudes.h" #include "MemoryAccessIflavorVec.h" @@ -27,16 +26,14 @@ #include "MemoryAccessMomenta.h" #include "MemoryAccessWavefunctions.h" #include "color_sum.h" -#include "processConfig.h" #include "MemoryAccessDenominators.h" #include "MemoryAccessNumerators.h" -#include "coloramps.h" +#include "ColorData.h" +#include "ProcessTables.h" #include #include -#include // for feenableexcept, fegetexcept and FE_XXX -#include // for FLT_MIN #include #include #include @@ -48,44 +45,5 @@ //-------------------------------------------------------------------------- -// Enable FPE traps (see #701, #733, #831 - except on MacOS where feenableexcept is not defined #730) -// [NB1: Fortran default is -ffpe-trap=none, i.e. FPE traps are not enabled, https://gcc.gnu.org/onlinedocs/gfortran/Debugging-Options.html] -// [NB2: Fortran default is -ffpe-summary=invalid,zero,overflow,underflow,denormal, i.e. warn at the end on STOP] -inline void -fpeEnable() -{ - static bool first = true; // FIXME: quick and dirty hack to do this only once (can be removed when separate C++/CUDA builds are implemented) - if( !first ) return; - first = false; -#ifndef __APPLE__ // on MacOS feenableexcept is not defined #730 - //int fpes = fegetexcept(); - //std::cout << "fpeEnable: analyse fegetexcept()=" << fpes << std::endl; - //std::cout << "fpeEnable: FE_DIVBYZERO is" << ( ( fpes & FE_DIVBYZERO ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_INEXACT is" << ( ( fpes & FE_INEXACT ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_INVALID is" << ( ( fpes & FE_INVALID ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_OVERFLOW is" << ( ( fpes & FE_OVERFLOW ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_UNDERFLOW is" << ( ( fpes & FE_UNDERFLOW ) ? " " : " NOT " ) << "enabled" << std::endl; - constexpr bool enableFPE = true; // this is hardcoded and no longer controlled by getenv( "CUDACPP_RUNTIME_ENABLEFPE" ) - if( enableFPE ) - { - std::cout << "INFO: The following Floating Point Exceptions will cause SIGFPE program aborts: FE_DIVBYZERO, FE_INVALID, FE_OVERFLOW" << std::endl; - feenableexcept( FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW ); // new strategy #831 (do not enable FE_UNDERFLOW) - //fpes = fegetexcept(); - //std::cout << "fpeEnable: analyse fegetexcept()=" << fpes << std::endl; - //std::cout << "fpeEnable: FE_DIVBYZERO is" << ( ( fpes & FE_DIVBYZERO ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_INEXACT is" << ( ( fpes & FE_INEXACT ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_INVALID is" << ( ( fpes & FE_INVALID ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_OVERFLOW is" << ( ( fpes & FE_OVERFLOW ) ? " " : " NOT " ) << "enabled" << std::endl; - //std::cout << "fpeEnable: FE_UNDERFLOW is" << ( ( fpes & FE_UNDERFLOW ) ? " " : " NOT " ) << "enabled" << std::endl; - } - else - { - //std::cout << "INFO: Do not enable SIGFPE traps for Floating Point Exceptions" << std::endl; - } -#else - //std::cout << "INFO: Keep default SIGFPE settings because feenableexcept is not available on MacOS" << std::endl; -#endif -} - %(process_function_definitions)s //========================================================================== diff --git a/madgraph/iolibs/template_files/madmatrix/process_class.inc b/madgraph/iolibs/template_files/madmatrix/process_class.inc index 59a6d07333..990f935f18 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_class.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_class.inc @@ -25,9 +25,6 @@ // Initialize process (read model parameters from file) virtual void initProc( const std::string& param_card_name ); - // Retrieve the compiler that was used to build this module - static const std::string getCompiler(); - // PDG id for a given flavor combination index and particle index (0-based). // Useful for tools that need to enumerate flavor combinations without going through UMAMI. static int flavorPDG( int iflavor, int ipar ); @@ -45,24 +42,16 @@ public: - // Process-independent compile-time constants - static constexpr int np4 = 4; // dimensions of 4-momenta (E,px,py,pz) - static constexpr int nw6 = %(nwave)d; // dimensions of each wavefunction (HELAS KEK 91-11): e.g. 6 for e+ e- -> mu+ mu- (fermions and vectors) - - // Process-dependent compile-time constants - static constexpr int npari = %(nincoming)d; // #particles in the initial state (incoming): e.g. 2 (e+ e-) for e+ e- -> mu+ mu- - static constexpr int nparf = %(noutcoming)d; // #particles in the final state (outgoing): e.g. 2 (mu+ mu-) for e+ e- -> mu+ mu- - static constexpr int npar = npari + nparf; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - static constexpr int ncomb = %(nbhel)d; // #helicity combinations: e.g. 16 for e+ e- -> mu+ mu- (2**4 = fermion spin up/down ** npar) - static constexpr int ndiagrams = %(ndiagrams)d; // #Feynman diagrams: e.g. 3 for e+ e- -> mu+ mu- - static constexpr int ncolor = %(ncolor)s; // the number of leading colors: e.g. 1 for e+ e- -> mu+ mu- - static constexpr int nmaxflavor = %(nmaxflavor)d; // the maximum number of flavor combinations - - // Hardcoded parameters for this process (constant class variables) - // [NB: this class assumes nprocesses==1 i.e. a single DSIG1 and no DSIG2 in Fortran (#272 and #343)] - // [NB: these parameters (e.g. nwf) are P1-specific, i.e. they are different for different P1 subdirectories (#644)] - // [NB: I am currently unable to get the right value of nwf in CPPProcess.h - will hardcode it in CPPProcess.cc instead (#644)] - //static const int nwf = ??; // #wavefunctions = #external (npar) + #internal: e.g. 5 for e+ e- -> mu+ mu- (1 internal is gamma or Z) + // Compile-time constants: single source of truth is ProcessData.h + static constexpr int np4 = ProcessData::np4; + static constexpr int nw6 = ProcessData::nw6; + static constexpr int npari = ProcessData::npari; + static constexpr int nparf = ProcessData::nparf; + static constexpr int npar = ProcessData::npar; + static constexpr int ncomb = ProcessData::ncomb; + static constexpr int ndiagrams = ProcessData::ndiagrams; + static constexpr int ncolor = ProcessData::ncolor; + static constexpr int nmaxflavor = ProcessData::nmaxflavor; private: /* clang-format on */ diff --git a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc index 082c373aa2..37ff7ec830 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_function_definitions.inc @@ -10,225 +10,22 @@ //========================================================================== // Class member functions for calculating the matrix elements for %(process_lines)s +// +// sigmaKin and everything it calls (calculate_jamps, good-helicity +// filtering, color/channel selection) are backend-owned, see +// backend/{cpu,simd,gpu}/SigmaKin.cc, which #includes the process-specific +// EvaluateDiagrams.inc generated alongside this file. This file only +// contains what's genuinely process-specific: the class constructor, +// physics-parameter/coupling setup (initProc), and the flavor accessor. +//========================================================================== -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { - __device__ INLINE unsigned int getChannelId( const unsigned int* allChannelIds -#ifndef MGONGPUCPP_GPUIMPL - , - const int ievt00, - bool sanityCheckMixedPrecision = true -#endif - ) - { - unsigned int channelId = 0; // disable multichannel single-diagram enhancement unless allChannelIds != nullptr -#ifdef MGONGPUCPP_GPUIMPL - using CID_ACCESS = DeviceAccessChannelIds; // non-trivial access: buffer includes all events - // SCALAR channelId for the current event (CUDA) or for the whole SIMD event page (C++) - // The cudacpp implementation ASSUMES (and checks! #898) that all channelIds are the same in a SIMD event page - if( allChannelIds != nullptr ) - { - const unsigned int* channelIds = allChannelIds; // fix #899 (distinguish channelIds and allChannelIds) - const uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) - // NB: channelIds_sv is a scalar in CUDA - channelId = channelIds_sv; - assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) - } -#else // Cuda or C++ - using CID_ACCESS = HostAccessChannelIds; // non-trivial access: buffer includes all events - // SCALAR channelId for the whole SIMD neppV2 event page (C++), i.e. one or two neppV event page(s) - // The cudacpp implementation ASSUMES (and checks! #898) that all channelIds are the same in a neppV2 SIMD event page - // **NB! in "mixed" precision, using SIMD, calculate_wavefunctions computes MEs for TWO neppV pages with a single channelId! #924 - if( allChannelIds != nullptr ) - { - // First - and/or only - neppV page of channels (iParity=0 => ievt0 = ievt00 + 0 * neppV) - const unsigned int* channelIds = CID_ACCESS::ieventAccessRecordConst( allChannelIds, ievt00 ); // fix bug #899/#911 - uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) -#ifndef MGONGPU_CPPSIMD - // NB: channelIds_sv is a scalar in no-SIMD C++ - channelId = channelIds_sv; -#else - // NB: channelIds_sv is a vector in SIMD C++ - channelId = channelIds_sv[0]; // element[0] - for( int i = 1; i < neppV; ++i ) // elements[1...neppV-1] - { - assert( channelId == channelIds_sv[i] ); // SANITY CHECK #898: check that all events in a SIMD vector have the same channelId - } -#endif - assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) - if( sanityCheckMixedPrecision ) - { -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - // Second neppV page of channels (iParity=1 => ievt0 = ievt00 + 1 * neppV) - const unsigned int* channelIds2 = CID_ACCESS::ieventAccessRecordConst( allChannelIds, ievt00 + neppV ); // fix bug #899/#911 - uint_v channelIds2_v = CID_ACCESS::kernelAccessConst( channelIds2 ); // fix #895 (compute this only once for all diagrams) - // **NB! in "mixed" precision, using SIMD, calculate_wavefunctions computes MEs for TWO neppV pages with a single channelId! #924 - for( int i = 0; i < neppV; ++i ) - { - assert( channelId == channelIds2_v[i] ); // SANITY CHECKS #898 #924: all events in the 2nd SIMD vector have the same channelId as that of the 1st SIMD vector - } -#endif - } - } -#endif // MGONGPUCPP_GPUIMPL - return channelId; - } - - constexpr int np4 = CPPProcess::np4; // dimensions of 4-momenta (E, px, py, pz) - constexpr int nw6 = CPPProcess::nw6; // dimensions of each wavefunction (HELAS KEK 91-11): e.g. 6 for e+ e- -> mu+ mu- (fermions and vectors) - constexpr int npar = CPPProcess::npar; // #particles in total (external = initial + final): e.g. 4 for e+ e- -> mu+ mu- - constexpr int ncomb = CPPProcess::ncomb; // #helicity combinations: e.g. 16 for e+ e- -> mu+ mu- (2**4 = fermion spin up/down ** npar) - constexpr int ncolor = CPPProcess::ncolor; // the number of leading colors - constexpr int nmaxflavor = CPPProcess::nmaxflavor; // the maximum number of flavor combinations - - // [NB: I am currently unable to get the right value of nwf in CPPProcess.h - will hardcode it in CPPProcess.cc instead (#644)] - //using CPPProcess::nwf; // #wavefunctions = #external (npar) + #internal: e.g. 5 for e+ e- -> mu+ mu- (1 internal is gamma or Z) - - using Parameters_dependentCouplings::ndcoup; // #couplings that vary event by event (depend on running alphas QCD) - using Parameters_independentCouplings::nicoup; // #couplings that are fixed for all events (do not depend on running alphas QCD) - - // The number of SIMD vectors of events processed by calculate_jamps -#if defined MGONGPU_CPPSIMD and defined MGONGPU_FPTYPE_DOUBLE and defined MGONGPU_FPTYPE2_FLOAT - constexpr int nParity = 2; -#else - constexpr int nParity = 1; -#endif - - // Physics parameters (masses, coupling, etc...) - // For CUDA performance, hardcoded constexpr's would be better: fewer registers and a tiny throughput increase - // However, physics parameters are user-defined through card files: use CUDA constant memory instead (issue #39) - // [NB if hardcoded parameters are used, it's better to define them here to avoid silent shadowing (issue #263)] - constexpr int nIPD = %(nipd)i; // SM independent parameters used in this CPPProcess.cc (FIXME? rename as sm_IndepParam?) - // Note: in the Python code generator, nIPD == nparam, while nIPC <= nicoup, because (see #823) - // nIPC may vary from one P*/CPPProcess.cc to another, while nicoup is defined in src/Param.h and is common to all P* - constexpr int nIPC = %(nipc)i; // SM independent couplings used in this CPPProcess.cc (FIXME? rename as sm_IndepCoupl?) - // nIPF are the number of SM independent flavor couplings, of type FLV_COUPLING - constexpr int nMF = FLV_COUPLING::max_flavor; - constexpr int nIPF = %(nipf)i; - // nDPF are the number of dependent (event-by-event, running-alphas) flavor couplings - constexpr int nDPF = %(ndpf)i; - static_assert( nIPC <= nicoup ); - static_assert( nIPD >= 0 ); // Hack to avoid build warnings when nIPD==0 is unused - static_assert( nIPC >= 0 ); // Hack to avoid build warnings when nIPC==0 is unused - static_assert( nMF >= 0 ); // Hack to avoid build warnings when nMF ==0 is unused - static_assert( nIPF >= 0 ); // Hack to avoid build warnings when nIPF==0 is unused - static_assert( nDPF >= 0 ); // Hack to avoid build warnings when nDPF==0 is unused -#ifdef MGONGPU_HARDCODE_PARAM - %(cipdhrdcod)s - %(cipchrdcod)s - %(cipfhrdcod)s -#else -#ifdef MGONGPUCPP_GPUIMPL - %(cipddevice)s - %(cipcdevice)s - %(cipfdevice)s -#else - %(cipdstatic)s - %(cipcstatic)s - %(cipfstatic)s -#endif -#endif - - // Dependent (event-by-event, running-alphas) flavor couplings: partner indices and - // the per-flavor idcoup are pure compile-time constants (the complex values are - // gathered per event page in calculate_jamps), so they are emitted the same way in - // all build modes (no CUDA constant memory copy needed). - %(cdpfdecl)s - - // AV Jan 2024 (PR #625): this ugly #define was the only way I found to avoid creating arrays[nBsm] in CPPProcess.cc if nBsm is 0 - // The problem is that nBsm is determined when generating Parameters.h, which happens after CPPProcess.cc has already been generated - // For simplicity, keep this code hardcoded also for SM processes (a nullptr is needed as in the case nBsm == 0) -#ifdef MGONGPUCPP_NBSMINDEPPARAM_GT_0 -#ifdef MGONGPU_HARDCODE_PARAM - __device__ const double* bsmIndepParam = Parameters::mdl_bsmIndepParam; -#else -#ifdef MGONGPUCPP_GPUIMPL - __device__ __constant__ double bsmIndepParam[Parameters::nBsmIndepParam]; -#else - static double bsmIndepParam[Parameters::nBsmIndepParam]; -#endif -#endif -#else -#ifdef MGONGPU_HARDCODE_PARAM - __device__ const double* bsmIndepParam = nullptr; -#else -#ifdef MGONGPUCPP_GPUIMPL - __device__ __constant__ double* bsmIndepParam = nullptr; -#else - static double* bsmIndepParam = nullptr; -#endif -#endif -#endif - - // Helicity combinations (and filtering of "good" helicity combinations) -#ifdef MGONGPUCPP_GPUIMPL - __device__ __constant__ short cHel[ncomb][npar]; - __device__ __constant__ int dcNGoodHel; - __device__ __constant__ int dcGoodHel[ncomb]; - __device__ __constant__ short cFlavors[nmaxflavor][npar]; -#else - static short cHel[ncomb][npar]; - static short cFlavors[nmaxflavor][npar]; -#endif - static int cNGoodHel; - static int cGoodHel[ncomb]; - - // Host-side flavor table: single source of truth for PDG ids (used by both the - // constructor copy into cFlavors and the public CPPProcess::flavorPDG accessor). -%(all_flavors)s - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - class DeviceAccessJamp2 - { - public: - static __device__ inline fptype& - kernelAccessIcol( fptype* buffer, const int icol ) - { - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - return buffer[icol * nevt + ievt]; - } - static __device__ inline const fptype& - kernelAccessIcolConst( const fptype* buffer, const int icol ) - { - const int nevt = gridDim.x * blockDim.x; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - return buffer[icol * nevt + ievt]; - } - }; -#endif + using namespace ProcessData; + using namespace ProcessTables; //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL - __device__ INLINE unsigned int - gpu_channelId( const unsigned int* allChannelIds ) - { - unsigned int channelId = 0; // disable multichannel single-diagram enhancement unless allChannelIds != nullptr - using CID_ACCESS = DeviceAccessChannelIds; // non-trivial access: buffer includes all events - // SCALAR channelId for the current event (CUDA) - if( allChannelIds != nullptr ) - { - const unsigned int* channelIds = allChannelIds; // fix #899 (distinguish channelIds and allChannelIds) - const uint_sv channelIds_sv = CID_ACCESS::kernelAccessConst( channelIds ); // fix #895 (compute this only once for all diagrams) - // NB: channelIds_sv is a scalar in CUDA - channelId = channelIds_sv; - assert( channelId > 0 ); // SANITY CHECK: scalar channelId must be > 0 if multichannel is enabled (allChannelIds != nullptr) - } - return channelId; - } -#endif - - //-------------------------------------------------------------------------- -%(all_sigmaKin)s - //-------------------------------------------------------------------------- - CPPProcess::CPPProcess( bool verbose, bool debug ) : m_verbose( verbose ) @@ -238,16 +35,7 @@ namespace mg5amcCpu #endif , m_masses() { - // Helicities for the process [NB do keep 'static' for this constexpr array, see issue #283] - // *** NB There is no automatic check yet that these are in the same order as Fortran! #569 *** -%(all_helicities)s -#ifdef MGONGPUCPP_GPUIMPL - gpuMemcpyToSymbol( cHel, tHel, ncomb * npar * sizeof( short ) ); - gpuMemcpyToSymbol( cFlavors, tFlavors, nmaxflavor * npar * sizeof( short ) ); -#else - memcpy( cHel, tHel, ncomb * npar * sizeof( short ) ); - memcpy( cFlavors, tFlavors, nmaxflavor * npar * sizeof( short ) ); -#endif + setHelicitiesAndFlavors( &tHel[0][0], &tFlavors[0][0] ); // Enable SIGFPE traps for Floating Point Exceptions #ifdef MGONGPUCPP_DEBUG @@ -290,33 +78,13 @@ namespace mg5amcCpu //m_pars->printDependentCouplings(); // now computed event-by-event (running alphas #373) } %(initProc_lines)s -#ifdef MGONGPUCPP_GPUIMPL - // Create the normalized color matrix in device memory - createNormalizedColorMatrix(); -#endif - // Read physics parameters like masses and couplings from user configuration files (static: initialize once) - // Then copy them to CUDA constant memory (issue #39) or its C++ emulation in file-scope static memory + createNormalizedColorMatrix(); // no-op on cpu/simd, pushes to device memory on gpu + // Read physics parameters like masses and couplings from user configuration files, + // then hand them to the backend-owned storage via the setters (see SigmaKin.cc). %(cipdassign)s %(cipcassign)s %(cipfassign)s -#ifdef MGONGPUCPP_GPUIMPL - %(cipd2tipdSym)s - %(cipc2tipcSym)s - %(cipf2tipfSym)s -#ifdef MGONGPUCPP_NBSMINDEPPARAM_GT_0 - if( Parameters::nBsmIndepParam > 0 ) - gpuMemcpyToSymbol( bsmIndepParam, m_pars->mdl_bsmIndepParam, Parameters::nBsmIndepParam * sizeof( double ) ); -#endif -#else - %(cipd2tipd)s - %(cipc2tipc)s - %(cipf2tipf)s -#ifdef MGONGPUCPP_NBSMINDEPPARAM_GT_0 - if( Parameters::nBsmIndepParam > 0 ) - memcpy( bsmIndepParam, m_pars->mdl_bsmIndepParam, Parameters::nBsmIndepParam * sizeof( double ) ); -#endif -#endif%(cipddump)s%(cipcdump)s%(cipfdump)s - //for ( int i=0; imdl_bsmIndepParam[i] = " << m_pars->mdl_bsmIndepParam[i] << std::endl; + %(bsmassign)s } #else // Initialize process (with hardcoded parameters) @@ -332,594 +100,13 @@ namespace mg5amcCpu //Parameters::printDependentCouplings(); // now computed event-by-event (running alphas #373) } %(hardcoded_initProc_lines)s -#ifdef MGONGPUCPP_GPUIMPL - // Create the normalized color matrix in device memory - createNormalizedColorMatrix(); -#endif - } -#endif - - //-------------------------------------------------------------------------- - - // Retrieve the compiler that was used to build this module - const std::string - CPPProcess::getCompiler() - { - std::stringstream out; - // HIP version (HIPCC) - // [Use __HIPCC__ instead of MGONGPUCPP_GPUIMPL here!] - // [This tests if 'hipcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] - // [Check 'hipcc -dM -E -x hip -I ../../src CPPProcess.cc | grep HIP'] -#ifdef __HIPCC__ -#if defined HIP_VERSION_MAJOR && defined HIP_VERSION_MINOR && defined HIP_VERSION_PATCH - out << "hipcc " << HIP_VERSION_MAJOR << "." << HIP_VERSION_MINOR << "." << HIP_VERSION_PATCH; -#else - out << "hipcc UNKNOWN"; -#endif - out << " ("; -#endif - // CUDA version (NVCC) - // [Use __NVCC__ instead of MGONGPUCPP_GPUIMPL here!] - // [This tests if 'nvcc' was used even to build a .cc file, even if not necessarily 'nvcc -x cu' for a .cu file] - // [Check 'nvcc --compiler-options -dM -E dummy.c | grep CUDA': see https://stackoverflow.com/a/53713712] -#ifdef __NVCC__ -#if defined __CUDACC_VER_MAJOR__ && defined __CUDACC_VER_MINOR__ && defined __CUDACC_VER_BUILD__ - out << "nvcc " << __CUDACC_VER_MAJOR__ << "." << __CUDACC_VER_MINOR__ << "." << __CUDACC_VER_BUILD__; -#else - out << "nvcc UNKNOWN"; -#endif - out << " ("; -#endif - // ICX version (either as CXX or as host compiler inside NVCC) -#if defined __INTEL_COMPILER -#error "icc is no longer supported: please use icx" -#elif defined __INTEL_LLVM_COMPILER // alternative: __INTEL_CLANG_COMPILER - out << "icx " << __INTEL_LLVM_COMPILER; -#ifdef __NVCC__ - out << ", "; -#else - out << " ("; -#endif -#endif - // CLANG version (either as CXX or as host compiler inside NVCC or inside ICX) -#if defined __clang__ -#if defined __clang_major__ && defined __clang_minor__ && defined __clang_patchlevel__ -#ifdef __APPLE__ - out << "Apple clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; -#else - out << "clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__; - /* - // === AV 26-Jan-2024 DISABLE THIS CODE (START) - // === AV 26-Jan-2024 First, it is totally wrong to assume that the CXX environment variable is used in the build! - // === AV 26-Jan-2024 Second and worse, here we need build time values, while CXX in this code is evaluated at runtime! - // GCC toolchain version inside CLANG - std::string tchainout; - std::string tchaincmd = "readelf -p .comment $(${CXX} -print-libgcc-file-name) |& grep 'GCC: (GNU)' | grep -v Warning | sort -u | awk '{print $5}'"; - std::unique_ptr tchainpipe( popen( tchaincmd.c_str(), "r" ), pclose ); - if( !tchainpipe ) throw std::runtime_error( "`readelf ...` failed?" ); - std::array tchainbuf; - while( fgets( tchainbuf.data(), tchainbuf.size(), tchainpipe.get() ) != nullptr ) tchainout += tchainbuf.data(); - tchainout.pop_back(); // remove trailing newline -#if defined __NVCC__ or defined __INTEL_LLVM_COMPILER - out << ", gcc " << tchainout; -#else - out << " (gcc " << tchainout << ")"; -#endif - // === AV 26-Jan-2024 DISABLE THIS CODE (END) - */ -#endif -#else - out << "clang UNKNOWKN"; -#endif -#else - // GCC version (either as CXX or as host compiler inside NVCC) -#if defined __GNUC__ && defined __GNUC_MINOR__ && defined __GNUC_PATCHLEVEL__ - out << "gcc " << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__; -#else - out << "gcc UNKNOWKN"; -#endif -#endif -#if defined __HIPCC__ or defined __NVCC__ or defined __INTEL_LLVM_COMPILER - out << ")"; -#endif - return out.str(); - } - - //-------------------------------------------------------------------------- - - __global__ void /* clang-format off */ - computeDependentCouplings( const fptype* allgs, // input: Gs[nevt] - fptype* allcouplings // output: couplings[nevt*ndcoup*2] -#ifndef MGONGPUCPP_GPUIMPL - , const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#endif - ) /* clang-format on */ - { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; - using G_ACCESS = DeviceAccessGs; - using C_ACCESS = DeviceAccessCouplings; - G2COUP( allgs, allcouplings, bsmIndepParam ); -#else - using namespace mg5amcCpu; - using G_ACCESS = HostAccessGs; - using C_ACCESS = HostAccessCouplings; - for( int ipagV = 0; ipagV < nevt / neppV; ++ipagV ) - { - const int ievt0 = ipagV * neppV; - const fptype* gs = MemoryAccessGs::ieventAccessRecordConst( allgs, ievt0 ); - fptype* couplings = MemoryAccessCouplings::ieventAccessRecord( allcouplings, ievt0 ); - G2COUP( gs, couplings, bsmIndepParam ); - } -#endif - } - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - void /* clang-format off */ - sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: indices of the flavor combinations - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - fptype* allNumerators, // output: multichannel numerators[nevt], running_sum_over_helicities - fptype* allDenominators, // output: multichannel denominators[nevt], running_sum_over_helicities - fptype_sv* allJamps, // tmp: jamp[ncolor*2*nevt] _for one helicity_ (reused in the getGoodHel helicity loop) - bool* isGoodHel, // output: isGoodHel[ncomb] - host array - const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) - { /* clang-format on */ - const int maxtry0 = 16; - fptype hstMEs[maxtry0]; - const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt16, e.g. in future VPUs) - constexpr int maxtry0 = std::max( 16, neppV ); // 16, but at least neppV (otherwise the npagV loop does not even start) - // Loop over only nevt events if nevt is < 16 (note that nevt is always >= neppV) - assert( nevt >= neppV ); - const int maxtry = std::min( maxtry0, nevt ); // 16, but at most nevt (avoid invalid memory access if nevt 1) - { - for (int i = comp_beg[icomp] - 1; i < comp_end[icomp]; i++) - { - if (pid_work[i] == 0) - continue; - int n_tot = 1; - for (int j = i + 1; j < comp_end[icomp]; j++) - { - if (pid_work[i] != pid_work[j]) - continue; - bool same_block = (block_len[i] == block_len[j]); - for (int k = 0; same_block && k < block_len[i]; k++) - { - if (cFlavors[iflavor][block_start[i] - 1 + k] != cFlavors[iflavor][block_start[j] - 1 + k]) - same_block = false; - } - if (same_block) - { - pid_work[j] = 0; - n_tot = n_tot + 1; - old_factor = old_factor / n_tot; - } - } - } - } - total_factor = total_factor * old_factor; - } - return total_factor; - } - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - __global__ void - normalise_output( fptype* allMEs, // output: allMEs[nevt], |M|^2 running_sum_over_helicities - const unsigned int* iflavorVec, - fptype* ghelAllNumerators, // input/tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - fptype* ghelAllDenominators, // input/tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable SDE enhancement (fix #899/#911) - bool storeChannelWeights, // if true, compute final multichannel weights - bool mulChannelWeight, // if true, multiply matrix element by channel weight - const fptype globaldenom) /* clang-format on */ - { - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) - allMEs[ievt] = allMEs[ievt] * broken_symmetry_factor(iflavorVec[ievt]) / globaldenom; - const int nevt = gridDim.x * blockDim.x; - if( storeChannelWeights ) // fix segfault #892 (not 'channelIds[0] != 0') - { - fptype* totAllNumerators = ghelAllNumerators; // reuse "helicity #0" buffer to compute the total over all helicities - fptype* totAllDenominators = ghelAllDenominators; // reuse "helicity #0" buffer to compute the total over all helicities - for( int ighel = 1; ighel < dcNGoodHel; ighel++ ) // NB: the loop starts at ighel=1 - { - fptype* hAllDenominators = ghelAllDenominators + ighel * nevt; - totAllDenominators[ievt] += hAllDenominators[ievt]; - fptype* hAllNumerators = ghelAllNumerators + ( ievt + ighel * nevt ) * processConfig::ndiagrams; - fptype* firstNumerator = ghelAllNumerators + ievt * processConfig::ndiagrams; - for( int idiag = 0; idiag < processConfig::ndiagrams; ++idiag ) - { - firstNumerator[idiag] += hAllNumerators[idiag]; - } - } - if( mulChannelWeight ) - { - unsigned int channelId = allChannelIds[ievt]; - allMEs[ievt] *= totAllNumerators[channelId - 1 + ievt * processConfig::ndiagrams] / totAllDenominators[ievt]; - } - } - return; + createNormalizedColorMatrix(); // no-op on cpu/simd, pushes to device memory on gpu + %(cipdhrdassign)s + %(cipchrdassign)s + %(cipfhrdassign)s + %(bsmhrdassign)s } #endif //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - add_and_select_hel( int* allselhel, // output: helicity selection[nevt] - const fptype* allrndhel, // input: random numbers[nevt] for helicity selection - fptype* ghelAllMEs, // input/tmp: allMEs for nGoodHel <= ncomb individual/runningsum helicities (index is ighel) - fptype* allMEs, // output: allMEs[nevt], final sum over helicities - const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) - { - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) - // Compute the sum of MEs over all good helicities (defer this after the helicity loop to avoid breaking streams parall> - for( int ighel = 0; ighel < dcNGoodHel; ighel++ ) - { - allMEs[ievt] += ghelAllMEs[ighel * nevt + ievt]; - ghelAllMEs[ighel * nevt + ievt] = allMEs[ievt]; // reuse the buffer to store the running sum for helicity selection - } - // Event-by-event random choice of helicity #403 - //printf( "select_hel: ievt=%%4d rndhel=%%f\n", ievt, allrndhel[ievt] ); - for( int ighel = 0; ighel < dcNGoodHel; ighel++ ) - { - if( allrndhel[ievt] < ( ghelAllMEs[ighel * nevt + ievt] / allMEs[ievt] ) ) - { - const int ihelF = dcGoodHel[ighel] + 1; // NB Fortran [1,ncomb], cudacpp [0,ncomb-1] - allselhel[ievt] = ihelF; - //printf( "select_hel: ievt=%%4d ihel=%%4d\n", ievt, ihelF ); - break; - } - } - return; - } -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - select_col_and_diag( int* allselcol, // output: color selection[nevt] - unsigned int* allDiagramIdsOut, // output: sampled diagram ids - const fptype* allrndcol, // input: random numbers[nevt] for color selection - const fptype* allrnddiagram, // input: random numbers[nevt] for diagram selection - const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable SDE enhancement (fix #899/#911) - const fptype_sv* allJamp2s, // input: jamp2[ncolor][nevt] for color choice (nullptr if disabled) - const fptype* allNumerators, // input: all numerators - const fptype* allDenominators, // input: all denominators - const int nevt ) // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) - { - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) - // SCALAR channelId for the current event (CUDA) - unsigned int channelId = gpu_channelId( allChannelIds ); - // Event-by-event random choice of color #402 - - // Event-by-event random choice of channel - if( allrnddiagram != nullptr ) - { - fptype numerator_sum = 0., normalization = 0.; - for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) - { - if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - normalization += allNumerators[ievt * processConfig::ndiagrams + ichan]; - } - channelId = mgOnGpu::nchannels; - for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) - { - if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - numerator_sum += allNumerators[ievt * processConfig::ndiagrams + ichan]; - if( allrnddiagram[ievt] < numerator_sum / normalization ) - { - channelId = ichan + 1; - break; - } - } - allDiagramIdsOut[ievt] = channelId; - } - - if( channelId != 0 ) // no event-by-event choice of color if channelId == 0 (fix FPE #783) - { - if( channelId > mgOnGpu::nchannels ) - { - printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%%d which is greater than nchannels=%%d\n", channelId, mgOnGpu::nchannels ); - assert( channelId <= mgOnGpu::nchannels ); // SANITY CHECK #919 #910 - } - // Determine the jamp2 for this event (TEMPORARY? could do this with a dedicated memory accessor instead...) - fptype_sv jamp2_sv[ncolor] = { 0 }; - assert( allJamp2s != nullptr ); // sanity check - using J2_ACCESS = DeviceAccessJamp2; - for( int icolC = 0; icolC < ncolor; icolC++ ) - jamp2_sv[icolC] = J2_ACCESS::kernelAccessIcolConst( allJamp2s, icolC ); - // NB (see #877): in the array channel2iconfig, the input index uses C indexing (channelId -1), the output index uses F indexing (iconfig) - // NB (see #917): mgOnGpu::channel2iconfig returns an int (which may be -1), not an unsigned int! - const int iconfig = mgOnGpu::channel2iconfig[channelId - 1]; // map N_diagrams to N_config <= N_diagrams configs (fix LHE color mismatch #856: see also #826, #852, #853) - if( iconfig <= 0 ) - { - printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%%d which has no associated SDE iconfig\n", channelId ); - assert( iconfig > 0 ); // SANITY CHECK #917 - } - else if( iconfig > (int)mgOnGpu::nconfigSDE ) - { - printf( "INTERNAL ERROR! Cannot choose an event-by-event random color for channelId=%%d (invalid SDE iconfig=%%d\n > nconfig=%%d)", channelId, iconfig, mgOnGpu::nconfigSDE ); - assert( iconfig <= (int)mgOnGpu::nconfigSDE ); // SANITY CHECK #917 - } - fptype targetamp[ncolor] = { 0 }; - // NB (see #877): explicitly use 'icolC' rather than 'icol' to indicate that icolC uses C indexing in [0, N_colors-1] - for( int icolC = 0; icolC < ncolor; icolC++ ) - { - if( icolC == 0 ) - targetamp[icolC] = 0; - else - targetamp[icolC] = targetamp[icolC - 1]; - // NB (see #877): in the array icolamp, the input index uses C indexing (iconfig -1) - if( mgOnGpu::icolamp[iconfig - 1][icolC] ) targetamp[icolC] += jamp2_sv[icolC]; - } - //printf( "sigmaKin: ievt=%%4d rndcol=%%f\n", ievt, allrndcol[ievt] ); - for( int icolC = 0; icolC < ncolor; icolC++ ) - { - if( allrndcol[ievt] < ( targetamp[icolC] / targetamp[ncolor - 1] ) ) - { - allselcol[ievt] = icolC + 1; // NB Fortran [1,ncolor], cudacpp [0,ncolor-1] - //printf( "sigmaKin: ievt=%%d icol=%%d\n", ievt, icolC+1 ); - break; - } - } - } - else - { - allselcol[ievt] = 0; // no color selected in Fortran range [1,ncolor] if channelId == 0 (see #931) - } - return; - } -#endif - - //-------------------------------------------------------------------------- - // Evaluate |M|^2, part independent of incoming flavour - - void /* clang-format off */ - sigmaKin( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: indices of the flavor combinations - const fptype* allrndhel, // input: random numbers[nevt] for helicity selection - const fptype* allrndcol, // input: random numbers[nevt] for color selection - const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable single-diagram enhancement (fix #899/#911) - const fptype* allrnddiagram, // input: random numbers[nevt] for diagram sampling - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - int* allselhel, // output: helicity selection[nevt] -#ifdef MGONGPUCPP_GPUIMPL - int* allselcol, // output: helicity selection[nevt] - fptype* colAllJamp2s, // tmp: allJamp2s super-buffer for ncolor individual colors, running sum over colors and helicities - fptype* ghelAllNumerators, // tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - fptype* ghelAllDenominators, // tmp: allDenominators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - unsigned int* allDiagramIdsOut, // output: multichannel channelIds[nevt] (1 to #diagrams) - bool mulChannelWeight, // if true, multiply channel weight to ME output - fptype* ghelAllMEs, // tmp: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - fptype* ghelAllJamps, // tmp: jamp[2*ncolor*nGoodHel*nevt] super-buffer for nGoodHel <= ncomb individual helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) - const bool async, - const int gpublocks, // input: cuda gpublocks - const int gputhreads // input: cuda gputhreads -#else - int* allselcol, // output: helicity selection[nevt] - fptype* allNumerators, // output: multichannel numerators[nevt], running_sum_over_helicities - fptype* allDenominators, // output: multichannel denominators[nevt], running_sum_over_helicities - unsigned int* allDiagramIdsOut, // output: multichannel channelIds[nevt] (1 to #diagrams) - bool mulChannelWeight, // if true, multiply channel weight to ME output - const int nevt // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#endif - ) /* clang-format on */ - { - mgDebugInitialise(); - - // SANITY CHECKS for cudacpp code generation (see issues #272 and #343 and PRs #619, #626, #360, #396 and #754) - // These variable are not used anywhere else in the code and their scope is limited to this sanity check - { - // nprocesses == 2 may happen for "mirror processes" such as P0_uux_ttx within pp_tt012j (see PR #754) - constexpr int nprocesses = %(nproc)i; - static_assert( nprocesses == 1 || nprocesses == 2, "Assume nprocesses == 1 or 2" ); - constexpr int process_id = %(proc_id)i; // code generation source: %(proc_id_source)s - static_assert( process_id == 1, "Assume process_id == 1" ); - } - - // Denominators: spins, colors and identical particles - constexpr int helcolDenominators[1] = { %(den_factors)s }; // assume nprocesses == 1 (#272 and #343) - -#ifndef MGONGPUCPP_GPUIMPL - //assert( (size_t)(allmomenta) %% mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment [COMMENT OUT TO TEST MISALIGNED ACCESS] - //assert( (size_t)(allMEs) %% mgOnGpu::cppAlign == 0 ); // SANITY CHECK: require SIMD-friendly alignment [COMMENT OUT TO TEST MISALIGNED ACCESS] - using E_ACCESS = HostAccessMatrixElements; // non-trivial access: buffer includes all events - using NUM_ACCESS = HostAccessNumerators; // non-trivial access: buffer includes all events - using DEN_ACCESS = HostAccessDenominators; // non-trivial access: buffer includes all events -#endif - - // Start sigmaKin_lines -%(sigmaKin_lines)s } // end namespace diff --git a/madgraph/iolibs/template_files/madmatrix/process_h.inc b/madgraph/iolibs/template_files/madmatrix/process_h.inc index fb715a8778..04668e1c35 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_h.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_h.inc @@ -16,111 +16,27 @@ #include "mgOnGpuConfig.h" -#include "mgOnGpuVectors.h" +#if __has_include("mgOnGpuVectors.h") // simd only +#include "mgOnGpuVectors.h" +#else +#include "mgOnGpuCxtypes.h" +#endif -#include "GpuAbstraction.h" #include "Parameters.h" +#include "ProcessData.h" +#include "SigmaKin.h" // for the setters CPPProcess's constructor/initProc call +%(helamps_h)s #include //-------------------------------------------------------------------------- -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { %(process_class_definitions)s //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL - __global__ void - computeDependentCouplings( const fptype* allgs, // input: Gs[nevt] - fptype* allcouplings ); // output: couplings[nevt*ndcoup*2] -#else - __global__ void - computeDependentCouplings( const fptype* allgs, // input: Gs[nevt] - fptype* allcouplings, // output: couplings[nevt*ndcoup*2] - const int nevt ); // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#endif - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - void - sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: index of the flavor combination - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - fptype* allNumerators, // output: multichannel numerators[nevt], running_sum_over_helicities - fptype* allDenominators, // output: multichannel denominators[nevt], running_sum_over_helicities - fptype_sv* allJamps, // output: jamp[ncolor*2*nevt] - bool* isGoodHel, // output: isGoodHel[ncomb] - device array (GPU device implementation) - const int nevt ); // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#else - void - sigmaKin_getGoodHel( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: index of the flavor combination - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - fptype* allNumerators, // output: multichannel numerators[nevt], running_sum_over_helicities - fptype* allDenominators, // output: multichannel denominators[nevt], running_sum_over_helicities - bool* isGoodHel, // output: isGoodHel[ncomb] - host array (C++ implementation) - const int nevt ); // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#endif /* clang-format on */ - - //-------------------------------------------------------------------------- - - int // output: nGoodHel (the number of good helicity combinations out of ncomb) - sigmaKin_setGoodHel( const bool* isGoodHel ); // input: isGoodHel[ncomb] - host array - - //-------------------------------------------------------------------------- - -#ifdef MGONGPUCPP_GPUIMPL /* clang-format off */ - void - sigmaKin( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: index of the flavor combination - const fptype* allrndhel, // input: random numbers[nevt] for helicity selection - const fptype* allrndcol, // input: random numbers[nevt] for color selection - const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable single-diagram enhancement (fix #899/#911) - const fptype* allrnddiagram, // input: random numbers[nevt] for channel sampling - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - int* allselhel, // output: helicity selection[nevt] - int* allselcol, // output: helicity selection[nevt] - fptype* colAllJamp2s, // tmp: allJamp2s super-buffer for ncolor individual colors, running sum over colors and helicities - fptype* ghelAllNumerators, // tmp: allNumerators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - fptype* ghelAllDenominators, // tmp: allDenominators super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - unsigned int* allDiagramIdsOut, // output: multichannel channelIds[nevt] (1 to #diagrams) - bool mulChannelWeight, // if true, multiply channel weight to ME output - fptype* ghelAllMEs, // tmp: allMEs super-buffer for nGoodHel <= ncomb individual helicities (index is ighel) - fptype* ghelAllJamps, // tmp: allJamps super-buffer[2][ncol][nGoodHel][nevt] for nGoodHel <= ncomb individual helicities - fptype2* ghelAllBlasTmp, // tmp: allBlasTmp super-buffer for nGoodHel <= ncomb individual helicities - gpuBlasHandle_t* pBlasHandle, // input: cuBLAS/hipBLAS handle - gpuStream_t* ghelStreams, // input: cuda streams (index is ighel: only the first nGoodHel <= ncomb are non-null) - const bool async, // input: if true, run everything asynchronously in first stream in ghelStreams - const int gpublocks, // input: cuda gpublocks - const int gputhreads ); // input: cuda gputhreads -#else - void - sigmaKin( const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: index of the flavor combination - const fptype* allrndhel, // input: random numbers[nevt] for helicity selection - const fptype* allrndcol, // input: random numbers[nevt] for color selection - const unsigned int* allChannelIds, // input: multichannel channelIds[nevt] (1 to #diagrams); nullptr to disable single-diagram enhancement (fix #899) - const fptype* allrnddiagram, // input: random numbers[nevt] for channel sampling - fptype* allMEs, // output: allMEs[nevt], |M|^2 final_avg_over_helicities - int* allselhel, // output: helicity selection[nevt] - int* allselcol, // output: helicity selection[nevt] - fptype* allNumerators, // tmp: multichannel numerators[nevt], running_sum_over_helicities - fptype* allDenominators, // tmp: multichannel denominators[nevt], running_sum_over_helicities - unsigned int* allDiagramIdsOut, // output: multichannel channelIds[nevt] (1 to #diagrams) - bool mulChannelWeight, // if true, multiply channel weight to ME output - const int nevt ); // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) -#endif /* clang-format on */ - + // sigmaKin, computeDependentCouplings and friends are backend-owned: + // see backend/{cpu,simd,gpu}/SigmaKin.h for their declarations. //-------------------------------------------------------------------------- } diff --git a/madgraph/iolibs/template_files/madmatrix/process_sigmaKin_function.inc b/madgraph/iolibs/template_files/madmatrix/process_sigmaKin_function.inc index 227301a6ea..7e7a754092 100644 --- a/madgraph/iolibs/template_files/madmatrix/process_sigmaKin_function.inc +++ b/madgraph/iolibs/template_files/madmatrix/process_sigmaKin_function.inc @@ -17,7 +17,7 @@ gpuMemset( allMEs, 0, nevt * sizeof( fptype ) ); gpuMemset( ghelAllJamps, 0, cNGoodHel * ncolor * mgOnGpu::nx2 * nevt * sizeof( fptype ) ); gpuMemset( colAllJamp2s, 0, ncolor * nevt * sizeof( fptype ) ); - gpuMemset( ghelAllNumerators, 0, cNGoodHel * processConfig::ndiagrams * nevt * sizeof( fptype ) ); + gpuMemset( ghelAllNumerators, 0, cNGoodHel * ProcessData::ndiagrams * nevt * sizeof( fptype ) ); gpuMemset( ghelAllDenominators, 0, cNGoodHel * nevt * sizeof( fptype ) ); gpuMemset( ghelAllMEs, 0, cNGoodHel * nevt * sizeof( fptype ) ); #else @@ -29,11 +29,11 @@ fptype* MEs = E_ACCESS::ieventAccessRecord( allMEs, ievt0 ); fptype_sv& MEs_sv = E_ACCESS::kernelAccess( MEs ); MEs_sv = fptype_sv{ 0 }; - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ProcessData::ndiagrams ); fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); - for( int i = 0; i < processConfig::ndiagrams; ++i ) + for( int i = 0; i < ProcessData::ndiagrams; ++i ) { numerators_sv[i] = fptype_sv{ 0 }; } @@ -60,7 +60,7 @@ { const int ihel = cGoodHel[ighel]; fptype* hAllJamps = ghelAllJamps + ighel * nevt; // HACK: bypass DeviceAccessJamp (consistent with layout defined there) - fptype* hAllNumerators = ghelAllNumerators + ighel * nevt * processConfig::ndiagrams; + fptype* hAllNumerators = ghelAllNumerators + ighel * nevt * ProcessData::ndiagrams; fptype* hAllDenominators = ghelAllDenominators + ighel * nevt; gpuLaunchKernelStream( calculate_jamps, gpublocks, gputhreads, ghelStreams[ighel], ihel, allmomenta, allcouplings, iflavorVec, hAllJamps, storeChannelWeights, hAllNumerators, hAllDenominators, colAllJamp2s, nevt, false ); } @@ -192,14 +192,14 @@ for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) { if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - normalization += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + normalization += allNumerators[ievt / neppV * neppV * ProcessData::ndiagrams + ichan * neppV + ieppV %% neppV]; } channelIdVec[ieppV] = mgOnGpu::nchannels; for( unsigned int ichan = 0; ichan < mgOnGpu::nchannels; ichan++ ) { if( mgOnGpu::channel2iconfig[ichan] == -1 ) continue; - numerator_sum += allNumerators[ievt / neppV * neppV * processConfig::ndiagrams + + numerator_sum += allNumerators[ievt / neppV * neppV * ProcessData::ndiagrams + ichan * neppV + ieppV %% neppV]; if( allrnddiagram[ievt] < numerator_sum / normalization ) { @@ -295,7 +295,7 @@ if( mulChannelWeight && allChannelIds != nullptr ) // fix segfault #892 (not 'channelIds[0] != 0') { const unsigned int channelId = getChannelId( allChannelIds, ievt0, false ); - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); + fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * ProcessData::ndiagrams ); fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); diff --git a/madgraph/iolibs/template_files/madmatrix/runTest.cc b/madgraph/iolibs/template_files/madmatrix/runTest.cc index 369d7197cc..4601fa3a8f 100644 --- a/madgraph/iolibs/template_files/madmatrix/runTest.cc +++ b/madgraph/iolibs/template_files/madmatrix/runTest.cc @@ -9,6 +9,11 @@ #include "mgOnGpuConfig.h" +#ifdef MGONGPUCPP_GPUIMPL +#include "GpuAbstraction.h" +#include "GpuRuntime.h" // for checkGpu(), used by gpuMemcpyFromSymbol() +#endif + #include "CPPProcess.h" #include "MadgraphTest.h" #include "MatrixElementKernels.h" @@ -18,15 +23,33 @@ #include "MemoryBuffers.h" #include "RamboSamplingKernels.h" #include "RandomNumberKernels.h" -#include "coloramps.h" -#include "epoch_process_id.h" +#include "ColorData.h" +#include "ProcessData.h" #include -#ifdef MGONGPUCPP_GPUIMPL -using namespace mg5amcGpu; +using namespace madmatrix; + +// Host-accessible copy of mgOnGpu::channel2iconfig, needed only by setChannelIds() +#ifndef MGONGPUCPP_GPUIMPL +inline const int* +getHostChannel2iconfig() +{ + return mgOnGpu::channel2iconfig; +} #else -using namespace mg5amcCpu; +inline const int* +getHostChannel2iconfig() +{ + static int hostCopy[mgOnGpu::nchannels]; + static bool first = true; + if( first ) + { + first = false; + gpuMemcpyFromSymbol( hostCopy, mgOnGpu::channel2iconfig, mgOnGpu::nchannels * sizeof( int ) ); + } + return hostCopy; +} #endif struct CUDA_CPU_TestBase : public TestDriverBase @@ -64,7 +87,7 @@ struct CUDA_CPU_TestBase : public TestDriverBase //for( unsigned int idiagram = 1; idiagram < CPPProcess::ndiagrams; idiagram++ ) // two bugs #920 and #919 for( unsigned int idiagram = 0; idiagram < mgOnGpu::nchannels; idiagram++ ) // fix #920 and work around #919 { - if( mgOnGpu::hostChannel2iconfig[idiagram] == iconfig ) + if( getHostChannel2iconfig()[idiagram] == iconfig ) { channelId = idiagram + 1; // fix #917 (NB add +1 because channelId uses F indexing) break; diff --git a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/src/rambo/CommonRandomNumberKernel.cc similarity index 96% rename from madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc rename to madgraph/iolibs/template_files/madmatrix/src/rambo/CommonRandomNumberKernel.cc index 16671b5686..4ca90fea38 100644 --- a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/src/rambo/CommonRandomNumberKernel.cc @@ -4,18 +4,18 @@ // Further modified by: J. Teig, A. Valassi (2021-2024). // Integrated with the MadGraph7 project in Feb 2026. +#include "mgOnGpuConfig.h" + #include "CommonRandomNumbers.h" +#ifdef MGONGPUCPP_GPUIMPL #include "GpuAbstraction.h" +#endif #include "MemoryBuffers.h" #include "RandomNumberKernels.h" #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/CommonRandomNumbers.h b/madgraph/iolibs/template_files/madmatrix/src/rambo/CommonRandomNumbers.h similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/CommonRandomNumbers.h rename to madgraph/iolibs/template_files/madmatrix/src/rambo/CommonRandomNumbers.h diff --git a/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/src/rambo/CurandRandomNumberKernel.cc similarity index 98% rename from madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc rename to madgraph/iolibs/template_files/madmatrix/src/rambo/CurandRandomNumberKernel.cc index 0c8e6e6445..da8ba6d7e2 100644 --- a/madgraph/iolibs/template_files/madmatrix/CurandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/src/rambo/CurandRandomNumberKernel.cc @@ -24,11 +24,7 @@ inline void assertCurand( curandStatus_t code, const char *file, int line, bool } #endif /* clang-format on */ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_CURAND diff --git a/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc b/madgraph/iolibs/template_files/madmatrix/src/rambo/HiprandRandomNumberKernel.cc similarity index 98% rename from madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc rename to madgraph/iolibs/template_files/madmatrix/src/rambo/HiprandRandomNumberKernel.cc index b806da2e09..1b2f2b2ec9 100644 --- a/madgraph/iolibs/template_files/madmatrix/HiprandRandomNumberKernel.cc +++ b/madgraph/iolibs/template_files/madmatrix/src/rambo/HiprandRandomNumberKernel.cc @@ -28,11 +28,7 @@ inline void assertHiprand( hiprandStatus_t code, const char *file, int line, boo } #endif /* clang-format on */ -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- #ifndef MGONGPU_HAS_NO_HIPRAND diff --git a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc b/madgraph/iolibs/template_files/madmatrix/src/rambo/RamboSamplingKernels.cc similarity index 99% rename from madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc rename to madgraph/iolibs/template_files/madmatrix/src/rambo/RamboSamplingKernels.cc index d40b1b4174..2505402c53 100644 --- a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.cc +++ b/madgraph/iolibs/template_files/madmatrix/src/rambo/RamboSamplingKernels.cc @@ -6,7 +6,9 @@ #include "RamboSamplingKernels.h" +#ifdef MGONGPUCPP_GPUIMPL #include "GpuRuntime.h" +#endif #include "MemoryAccessMomenta.h" #include "MemoryAccessRandomNumbers.h" #include "MemoryAccessWeights.h" @@ -16,11 +18,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h b/madgraph/iolibs/template_files/madmatrix/src/rambo/RamboSamplingKernels.h similarity index 98% rename from madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h rename to madgraph/iolibs/template_files/madmatrix/src/rambo/RamboSamplingKernels.h index 6244d55618..4e0253a852 100644 --- a/madgraph/iolibs/template_files/madmatrix/RamboSamplingKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/src/rambo/RamboSamplingKernels.h @@ -13,11 +13,7 @@ #include -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h b/madgraph/iolibs/template_files/madmatrix/src/rambo/RandomNumberKernels.h similarity index 98% rename from madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h rename to madgraph/iolibs/template_files/madmatrix/src/rambo/RandomNumberKernels.h index 0fb880bb36..afbab3d408 100644 --- a/madgraph/iolibs/template_files/madmatrix/RandomNumberKernels.h +++ b/madgraph/iolibs/template_files/madmatrix/src/rambo/RandomNumberKernels.h @@ -18,11 +18,7 @@ struct curandGenerator_st; struct rocrand_generator_base_type; typedef rocrand_generator_base_type hiprandGenerator_st; -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/massless_rambo.h b/madgraph/iolibs/template_files/madmatrix/src/rambo/massless_rambo.h similarity index 99% rename from madgraph/iolibs/template_files/madmatrix/massless_rambo.h rename to madgraph/iolibs/template_files/madmatrix/src/rambo/massless_rambo.h index bcd3990bed..4f64d1aead 100644 --- a/madgraph/iolibs/template_files/madmatrix/massless_rambo.h +++ b/madgraph/iolibs/template_files/madmatrix/src/rambo/massless_rambo.h @@ -22,11 +22,7 @@ #include // Simplified rambo version for 2 to N (with N>=2) processes with massless particles -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { namespace massless_rambo { diff --git a/madgraph/iolibs/template_files/madmatrix/rambo.h b/madgraph/iolibs/template_files/madmatrix/src/rambo/rambo.h similarity index 100% rename from madgraph/iolibs/template_files/madmatrix/rambo.h rename to madgraph/iolibs/template_files/madmatrix/src/rambo/rambo.h diff --git a/madgraph/iolibs/template_files/madmatrix/testmisc.cc b/madgraph/iolibs/template_files/madmatrix/testmisc.cc index 74c4902dc5..7e0b2f5ef0 100644 --- a/madgraph/iolibs/template_files/madmatrix/testmisc.cc +++ b/madgraph/iolibs/template_files/madmatrix/testmisc.cc @@ -12,7 +12,7 @@ #include "mgOnGpuVectors.h" #include "constexpr_math.h" -#include "epoch_process_id.h" +#include "ProcessData.h" #include "valgrind.h" #include @@ -31,12 +31,8 @@ #define XTESTID( s ) TESTID( s ) -// NB: namespaces mg5amcGpu and mg5amcCpu includes types which are defined in different ways for CPU and GPU builds (see #318 and #725) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +//One namespace. Split ber backend. +namespace madmatrix { #ifdef MGONGPU_CPPSIMD /* clang-format off */ #define EXPECT_TRUE_sv( cond ) { bool_v mask( cond ); EXPECT_TRUE( maskand( mask ) ); } @@ -65,11 +61,7 @@ namespace mg5amcCpu TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testmisc ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif + using namespace madmatrix; //-------------------------------------------------------------------------- diff --git a/madgraph/iolibs/template_files/madmatrix/testxxx.cc b/madgraph/iolibs/template_files/madmatrix/testxxx.cc index 4fcf7d26b6..b9af47f014 100644 --- a/madgraph/iolibs/template_files/madmatrix/testxxx.cc +++ b/madgraph/iolibs/template_files/madmatrix/testxxx.cc @@ -14,7 +14,7 @@ #include "MemoryAccessMomenta.h" #include "MemoryAccessWavefunctions.h" #include "MemoryBuffers.h" -#include "epoch_process_id.h" +#include "ProcessData.h" #include @@ -33,11 +33,7 @@ #define XTESTID( s ) TESTID( s ) -#ifdef MGONGPUCPP_GPUIMPL -namespace mg5amcGpu -#else -namespace mg5amcCpu -#endif +namespace madmatrix { std::string fpeHandlerMessage = "unknown"; int fpeHandlerIevt = -1; @@ -54,11 +50,7 @@ namespace mg5amcCpu TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif + using namespace madmatrix; #ifndef __APPLE__ // test #701 (except on MacOS where feenableexcept is not defined #730) auto fpeHandlerDefault = signal( SIGFPE, fpeHandlerTestxxx ); #endif @@ -73,9 +65,9 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) assert( nevt %% neppV == 0 ); // nevt must be a multiple of neppV // Fill in the input momenta #ifdef MGONGPUCPP_GPUIMPL - mg5amcGpu::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] + madmatrix::PinnedHostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] #else - mg5amcCpu::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] + madmatrix::HostBufferMomenta hstMomenta( nevt ); // AOSOA[npagM][npar=4][np4=4][neppM] #endif /* clang-format off */ // NB NEW TESTS FOR DEBUGGING #701: KEEP TWO SEPARATE SETS (16-SIMD-VECTORS!) OF TESTS FOR M==0 AND M!=0! const fptype par0[np4 * nevt] = // AOS[nevt][np4] @@ -317,11 +309,7 @@ TEST( XTESTID( MG_EPOCH_PROCESS_ID ), testxxx ) { for( int ievt = 0; ievt < nevt; ievt++ ) { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; -#else - using namespace mg5amcCpu; -#endif + using namespace madmatrix; if( debug ) { std::cout << std::endl; diff --git a/madmatrix/model_handling.py b/madmatrix/model_handling.py index b4d1bac787..5b84bfe19a 100644 --- a/madmatrix/model_handling.py +++ b/madmatrix/model_handling.py @@ -1530,15 +1530,17 @@ def get_process_class_definitions(self, write=True): return replace_dict # AV - replace export_cpp.OneProcessExporterCPP method (fix CPPProcess.cc) + # backend_separation: cIPD/cIPC/cIPF/bsmIndepParam storage now lives in + # backend/{cpu,simd,gpu}/SigmaKin.cc. This method still computes the local + # tIPD/tIPC/tIPF assignment text (genuinely process-specific: which SM + # parameters/couplings this process uses), but ends each with a call to + # the corresponding backend setter instead of a storage-declaration + # variant + direct memcpy/gpuMemcpyToSymbol. def get_process_function_definitions(self, write=True): """The complete class definition for the process""" replace_dict = super().get_process_function_definitions(write=False) # defines replace_dict['initProc_lines'] - replace_dict['hardcoded_initProc_lines'] = replace_dict['initProc_lines'].replace( 'm_pars->', 'Parameters::') - couplings2order_indep = [] - ###replace_dict['ncouplings'] = len(self.couplings2order) - ###replace_dict['ncouplingstimes2'] = 2 * replace_dict['ncouplings'] + replace_dict['hardcoded_initProc_lines'] = self.get_hardcoded_initProc_lines(self.matrix_elements[0]) replace_dict['nparams'] = len(self.params2order) - ###replace_dict['nmodels'] = replace_dict['nparams'] + replace_dict['ncouplings'] # AV unused??? replace_dict['coupling_list'] = ' ' replace_dict['hel_amps_cc'] = '#include \"HelAmps_%s.cc\"' % self.model_name # AV coupling = [''] * len(self.couplings2order) @@ -1556,59 +1558,52 @@ def get_process_function_definitions(self, write=True): if "aS" in key and coup in coup_list: keep = False if keep: coupling_indep.append( coup ) # AV only indep! replace_dict['ncouplings'] = len(coupling_indep) # AV only indep! - replace_dict['nipc'] = len(coupling_indep) + + # dependent (running-alphas, event-by-event) flavor couplings -> for ProcessTables.h (Step 3). + flv_couplings_dep = [''] * len(self.couporderflv_dep) + for flv_coup, pos in self.couporderflv_dep.items(): + flv_couplings_dep[pos] = flv_coup + + # Cache counts for edit_processdata()/edit_processtables(), which run + # after generate_process_files() has populated couplings2order etc. + self._nipc = len(coupling_indep) + self._nipd = len(params) + self._nipf = len(flv_couplings) + self._ndpf = len(flv_couplings_dep) + if len(coupling_indep) > 0: - replace_dict['cipcassign'] = 'const cxtype tIPC[nIPC] = { cxmake( m_pars->%s ) };'\ - % ( ' ), cxmake( m_pars->'.join(coupling_indep) ) # AV only indep! - replace_dict['cipcdevice'] = '__device__ __constant__ fptype cIPC[nIPC * 2];' - replace_dict['cipcstatic'] = 'static fptype cIPC[nIPC * 2];' - replace_dict['cipc2tipcSym'] = 'gpuMemcpyToSymbol( cIPC, tIPC, nIPC * sizeof( cxtype ) );' - replace_dict['cipc2tipc'] = 'memcpy( cIPC, tIPC, nIPC * sizeof( cxtype ) );' - replace_dict['cipcdump'] = '\n //for ( int i=0; i'.join(flv_couplings) ) - replace_dict['cipfdevice'] = """__device__ __constant__ int cIPF_partner1[nMF * nIPF]; - __device__ __constant__ int cIPF_partner2[nMF * nIPF]; - __device__ __constant__ fptype cIPF_value[nMF * nIPF * 2];""" - replace_dict['cipfstatic'] = """static int cIPF_partner1[nMF * nIPF]; - static int cIPF_partner2[nMF * nIPF]; - static fptype cIPF_value[nMF * nIPF * 2];""" - replace_dict['cipf2tipfSym'] = """gpuMemcpyToSymbol( cIPF_partner1, tIPF_partner1, nMF * nIPF * sizeof( int ) ); - gpuMemcpyToSymbol( cIPF_partner2, tIPF_partner2, nMF * nIPF * sizeof( int ) ); - gpuMemcpyToSymbol( cIPF_value , tIPF_value , nMF * nIPF * sizeof( cxtype ) );""" - replace_dict['cipf2tipf'] = """memcpy( cIPF_partner1, tIPF_partner1, nMF * nIPF * sizeof( int ) ); - memcpy( cIPF_partner2, tIPF_partner2, nMF * nIPF * sizeof( int ) ); - memcpy( cIPF_value , tIPF_value , nMF * nIPF * sizeof( cxtype ) );""" - replace_dict['cipfdump'] = ''' - //for ( int i=0; i < nIPD; i++ ) { - // std::cout << std::setprecision(17) << "tIPF[i].partner1 = { "; - // for ( int j=0; j < nMF-1; j++ ) std::cout << std::setprecision(17) << tIPF[i].partner1[j] << ", "; - // std::cout << std::setprecision(17) << tIPF[i].partner1[nMF-1] << " }" << std::endl; - // std::cout << std::setprecision(17) << "tIPF[i].partner2 = { "; - // for ( int j=0; j < nMF-1; j++ ) std::cout << std::setprecision(17) << tIPF[i].partner2[j] << ", "; - // std::cout << std::setprecision(17) << tIPF[i].partner2[nMF-1] << " }" << std::endl; - // std::cout << std::setprecision(17) << "tIPF[i].value = { "; - // for ( int j=0; j < nMF-1; j++ ) std::cout << std::setprecision(17) << tIPF[i].value[j] << ", "; - // std::cout << std::setprecision(17) << tIPF[i].value[nMF-1] << " }" << std::endl; - //} -''' - coup_str_hrd_partner1 = '__device__ const int cIPF_partner1[nMF * nIPF] = { ' - coup_str_hrd_partner2 = '__device__ const int cIPF_partner2[nMF * nIPF] = { ' - coup_str_hrd_value = '__device__ const fptype cIPF_value[nMF * nIPF * 2] = { ' - for flv_coup in flv_couplings: - coup_str_hrd_partner1 += ( ('Parameters_%(model_name)s::%(coup)s.param1' % {"model_name": self.model_name, "coup": flv_coup} + '[%d], ') * nMF) % ( *range(nMF), ) - coup_str_hrd_partner2 += ( ('Parameters_%(model_name)s::%(coup)s.param2' % {"model_name": self.model_name, "coup": flv_coup} + '[%d], ') * nMF) % ( *range(nMF), ) - # Guard against null value[] slots: flavor combinations with no - # coupling are left null by the FLV_COUPLING constructor, so the - # hardcoded cIPF_value read must not dereference an uninitialised - # pointer. Mirrors the runtime path (value[j] ? *value[j] : 0). - value_base = 'Parameters_%(model_name)s::%(coup)s.value' % {"model_name": self.model_name, "coup": flv_coup} - for i in range(nMF): - coup_str_hrd_value += '(fptype)( %(b)s[%(i)d] ? %(b)s[%(i)d]->real() : 0. ), ' % {'b': value_base, 'i': i} - coup_str_hrd_value += '(fptype)( %(b)s[%(i)d] ? %(b)s[%(i)d]->imag() : 0. ), ' % {'b': value_base, 'i': i} - coup_str_hrd_partner1 = coup_str_hrd_partner1[:-2] + ' };' - coup_str_hrd_partner2 = coup_str_hrd_partner2[:-2] + ' };' - coup_str_hrd_value = coup_str_hrd_value[:-2] + ' };' - replace_dict['cipfhrdcod'] = '%s\n %s\n %s' % (coup_str_hrd_partner1, coup_str_hrd_partner2, coup_str_hrd_value) + } + setFlavorCouplings( tIPF_partner1, tIPF_partner2, tIPF_value );""" % ( ', m_pars->'.join(flv_couplings) ) + # Hardcoded variant: same shape, values come from Parameters:: instead of m_pars-> + hrd_lines = ['int tIPF_partner1[nMF * nIPF];', ' int tIPF_partner2[nMF * nIPF];', ' cxtype tIPF_value[nMF * nIPF];'] + for i, flv_coup in enumerate(flv_couplings): + base = 'Parameters_%s::%s' % (self.model_name, flv_coup) + for j in range(nMF): + hrd_lines.append(' tIPF_partner1[%d] = %s.param1[%d];' % (i * nMF + j, base, j)) + hrd_lines.append(' tIPF_partner2[%d] = %s.param2[%d];' % (i * nMF + j, base, j)) + hrd_lines.append(' tIPF_value[%d] = %s.value[%d] ? *%s.value[%d] : cxtype{};' % (i * nMF + j, base, j, base, j)) + hrd_lines.append(' setFlavorCouplings( tIPF_partner1, tIPF_partner2, tIPF_value );') + replace_dict['cipfhrdassign'] = '\n '.join(hrd_lines) else: replace_dict['cipfassign'] = '' - replace_dict['cipfdevice'] = """__device__ __constant__ int* cIPF_partner1 = nullptr; // unused as nIPF=0' - __device__ __constant__ int* cIPF_partner2 = nullptr; // unused as nIPF=0' - __device__ __constant__ fptype* cIPF_value = nullptr; // unused as nIPF=0'""" - replace_dict['cipfstatic'] = """static int* cIPF_partner1 = nullptr; // unused as nIPF=0' - static int* cIPF_partner2 = nullptr; // unused as nIPF=0' - static fptype* cIPF_value = nullptr; // unused as nIPF=0'""" - replace_dict['cipf2tipfSym'] = '' - replace_dict['cipf2tipf'] = '' - replace_dict['cipfdump'] = '' - replace_dict['cipfhrdcod'] = """__device__ const int* cIPF_partner1 = nullptr; // unused as nIPF=0' - __device__ const int* cIPF_partner2 = nullptr; // unused as nIPF=0' - __device__ const fptype* cIPF_value = nullptr; // unused as nIPF=0'""" - - # dependent (running-alphas, event-by-event) flavor couplings -> cDPF_* (Step 3). - # Unlike cIPF, these have NO baked-in value array: partner1/partner2 and the - # per-flavor idcoup (the index of the underlying dependent coupling in the - # event-by-event allcouplings buffer) are pure codegen constants. The actual - # complex values are gathered per event page in calculate_jamps (see - # super_get_matrix_element_calls). The single-leg serialization mirrors the - # Fortran side / write_flv_couplings (the unmerged partner has flavor index 1). - flv_couplings_dep = [''] * len(self.couporderflv_dep) - for flv_coup, pos in self.couporderflv_dep.items(): - flv_couplings_dep[pos] = flv_coup - replace_dict['ndpf'] = len(flv_couplings_dep) - if len(flv_couplings_dep): - nMF = max(len(ids) for ids in self.model['merged_particles'].values()) - flv_map = self.helas_call_writer.flv_couplings_map - partner1_vals, partner2_vals, idcoup_vals = [], [], [] - for name in flv_couplings_dep: - coupl = flv_map[name] - p1 = [-1] * nMF - p2 = [-1] * nMF - idc = ['-1'] * nMF - for key, gc in coupl.flavors.items(): - nonzero = [i for i in key if i != 0] - if len(nonzero) == 2: - k1, k2 = nonzero - else: - # single merged leg: unmerged partner has flavor index 1 - k1 = nonzero[0]; k2 = 1 - p1[k1-1] = k2-1 - p2[k2-1] = k1-1 - # symbolic idcoup: resolves to the position of this dependent coupling - # in the event-by-event allcouplings buffer (== COUPs index), defined in - # Parameters_dependentCouplings (Parameters_.h) - idc[k1-1] = '(int)Parameters_dependentCouplings::idcoup_%s' % gc - partner1_vals += [str(v) for v in p1] - partner2_vals += [str(v) for v in p2] - idcoup_vals += idc - cdpfdecl = '__device__ const int cDPF_partner1[nMF * nDPF] = { %s };\n' % ', '.join(partner1_vals) - cdpfdecl += ' __device__ const int cDPF_partner2[nMF * nDPF] = { %s };\n' % ', '.join(partner2_vals) - cdpfdecl += ' __device__ const int cDPF_idcoup[nMF * nDPF] = { %s };' % ', '.join(idcoup_vals) - replace_dict['cdpfdecl'] = cdpfdecl - else: - replace_dict['cdpfdecl'] = """__device__ const int* cDPF_partner1 = nullptr; // unused as nDPF=0 - __device__ const int* cDPF_partner2 = nullptr; // unused as nDPF=0 - __device__ const int* cDPF_idcoup = nullptr; // unused as nDPF=0""" - - # FIXME! Here there should be different code generated depending on MGONGPUCPP_NBSMINDEPPARAM_GT_0 (issue #827) - replace_dict['all_helicities'] = self.get_helicity_matrix(self.matrix_elements[0]) - replace_dict['all_helicities'] = replace_dict['all_helicities'] .replace('helicities', 'tHel') - replace_dict['all_flavors'] = self.get_flavor_matrix(self.matrix_elements[0]) - replace_dict['all_flavors'] = replace_dict['all_flavors'].replace('flavors', 'tFlavors') - color_amplitudes = [me.get_color_amplitudes() for me in self.matrix_elements] # as in OneProcessExporterCPP.get_process_function_definitions - replace_dict['ncolor'] = len(color_amplitudes[0]) - # broken_symmetry_factor function: use the shared decay-aware symmetry - # data (same as the Fortran / standalone_cpp exporters) instead of the - # old simple PID-count version, so identical-particle and decay-chain - # symmetry factors match across backends. - _, nincoming = self.matrix_elements[0].get_nexternal_ninitial() - replace_dict['nincoming'] = nincoming - process = self.matrix_elements[0].get('processes')[0] - sym_data = export_v4.ProcessExporterFortran._get_broken_symmetry_data( - process, nincoming) - export_v4.ProcessExporterFortran._fill_broken_sym_replace_dict( - replace_dict, sym_data) + replace_dict['cipfhrdassign'] = '' + + # mdl_bsmIndepParam only exists as a symbol at all when the model has + # BSM params (see MGONGPUCPP_NBSMINDEPPARAM_GT_0, PR #625) - this is a + # pre-existing macro, not new, and must stay a #ifdef (not a runtime + # check) since the symbol itself may not exist to name-lookup. + replace_dict['bsmassign'] = '''#ifdef MGONGPUCPP_NBSMINDEPPARAM_GT_0 + if( Parameters::nBsmIndepParam > 0 ) setBsmIndepParam( m_pars->mdl_bsmIndepParam, Parameters::nBsmIndepParam ); +#endif''' + replace_dict['bsmhrdassign'] = '''#ifdef MGONGPUCPP_NBSMINDEPPARAM_GT_0 + if( Parameters::nBsmIndepParam > 0 ) setBsmIndepParam( Parameters::mdl_bsmIndepParam, Parameters::nBsmIndepParam ); +#endif''' file = self.read_template_file(self.process_definition_template) % replace_dict # HACK! ignore write=False case if len(params) == 0: # remove cIPD from OpenMP pragma (issue #349) @@ -1779,158 +1678,47 @@ def get_sigmaKin_lines(self, color_amplitudes, write=True): return replace_dict # AV - modify export_cpp.OneProcessExporterCPP method (fix CPPProcess.cc) + # backend_separation: calculate_jamps' prologue (signature, memory-access + # typedefs) and epilogue (color-choice bookkeeping, jamp output copy - was + # process_matrix.inc) are backend-conditional but process-independent, so + # they now live as real files in backend/{cpu,simd,gpu}/CalculateJamps.cc. + # Only the diagram/vertex-call sequence (helas_calls) is process-specific; + # it is written here to EvaluateDiagrams.inc, which that file #includes. def get_all_sigmaKin_lines(self, color_amplitudes, class_name): - """Get sigmaKin_process for all subprocesses for CPPProcess.cc""" - ret_lines = [] + """Write EvaluateDiagrams.inc for CPPProcess.cc""" if self.single_helicities: - ###misc.sprint(type(self.helas_call_writer)) - ###misc.sprint( 'before get_matrix_element_calls', self.matrix_elements[0].get_number_of_wavefunctions() ) # WRONG value of nwf, eg 7 for gg_tt helas_calls = self.helas_call_writer.get_matrix_element_calls(\ self.matrix_elements[0], color_amplitudes[0], multi_channel_map = self.multi_channel_map ) - ###misc.sprint( 'after get_matrix_element_calls', self.matrix_elements[0].get_number_of_wavefunctions() ) # CORRECT value of nwf, eg 5 for gg_tt assert len(self.matrix_elements) == 1 or len(self.matrix_elements) == 2 # how to handle if this is not true? self.couplings2order = self.helas_call_writer.couplings2order self.couporderflv = self.helas_call_writer.couporderflv self.couporderflv_dep = self.helas_call_writer.couporderflv_dep self.params2order = self.helas_call_writer.params2order - ret_lines.append(""" - // Evaluate QCD partial amplitudes jamps for this given helicity from Feynman diagrams - // Also compute running sums over helicities adding jamp2, numerator, denominator - // (NB: this function no longer handles matrix elements as the color sum has now been moved to a separate function/kernel) - // In CUDA, this function processes a single event - // ** NB1: NEW Nov2024! In CUDA this is now a kernel function (it used to be a device function) - // ** NB2: NEW Nov2024! in CUDA this now takes a channelId array as input (it used to take a scalar channelId as input) - // In C++, this function processes a single event "page" or SIMD vector (or for two in "mixed" precision mode, nParity=2) - // *** NB: in C++, calculate_jamps accepts a SCALAR channelId because it is GUARANTEED that all events in a SIMD vector have the same channelId #898 - __global__ void /* clang-format off */ - calculate_jamps( int ihel, - const fptype* allmomenta, // input: momenta[nevt*npar*4] - const fptype* allcouplings, // input: couplings[nevt*ndcoup*2] - const unsigned int* iflavorVec, // input: indices of the flavor combinations -#ifdef MGONGPUCPP_GPUIMPL - fptype* allJamps, // output: jamp[2*ncolor*nevt] buffer for one helicity _within a super-buffer for dcNGoodHel helicities_ - bool storeChannelWeights, - fptype* allNumerators, // input/output: multichannel numerators[nevt], add helicity ihel - fptype* allDenominators, // input/output: multichannel denominators[nevt], add helicity ihel - fptype* colAllJamp2s, // output: allJamp2s[ncolor][nevt] super-buffer, sum over col/hel (nullptr to disable) - const int nevt, // input: #events (for cuda: nevt == ndim == gpublocks*gputhreads) - const bool processAllHelicities // input: if true, use blockIdx.y to index helicities -#else - cxtype_sv* allJamp_sv, // output: jamp_sv[ncolor] (float/double) or jamp_sv[2*ncolor] (mixed) for this helicity - bool storeChannelWeights, - fptype* allNumerators, // input/output: multichannel numerators[nevt], add helicity ihel - fptype* allDenominators, // input/output: multichannel denominators[nevt], add helicity ihel - fptype_sv* jamp2_sv, // output: jamp2[nParity][ncolor][neppV] for color choice (nullptr if disabled) - const int ievt00 // input: first event number in current C++ event page (for CUDA, ievt depends on threadid) -#endif - ) - //ALWAYS_INLINE // attributes are not permitted in a function definition - { -#ifdef MGONGPUCPP_GPUIMPL - using namespace mg5amcGpu; - using M_ACCESS = DeviceAccessMomenta; // non-trivial access: buffer includes all events - using W_ACCESS = DeviceAccessWavefunctions; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event - using A_ACCESS = DeviceAccessAmplitudes; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event - using CD_ACCESS = DeviceAccessCouplings; // non-trivial access (dependent couplings): buffer includes all events - using CI_ACCESS = DeviceAccessCouplingsFixed; // TRIVIAL access (independent couplings): buffer for one event - using F_ACCESS = DeviceAccessIflavorVec; // non-trivial access: buffer includes all events - using NUM_ACCESS = DeviceAccessNumerators; // non-trivial access: buffer includes all events - using DEN_ACCESS = DeviceAccessDenominators; // non-trivial access: buffer includes all events -#else - using namespace mg5amcCpu; - using M_ACCESS = HostAccessMomenta; // non-trivial access: buffer includes all events - using W_ACCESS = HostAccessWavefunctions; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event - using A_ACCESS = HostAccessAmplitudes; // TRIVIAL ACCESS (no kernel splitting yet): buffer for one event - using CD_ACCESS = HostAccessCouplings; // non-trivial access (dependent couplings): buffer includes all events - using CI_ACCESS = HostAccessCouplingsFixed; // TRIVIAL access (independent couplings): buffer for one event - using F_ACCESS = HostAccessIflavorVec; // non-trivial access: buffer includes all events - using NUM_ACCESS = HostAccessNumerators; // non-trivial access: buffer includes all events - using DEN_ACCESS = HostAccessDenominators; // non-trivial access: buffer includes all events -#endif - mgDebug( 0, __FUNCTION__ ); - //bool debug = true; -#ifndef MGONGPUCPP_GPUIMPL - //debug = ( ievt00 >= 64 && ievt00 < 80 && ihel == 3 ); // example: debug #831 - //if( debug ) printf( \"calculate_jamps: ievt00=%d ihel=%2d\\n\", ievt00, ihel ); -#else - //const int ievt = blockDim.x * blockIdx.x + threadIdx.x; - //debug = ( ievt == 0 ); - //if( debug ) printf( \"calculate_jamps: ievt=%6d ihel=%2d\\n\", ievt, ihel ); - if (processAllHelicities) { - int ighel = blockIdx.y; - ihel = dcGoodHel[ighel]; - allJamps = allJamps + ighel * nevt; - allNumerators = allNumerators + ighel * nevt * processConfig::ndiagrams; - allDenominators = allDenominators + ighel * nevt; - } -#endif /* clang-format on */""") - nwavefuncs = self.matrix_elements[0].get_number_of_wavefunctions() - ret_lines.append(""" - // The variable nwf (which is specific to each P1 subdirectory, #644) is only used here - // It is hardcoded here because various attempts to hardcode it in CPPProcess.h at generation time gave the wrong result... - static const int nwf = %i; // #wavefunctions = #external (npar) + #internal: e.g. 5 for e+ e- -> mu+ mu- (1 internal is gamma or Z)"""%nwavefuncs ) - ret_lines.append(""" - // Local TEMPORARY variables for a subset of Feynman diagrams in the given CUDA event (ievt) or C++ event page (ipagV) - // [NB these variables are reused several times (and re-initialised each time) within the same event or event page] - // ** NB: in other words, amplitudes and wavefunctions still have TRIVIAL ACCESS: there is currently no need - // ** NB: to have large memory structurs for wavefunctions/amplitudes in all events (no kernel splitting yet)! - //MemoryBufferWavefunctions w_buffer[nwf]{ neppV }; - // Create memory for both momenta and wavefunctions separately, and later wrap them in ALOHAOBJ - fptype_sv pvec_sv[nwf][np4]; - cxtype_sv w_sv[nwf][nw6]; // particle wavefunctions within Feynman diagrams (nw6 is 4: spin wavefunctions, momenta are no more included, see before) - cxtype_sv amp_sv[1]; // invariant amplitude for one given Feynman diagram - - // Wrap the memory into ALOHAOBJ - ALOHAOBJ aloha_obj[nwf]; - for( int iwf = 0; iwf < nwf; iwf++ ) aloha_obj[iwf] = ALOHAOBJ{pvec_sv[iwf], w_sv[iwf]}; - fptype* amp_fp; - amp_fp = reinterpret_cast( amp_sv );""") + content = [] if fd_gauge: - ret_lines.append(""" + content.append(""" // special temporary ALOHAOBJ to hold F/Vtmp values in the combined vertex functions while using the FD gauge fptype_sv pvec_sv_tmp[1][np4]; - cxtype_sv w_sv_tmp[1][nw6]; + cxtype_sv w_sv_tmp[1][nw6]; ALOHAOBJ aloha_obj_tmp[1]; aloha_obj_tmp[0] = ALOHAOBJ{pvec_sv_tmp[0], w_sv_tmp[0]}; - + // special one value to hold tmp vertex value inside the combined vertex functions while using the FD gauge cxtype_sv amp_tmp_sv[1]; //to ensure proper aligment for vector instructions fptype* amp_tmp_fp; amp_tmp_fp = reinterpret_cast( amp_tmp_sv ); """) - ret_lines.append(""" - // Local variables for the given CUDA event (ievt) or C++ event page (ipagV) - // [jamp: sum (for one event or event page) of the invariant amplitudes for all Feynman diagrams in a given color combination] - cxtype_sv jamp_sv[ncolor] = {}; // all zeros (NB: vector cxtype_v IS initialized to 0, but scalar cxtype is NOT, if "= {}" is missing!) - - // === Calculate wavefunctions and amplitudes for all diagrams in all processes === - // === (for one event in CUDA, for one - or two in mixed mode - SIMD event pages in C++ === - - // START LOOP ON IPARITY - for( int iParity = 0; iParity < nParity; ++iParity ) - { -#ifndef MGONGPUCPP_GPUIMPL - const int ievt0 = ievt00 + iParity * neppV; -#endif""") - ret_lines += helas_calls + content += helas_calls else: - ret_lines.extend([self.get_sigmaKin_single_process(i, me) \ - for i, me in enumerate(self.matrix_elements)]) - #ret_lines.extend([self.get_matrix_single_process(i, me, - # color_amplitudes[i], - # class_name) \ - # for i, me in enumerate(self.matrix_elements)]) - file_extend = [] - for i, me in enumerate(self.matrix_elements): - file = self.get_matrix_single_process( i, me, color_amplitudes[i], class_name ) - file = strip_banner(file, banner_mark = "!") # skip first 8 lines in process_matrix.inc (copyright) - file_extend.append( file ) - assert i == 0, "more than one ME in get_all_sigmaKin_lines" # AV sanity check (added for color_sum.cc but valid independently) - ret_lines.extend( file_extend ) - return '\n'.join(ret_lines) + content = [self.get_sigmaKin_single_process(i, me) \ + for i, me in enumerate(self.matrix_elements)] + ff = open(pjoin(self.path, 'EvaluateDiagrams.inc'), 'w') + ff.write('\n'.join(content)) + ff.close() + return '' # AV - modify export_cpp.OneProcessExporterCPP method (replace '# Process' by '// Process') def get_process_info_lines(self, matrix_element): @@ -1944,18 +1732,109 @@ def get_process_info_lines(self, matrix_element): def generate_process_files(self): """Generate mgOnGpuConfig.h, CPPProcess.cc, CPPProcess.h, check_sa.cc, gXXX.cu links""" ###misc.sprint('Entering OneProcessExporterMadMatrix.generate_process_files') - self.edit_mgonGPU() - self.edit_processidfile() # AV new file (NB this is Sigma-specific, should not be a symlink to Subprocesses) - self.edit_processConfig() # sub process specific, not to be symlinked from the Subprocesses directory - self.edit_colorsum() # AV new file (NB this is Sigma-specific, should not be a symlink to Subprocesses) - self.edit_coloramps() - self.edit_memorybuffers() # AV new file (NB this is generic in Subprocesses and then linked in Sigma-specific) - self.edit_memoryaccesscouplings() # AV new file (NB this is generic in Subprocesses and then linked in Sigma-specific) + self.edit_colordata() # AV new file (NB this is Sigma-specific, should not be a symlink to Subprocesses) super().generate_process_files() + # needs to be after get_matrix_element_calls to have nwf ready + self.edit_processdata() + self.edit_processtables() # NB: symlink of cudacpp.mk to makefile is overwritten by madevent makefile if this exists (#480) # NB: this relies on the assumption that cudacpp code is generated before madevent code files.ln(pjoin(self.path, "..", "makefile"), self.path, "makefile") + # seperate process constants to one truth file + def edit_processdata(self): + """Generate ProcessData.h""" + template = open(pjoin(self.template_path, 'madmatrix', 'ProcessData.h'), 'r').read() + me = self.matrix_elements[0] + replace_dict = {} + nexternal, nincoming = me.get_nexternal_ninitial() + replace_dict['nincoming'] = nincoming + replace_dict['noutcoming'] = nexternal - nincoming + replace_dict['nbhel'] = me.get_helicity_combinations() + replace_dict['ndiagrams'] = len(me.get('diagrams')) + replace_dict['nmaxflavor'] = len(me.get_external_flavors_with_iden()) + replace_dict['nwave'] = 4 + (1 if fd_gauge else 0) + replace_dict['ncolor'] = len(me.get_color_amplitudes()) + replace_dict['nwf'] = me.get_number_of_wavefunctions() + replace_dict['nproc'] = sum(2 if m.get('has_mirror_process') else 1 for m in self.matrix_elements) + replace_dict['proc_id'] = self.proc_id if self.proc_id > 0 else 1 + den_factors = [str(m.get_denominator_factor()) for m in self.matrix_elements] + replace_dict['den_factors'] = ",".join(den_factors) + # cached by get_process_function_definitions(), which runs earlier in + # super().generate_process_files() + replace_dict['nipd'] = self._nipd + replace_dict['nipc'] = self._nipc + replace_dict['nipf'] = self._nipf + replace_dict['ndpf'] = self._ndpf + replace_dict['processid'] = self.name + replace_dict['processid_uppercase'] = self.name.upper() + replace_dict['thel_lines'] = self.get_helicity_matrix(me).replace('helicities', 'tHel') + replace_dict['tflavors_lines'] = self.get_flavor_matrix(me).replace('flavors', 'tFlavors') + ff = open(pjoin(self.path, 'ProcessData.h'), 'w') + ff.write(template % replace_dict) + ff.close() + + # backend_separation: process-specific compile-time DATA (arrays, not + # scalars) that backend-owned code needs but can't take as a runtime + # parameter without losing constexpr-ness (see ProcessTables.h). + def edit_processtables(self): + """Generate ProcessTables.h""" + template = open(pjoin(self.template_path, 'madmatrix', 'ProcessTables.h'), 'r').read() + replace_dict = {} + + # Dependent (event-by-event, running-alphas) flavor couplings: partner + # indices and the per-flavor idcoup are pure compile-time constants + # (the complex values are gathered per event page in calculate_jamps). + flv_couplings_dep = [''] * len(self.couporderflv_dep) + for flv_coup, pos in self.couporderflv_dep.items(): + flv_couplings_dep[pos] = flv_coup + if len(flv_couplings_dep): + nMF = max(len(ids) for ids in self.model['merged_particles'].values()) + flv_map = self.helas_call_writer.flv_couplings_map + partner1_vals, partner2_vals, idcoup_vals = [], [], [] + for name in flv_couplings_dep: + coupl = flv_map[name] + p1 = [-1] * nMF + p2 = [-1] * nMF + idc = ['-1'] * nMF + for key, gc in coupl.flavors.items(): + nonzero = [i for i in key if i != 0] + if len(nonzero) == 2: + k1, k2 = nonzero + else: + # single merged leg: unmerged partner has flavor index 1 + k1 = nonzero[0]; k2 = 1 + p1[k1-1] = k2-1 + p2[k2-1] = k1-1 + # symbolic idcoup: resolves to the position of this dependent coupling + # in the event-by-event allcouplings buffer (== COUPs index), defined in + # Parameters_dependentCouplings (Parameters_.h) + idc[k1-1] = '(int)Parameters_dependentCouplings::idcoup_%s' % gc + partner1_vals += [str(v) for v in p1] + partner2_vals += [str(v) for v in p2] + idcoup_vals += idc + cdpfdecl = '__device__ constexpr int cDPF_partner1[nMF * nDPF] = { %s };\n' % ', '.join(partner1_vals) + cdpfdecl += ' __device__ constexpr int cDPF_partner2[nMF * nDPF] = { %s };\n' % ', '.join(partner2_vals) + cdpfdecl += ' __device__ constexpr int cDPF_idcoup[nMF * nDPF] = { %s };' % ', '.join(idcoup_vals) + replace_dict['cdpfdecl'] = cdpfdecl + else: + replace_dict['cdpfdecl'] = """__device__ constexpr const int* cDPF_partner1 = nullptr; // unused as nDPF=0 + __device__ constexpr const int* cDPF_partner2 = nullptr; // unused as nDPF=0 + __device__ constexpr const int* cDPF_idcoup = nullptr; // unused as nDPF=0""" + + # broken_symmetry_factor data: same shared decay-aware symmetry data + # as the Fortran / standalone_cpp exporters. + _, nincoming = self.matrix_elements[0].get_nexternal_ninitial() + process = self.matrix_elements[0].get('processes')[0] + sym_data = export_v4.ProcessExporterFortran._get_broken_symmetry_data( + process, nincoming) + export_v4.ProcessExporterFortran._fill_broken_sym_replace_dict( + replace_dict, sym_data) + + ff = open(pjoin(self.path, 'ProcessTables.h'), 'w') + ff.write(template % replace_dict) + ff.close() + # AV - replace the export_cpp.OneProcessExporterCPP method (add debug printouts and multichannel handling #473) def edit_mgonGPU(self): """Generate mgOnGpuConfig.h""" @@ -1972,44 +1851,14 @@ def edit_mgonGPU(self): ff.write(template % replace_dict) ff.close() - # AV - new method - def edit_processidfile(self): - """Generate epoch_process_id.h""" - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_processidfile') - template = open(pjoin(self.template_path,'madmatrix','epoch_process_id.h'),'r').read() - replace_dict = {} - replace_dict['processid'] = self.name - replace_dict['processid_uppercase'] = self.name.upper() - ff = open(pjoin(self.path, 'epoch_process_id.h'),'w') - ff.write(template % replace_dict) - ff.close() - - # AV - new method - def edit_colorsum(self): - """Generate color_sum.cc""" - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_colorsum') - template = open(pjoin(self.template_path,'madmatrix','color_sum.cc'),'r').read() + # generate process specific color matrix + channel/config maps - algo is backend owned + def edit_colordata(self): + """Generate ColorData.h""" + ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_colordata') + template = open(pjoin(self.template_path,'madmatrix','ColorData.h'),'r').read() replace_dict = {} # Extract color matrix again (this was also in get_matrix_single_process called within get_all_sigmaKin_lines) replace_dict['color_matrix_lines'] = self.get_color_matrix_lines(self.matrix_elements[0]) - ff = open(pjoin(self.path, 'color_sum.cc'),'w') - ff.write(template % replace_dict) - ff.close() - - def edit_processConfig(self): - """Generate process_config.h""" - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_processConfig') - template = open(pjoin(self.template_path,'madmatrix','processConfig.h'),'r').read() - replace_dict = {} - replace_dict['ndiagrams'] = len(self.matrix_elements[0].get('diagrams')) - replace_dict['processid_uppercase'] = self.name.upper() - ff = open(pjoin(self.path, 'processConfig.h'),'w') - ff.write(template % replace_dict) - ff.close() - - # AV - new method - def edit_coloramps(self): - """Generate coloramps.h""" # we don't sort self.multi_channel_map, and we rely on MadSpace sorting # so, diagrams there may be unsorted @@ -2018,12 +1867,7 @@ def edit_coloramps(self): for config in config_subproc_map_C: config_subproc_map.append([c+1 for c in config]) - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_coloramps') - template = open(pjoin(self.template_path,'madmatrix','coloramps.h'),'r').read() - ff = open(pjoin(self.path, 'coloramps.h'),'w') # The following five lines from OneProcessExporterCPP.get_sigmaKin_lines (using OneProcessExporterCPP.get_icolamp_lines) - replace_dict={} - iconfig_to_diag = {} diag_to_iconfig = {} iconfig = 0 @@ -2069,28 +1913,8 @@ def edit_coloramps(self): icolamp_text += text % (iconfigc+1, iconfig_to_diag[iconfigc+1]-1) # diag - 1 is to follow MadSpace indexing icolamp.append(icolamp_text) replace_dict['is_LC'] = '\n'.join(icolamp) - ff.write(template % replace_dict) - ff.close() - # AV - new method - def edit_memorybuffers(self): - """Generate MemoryBuffers.h""" - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_memorybuffers') - template = open(pjoin(self.template_path,'madmatrix','MemoryBuffers.h'),'r').read() - replace_dict = {} - replace_dict['model_name'] = self.model_name - ff = open(pjoin(self.path, '..', 'MemoryBuffers.h'),'w') - ff.write(template % replace_dict) - ff.close() - - # AV - new method - def edit_memoryaccesscouplings(self): - """Generate MemoryAccessCouplings.h""" - ###misc.sprint('Entering OneProcessExporterMadMatrix.edit_memoryaccesscouplings') - template = open(pjoin(self.template_path,'madmatrix','MemoryAccessCouplings.h'),'r').read() - replace_dict = {} - replace_dict['model_name'] = self.model_name - ff = open(pjoin(self.path, '..', 'MemoryAccessCouplings.h'),'w') + ff = open(pjoin(self.path, 'ColorData.h'),'w') ff.write(template % replace_dict) ff.close() @@ -2157,24 +1981,31 @@ def get_color_matrix_lines(self, matrix_element): # AV - replace the export_cpp.OneProcessExporterCPP method (improve formatting) def get_initProc_lines(self, matrix_element, color_amplitudes): - """Get initProc_lines for function definition for CPPProcess::initProc""" - initProc_lines = [] - initProc_lines.append('// Set external particle masses for this matrix element') + """initProc_lines for CPPProcess::initProc (non-hardcoded branch): a generated pointer-to-member table read by gatherFptype() (see Parameters.h).""" + masses = [part.get('mass') for part in matrix_element.get_external_wavefunctions()] + return ('// Set external particle masses for this matrix element\n' + ' static constexpr double Parameters::* const massMembers[npar] = {\n' + ' &Parameters::' + ',\n &Parameters::'.join(masses) + '\n' + ' };\n' + ' fptype tMasses[npar];\n' + ' gatherFptype( m_pars, massMembers, tMasses );\n' + ' m_masses.assign( tMasses, tMasses + npar );') + + def get_hardcoded_initProc_lines(self, matrix_element): + """initProc_lines for CPPProcess::initProc, MGONGPU_HARDCODE_PARAM branch: Parameters has no instance here, so this stays imperative.""" + initProc_lines = ['// Set external particle masses for this matrix element'] for part in matrix_element.get_external_wavefunctions(): - ###initProc_lines.append('mME.push_back(pars->%s);' % part.get('mass')) - initProc_lines.append(' m_masses.push_back( m_pars->%s );' % part.get('mass')) # AV - ###for i, colamp in enumerate(color_amplitudes): - ### initProc_lines.append('jamp2_sv[%d] = new double[%d];' % (i, len(colamp))) # AV - this was commented out already + initProc_lines.append(' m_masses.push_back( Parameters::%s );' % part.get('mass')) return '\n'.join(initProc_lines) # AV - replace the export_cpp.OneProcessExporterCPP method (fix helicity order and improve formatting) def get_helicity_matrix(self, matrix_element): """Return the Helicity matrix definition lines for this matrix element""" - helicity_line = ' static constexpr short helicities[ncomb][npar] = {\n '; # AV (this is tHel) + helicity_line = ' static constexpr short helicities[ncomb][npar] = {\n '; # AV (this is tHel) helicity_line_list = [] for helicities in matrix_element.get_helicity_matrix(allow_reverse=True): # AV was False: different order in Fortran and cudacpp! #569 helicity_line_list.append( '{ ' + ', '.join(['%d'] * len(helicities)) % tuple(helicities) + ' }' ) # AV - return helicity_line + ',\n '.join(helicity_line_list) + ' };' # AV + return helicity_line + ',\n '.join(helicity_line_list) + ' };' # AV def get_flavor_matrix(self, matrix_element): """Return the flavor matrix definition lines for this matrix element""" @@ -2427,87 +2258,6 @@ def super_get_matrix_element_calls(self, matrix_element, color_amplitudes, multi ###misc.sprint(multi_channel_map) res = [] ###res.append('for(int i=0;i<%s;i++){jamp[i] = cxtype(0.,0.);}' % len(color_amplitudes)) - res.append("""//constexpr size_t nxcoup = ndcoup + nicoup; // both dependent and independent couplings (BUG #823) - constexpr size_t nxcoup = ndcoup + nIPC; // both dependent and independent couplings (FIX #823) - const fptype* allCOUPs[nxcoup]; -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#pragma nv_diagnostic push -#pragma nv_diag_suppress 186 // e.g. <> -#endif - for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) - allCOUPs[idcoup] = CD_ACCESS::idcoupAccessBufferConst( allcouplings, idcoup ); // dependent couplings, vary event-by-event - //for( size_t iicoup = 0; iicoup < nicoup; iicoup++ ) // BUG #823 - for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) // FIX #823 - allCOUPs[ndcoup + iicoup] = CI_ACCESS::iicoupAccessBufferConst( cIPC, iicoup ); // independent couplings, fixed for all events -#ifdef MGONGPUCPP_GPUIMPL -#ifdef __CUDACC__ // this must be __CUDACC__ (not MGONGPUCPP_GPUIMPL) -#pragma nv_diagnostic pop -#endif - // CUDA kernels take input/output buffers with momenta/MEs for all events - const fptype* momenta = allmomenta; - const fptype* COUPs[nxcoup]; - for( size_t ixcoup = 0; ixcoup < nxcoup; ixcoup++ ) COUPs[ixcoup] = allCOUPs[ixcoup]; - const int ievt = blockDim.x * blockIdx.x + threadIdx.x; // index of event (thread) in grid - fptype* numerators = &allNumerators[ievt * processConfig::ndiagrams]; - fptype* denominators = allDenominators; -#else - // C++ kernels take input/output buffers with momenta/MEs for one specific event (the first in the current event page) - const fptype* momenta = M_ACCESS::ieventAccessRecordConst( allmomenta, ievt0 ); - const fptype* COUPs[nxcoup]; - for( size_t idcoup = 0; idcoup < ndcoup; idcoup++ ) - COUPs[idcoup] = CD_ACCESS::ieventAccessRecordConst( allCOUPs[idcoup], ievt0 ); // dependent couplings, vary event-by-event - //for( size_t iicoup = 0; iicoup < nicoup; iicoup++ ) // BUG #823 - for( size_t iicoup = 0; iicoup < nIPC; iicoup++ ) // FIX #823 - COUPs[ndcoup + iicoup] = allCOUPs[ndcoup + iicoup]; // independent couplings, fixed for all events - fptype* numerators = NUM_ACCESS::ieventAccessRecord( allNumerators, ievt0 * processConfig::ndiagrams ); - fptype* denominators = DEN_ACCESS::ieventAccessRecord( allDenominators, ievt0 ); -#endif - // Create an array of views over the Flavor Couplings - FLV_COUPLING_ARRAY flvCOUPs{ cIPF_partner1, cIPF_partner2, cIPF_value }; - - // Dependent (event-by-event, running-alphas) flavor couplings (Step 3): the per-flavor - // values are NOT baked in (they run per event). Gather the current values of the - // underlying dependent couplings for this event page into an AOSOA buffer dpf_value - // (one nx2*neppC SIMD record per (coupling,flavor) slot, matching CD_ACCESS), then build - // an ordinary value-based view over it. The flavor index is constant across a SIMD lane - // (guaranteed by the phase-space integrator), so each lane gets its own running value - // while sharing the same flavor selection. This is the direct analogue of Fortran's - // FLV_xx%VAL(k)%P => GC_yyy(J). The vertex routines are instantiated with CD_ACCESS so - // get_coupling_def reads dpf_value with the right per-flavor stride (CD_ACCESS::flv_stride). - constexpr int ndpfbuf = ( nDPF > 0 ? nDPF * nMF * CD_ACCESS::flv_stride : 1 ); -#ifndef MGONGPUCPP_GPUIMPL - // cppAlign is only defined for SIMD - alignas( mgOnGpu::cppAlign ) fptype dpf_value[ndpfbuf]{}; -#else - fptype dpf_value[ndpfbuf]{}; -#endif - for( int idpf = 0; idpf < nDPF; idpf++ ) - for( int imf = 0; imf < nMF; imf++ ) - { - const int idc = cDPF_idcoup[idpf * nMF + imf]; - if( idc >= 0 ) - CD_ACCESS::kernelAccess( dpf_value + ( idpf * nMF + imf ) * CD_ACCESS::flv_stride ) = - CD_ACCESS::kernelAccessConst( COUPs[idc] ); - } - FLV_COUPLING_ARRAY flvCOUPs_dep{ cDPF_partner1, cDPF_partner2, dpf_value }; - - // Reset color flows (reset jamp_sv) at the beginning of a new event or event page - for( int i = 0; i < ncolor; i++ ) { jamp_sv[i] = cxzero_sv(); } - - // Numerators and denominators for the current event (CUDA) or SIMD event page (C++) - fptype_sv* numerators_sv = NUM_ACCESS::kernelAccessP( numerators ); - fptype_sv& denominators_sv = DEN_ACCESS::kernelAccess( denominators ); - // Scalar iflavor for the current event - // for GPU it is an int - // for SIMD it is also an int, since it is constant across the SIMD vector -#ifdef MGONGPUCPP_GPUIMPL - const unsigned int iflavor = F_ACCESS::kernelAccessConst( iflavorVec ); -#else - const unsigned int* iflavor_rec = F_ACCESS::ieventAccessRecordConst( iflavorVec, ievt0 ); - const uint_sv iflavor_sv = F_ACCESS::kernelAccessConst( iflavor_rec ); - const unsigned int iflavor = reinterpret_cast(&iflavor_sv)[0]; -#endif -""") diagrams = matrix_element.get('diagrams') diag_to_config = {} for config in sorted(multi_channel_map.keys()): diff --git a/madmatrix/output.py b/madmatrix/output.py index 1820b10e96..5a6c30d25b 100644 --- a/madmatrix/output.py +++ b/madmatrix/output.py @@ -83,21 +83,16 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): from_template = {'.': relative_path_list(home_path, ['COPYRIGHT', 'COPYING', 'COPYING.LESSER']), 'src': relative_path_list(madmatrix_templates, [ - 'mgOnGpuFptypes.h', 'mgOnGpuCxtypes.h', 'mgOnGpuVectors.h', - 'constexpr_math.h', 'read_slha.h', 'read_slha.cc' + 'read_slha.h', 'read_slha.cc' ]), - 'SubProcesses': relative_path_list(madmatrix_templates, ['nvtx.h', 'GpuRuntime.h', 'GpuAbstraction.h', 'color_sum.h', 'color_sum.cc', - 'MemoryAccessHelpers.h', 'MemoryAccessVectors.h', - 'MemoryAccessMatrixElements.h', 'MemoryAccessMomenta.h', - 'MemoryAccessRandomNumbers.h', 'MemoryAccessWeights.h', - 'MemoryAccessAmplitudes.h', 'MemoryAccessWavefunctions.h', - 'MemoryAccessGs.h', 'MemoryAccessCouplingsFixed.h', - 'MemoryAccessNumerators.h', 'MemoryAccessDenominators.h', - 'MemoryAccessChannelIds.h', 'MemoryAccessIflavorVec.h', - 'CrossSectionKernels.cc', 'CrossSectionKernels.h', - 'MatrixElementKernels.cc', 'MatrixElementKernels.h', - 'EventStatistics.h', - 'umami.h', 'umami.cc', 'rambo.h']), + # Backend-owned skeleton files live only under backend// now + # (see backend_variants below); only genuinely backend-agnostic files + # (no backend/ counterpart) are copied flat into SubProcesses/. umami.h + # is the only one needed outside standalone mode too (it's the header + # for backend//umami.cc's UMAMI API); nvtx.h is + # standalone-driver-only (see _standalone_extra_files below). The + # rambo/random-number sources live once in src/rambo/ + 'SubProcesses': relative_path_list(madmatrix_templates, ['umami.h']), # run_card.toml is generated in finalize() (ProcessExporterMG7.create_run_card) # from the template, not copied verbatim. # Default cards for the optional post-processing tools @@ -113,25 +108,38 @@ class ProcessExporterMadMatrix(export_cpp.ProcessExporterMG7): 'madanalysis5_hadron_card_default.dat', 'rivet_card_default.dat'])} - to_link_in_P = ['nvtx.h', 'GpuRuntime.h', 'GpuAbstraction.h', 'color_sum.h', - 'MemoryAccessHelpers.h', 'MemoryAccessVectors.h', - 'MemoryAccessMatrixElements.h', 'MemoryAccessMomenta.h', - 'MemoryAccessRandomNumbers.h', 'MemoryAccessWeights.h', - 'MemoryAccessAmplitudes.h', 'MemoryAccessWavefunctions.h', - 'MemoryAccessGs.h', 'MemoryAccessCouplingsFixed.h', - 'MemoryAccessNumerators.h', 'MemoryAccessDenominators.h', - 'MemoryAccessChannelIds.h', 'MemoryAccessIflavorVec.h', - 'CrossSectionKernels.cc', 'CrossSectionKernels.h', - 'MatrixElementKernels.cc', 'MatrixElementKernels.h', - 'EventStatistics.h', - 'MemoryBuffers.h', # this is generated from a template in Subprocesses but we still link it in P1 - 'MemoryAccessCouplings.h', # this is generated from a template in Subprocesses but we still link it in P1 - 'umami.h', 'umami.cc', 'rambo.h'] + # Backend split (step 1, not yet wired into the build): mirror + # template_files/madmatrix/backend/{cpu,simd,gpu}/ as a top-level + # backend// dir, sibling of SubProcesses/src/lib. + backend_variants = ('cpu', 'simd', 'gpu') + backend_template_dir = pjoin(madmatrix_templates, 'backend') + for _backend_variant in backend_variants: # plain loop: comprehension wouldn't see the locals above + from_template[pjoin('backend', _backend_variant)] = relative_path_list( + pjoin(backend_template_dir, _backend_variant), + sorted(os.listdir(pjoin(backend_template_dir, _backend_variant)))) + del _backend_variant + + # Rambo/random-number files copy in src/rambo/ + rambo_template_dir = pjoin(madmatrix_templates, 'src', 'rambo') + from_template['src/rambo'] = relative_path_list( + rambo_template_dir, sorted(os.listdir(rambo_template_dir))) + + # Backend-owned skeleton files (GpuRuntime.h, color_sum.{h,cc}, the + # MemoryAccess*.h family, MatrixElementKernels/CrossSectionKernels/umami.cc, + # etc.) are NOT linked into P* at all: they are compiled straight from the + # single top-level backend// dir via the Makefile's INCFLAGS/vpath + # (see BACKENDDIR in madmatrix.mk). Only files with no backend/ counterpart + # - genuinely backend-agnostic - stay here. + to_link_in_P = ['umami.h'] template_src_make = pjoin(madmatrix_templates, 'madmatrix_src.mk') template_Sub_make = pjoin(madmatrix_templates, 'madmatrix.mk') - dirs_to_create = ['bin', 'src', 'lib', 'Cards', 'SubProcesses'] + dirs_to_create = ['bin', 'src', 'src/rambo', 'lib', 'Cards', 'SubProcesses', + 'backend', + 'backend/cpu', + 'backend/simd', + 'backend/gpu'] # AV - use a custom UFOModelConverter (model/aloha exporter) create_model_class = model_handling.MadMatrixUFOModelConverter @@ -180,8 +188,7 @@ def generate_subprocess_directory(self, matrix_element, cpp_helas_call_writer, p # guarded wavefunction/amplitude calls. if cpp_helas_call_writer is not None: cpp_helas_call_writer.use_flavor_mask = self.use_flavor_mask - out = super().generate_subprocess_directory(matrix_element, cpp_helas_call_writer, proc_number) - return out + return super().generate_subprocess_directory(matrix_element, cpp_helas_call_writer, proc_number) # AV (default from OM's tutorial) - add a debug printout def convert_model(self, model, wanted_lorentz=[], wanted_couplings=[]): @@ -210,11 +217,7 @@ class ProcessExporterMadMatrixStandalone(ProcessExporterMadMatrix): template_Sub_make = pjoin(ProcessExporterMadMatrix.madmatrix_templates, 'madmatrix_standalone.mk') # Standalone-only template files needed to build check_sa.exe - _standalone_extra_files = ['check_sa.cc', - 'RamboSamplingKernels.cc', 'RamboSamplingKernels.h', - 'CommonRandomNumberKernel.cc', 'CommonRandomNumbers.h', - 'RandomNumberKernels.h', - 'massless_rambo.h', 'timer.h', 'timermap.h'] + _standalone_extra_files = ['check_sa.cc', 'nvtx.h', 'timer.h', 'timermap.h'] from_template = dict(ProcessExporterMadMatrix.from_template) from_template['SubProcesses'] = (ProcessExporterMadMatrix.from_template['SubProcesses'] diff --git a/madmatrix/trex.py b/madmatrix/trex.py index 9809fd01cf..f903569fb6 100644 --- a/madmatrix/trex.py +++ b/madmatrix/trex.py @@ -187,6 +187,8 @@ def edit_rwgt_runner(self): def generate_process_files(self): """Generate mgOnGpuConfig.h, CPPProcess.cc, CPPProcess.h, check_sa.cc, gXXX.cu links""" super().generate_process_files() + # for TREX for now no /backend + self.edit_mgonGPU() self.edit_rwgt_header() self.edit_rwgt_runner() @@ -202,9 +204,13 @@ class TREX_ProcessExporter(output.ProcessExporterMadMatrix): r = PLUGINDIR + '/MadtRex/template_files/' m = PLUGINDIR + '/MadtRex/makefiles/' from_template = dict(output.ProcessExporterMadMatrix.from_template) - from_template['src'] = from_template['src'] + [t+'librex.so', t+'libtearex.so', - t+'Rex.h', t+'teaRex.h', - r+'rwgt_instance.h', r+'rwgt_instance.cc'] + # for TREX for now no /backend + from_template['src'] = from_template['src'] + output.relative_path_list( + output.ProcessExporterMadMatrix.madmatrix_templates, + ['mgOnGpuFptypes.h', 'mgOnGpuCxtypes.h', 'mgOnGpuVectors.h', 'constexpr_math.h'] + ) + [t+'librex.so', t+'libtearex.so', + t+'Rex.h', t+'teaRex.h', + r+'rwgt_instance.h', r+'rwgt_instance.cc'] from_template['SubProcesses'] = from_template['SubProcesses'] + [m+'cudacpp_driver.mk', r+'rwgt_instance.h', t+'Rex.h', t+'teaRex.h'] @@ -269,7 +275,8 @@ def generate_subprocess_directory(self, matrix_element, cpp_helas_call_writer, # Create the process .h and .cc files process_exporter_cpp.generate_process_files() for file in self.to_link_in_P: - files.ln('../%s' % file) + files.ln('../%s' % file) + self._link_backend_dirs_in_P(matrix_element) return def export_driver(self):