From 5bb31957dbcf69cfb34d74022b6a31f9f5dee99b Mon Sep 17 00:00:00 2001 From: SY Wang Date: Fri, 30 Oct 2026 17:55:55 +0800 Subject: [PATCH] Format source_hsolver (Part II) --- source/source_hsolver/module_genelpa/cblacs.h | 76 +- .../module_genelpa/elpa_generic.h | 1074 +++++++++++++---- .../module_genelpa/elpa_new.cpp | 384 +++++- .../source_hsolver/module_genelpa/elpa_new.h | 3 - .../module_genelpa/elpa_new_complex.cpp | 966 +++++++++++---- .../module_genelpa/elpa_new_real.cpp | 663 ++++++++-- .../module_genelpa/elpa_runtime_check.cpp | 178 ++- .../module_genelpa/elpa_runtime_check.h | 1 - .../module_genelpa/elpa_solver.h | 116 +- source/source_hsolver/module_genelpa/pblas.h | 226 +++- .../source_hsolver/module_genelpa/scalapack.h | 83 +- .../source_hsolver/module_genelpa/utils.cpp | 245 +++- source/source_hsolver/module_genelpa/utils.h | 52 +- .../module_linalg/cn_subspace.cpp | 83 +- .../module_linalg/cn_subspace.h | 31 +- .../module_linalg/exp_action.cpp | 24 +- .../source_hsolver/module_linalg/exp_action.h | 28 +- .../module_linalg/exp_lanczos.cpp | 394 ++++-- .../module_linalg/exp_lanczos.h | 82 +- .../module_linalg/exp_taylor.cpp | 119 +- .../source_hsolver/module_linalg/exp_taylor.h | 28 +- .../module_linalg/hsolver_linear.cpp | 100 +- .../module_linalg/hsolver_linear.h | 52 +- .../module_linalg/kernels/cuda/linear_op.cu | 712 ++++++++--- .../module_linalg/kernels/linear_op.cpp | 190 ++- .../module_linalg/kernels/linear_op.h | 322 ++++- .../kernels/rocm/linear_op.hip.cu | 712 ++++++++--- .../module_linalg/linear_algebra.cpp | 223 +++- .../module_linalg/linear_algebra.h | 75 +- .../module_linalg/linear_bicgstab.cpp | 262 +++- .../module_linalg/linear_bicgstab.h | 30 +- .../module_linalg/linear_cgs.cpp | 216 +++- .../source_hsolver/module_linalg/linear_cgs.h | 33 +- .../module_linalg/linear_gmres.cpp | 346 ++++-- .../module_linalg/linear_gmres.h | 77 +- .../module_linalg/linear_low_rank.cpp | 253 +++- .../module_linalg/linear_low_rank.h | 98 +- .../module_linalg/linear_operator.h | 14 +- .../module_linalg/linear_solver_types.h | 2 +- .../module_linalg/linear_workspace.cpp | 441 +++++-- .../module_linalg/linear_workspace.h | 183 ++- .../module_linalg/orb_buffer.cpp | 45 +- .../source_hsolver/module_linalg/orb_buffer.h | 5 +- .../module_linalg/orthonormal.cpp | 204 +++- .../module_linalg/orthonormal.h | 87 +- .../module_linalg/orthonormal_small.cpp | 125 +- .../unittests/linear_test_utils.h | 42 +- .../unittests/test_cn_subspace.cpp | 53 +- .../unittests/test_exp_action.cpp | 180 ++- .../unittests/test_hsolver_linear.cpp | 233 +++- .../unittests/test_linear_low_rank.cpp | 115 +- .../unittests/test_orthonormal.cpp | 74 +- .../module_pexsi/dist_bcd_matrix.cpp | 71 +- .../module_pexsi/dist_bcd_matrix.h | 41 +- .../module_pexsi/dist_ccs_matrix.cpp | 32 +- .../module_pexsi/dist_ccs_matrix.h | 62 +- .../module_pexsi/dist_mat_tf.cpp | 472 +++++--- .../source_hsolver/module_pexsi/dist_mat_tf.h | 125 +- .../module_pexsi/pexsi_solver.cpp | 92 +- .../module_pexsi/pexsi_solver.h | 132 +- .../module_pexsi/simple_pexsi.cpp | 185 +-- .../module_pexsi/simple_pexsi.h | 41 +- 62 files changed, 8600 insertions(+), 3013 deletions(-) diff --git a/source/source_hsolver/module_genelpa/cblacs.h b/source/source_hsolver/module_genelpa/cblacs.h index 3b671ef1f9f..8b2de95671c 100644 --- a/source/source_hsolver/module_genelpa/cblacs.h +++ b/source/source_hsolver/module_genelpa/cblacs.h @@ -2,28 +2,74 @@ #define ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_CBLACS_H // blacs - // Initialization +// Initialization #include "mpi.h" #include int Csys2blacs_handle(MPI_Comm SysCtxt); void Cblacs_pinfo(int *myid, int *nprocs); void Cblacs_get(int icontxt, int what, int *val); -void Cblacs_gridinit(int* icontxt, char *layout, int nprow, int npcol); -void Cblacs_gridmap(int* icontxt, int *usermap, int ldumap, int nprow, int npcol); - // Destruction +void Cblacs_gridinit( + int *icontxt, + char *layout, + int nprow, + int npcol); +void Cblacs_gridmap( + int *icontxt, + int *usermap, + int ldumap, + int nprow, + int npcol); +// Destruction void Cblacs_gridexit(int icontxt); - // Informational and Miscellaneous -void Cblacs_gridinfo(int icontxt, int* nprow, int *npcol, int *myprow, int *mypcol); +// Informational and Miscellaneous +void Cblacs_gridinfo( + int icontxt, + int *nprow, + int *npcol, + int *myprow, + int *mypcol); int Cblacs_pnum(int icontxt, int prow, int pcol); -void Cblacs_pcoord(int icontxt, int pnum, int *prow, int *pcol); +void Cblacs_pcoord( + int icontxt, + int pnum, + int *prow, + int *pcol); void Cblacs_barrier(int icontxt, char *scope); - // Point to Point -void Cdgesd2d(int icontxt, int m, int n, double *a, int lda, int rdest, int cdest); -void Cdgerv2d(int icontxt, int m, int n, double *a, int lda, int rsrc, int csrc); -void Czgesd2d(int icontxt, int m, int n, std::complex *a, int lda, int rdest, int cdest); -void Czgerv2d(int icontxt, int m, int n, std::complex *a, int lda, int rsrc, int csrc); - // Combine -//void Cdgamx2d(int icontxt, int scope, int top, int m, int n, -// double *a, int lda, int *ra, int *ca, int rcflag, int rdest, int cdest); +// Point to Point +void Cdgesd2d( + int icontxt, + int m, + int n, + double *a, + int lda, + int rdest, + int cdest); +void Cdgerv2d( + int icontxt, + int m, + int n, + double *a, + int lda, + int rsrc, + int csrc); +void Czgesd2d( + int icontxt, + int m, + int n, + std::complex *a, + int lda, + int rdest, + int cdest); +void Czgerv2d( + int icontxt, + int m, + int n, + std::complex *a, + int lda, + int rsrc, + int csrc); +// Combine +// void Cdgamx2d(int icontxt, int scope, int top, int m, int n, +// double *a, int lda, int *ra, int *ca, int rcflag, int rdest, int cdest); #endif // ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_CBLACS_H diff --git a/source/source_hsolver/module_genelpa/elpa_generic.h b/source/source_hsolver/module_genelpa/elpa_generic.h index 1638dee751b..c597e5760e7 100644 --- a/source/source_hsolver/module_genelpa/elpa_generic.h +++ b/source/source_hsolver/module_genelpa/elpa_generic.h @@ -12,13 +12,29 @@ * \param error on return the error code, which can be queried with elpa_strerr() * \result void */ -inline void elpa_set(elpa_t handle, const char *name, int value, int *error) -{ - elpa_set_integer(handle, name, value, error); -} -inline void elpa_set(elpa_t handle, const char *name, double value, int *error) -{ - elpa_set_double(handle, name, value, error); +inline void elpa_set( + elpa_t handle, + const char *name, + int value, + int *error) +{ + elpa_set_integer( + handle, + name, + value, + error); +} +inline void elpa_set( + elpa_t handle, + const char *name, + double value, + int *error) +{ + elpa_set_double( + handle, + name, + value, + error); } /*! \brief generic C method for elpa_get @@ -30,13 +46,29 @@ inline void elpa_set(elpa_t handle, const char *name, double value, int *error) * \param error on return the error code, which can be queried with elpa_strerr() * \result void */ -inline void elpa_get(elpa_t handle, const char *name, int *value, int *error) -{ - elpa_get_integer(handle, name, value, error); -} -inline void elpa_get(elpa_t handle, const char *name, double *value, int *error) -{ - elpa_get_double(handle, name, value, error); +inline void elpa_get( + elpa_t handle, + const char *name, + int *value, + int *error) +{ + elpa_get_integer( + handle, + name, + value, + error); +} +inline void elpa_get( + elpa_t handle, + const char *name, + double *value, + int *error) +{ + elpa_get_double( + handle, + name, + value, + error); } /*! \brief generic C method for elpa_eigenvectors @@ -50,68 +82,184 @@ inline void elpa_get(elpa_t handle, const char *name, double *value, int *error) * \result void */ #if ELPA_API_VERSION <= 20210502 // ELPA 2021.05.002 and earlier versions -inline void elpa_eigenvectors(const elpa_t handle, double *a, double *ev, double *q, int *error) -{ - elpa_eigenvectors_d(handle, a, ev, q, error); +inline void elpa_eigenvectors( + const elpa_t handle, + double *a, + double *ev, + double *q, + int *error) +{ + elpa_eigenvectors_d( + handle, + a, + ev, + q, + error); } -inline void elpa_eigenvectors(const elpa_t handle, float *a, float *ev, float *q, int *error) -{ - elpa_eigenvectors_f(handle, a, ev, q, error); +inline void elpa_eigenvectors( + const elpa_t handle, + float *a, + float *ev, + float *q, + int *error) +{ + elpa_eigenvectors_f( + handle, + a, + ev, + q, + error); } -inline void elpa_eigenvectors(const elpa_t handle, std::complex *a, double *ev, std::complex *q, int *error) -{ - elpa_eigenvectors_dc(handle, reinterpret_cast(a), ev, reinterpret_cast(q), error); +inline void elpa_eigenvectors( + const elpa_t handle, + std::complex *a, + double *ev, + std::complex *q, + int *error) +{ + elpa_eigenvectors_dc( + handle, + reinterpret_cast(a), + ev, + reinterpret_cast(q), + error); } -inline void elpa_eigenvectors(const elpa_t handle, std::complex *a, float *ev, std::complex *q, int *error) -{ - elpa_eigenvectors_fc(handle, reinterpret_cast(a), ev, reinterpret_cast(q), error); -} -#elif ELPA_API_VERSION < 20220501 // ELPA version between 2021.11.001 and 2022.05.001 -inline void elpa_eigenvectors(const elpa_t handle, double *a, double *ev, double *q, int *error) -{ - elpa_eigenvectors_all_host_arrays_d(handle, a, ev, q, error); +inline void elpa_eigenvectors( + const elpa_t handle, + std::complex *a, + float *ev, + std::complex *q, + int *error) +{ + elpa_eigenvectors_fc( + handle, + reinterpret_cast(a), + ev, + reinterpret_cast(q), + error); +} +#elif ELPA_API_VERSION < 20220501 // ELPA version between 2021.11.001 and 2022.05.001 +inline void elpa_eigenvectors( + const elpa_t handle, + double *a, + double *ev, + double *q, + int *error) +{ + elpa_eigenvectors_all_host_arrays_d( + handle, + a, + ev, + q, + error); } -inline void elpa_eigenvectors(const elpa_t handle, float *a, float *ev, float *q, int *error) -{ - elpa_eigenvectors_all_host_arrays_f(handle, a, ev, q, error); +inline void elpa_eigenvectors( + const elpa_t handle, + float *a, + float *ev, + float *q, + int *error) +{ + elpa_eigenvectors_all_host_arrays_f( + handle, + a, + ev, + q, + error); } -inline void elpa_eigenvectors(const elpa_t handle, std::complex *a, double *ev, std::complex *q, int *error) -{ - elpa_eigenvectors_all_host_arrays_dc(handle, reinterpret_cast(a), - ev, reinterpret_cast(q), error); +inline void elpa_eigenvectors( + const elpa_t handle, + std::complex *a, + double *ev, + std::complex *q, + int *error) +{ + elpa_eigenvectors_all_host_arrays_dc( + handle, + reinterpret_cast(a), + ev, + reinterpret_cast(q), + error); } -inline void elpa_eigenvectors(const elpa_t handle, std::complex *a, float *ev, std::complex *q, int *error) -{ - elpa_eigenvectors_all_host_arrays_fc(handle, reinterpret_cast(a), - ev, reinterpret_cast(q), error); -} -#else // ELPA version 2022.05.001, ELPA has its own c++ interface from version 2022.11.001 -inline void elpa_eigenvectors(const elpa_t handle, double *a, double *ev, double *q, int *error) -{ - elpa_eigenvectors_a_h_a_d(handle, a, ev, q, error); +inline void elpa_eigenvectors( + const elpa_t handle, + std::complex *a, + float *ev, + std::complex *q, + int *error) +{ + elpa_eigenvectors_all_host_arrays_fc( + handle, + reinterpret_cast(a), + ev, + reinterpret_cast(q), + error); +} +#else // ELPA version 2022.05.001, ELPA has its own c++ interface from version 2022.11.001 +inline void elpa_eigenvectors( + const elpa_t handle, + double *a, + double *ev, + double *q, + int *error) +{ + elpa_eigenvectors_a_h_a_d( + handle, + a, + ev, + q, + error); } -inline void elpa_eigenvectors(const elpa_t handle, float *a, float *ev, float *q, int *error) -{ - elpa_eigenvectors_a_h_a_f(handle, a, ev, q, error); +inline void elpa_eigenvectors( + const elpa_t handle, + float *a, + float *ev, + float *q, + int *error) +{ + elpa_eigenvectors_a_h_a_f( + handle, + a, + ev, + q, + error); } -inline void elpa_eigenvectors(const elpa_t handle, std::complex *a, double *ev, std::complex *q, int *error) -{ - elpa_eigenvectors_a_h_a_dc(handle, reinterpret_cast(a), - ev, reinterpret_cast(q), error); +inline void elpa_eigenvectors( + const elpa_t handle, + std::complex *a, + double *ev, + std::complex *q, + int *error) +{ + elpa_eigenvectors_a_h_a_dc( + handle, + reinterpret_cast(a), + ev, + reinterpret_cast(q), + error); } -inline void elpa_eigenvectors(const elpa_t handle, std::complex *a, float *ev, std::complex *q, int *error) -{ - elpa_eigenvectors_a_h_a_fc(handle, reinterpret_cast(a), - ev, reinterpret_cast(q), error); +inline void elpa_eigenvectors( + const elpa_t handle, + std::complex *a, + float *ev, + std::complex *q, + int *error) +{ + elpa_eigenvectors_a_h_a_fc( + handle, + reinterpret_cast(a), + ev, + reinterpret_cast(q), + error); } #endif @@ -126,39 +274,97 @@ inline void elpa_eigenvectors(const elpa_t handle, std::complex *a, floa * \result void */ #if ELPA_API_VERSION <= 20210502 // ELPA 2021.05.002 and earlier versions -inline void elpa_skew_eigenvectors(const elpa_t handle, double *a, double *ev, double *q, int *error) -{ - elpa_eigenvectors_d(handle, a, ev, q, error); +inline void elpa_skew_eigenvectors( + const elpa_t handle, + double *a, + double *ev, + double *q, + int *error) +{ + elpa_eigenvectors_d( + handle, + a, + ev, + q, + error); } -inline void elpa_skew_eigenvectors(const elpa_t handle, float *a, float *ev, float *q, int *error) -{ - elpa_eigenvectors_f(handle, a, ev, q, error); -} -#elif ELPA_API_VERSION < 20220501 // ELPA version between 2021.11.001 and 2022.05.001 -inline void elpa_skew_eigenvectors(const elpa_t handle, double *a, double *ev, double *q, int *error) -{ - elpa_skew_eigenvectors_all_host_arrays_d(handle, a, ev, q, error); +inline void elpa_skew_eigenvectors( + const elpa_t handle, + float *a, + float *ev, + float *q, + int *error) +{ + elpa_eigenvectors_f( + handle, + a, + ev, + q, + error); +} +#elif ELPA_API_VERSION < 20220501 // ELPA version between 2021.11.001 and 2022.05.001 +inline void elpa_skew_eigenvectors( + const elpa_t handle, + double *a, + double *ev, + double *q, + int *error) +{ + elpa_skew_eigenvectors_all_host_arrays_d( + handle, + a, + ev, + q, + error); } -inline void elpa_skew_eigenvectors(const elpa_t handle, float *a, float *ev, float *q, int *error) -{ - elpa_skew_eigenvectors_all_host_arrays_f(handle, a, ev, q, error); -} -#else // ELPA version 2022.05.001, ELPA has its own c++ interface from version 2022.11.001 -inline void elpa_skew_eigenvectors(const elpa_t handle, double *a, double *ev, double *q, int *error) -{ - elpa_skew_eigenvectors_a_h_a_d(handle, a, ev, q, error); +inline void elpa_skew_eigenvectors( + const elpa_t handle, + float *a, + float *ev, + float *q, + int *error) +{ + elpa_skew_eigenvectors_all_host_arrays_f( + handle, + a, + ev, + q, + error); +} +#else // ELPA version 2022.05.001, ELPA has its own c++ interface from version 2022.11.001 +inline void elpa_skew_eigenvectors( + const elpa_t handle, + double *a, + double *ev, + double *q, + int *error) +{ + elpa_skew_eigenvectors_a_h_a_d( + handle, + a, + ev, + q, + error); } -inline void elpa_skew_eigenvectors(const elpa_t handle, float *a, float *ev, float *q, int *error) -{ - elpa_skew_eigenvectors_a_h_a_f(handle, a, ev, q, error); +inline void elpa_skew_eigenvectors( + const elpa_t handle, + float *a, + float *ev, + float *q, + int *error) +{ + elpa_skew_eigenvectors_a_h_a_f( + handle, + a, + ev, + q, + error); } #endif - - /*! \brief generic C method for elpa_generalized_eigenvectors * * \details @@ -171,26 +377,80 @@ inline void elpa_skew_eigenvectors(const elpa_t handle, float *a, float *ev, f * \param error on return the error code, which can be queried with elpa_strerr() * \result void */ -inline void elpa_generalized_eigenvectors(elpa_t handle, double *a, double *b, double *ev, double *q, int is_already_decomposed, int *error) -{ - elpa_generalized_eigenvectors_d(handle, a, b, ev, q, is_already_decomposed, error); +inline void elpa_generalized_eigenvectors( + elpa_t handle, + double *a, + double *b, + double *ev, + double *q, + int is_already_decomposed, + int *error) +{ + elpa_generalized_eigenvectors_d( + handle, + a, + b, + ev, + q, + is_already_decomposed, + error); } -inline void elpa_generalized_eigenvectors(elpa_t handle, float *a, float *b, float *ev, float *q, int is_already_decomposed, int *error) -{ - elpa_generalized_eigenvectors_f(handle, a, b, ev, q, is_already_decomposed, error); +inline void elpa_generalized_eigenvectors( + elpa_t handle, + float *a, + float *b, + float *ev, + float *q, + int is_already_decomposed, + int *error) +{ + elpa_generalized_eigenvectors_f( + handle, + a, + b, + ev, + q, + is_already_decomposed, + error); } -inline void elpa_generalized_eigenvectors(elpa_t handle, std::complex *a, std::complex *b, double *ev, std::complex *q, int is_already_decomposed, int *error) -{ - elpa_generalized_eigenvectors_dc(handle, reinterpret_cast(a), reinterpret_cast(b), - ev, reinterpret_cast(q), is_already_decomposed, error); +inline void elpa_generalized_eigenvectors( + elpa_t handle, + std::complex *a, + std::complex *b, + double *ev, + std::complex *q, + int is_already_decomposed, + int *error) +{ + elpa_generalized_eigenvectors_dc( + handle, + reinterpret_cast(a), + reinterpret_cast(b), + ev, + reinterpret_cast(q), + is_already_decomposed, + error); } -inline void elpa_generalized_eigenvectors(elpa_t handle, std::complex *a, std::complex *b, float *ev, std::complex *q, int is_already_decomposed, int *error) -{ - elpa_generalized_eigenvectors_fc(handle, reinterpret_cast(a), reinterpret_cast(b), - ev, reinterpret_cast(q), is_already_decomposed, error); +inline void elpa_generalized_eigenvectors( + elpa_t handle, + std::complex *a, + std::complex *b, + float *ev, + std::complex *q, + int is_already_decomposed, + int *error) +{ + elpa_generalized_eigenvectors_fc( + handle, + reinterpret_cast(a), + reinterpret_cast(b), + ev, + reinterpret_cast(q), + is_already_decomposed, + error); } /*! \brief generic C method for elpa_eigenvalues @@ -203,55 +463,151 @@ inline void elpa_generalized_eigenvectors(elpa_t handle, std::complex *a * \result void */ #if ELPA_API_VERSION <= 20210502 // ELPA 2021.05.002 and earlier versions -inline void elpa_eigenvalues(elpa_t handle, double *a, double *ev, int *error) -{ - elpa_eigenvalues_d(handle, a, ev, error); -} -inline void elpa_eigenvalues(elpa_t handle, float *a, float *ev, int *error) -{ - elpa_eigenvalues_f(handle, a, ev, error); -} -inline void elpa_eigenvalues(elpa_t handle, std::complex *a, double *ev, int *error) -{ - elpa_eigenvalues_dc(handle, reinterpret_cast(a), ev, error); -} -inline void elpa_eigenvalues(elpa_t handle, std::complex *a, float *ev, int *error) -{ - elpa_eigenvalues_fc (handle, reinterpret_cast(a), ev, error); -} -#elif ELPA_API_VERSION < 20220501 // ELPA version between 2021.11.001 and 2022.05.001 -inline void elpa_eigenvalues(elpa_t handle, double *a, double *ev, int *error) -{ - elpa_eigenvalues_all_host_arrays_d(handle, a, ev, error); -} -inline void elpa_eigenvalues(elpa_t handle, float *a, float *ev, int *error) -{ - elpa_eigenvalues_all_host_arrays_f(handle, a, ev, error); -} -inline void elpa_eigenvalues(elpa_t handle, std::complex *a, double *ev, int *error) -{ - elpa_eigenvalues_all_host_arrays_dc(handle, reinterpret_cast(a), ev, error); -} -inline void elpa_eigenvalues(elpa_t handle, std::complex *a, float *ev, int *error) -{ - elpa_eigenvalues_all_host_arrays_fc(handle, reinterpret_cast(a), ev, error); -} -#else // ELPA version 2022.05.001, ELPA has its own c++ interface from version 2022.11.001 -inline void elpa_eigenvalues(elpa_t handle, double *a, double *ev, int *error) -{ - elpa_eigenvalues_a_h_a_d(handle, a, ev, error); -} -inline void elpa_eigenvalues(elpa_t handle, float *a, float *ev, int *error) -{ - elpa_eigenvalues_a_h_a_f(handle, a, ev, error); -} -inline void elpa_eigenvalues(elpa_t handle, std::complex *a, double *ev, int *error) -{ - elpa_eigenvalues_a_h_a_dc(handle, reinterpret_cast(a), ev, error); -} -inline void elpa_eigenvalues(elpa_t handle, std::complex *a, float *ev, int *error) -{ - elpa_eigenvalues_a_h_a_fc(handle, reinterpret_cast(a), ev, error); +inline void elpa_eigenvalues( + elpa_t handle, + double *a, + double *ev, + int *error) +{ + elpa_eigenvalues_d( + handle, + a, + ev, + error); +} +inline void elpa_eigenvalues( + elpa_t handle, + float *a, + float *ev, + int *error) +{ + elpa_eigenvalues_f( + handle, + a, + ev, + error); +} +inline void elpa_eigenvalues( + elpa_t handle, + std::complex *a, + double *ev, + int *error) +{ + elpa_eigenvalues_dc( + handle, + reinterpret_cast(a), + ev, + error); +} +inline void elpa_eigenvalues( + elpa_t handle, + std::complex *a, + float *ev, + int *error) +{ + elpa_eigenvalues_fc( + handle, + reinterpret_cast(a), + ev, + error); +} +#elif ELPA_API_VERSION < 20220501 // ELPA version between 2021.11.001 and 2022.05.001 +inline void elpa_eigenvalues( + elpa_t handle, + double *a, + double *ev, + int *error) +{ + elpa_eigenvalues_all_host_arrays_d( + handle, + a, + ev, + error); +} +inline void elpa_eigenvalues( + elpa_t handle, + float *a, + float *ev, + int *error) +{ + elpa_eigenvalues_all_host_arrays_f( + handle, + a, + ev, + error); +} +inline void elpa_eigenvalues( + elpa_t handle, + std::complex *a, + double *ev, + int *error) +{ + elpa_eigenvalues_all_host_arrays_dc( + handle, + reinterpret_cast(a), + ev, + error); +} +inline void elpa_eigenvalues( + elpa_t handle, + std::complex *a, + float *ev, + int *error) +{ + elpa_eigenvalues_all_host_arrays_fc( + handle, + reinterpret_cast(a), + ev, + error); +} +#else // ELPA version 2022.05.001, ELPA has its own c++ interface from version 2022.11.001 +inline void elpa_eigenvalues( + elpa_t handle, + double *a, + double *ev, + int *error) +{ + elpa_eigenvalues_a_h_a_d( + handle, + a, + ev, + error); +} +inline void elpa_eigenvalues( + elpa_t handle, + float *a, + float *ev, + int *error) +{ + elpa_eigenvalues_a_h_a_f( + handle, + a, + ev, + error); +} +inline void elpa_eigenvalues( + elpa_t handle, + std::complex *a, + double *ev, + int *error) +{ + elpa_eigenvalues_a_h_a_dc( + handle, + reinterpret_cast(a), + ev, + error); +} +inline void elpa_eigenvalues( + elpa_t handle, + std::complex *a, + float *ev, + int *error) +{ + elpa_eigenvalues_a_h_a_fc( + handle, + reinterpret_cast(a), + ev, + error); } #endif @@ -265,31 +621,79 @@ inline void elpa_eigenvalues(elpa_t handle, std::complex *a, float *ev, * \result void */ #if ELPA_API_VERSION <= 20210502 // ELPA 2021.05.002 and earlier versions -inline void elpa_skew_eigenvalues(elpa_t handle, double *a, double *ev, int *error) -{ - elpa_eigenvalues_d(handle, a, ev, error); -} -inline void elpa_skew_eigenvalues(elpa_t handle, float *a, float *ev, int *error) -{ - elpa_eigenvalues_f(handle, a, ev, error); -} -#elif ELPA_API_VERSION < 20220501 // ELPA version between 2021.11.001 and 2022.05.001 -inline void elpa_skew_eigenvalues(elpa_t handle, double *a, double *ev, int *error) -{ - elpa_eigenvalues_all_host_arrays_d(handle, a, ev, error); -} -inline void elpa_skew_eigenvalues(elpa_t handle, float *a, float *ev, int *error) -{ - elpa_eigenvalues_all_host_arrays_f(handle, a, ev, error); -} -#else // ELPA version 2022.05.001, ELPA has its own c++ interface from version 2022.11.001 -inline void elpa_skew_eigenvalues(elpa_t handle, double *a, double *ev, int *error) -{ - elpa_eigenvalues_a_h_a_d(handle, a, ev, error); -} -inline void elpa_skew_eigenvalues(elpa_t handle, float *a, float *ev, int *error) -{ - elpa_eigenvalues_a_h_a_f(handle, a, ev, error); +inline void elpa_skew_eigenvalues( + elpa_t handle, + double *a, + double *ev, + int *error) +{ + elpa_eigenvalues_d( + handle, + a, + ev, + error); +} +inline void elpa_skew_eigenvalues( + elpa_t handle, + float *a, + float *ev, + int *error) +{ + elpa_eigenvalues_f( + handle, + a, + ev, + error); +} +#elif ELPA_API_VERSION < 20220501 // ELPA version between 2021.11.001 and 2022.05.001 +inline void elpa_skew_eigenvalues( + elpa_t handle, + double *a, + double *ev, + int *error) +{ + elpa_eigenvalues_all_host_arrays_d( + handle, + a, + ev, + error); +} +inline void elpa_skew_eigenvalues( + elpa_t handle, + float *a, + float *ev, + int *error) +{ + elpa_eigenvalues_all_host_arrays_f( + handle, + a, + ev, + error); +} +#else // ELPA version 2022.05.001, ELPA has its own c++ interface from version 2022.11.001 +inline void elpa_skew_eigenvalues( + elpa_t handle, + double *a, + double *ev, + int *error) +{ + elpa_eigenvalues_a_h_a_d( + handle, + a, + ev, + error); +} +inline void elpa_skew_eigenvalues( + elpa_t handle, + float *a, + float *ev, + int *error) +{ + elpa_eigenvalues_a_h_a_f( + handle, + a, + ev, + error); } #endif @@ -303,39 +707,27 @@ inline void elpa_skew_eigenvalues(elpa_t handle, float *a, float *ev, int *err * \result void */ -#if ELPA_API_VERSION < 20220501 // ELPA version before 2022.05.001 -inline void elpa_cholesky(elpa_t handle, double *a, int *error) -{ - elpa_cholesky_d(handle, a, error); -} -inline void elpa_cholesky(elpa_t handle, float *a, int *error) -{ - elpa_cholesky_f(handle, a, error); -} +#if ELPA_API_VERSION < 20220501 // ELPA version before 2022.05.001 +inline void elpa_cholesky(elpa_t handle, double *a, int *error) { elpa_cholesky_d(handle, a, error); } +inline void elpa_cholesky(elpa_t handle, float *a, int *error) { elpa_cholesky_f(handle, a, error); } inline void elpa_cholesky(elpa_t handle, std::complex *a, int *error) { - elpa_cholesky_dc(handle, reinterpret_cast(a), error); + elpa_cholesky_dc(handle, reinterpret_cast(a), error); } -inline void elpa_cholesky(elpa_t handle, std::complex *a, int *error) +inline void elpa_cholesky(elpa_t handle, std::complex *a, int *error) { - elpa_cholesky_fc(handle, reinterpret_cast(a), error); + elpa_cholesky_fc(handle, reinterpret_cast(a), error); } #else -inline void elpa_cholesky(elpa_t handle, double *a, int *error) -{ - elpa_cholesky_a_h_a_d(handle, a, error); -} -inline void elpa_cholesky(elpa_t handle, float *a, int *error) -{ - elpa_cholesky_a_h_a_f(handle, a, error); -} +inline void elpa_cholesky(elpa_t handle, double *a, int *error) { elpa_cholesky_a_h_a_d(handle, a, error); } +inline void elpa_cholesky(elpa_t handle, float *a, int *error) { elpa_cholesky_a_h_a_f(handle, a, error); } inline void elpa_cholesky(elpa_t handle, std::complex *a, int *error) { - elpa_cholesky_a_h_a_dc(handle, reinterpret_cast(a), error); + elpa_cholesky_a_h_a_dc(handle, reinterpret_cast(a), error); } -inline void elpa_cholesky(elpa_t handle, std::complex *a, int *error) +inline void elpa_cholesky(elpa_t handle, std::complex *a, int *error) { - elpa_cholesky_a_h_a_fc(handle, reinterpret_cast(a), error); + elpa_cholesky_a_h_a_fc(handle, reinterpret_cast(a), error); } #endif @@ -356,47 +748,231 @@ inline void elpa_cholesky(elpa_t handle, std::complex *a, int *error) * \param error on return the error code, which can be queried with elpa_strerr() * \result void */ -#if ELPA_API_VERSION < 20220501 // ELPA version before 2022.05.001 -inline void elpa_hermitian_multiply(elpa_t handle, char uplo_a, char uplo_c, int ncb, double *a, double *b, int nrows_b, int ncols_b, double *c, int nrows_c, int ncols_c, int *error) -{ - elpa_hermitian_multiply_d(handle, uplo_a, uplo_c, ncb, a, b, nrows_b, ncols_b, c, nrows_c, ncols_c, error); -} -inline void elpa_hermitian_multiply(elpa_t handle, char uplo_a, char uplo_c, int ncb, float *a, float *b, int nrows_b, int ncols_b, float *c, int nrows_c, int ncols_c, int *error) -{ - elpa_hermitian_multiply_df(handle, uplo_a, uplo_c, ncb, a, b, nrows_b, ncols_b, c, nrows_c, ncols_c, error); -} -inline void elpa_hermitian_multiply(elpa_t handle, char uplo_a, char uplo_c, int ncb, std::complex *a, std::complex *b, int nrows_b, int ncols_b, std::complex *c, int nrows_c, int ncols_c, int *error) -{ - elpa_hermitian_multiply_dc(handle, uplo_a, uplo_c, ncb, reinterpret_cast(a), - reinterpret_cast(b), nrows_b, ncols_b, - reinterpret_cast(c), nrows_c, ncols_c, error); -} -inline void elpa_hermitian_multiply(elpa_t handle, char uplo_a, char uplo_c, int ncb, std::complex *a, std::complex *b, int nrows_b, int ncols_b, std::complex *c, int nrows_c, int ncols_c, int *error) -{ - elpa_hermitian_multiply_fc(handle, uplo_a, uplo_c, ncb, reinterpret_cast(a), - reinterpret_cast(b), nrows_b, ncols_b, - reinterpret_cast(c), nrows_c, ncols_c, error); +#if ELPA_API_VERSION < 20220501 // ELPA version before 2022.05.001 +inline void elpa_hermitian_multiply( + elpa_t handle, + char uplo_a, + char uplo_c, + int ncb, + double *a, + double *b, + int nrows_b, + int ncols_b, + double *c, + int nrows_c, + int ncols_c, + int *error) +{ + elpa_hermitian_multiply_d( + handle, + uplo_a, + uplo_c, + ncb, + a, + b, + nrows_b, + ncols_b, + c, + nrows_c, + ncols_c, + error); +} +inline void elpa_hermitian_multiply( + elpa_t handle, + char uplo_a, + char uplo_c, + int ncb, + float *a, + float *b, + int nrows_b, + int ncols_b, + float *c, + int nrows_c, + int ncols_c, + int *error) +{ + elpa_hermitian_multiply_df( + handle, + uplo_a, + uplo_c, + ncb, + a, + b, + nrows_b, + ncols_b, + c, + nrows_c, + ncols_c, + error); +} +inline void elpa_hermitian_multiply( + elpa_t handle, + char uplo_a, + char uplo_c, + int ncb, + std::complex *a, + std::complex *b, + int nrows_b, + int ncols_b, + std::complex *c, + int nrows_c, + int ncols_c, + int *error) +{ + elpa_hermitian_multiply_dc( + handle, + uplo_a, + uplo_c, + ncb, + reinterpret_cast(a), + reinterpret_cast(b), + nrows_b, + ncols_b, + reinterpret_cast(c), + nrows_c, + ncols_c, + error); +} +inline void elpa_hermitian_multiply( + elpa_t handle, + char uplo_a, + char uplo_c, + int ncb, + std::complex *a, + std::complex *b, + int nrows_b, + int ncols_b, + std::complex *c, + int nrows_c, + int ncols_c, + int *error) +{ + elpa_hermitian_multiply_fc( + handle, + uplo_a, + uplo_c, + ncb, + reinterpret_cast(a), + reinterpret_cast(b), + nrows_b, + ncols_b, + reinterpret_cast(c), + nrows_c, + ncols_c, + error); } #else -inline void elpa_hermitian_multiply(elpa_t handle, char uplo_a, char uplo_c, int ncb, double *a, double *b, int nrows_b, int ncols_b, double *c, int nrows_c, int ncols_c, int *error) -{ - elpa_hermitian_multiply_a_h_a_d(handle, uplo_a, uplo_c, ncb, a, b, nrows_b, ncols_b, c, nrows_c, ncols_c, error); -} -inline void elpa_hermitian_multiply(elpa_t handle, char uplo_a, char uplo_c, int ncb, float *a, float *b, int nrows_b, int ncols_b, float *c, int nrows_c, int ncols_c, int *error) -{ - elpa_hermitian_multiply_a_h_a_f(handle, uplo_a, uplo_c, ncb, a, b, nrows_b, ncols_b, c, nrows_c, ncols_c, error); -} -inline void elpa_hermitian_multiply(elpa_t handle, char uplo_a, char uplo_c, int ncb, std::complex *a, std::complex *b, int nrows_b, int ncols_b, std::complex *c, int nrows_c, int ncols_c, int *error) -{ - elpa_hermitian_multiply_a_h_a_dc(handle, uplo_a, uplo_c, ncb, reinterpret_cast(a), - reinterpret_cast(b), nrows_b, ncols_b, - reinterpret_cast(c), nrows_c, ncols_c, error); -} -inline void elpa_hermitian_multiply(elpa_t handle, char uplo_a, char uplo_c, int ncb, std::complex *a, std::complex *b, int nrows_b, int ncols_b, std::complex *c, int nrows_c, int ncols_c, int *error) -{ - elpa_hermitian_multiply_a_h_a_fc(handle, uplo_a, uplo_c, ncb, reinterpret_cast(a), - reinterpret_cast(b), nrows_b, ncols_b, - reinterpret_cast(c), nrows_c, ncols_c, error); +inline void elpa_hermitian_multiply( + elpa_t handle, + char uplo_a, + char uplo_c, + int ncb, + double *a, + double *b, + int nrows_b, + int ncols_b, + double *c, + int nrows_c, + int ncols_c, + int *error) +{ + elpa_hermitian_multiply_a_h_a_d( + handle, + uplo_a, + uplo_c, + ncb, + a, + b, + nrows_b, + ncols_b, + c, + nrows_c, + ncols_c, + error); +} +inline void elpa_hermitian_multiply( + elpa_t handle, + char uplo_a, + char uplo_c, + int ncb, + float *a, + float *b, + int nrows_b, + int ncols_b, + float *c, + int nrows_c, + int ncols_c, + int *error) +{ + elpa_hermitian_multiply_a_h_a_f( + handle, + uplo_a, + uplo_c, + ncb, + a, + b, + nrows_b, + ncols_b, + c, + nrows_c, + ncols_c, + error); +} +inline void elpa_hermitian_multiply( + elpa_t handle, + char uplo_a, + char uplo_c, + int ncb, + std::complex *a, + std::complex *b, + int nrows_b, + int ncols_b, + std::complex *c, + int nrows_c, + int ncols_c, + int *error) +{ + elpa_hermitian_multiply_a_h_a_dc( + handle, + uplo_a, + uplo_c, + ncb, + reinterpret_cast(a), + reinterpret_cast(b), + nrows_b, + ncols_b, + reinterpret_cast(c), + nrows_c, + ncols_c, + error); +} +inline void elpa_hermitian_multiply( + elpa_t handle, + char uplo_a, + char uplo_c, + int ncb, + std::complex *a, + std::complex *b, + int nrows_b, + int ncols_b, + std::complex *c, + int nrows_c, + int ncols_c, + int *error) +{ + elpa_hermitian_multiply_a_h_a_fc( + handle, + uplo_a, + uplo_c, + ncb, + reinterpret_cast(a), + reinterpret_cast(b), + nrows_b, + ncols_b, + reinterpret_cast(c), + nrows_c, + ncols_c, + error); } #endif @@ -409,39 +985,27 @@ inline void elpa_hermitian_multiply(elpa_t handle, char uplo_a, char uplo_c, int * \param error on return the error code, which can be queried with elpa_strerr() * \result void */ -#if ELPA_API_VERSION < 20220501 // ELPA version before 2022.05.001 -inline void elpa_invert_triangular(elpa_t handle, double *a, int *error) -{ - elpa_invert_trm_d(handle, a, error); -} -inline void elpa_invert_triangular(elpa_t handle, float *a, int *error) -{ - elpa_invert_trm_f(handle, a, error); -} +#if ELPA_API_VERSION < 20220501 // ELPA version before 2022.05.001 +inline void elpa_invert_triangular(elpa_t handle, double *a, int *error) { elpa_invert_trm_d(handle, a, error); } +inline void elpa_invert_triangular(elpa_t handle, float *a, int *error) { elpa_invert_trm_f(handle, a, error); } inline void elpa_invert_triangular(elpa_t handle, std::complex *a, int *error) { - elpa_invert_trm_dc(handle, reinterpret_cast(a), error); + elpa_invert_trm_dc(handle, reinterpret_cast(a), error); } -inline void elpa_invert_triangular(elpa_t handle, std::complex *a, int *error) +inline void elpa_invert_triangular(elpa_t handle, std::complex *a, int *error) { - elpa_invert_trm_fc(handle, reinterpret_cast(a), error); + elpa_invert_trm_fc(handle, reinterpret_cast(a), error); } #else -inline void elpa_invert_triangular(elpa_t handle, double *a, int *error) -{ - elpa_invert_trm_a_h_a_d(handle, a, error); -} -inline void elpa_invert_triangular(elpa_t handle, float *a, int *error) -{ - elpa_invert_trm_a_h_a_f(handle, a, error); -} +inline void elpa_invert_triangular(elpa_t handle, double *a, int *error) { elpa_invert_trm_a_h_a_d(handle, a, error); } +inline void elpa_invert_triangular(elpa_t handle, float *a, int *error) { elpa_invert_trm_a_h_a_f(handle, a, error); } inline void elpa_invert_triangular(elpa_t handle, std::complex *a, int *error) { - elpa_invert_trm_a_h_a_dc(handle, reinterpret_cast(a), error); + elpa_invert_trm_a_h_a_dc(handle, reinterpret_cast(a), error); } -inline void elpa_invert_triangular(elpa_t handle, std::complex *a, int *error) +inline void elpa_invert_triangular(elpa_t handle, std::complex *a, int *error) { - elpa_invert_trm_a_h_a_fc(handle, reinterpret_cast(a), error); + elpa_invert_trm_a_h_a_fc(handle, reinterpret_cast(a), error); } #endif diff --git a/source/source_hsolver/module_genelpa/elpa_new.cpp b/source/source_hsolver/module_genelpa/elpa_new.cpp index fa7bb917e4e..efe93e771c1 100644 --- a/source/source_hsolver/module_genelpa/elpa_new.cpp +++ b/source/source_hsolver/module_genelpa/elpa_new.cpp @@ -25,13 +25,14 @@ extern "C" std::map NEW_ELPA_HANDLE_POOL; -ELPA_Solver::ELPA_Solver(const bool isReal, - const MPI_Comm comm, - const int nev, - const int narows, - const int nacols, - const int* desc, - const bool reuse_handle_0) +ELPA_Solver::ELPA_Solver( + const bool isReal, + const MPI_Comm comm, + const int nev, + const int narows, + const int nacols, + const int *desc, + const bool reuse_handle_0) { this->isReal = isReal; this->comm = comm; @@ -43,21 +44,32 @@ ELPA_Solver::ELPA_Solver(const bool isReal, this->mpi_comm_cols = 0; this->mpi_comm_rows = 0; for (int i = 0; i < 9; ++i) + { this->desc[i] = desc[i]; + } cblacs_ctxt = desc[1]; nFull = desc[2]; nblk = desc[4]; lda = desc[8]; // cout<<"parameters are passed\n"; MPI_Comm_rank(comm, &myid); - Cblacs_gridinfo(cblacs_ctxt, &nprows, &npcols, &myprow, &mypcol); + Cblacs_gridinfo( + cblacs_ctxt, + &nprows, + &npcols, + &myprow, + &mypcol); // cout<<"blacs grid is inited\n"; allocate_work(); // cout<<"work array is inited\n"; if (isReal) + { kernel_id = read_real_kernel(); + } else + { kernel_id = read_complex_kernel(); + } // cout<<"kernel id is inited as "<0) + // delete the old elpa_handle and reuse the handle_id=0 + if (reuse_handle_0 && total_handle > 0) { NEW_ELPA_HANDLE_POOL.erase(0); handle_id = 0; @@ -86,24 +98,68 @@ ELPA_Solver::ELPA_Solver(const bool isReal, NEW_ELPA_HANDLE_POOL[handle_id] = handle; #ifdef _OPENMP - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "omp_threads", num_threads, &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "omp_threads", + num_threads, + &error); #endif - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "na", nFull, &error); - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "nev", nev, &error); - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "local_nrows", narows, &error); - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "local_ncols", nacols, &error); - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "nblk", nblk, &error); - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "mpi_comm_parent", MPI_Comm_c2f(comm), &error); - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "process_row", myprow, &error); - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "process_col", mypcol, &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "na", + nFull, + &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "nev", + nev, + &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "local_nrows", + narows, + &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "local_ncols", + nacols, + &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "nblk", + nblk, + &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "mpi_comm_parent", + MPI_Comm_c2f(comm), + &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "process_row", + myprow, + &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "process_col", + mypcol, + &error); // blacs_context is required by ELPA for internal MPI operations // (e.g. MPI_Bcast in complex Cholesky/invert_triangular); // previously missing in this constructor but present in the otherParameter one - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "blacs_context", cblacs_ctxt, &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "blacs_context", + cblacs_ctxt, + &error); error = elpa_setup(NEW_ELPA_HANDLE_POOL[handle_id]); // cout<<"elpa handle is setup\n"; - elpa_set_integer(NEW_ELPA_HANDLE_POOL[handle_id], "solver", ELPA_SOLVER_2STAGE, &error); + elpa_set_integer( + NEW_ELPA_HANDLE_POOL[handle_id], + "solver", + ELPA_SOLVER_2STAGE, + &error); this->setQR(0); this->setKernel(isReal, kernel_id); // cout<<"elpa kernel is setup\n"; @@ -111,13 +167,14 @@ ELPA_Solver::ELPA_Solver(const bool isReal, // cout<<"log level is setup\n"; } -ELPA_Solver::ELPA_Solver(const bool isReal, - const MPI_Comm comm, - const int nev, - const int narows, - const int nacols, - const int* desc, - const int* otherParameter) +ELPA_Solver::ELPA_Solver( + const bool isReal, + const MPI_Comm comm, + const int nev, + const int narows, + const int nacols, + const int *desc, + const int *otherParameter) { this->isReal = isReal; this->comm = comm; @@ -129,7 +186,9 @@ ELPA_Solver::ELPA_Solver(const bool isReal, this->mpi_comm_cols = 0; this->mpi_comm_rows = 0; for (int i = 0; i < 9; ++i) + { this->desc[i] = desc[i]; + } kernel_id = otherParameter[0]; useQR = otherParameter[1]; @@ -140,7 +199,12 @@ ELPA_Solver::ELPA_Solver(const bool isReal, nblk = desc[4]; lda = desc[8]; MPI_Comm_rank(comm, &myid); - Cblacs_gridinfo(cblacs_ctxt, &nprows, &npcols, &myprow, &mypcol); + Cblacs_gridinfo( + cblacs_ctxt, + &nprows, + &npcols, + &myprow, + &mypcol); allocate_work(); int error; @@ -161,20 +225,72 @@ ELPA_Solver::ELPA_Solver(const bool isReal, NEW_ELPA_HANDLE_POOL[handle_id] = handle; #ifdef _OPENMP - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "omp_threads", num_threads, &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "omp_threads", + num_threads, + &error); #endif - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "na", nFull, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "nev", nev, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "local_nrows", narows, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "local_ncols", nacols, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "nblk", nblk, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "mpi_comm_parent", MPI_Comm_c2f(comm), &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "process_row", myprow, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "process_col", mypcol, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "blacs_context", cblacs_ctxt, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "solver", ELPA_SOLVER_2STAGE, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "debug", wantDebug, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "qr", useQR, &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "na", + nFull, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "nev", + nev, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "local_nrows", + narows, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "local_ncols", + nacols, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "nblk", + nblk, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "mpi_comm_parent", + MPI_Comm_c2f(comm), + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "process_row", + myprow, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "process_col", + mypcol, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "blacs_context", + cblacs_ctxt, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "solver", + ELPA_SOLVER_2STAGE, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "debug", + wantDebug, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "qr", + useQR, + &error); this->setQR(useQR); this->setKernel(isReal, kernel_id); this->setLoglevel(loglevel); @@ -189,8 +305,16 @@ void ELPA_Solver::setLoglevel(int loglevel) if (loglevel >= 2) { wantDebug = 1; - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "verbose", 1, &error); - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "debug", wantDebug, &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "verbose", + 1, + &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "debug", + wantDebug, + &error); if (!isLogfileInited) { std::stringstream logfilename; @@ -212,10 +336,20 @@ void ELPA_Solver::setKernel(bool isReal, int kernel) this->kernel_id = kernel; int error; if (isReal) - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "real_kernel", kernel, &error); + { + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "real_kernel", + kernel, + &error); + } else { - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "complex_kernel", kernel, &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "complex_kernel", + kernel, + &error); validate_elpa_complex_kernel(kernel, error, comm); } } @@ -224,7 +358,11 @@ void ELPA_Solver::setQR(int useQR) { this->useQR = useQR; int error; - elpa_set(NEW_ELPA_HANDLE_POOL[handle_id], "qr", useQR, &error); + elpa_set( + NEW_ELPA_HANDLE_POOL[handle_id], + "qr", + useQR, + &error); } void ELPA_Solver::exit() @@ -232,7 +370,9 @@ void ELPA_Solver::exit() // delete[] dwork; // delete[] zwork; if (loglevel > 2) + { logfile.close(); + } int error; elpa_deallocate(NEW_ELPA_HANDLE_POOL[handle_id], &error); elpa_uninit(&error); @@ -281,82 +421,156 @@ int ELPA_Solver::read_real_kernel() { int kernel_id = 0; - if (const char* env = getenv("ELPA_DEFAULT_real_kernel")) + if (const char *env = getenv("ELPA_DEFAULT_real_kernel")) { if (strcmp(env, "ELPA_2STAGE_REAL_GENERIC_SIMPLE") == 0) + { kernel_id = ELPA_2STAGE_REAL_GENERIC_SIMPLE; + } else if (strcmp(env, "ELPA_2STAGE_REAL_BGP") == 0) + { kernel_id = ELPA_2STAGE_REAL_BGP; + } else if (strcmp(env, "ELPA_2STAGE_REAL_BGQ") == 0) + { kernel_id = ELPA_2STAGE_REAL_BGQ; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SSE_ASSEMBLY") == 0) + { kernel_id = ELPA_2STAGE_REAL_SSE_ASSEMBLY; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SSE_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_REAL_SSE_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SSE_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_SSE_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SSE_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_SSE_BLOCK6; + } else if (strcmp(env, "ELPA_2STAGE_REAL_AVX_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_REAL_AVX_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_REAL_AVX_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_AVX_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_AVX_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_AVX_BLOCK6; + } else if (strcmp(env, "ELPA_2STAGE_REAL_AVX2_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_REAL_AVX2_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_REAL_AVX2_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_AVX2_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_AVX2_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_AVX2_BLOCK6; + } else if (strcmp(env, "ELPA_2STAGE_REAL_AVX512_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_REAL_AVX512_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_REAL_AVX512_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_AVX512_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_AVX512_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_AVX512_BLOCK6; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SPARC64_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_REAL_SPARC64_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SPARC64_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_SPARC64_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SPARC64_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_SPARC64_BLOCK6; + } else if (strcmp(env, "ELPA_2STAGE_REAL_NEON_ARCH64_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_REAL_NEON_ARCH64_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_REAL_NEON_ARCH64_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_NEON_ARCH64_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_NEON_ARCH64_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_NEON_ARCH64_BLOCK6; + } else if (strcmp(env, "ELPA_2STAGE_REAL_VSX_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_REAL_VSX_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_REAL_VSX_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_VSX_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_VSX_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_VSX_BLOCK6; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SVE128_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_REAL_SVE128_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SVE128_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_SVE128_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SVE128_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_SVE128_BLOCK6; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SVE256_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_REAL_SVE256_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SVE256_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_SVE256_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SVE256_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_SVE256_BLOCK6; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SVE512_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_REAL_SVE512_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SVE512_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_SVE512_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_SVE512_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_SVE512_BLOCK6; + } else if (strcmp(env, "ELPA_2STAGE_REAL_GENERIC_SIMPLE_BLOCK4") == 0) + { kernel_id = ELPA_2STAGE_REAL_GENERIC_SIMPLE_BLOCK4; + } else if (strcmp(env, "ELPA_2STAGE_REAL_GENERIC_SIMPLE_BLOCK6") == 0) + { kernel_id = ELPA_2STAGE_REAL_GENERIC_SIMPLE_BLOCK6; + } else + { kernel_id = ELPA_2STAGE_REAL_GENERIC; + } } else { @@ -386,54 +600,100 @@ int ELPA_Solver::read_real_kernel() int ELPA_Solver::read_complex_kernel() { int kernel_id; - if (const char* env = getenv("ELPA_DEFAULT_complex_kernel")) + if (const char *env = getenv("ELPA_DEFAULT_complex_kernel")) { if (strcmp(env, "ELPA_2STAGE_COMPLEX_GENERIC_SIMPLE") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_GENERIC_SIMPLE; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_BGP") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_BGP; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_BGQ") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_BGQ; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_SSE_ASSEMBLY") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_SSE_ASSEMBLY; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_SSE_BLOCK1") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_SSE_BLOCK1; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_SSE_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_SSE_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_AVX_BLOCK1") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_AVX_BLOCK1; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_AVX_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_AVX_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_AVX2_BLOCK1") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_AVX2_BLOCK1; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_AVX2_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_AVX2_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_AVX512_BLOCK1") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_AVX512_BLOCK1; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_AVX512_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_AVX512_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_SVE128_BLOCK1") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_SVE128_BLOCK1; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_SVE128_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_SVE128_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_SVE256_BLOCK1") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_SVE256_BLOCK1; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_SVE256_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_SVE256_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_SVE512_BLOCK1") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_SVE512_BLOCK1; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_SVE512_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_SVE512_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_NEON_ARCH64_BLOCK1") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_NEON_ARCH64_BLOCK1; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_NEON_ARCH64_BLOCK2") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_NEON_ARCH64_BLOCK2; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_NVIDIA_GPU") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_NVIDIA_GPU; + } else if (strcmp(env, "ELPA_2STAGE_COMPLEX_AMD_GPU") == 0) + { kernel_id = ELPA_2STAGE_COMPLEX_AMD_GPU; + } else + { kernel_id = ELPA_2STAGE_COMPLEX_GENERIC; + } } else { @@ -463,30 +723,44 @@ int ELPA_Solver::read_complex_kernel() int ELPA_Solver::allocate_work() { unsigned long nloc = static_cast(narows) * nacols; // local size - unsigned long maxloc = 0; // maximum local size - MPI_Allreduce(&nloc, &maxloc, 1, MPI_UNSIGNED_LONG, MPI_MAX, comm); + unsigned long maxloc = 0; // maximum local size + MPI_Allreduce( + &nloc, + &maxloc, + 1, + MPI_UNSIGNED_LONG, + MPI_MAX, + comm); maxloc = nloc; if (isReal) + { dwork.resize(maxloc); + } else + { zwork.resize(maxloc); + } return 0; } -void ELPA_Solver::timer(int myid, const char function[], const char step[], double& t0) +void ELPA_Solver::timer( + int myid, + const char function[], + const char step[], + double &t0) { double t1 = 0.0; if (t0 < 0) // t0 < 0 means this is the init call before the function { t0 = MPI_Wtime(); - t0 = (double)clock()/CLOCKS_PER_SEC; + t0 = (double)clock() / CLOCKS_PER_SEC; logfile << "DEBUG: Process " << myid << " Call " << function << std::endl; } else { t1 = MPI_Wtime(); - t1 = (double)clock()/CLOCKS_PER_SEC; + t1 = (double)clock() / CLOCKS_PER_SEC; logfile << "DEBUG: Process " << myid << " Step " << step << " " << function << " time: " << t1 - t0 << " s" << std::endl; } @@ -503,7 +777,9 @@ void ELPA_Solver::outputParameters() logfile << "myid " << myid << ": blacs ctxt:" << cblacs_ctxt << std::endl; logfile << "myid " << myid << ": desc: "; for (int i = 0; i < 9; ++i) + { logfile << desc[i] << " "; + } logfile << std::endl; logfile << "myid " << myid << ": nblk: " << nblk << " lda: " << lda << std::endl; logfile << "myid " << myid << ": useQR: " << useQR << " kernel:" << kernel_id << std::endl; diff --git a/source/source_hsolver/module_genelpa/elpa_new.h b/source/source_hsolver/module_genelpa/elpa_new.h index df36ad37a52..8c26f377d36 100644 --- a/source/source_hsolver/module_genelpa/elpa_new.h +++ b/source/source_hsolver/module_genelpa/elpa_new.h @@ -1,11 +1,8 @@ #ifndef ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_ELPA_NEW_H #define ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_ELPA_NEW_H - #include - - #if ELPA_API_VERSION >= 20221101 #include #else diff --git a/source/source_hsolver/module_genelpa/elpa_new_complex.cpp b/source/source_hsolver/module_genelpa/elpa_new_complex.cpp index 9aec04fa72b..798b5e565ee 100644 --- a/source/source_hsolver/module_genelpa/elpa_new_complex.cpp +++ b/source/source_hsolver/module_genelpa/elpa_new_complex.cpp @@ -1,10 +1,10 @@ -#include -#include -#include -#include #include +#include #include +#include +#include #include +#include #include "elpa_new.h" #include "elpa_solver.h" @@ -14,335 +14,688 @@ #include "source_base/tool_quit.h" - extern std::map NEW_ELPA_HANDLE_POOL; -int ELPA_Solver::eigenvector(std::complex* A, double* EigenValue, std::complex* EigenVector) +int ELPA_Solver::eigenvector(std::complex *A, double *EigenValue, std::complex *EigenVector) { int info = 0; int allinfo = 0; double t = 0.0; - if((loglevel>0 && myid==0) || loglevel>1) + if ((loglevel > 0 && myid == 0) || loglevel > 1) { - t=-1; - timer(myid, "elpa_eigenvectors_dc", "1", t); + t = -1; + timer( + myid, + "elpa_eigenvectors_dc", + "1", + t); } - elpa_eigenvectors(NEW_ELPA_HANDLE_POOL[handle_id], A, EigenValue, EigenVector, &info); - if((loglevel>0 && myid==0) || loglevel>1) + elpa_eigenvectors( + NEW_ELPA_HANDLE_POOL[handle_id], + A, + EigenValue, + EigenVector, + &info); + if ((loglevel > 0 && myid == 0) || loglevel > 1) { - timer(myid, "elpa_eigenvectors_dc", "1", t); + timer( + myid, + "elpa_eigenvectors_dc", + "1", + t); } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; return allinfo; } -int ELPA_Solver::generalized_eigenvector(std::complex* A, std::complex* B, int& DecomposedState, - double* EigenValue, std::complex* EigenVector) +int ELPA_Solver::generalized_eigenvector( + std::complex *A, + std::complex *B, + int &DecomposedState, + double *EigenValue, + std::complex *EigenVector) { int info, allinfo; double t = 0.0; - if((loglevel>0 && myid==0) || loglevel>1) + if ((loglevel > 0 && myid == 0) || loglevel > 1) + { + t = -1; + timer( + myid, + "decomposeRightMatrix", + "1", + t); + } + if (DecomposedState == 0) // B is not decomposed { - t=-1; - timer(myid, "decomposeRightMatrix", "1", t); + allinfo = decomposeRightMatrix( + B, + EigenValue, + EigenVector, + DecomposedState); } - if(DecomposedState==0) // B is not decomposed - allinfo=decomposeRightMatrix(B, EigenValue, EigenVector, DecomposedState); else - allinfo=0; + { + allinfo = 0; + } - if((loglevel>0 && myid==0) || loglevel>1) + if ((loglevel > 0 && myid == 0) || loglevel > 1) { - timer(myid, "decomposeRightMatrix", "1", t); + timer( + myid, + "decomposeRightMatrix", + "1", + t); } - if(allinfo != 0){ + if (allinfo != 0) + { // if allinfo is still not 0 anyway, report error and quit - if(myid == 0){ - ModuleBase::WARNING_QUIT("ELPA_Solver::generalized_eigenvector", "decomposeRightMatrix failed to decompose right matrix!\n info = " + std::to_string(allinfo)); + if (myid == 0) + { + ModuleBase::WARNING_QUIT( + "ELPA_Solver::generalized_eigenvector", + "decomposeRightMatrix failed to decompose right matrix!\n info = " + std::to_string(allinfo)); } } // transform A to A~ - if((loglevel>0 && myid==0) || loglevel>1) + if ((loglevel > 0 && myid == 0) || loglevel > 1) { - t=-1; - timer(myid, "transform A to A~", "2", t); + t = -1; + timer( + myid, + "transform A to A~", + "2", + t); } - if(DecomposedState == 1 || DecomposedState == 2) + if (DecomposedState == 1 || DecomposedState == 2) { // Form H * U^{-1} in work. Only the upper triangle of H is // authoritative, so use the Hermitian matrix-matrix product. - if(loglevel>1) + if (loglevel > 1) { - t=-1; - timer(myid, "A*U^-1", "2.1a", t); - } - ScalapackConnector::hemm('L', 'U', nFull, 1.0, A, B, 0.0, zwork.data(), desc); - if(loglevel>1) + t = -1; + timer( + myid, + "A*U^-1", + "2.1a", + t); + } + ScalapackConnector::hemm( + 'L', + 'U', + nFull, + 1.0, + A, + B, + 0.0, + zwork.data(), + desc); + if (loglevel > 1) { - timer(myid, "A*U^-1", "2.1a", t); + timer( + myid, + "A*U^-1", + "2.1a", + t); } // Form U^{-H} * (H * U^{-1}) in A. - if(loglevel>1) + if (loglevel > 1) { - t=-1; - timer(myid, "U^-H*(H*U^-1)", "2.2a", t); - } - ScalapackConnector::gemm('C', 'N', nFull, nFull, nFull, 1.0, B, zwork.data(), 0.0, A, desc); - if(loglevel>1) + t = -1; + timer( + myid, + "U^-H*(H*U^-1)", + "2.2a", + t); + } + ScalapackConnector::gemm( + 'C', + 'N', + nFull, + nFull, + nFull, + 1.0, + B, + zwork.data(), + 0.0, + A, + desc); + if (loglevel > 1) { - timer(myid, "U^-H*(H*U^-1)", "2.2a", t); + timer( + myid, + "U^-H*(H*U^-1)", + "2.2a", + t); } } else { // Here B stores S^{-1/2}. Form B * H in work. A is still the // original Hermitian H, whose authoritative data is its upper triangle. - if(loglevel>1) + if (loglevel > 1) { - t=-1; - timer(myid, "B*H", "2.1b", t); - } - ScalapackConnector::hemm('R', 'U', nFull, 1.0, A, B, 0.0, zwork.data(), desc); - if(loglevel>1) + t = -1; + timer( + myid, + "B*H", + "2.1b", + t); + } + ScalapackConnector::hemm( + 'R', + 'U', + nFull, + 1.0, + A, + B, + 0.0, + zwork.data(), + desc); + if (loglevel > 1) { - timer(myid, "B*H", "2.1b", t); + timer( + myid, + "B*H", + "2.1b", + t); } // Form B * work^H = B * H * B^H in A. - if(loglevel>1) + if (loglevel > 1) { - t=-1; - timer(myid, "B*H*B^H", "2.2b", t); - } - ScalapackConnector::gemm('N', 'C', nFull, nFull, nFull, 1.0, B, zwork.data(), 0.0, A, desc); - if(loglevel>1) + t = -1; + timer( + myid, + "B*H*B^H", + "2.2b", + t); + } + ScalapackConnector::gemm( + 'N', + 'C', + nFull, + nFull, + nFull, + 1.0, + B, + zwork.data(), + 0.0, + A, + desc); + if (loglevel > 1) { - timer(myid, "B*H*B^H", "2.2b", t); + timer( + myid, + "B*H*B^H", + "2.2b", + t); } } - if((loglevel>0 && myid==0) || loglevel>1) + if ((loglevel > 0 && myid == 0) || loglevel > 1) { - timer(myid, "transform A to A~", "2", t); + timer( + myid, + "transform A to A~", + "2", + t); } // calculate the eigenvalue and eigenvector of A~ - if((loglevel>0 && myid==0) || loglevel>1) + if ((loglevel > 0 && myid == 0) || loglevel > 1) + { + t = -1; + timer( + myid, + "elpa_eigenvectors", + "3", + t); + } + if (loglevel > 2) { - t=-1; - timer(myid, "elpa_eigenvectors", "3", t); + saveMatrix( + "A_tilde.dat", + nFull, + A, + desc, + cblacs_ctxt); } - if(loglevel>2) saveMatrix("A_tilde.dat", nFull, A, desc, cblacs_ctxt); - info=eigenvector(A, EigenValue, EigenVector); - if((loglevel>0 && myid==0) || loglevel>1) + info = eigenvector(A, EigenValue, EigenVector); + if ((loglevel > 0 && myid == 0) || loglevel > 1) { - timer(myid, "elpa_eigenvectors", "3", t); + timer( + myid, + "elpa_eigenvectors", + "3", + t); } #ifdef __MPI - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); #else allinfo = info; #endif - if(loglevel>2) saveMatrix("EigenVector_tilde.dat", nFull, EigenVector, desc, cblacs_ctxt); + if (loglevel > 2) + { + saveMatrix( + "EigenVector_tilde.dat", + nFull, + EigenVector, + desc, + cblacs_ctxt); + } - if((loglevel>0 && myid==0) || loglevel>1) + if ((loglevel > 0 && myid == 0) || loglevel > 1) { - t=-1; - timer(myid, "composeEigenVector", "4", t); + t = -1; + timer( + myid, + "composeEigenVector", + "4", + t); } // transform eigenvector c~ to original eigenvector c - allinfo=composeEigenVector(DecomposedState, B, EigenVector); - if((loglevel>0 && myid==0) || loglevel>1) + allinfo = composeEigenVector(DecomposedState, B, EigenVector); + if ((loglevel > 0 && myid == 0) || loglevel > 1) { - timer(myid, "composeEigenVector", "4", t); + timer( + myid, + "composeEigenVector", + "4", + t); } return allinfo; } -int ELPA_Solver::decomposeRightMatrix(std::complex* B, double* EigenValue, std::complex* EigenVector, int& DecomposedState) +int ELPA_Solver::decomposeRightMatrix( + std::complex *B, + double *EigenValue, + std::complex *EigenVector, + int &DecomposedState) { - int info=0; - int allinfo=0; + int info = 0; + int allinfo = 0; double t = 0.0; // first try cholesky decomposing - if(nFull1) + DecomposedState = 1; + if (loglevel > 1) { - t=-1; - timer(myid, "pzpotrf_", "1", t); - } - ScalapackConnector::potrf('U', nFull, B, desc); - if(loglevel>1) + t = -1; + timer( + myid, + "pzpotrf_", + "1", + t); + } + ScalapackConnector::potrf( + 'U', + nFull, + B, + desc); + if (loglevel > 1) { - timer(myid, "pzpotrf_", "1", t); - } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + timer( + myid, + "pzpotrf_", + "1", + t); + } + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; - if(allinfo != 0) //if pdpotrf fail, try elpa_cholesky_real + if (allinfo != 0) // if pdpotrf fail, try elpa_cholesky_real { - DecomposedState=2; - if(loglevel>1) + DecomposedState = 2; + if (loglevel > 1) { - t=-1; - timer(myid, "elpa_cholesky_dc", "2", t); + t = -1; + timer( + myid, + "elpa_cholesky_dc", + "2", + t); } elpa_cholesky(NEW_ELPA_HANDLE_POOL[handle_id], B, &info); - if(loglevel>1) + if (loglevel > 1) { - timer(myid, "elpa_cholesky_dc", "2", t); + timer( + myid, + "elpa_cholesky_dc", + "2", + t); } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; } - } else + } + else { - DecomposedState=2; - if(loglevel>1) + DecomposedState = 2; + if (loglevel > 1) { - t=-1; - timer(myid, "elpa_cholesky_dc", "1", t); + t = -1; + timer( + myid, + "elpa_cholesky_dc", + "1", + t); } elpa_cholesky(NEW_ELPA_HANDLE_POOL[handle_id], B, &info); - if(loglevel>1) + if (loglevel > 1) { - timer(myid, "elpa_cholesky_dc", "1", t); - } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + timer( + myid, + "elpa_cholesky_dc", + "1", + t); + } + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; - if(allinfo != 0) + if (allinfo != 0) { - DecomposedState=1; - if(loglevel>1) + DecomposedState = 1; + if (loglevel > 1) { - t=-1; - timer(myid, "pzpotrf_", "2", t); + t = -1; + timer( + myid, + "pzpotrf_", + "2", + t); } - ScalapackConnector::potrf('U', nFull, B, desc); - if(loglevel>1) + ScalapackConnector::potrf( + 'U', + nFull, + B, + desc); + if (loglevel > 1) { - timer(myid, "pzpotrf_", "2", t); + timer( + myid, + "pzpotrf_", + "2", + t); } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; } } - if(allinfo==0) // calculate U^{-1} + if (allinfo == 0) // calculate U^{-1} { - if(loglevel>1) + if (loglevel > 1) { - t=-1; - timer(myid, "clear low triangle", "1", t); - } - for(int j=0; jjGlobal) B[i+j*narows]=0; + int iGlobal = globalIndex( + i, + nblk, + nprows, + myprow); + if (iGlobal > jGlobal) + { + B[i + j * narows] = 0; + } } } - if(loglevel>1) + if (loglevel > 1) { - timer(myid, "clear low triangle", "1", t); + timer( + myid, + "clear low triangle", + "1", + t); } - if(loglevel>2) saveMatrix("U.dat", nFull, B, desc, cblacs_ctxt); - if(loglevel>1) + if (loglevel > 2) { - t=-1; - timer(myid, "invert U", "1", t); + saveMatrix( + "U.dat", + nFull, + B, + desc, + cblacs_ctxt); + } + if (loglevel > 1) + { + t = -1; + timer( + myid, + "invert U", + "1", + t); } elpa_invert_triangular(NEW_ELPA_HANDLE_POOL[handle_id], B, &info); - if(loglevel>1) + if (loglevel > 1) + { + timer( + myid, + "invert U", + "1", + t); + } + if (loglevel > 2) { - timer(myid, "invert U", "1", t); + saveMatrix( + "U_inv.dat", + nFull, + B, + desc, + cblacs_ctxt); } - if(loglevel>2) saveMatrix("U_inv.dat", nFull, B, desc, cblacs_ctxt); - } else { + } + else + { // if cholesky decomposing failed, try diagonalize // Form B^{-1/2} = Q * diag(ev^{-1/2}) * Q^H in B. - DecomposedState=3; - if(loglevel>1) + DecomposedState = 3; + if (loglevel > 1) { - t=-1; - timer(myid, "calculate eigenvalue and eigenvector of B", "1", t); - } - //elpa_eigenvectors_all_host_arrays_dc(NEW_ELPA_HANDLE_POOL[handle_id], b, - // EigenValue, q, &info); - info=eigenvector(B, EigenValue, EigenVector); - if(loglevel>1) + t = -1; + timer( + myid, + "calculate eigenvalue and eigenvector of B", + "1", + t); + } + // elpa_eigenvectors_all_host_arrays_dc(NEW_ELPA_HANDLE_POOL[handle_id], b, + // EigenValue, q, &info); + info = eigenvector(B, EigenValue, EigenVector); + if (loglevel > 1) { - timer(myid, "calculate eigenvalue and eigenvector of B", "1", t); - } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + timer( + myid, + "calculate eigenvalue and eigenvector of B", + "1", + t); + } + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; // calculate q*ev and put to work - for(int i=0; iDBL_MIN?1.0/sqrt(EigenValue[eidx]):0; - for(int j=0; j DBL_MIN ? 1.0 / sqrt(EigenValue[eidx]) : 0; + for (int j = 0; j < narows; ++j) + { + zwork[i * lda + j] = EigenVector[i * lda + j] * ev_sqrt; + } } // Form Q * diag(ev^{-1/2}) * Q^H in B. - if(loglevel>1) + if (loglevel > 1) { - t=-1; - timer(myid, "qevq=qev*q^T", "2", t); - } - ScalapackConnector::gemm('N', 'C', nFull, nFull, nFull, 1.0, zwork.data(), EigenVector, 0.0, B, desc); - if(loglevel>1) + t = -1; + timer( + myid, + "qevq=qev*q^T", + "2", + t); + } + ScalapackConnector::gemm( + 'N', + 'C', + nFull, + nFull, + nFull, + 1.0, + zwork.data(), + EigenVector, + 0.0, + B, + desc); + if (loglevel > 1) { - timer(myid, "qevq=qev*q^T", "2", t); + timer( + myid, + "qevq=qev*q^T", + "2", + t); } } // if allinfo is still not 0 anyway, report error and quit - if(allinfo != 0) + if (allinfo != 0) { - if(myid == 0){ - ModuleBase::WARNING_QUIT("decomposeRightMatrix", - "Failed to decompose right matrix!\n info = " + std::to_string(allinfo)); + if (myid == 0) + { + ModuleBase::WARNING_QUIT( + "decomposeRightMatrix", "Failed to decompose right matrix!\n info = " + std::to_string(allinfo)); } } return allinfo; } -int ELPA_Solver::composeEigenVector(int DecomposedState, std::complex* B, std::complex* EigenVector) +int ELPA_Solver::composeEigenVector(int DecomposedState, std::complex *B, std::complex *EigenVector) { double t = 0.0; - if(DecomposedState==1 || DecomposedState==2) + if (DecomposedState == 1 || DecomposedState == 2) { // transform the eigenvectors to original general equation, let U^-1*q, and put to q - if(loglevel>1) + if (loglevel > 1) { - t=-1; - timer(myid, "Cpztrmm", "1", t); - } - ScalapackConnector::trmm('L', 'U', 'N', 'N', nFull, nev, 1.0, B, EigenVector, desc); - if(loglevel>1) + t = -1; + timer( + myid, + "Cpztrmm", + "1", + t); + } + ScalapackConnector::trmm( + 'L', + 'U', + 'N', + 'N', + nFull, + nev, + 1.0, + B, + EigenVector, + desc); + if (loglevel > 1) { - timer(myid, "Cpztrmm", "1", t); + timer( + myid, + "Cpztrmm", + "1", + t); } - } else { + } + else + { // transform the eigenvectors to original general equation, let b^C*q, and put to q - if(loglevel>1) + if (loglevel > 1) { - t=-1; - timer(myid, "Cpzgemm", "1", t); - } - ScalapackConnector::gemm('C', 'N', nFull, nev, nFull, 1.0, B, zwork.data(), 0.0, EigenVector, desc); - if(loglevel>1) + t = -1; + timer( + myid, + "Cpzgemm", + "1", + t); + } + ScalapackConnector::gemm( + 'C', + 'N', + nFull, + nev, + nFull, + 1.0, + B, + zwork.data(), + 0.0, + EigenVector, + desc); + if (loglevel > 1) { - timer(myid, "Cpzgemm", "1", t); + timer( + myid, + "Cpzgemm", + "1", + t); } } return 0; @@ -356,53 +709,111 @@ int ELPA_Solver::composeEigenVector(int DecomposedState, std::complex* B // D: Diagonal matrix of eigenvalue // maxError: maximum absolute value of error // meanError: mean absolute value of error -void ELPA_Solver::verify(std::complex* A, double* EigenValue, std::complex* EigenVector, - double &maxError, double &meanError) +void ELPA_Solver::verify( + std::complex *A, + double *EigenValue, + std::complex *EigenVector, + double &maxError, + double &meanError) { - std::complex* V=EigenVector; - const int naloc=narows*nacols; - std::complex* D=new std::complex[naloc]; - std::complex* R=zwork.data(); + std::complex *V = EigenVector; + const int naloc = narows * nacols; + std::complex *D = new std::complex[naloc]; + std::complex *R = zwork.data(); - for(int i=0; i2) saveMatrix("VD.dat", nFull, R, desc, cblacs_ctxt); + ScalapackConnector::hemm( + 'R', + 'U', + nFull, + 1.0, + D, + V, + 0.0, + R, + desc); + if (loglevel > 2) + { + saveMatrix( + "VD.dat", + nFull, + R, + desc, + cblacs_ctxt); + } // R=A*V-V*D=A*V-R - ScalapackConnector::hemm('L', 'U', nFull, 1.0, A, V, -1.0, R, desc); - if(loglevel>2) saveMatrix("AV-VD.dat", nFull, R, desc, cblacs_ctxt); + ScalapackConnector::hemm( + 'L', + 'U', + nFull, + 1.0, + A, + V, + -1.0, + R, + desc); + if (loglevel > 2) + { + saveMatrix( + "AV-VD.dat", + nFull, + R, + desc, + cblacs_ctxt); + } // calculate the maximum and mean value of sum_i{R(:,i)*R(:,i)} - double sumError=0; - maxError=0; - for(int i=1; i<=nev; ++i) + double sumError = 0; + maxError = 0; + for (int i = 1; i <= nev; ++i) { std::complex E; - ScalapackConnector::dot(nFull, E, R, 1, i, 1, - R, 1, i, 1, desc); - double abs_E=std::abs(E); - sumError+=abs_E; - maxError=std::max(maxError, abs_E); + ScalapackConnector::dot( + nFull, + E, + R, + 1, + i, + 1, + R, + 1, + i, + 1, + desc); + double abs_E = std::abs(E); + sumError += abs_E; + maxError = std::max(maxError, abs_E); } - meanError=sumError/nFull; + meanError = sumError / nFull; delete[] D; } @@ -414,57 +825,132 @@ void ELPA_Solver::verify(std::complex* A, double* EigenValue, std::compl // D: Diagonal matrix of eigenvalue // maxError: maximum absolute value of error // meanError: mean absolute value of error -void ELPA_Solver::verify(std::complex* A, std::complex* B, - double* EigenValue, std::complex* EigenVector, - double &maxError, double &meanError) +void ELPA_Solver::verify( + std::complex *A, + std::complex *B, + double *EigenValue, + std::complex *EigenVector, + double &maxError, + double &meanError) { - std::complex* V=EigenVector; - const int naloc=narows*nacols; - std::complex* D=new std::complex[naloc]; - std::complex* R=new std::complex[naloc]; + std::complex *V = EigenVector; + const int naloc = narows * nacols; + std::complex *D = new std::complex[naloc]; + std::complex *R = new std::complex[naloc]; - for(int i=0; i2) saveMatrix("BV.dat", nFull, zwork.data(), desc, cblacs_ctxt); + ScalapackConnector::hemm( + 'L', + 'U', + nFull, + 1.0, + B, + V, + 0.0, + zwork.data(), + desc); + if (loglevel > 2) + { + saveMatrix( + "BV.dat", + nFull, + zwork.data(), + desc, + cblacs_ctxt); + } // R=B*V*D=zwork*D - ScalapackConnector::hemm('L', 'U', nFull, 1.0, B, V, 0.0, zwork.data(), desc); - if(loglevel>2) saveMatrix("BVD.dat", nFull, R, desc, cblacs_ctxt); + ScalapackConnector::hemm( + 'L', + 'U', + nFull, + 1.0, + B, + V, + 0.0, + zwork.data(), + desc); + if (loglevel > 2) + { + saveMatrix( + "BVD.dat", + nFull, + R, + desc, + cblacs_ctxt); + } // R=A*V-B*V*D=A*V-R - ScalapackConnector::hemm('L', 'U', nFull, 1.0, B, V, 0.0, zwork.data(), desc); - if(loglevel>2) saveMatrix("AV-BVD.dat", nFull, R, desc, cblacs_ctxt); + ScalapackConnector::hemm( + 'L', + 'U', + nFull, + 1.0, + B, + V, + 0.0, + zwork.data(), + desc); + if (loglevel > 2) + { + saveMatrix( + "AV-BVD.dat", + nFull, + R, + desc, + cblacs_ctxt); + } // calculate the maximum and mean value of sum_i{R(:,i)*R(:,i)} - double sumError=0; - maxError=0; - for(int i=1; i<=nev; ++i) + double sumError = 0; + maxError = 0; + for (int i = 1; i <= nev; ++i) { std::complex E; - ScalapackConnector::dot(nFull, E, R, 1, i, 1, - R, 1, i, 1, desc); - double abs_E=std::abs(E); - sumError+=abs_E; - maxError=std::max(maxError, abs_E); + ScalapackConnector::dot( + nFull, + E, + R, + 1, + i, + 1, + R, + 1, + i, + 1, + desc); + double abs_E = std::abs(E); + sumError += abs_E; + maxError = std::max(maxError, abs_E); } - meanError=sumError/nFull; + meanError = sumError / nFull; delete[] D; delete[] R; diff --git a/source/source_hsolver/module_genelpa/elpa_new_real.cpp b/source/source_hsolver/module_genelpa/elpa_new_real.cpp index cc6d0242bd0..d3e525c3120 100644 --- a/source/source_hsolver/module_genelpa/elpa_new_real.cpp +++ b/source/source_hsolver/module_genelpa/elpa_new_real.cpp @@ -16,7 +16,7 @@ extern std::map NEW_ELPA_HANDLE_POOL; -int ELPA_Solver::eigenvector(double* A, double* EigenValue, double* EigenVector) +int ELPA_Solver::eigenvector(double *A, double *EigenValue, double *EigenVector) { int info = 0; double t = 0.0; @@ -24,21 +24,35 @@ int ELPA_Solver::eigenvector(double* A, double* EigenValue, double* EigenVector) if (loglevel > 0 && myid == 0) { t = -1; - timer(myid, "elpa_eigenvectors_all_host_arrays_d", "1", t); + timer( + myid, + "elpa_eigenvectors_all_host_arrays_d", + "1", + t); } - elpa_eigenvectors(NEW_ELPA_HANDLE_POOL[handle_id], A, EigenValue, EigenVector, &info); + elpa_eigenvectors( + NEW_ELPA_HANDLE_POOL[handle_id], + A, + EigenValue, + EigenVector, + &info); if (loglevel > 0 && myid == 0) { - timer(myid, "elpa_eigenvectors_all_host_arrays_d", "1", t); + timer( + myid, + "elpa_eigenvectors_all_host_arrays_d", + "1", + t); } return info; } -int ELPA_Solver::generalized_eigenvector(double* A, - double* B, - int& DecomposedState, - double* EigenValue, - double* EigenVector) +int ELPA_Solver::generalized_eigenvector( + double *A, + double *B, + int &DecomposedState, + double *EigenValue, + double *EigenVector) { int info, allinfo; double t = 0.0; @@ -46,20 +60,40 @@ int ELPA_Solver::generalized_eigenvector(double* A, if (loglevel > 0 && myid == 0) { t = -1; - timer(myid, "decomposeRightMatrix", "1", t); + timer( + myid, + "decomposeRightMatrix", + "1", + t); } if (DecomposedState == 0) - allinfo = decomposeRightMatrix(B, EigenValue, EigenVector, DecomposedState); + { + allinfo = decomposeRightMatrix( + B, + EigenValue, + EigenVector, + DecomposedState); + } else + { allinfo = 0; + } if (loglevel > 0 && myid == 0) { - timer(myid, "decomposeRightMatrix", "1", t); + timer( + myid, + "decomposeRightMatrix", + "1", + t); } - if (allinfo != 0){ + if (allinfo != 0) + { // if allinfo is still not 0 anyway, report error and quit - if(myid == 0){ - ModuleBase::WARNING_QUIT("ELPA_Solver::generalized_eigenvector", "decomposeRightMatrix failed to decompose right matrix!\n info = " + std::to_string(allinfo)); + if (myid == 0) + { + ModuleBase::WARNING_QUIT( + "ELPA_Solver::generalized_eigenvector", + "decomposeRightMatrix failed to decompose right matrix!\n info = " + std::to_string(allinfo)); } } @@ -67,7 +101,11 @@ int ELPA_Solver::generalized_eigenvector(double* A, if ((loglevel > 0 && myid == 0) || loglevel > 1) { t = -1; - timer(myid, "transform A to A~", "2", t); + timer( + myid, + "transform A to A~", + "2", + t); } if (DecomposedState == 1 || DecomposedState == 2) { @@ -75,24 +113,62 @@ int ELPA_Solver::generalized_eigenvector(double* A, if (loglevel > 1) { t = -1; - timer(myid, "A*U^-1", "2", t); - } - ScalapackConnector::gemm('T', 'N', nFull, nFull, nFull, 1.0, A, B, 0.0, dwork.data(), desc); + timer( + myid, + "A*U^-1", + "2", + t); + } + ScalapackConnector::gemm( + 'T', + 'N', + nFull, + nFull, + nFull, + 1.0, + A, + B, + 0.0, + dwork.data(), + desc); if (loglevel > 1) { - timer(myid, "A*U^-1", "2", t); + timer( + myid, + "A*U^-1", + "2", + t); } // calculate U^-T^(A*U^-1), put to a if (loglevel > 1) { t = -1; - timer(myid, "U^-T^(A*U^-1)", "3", t); - } - ScalapackConnector::gemm('T', 'N', nFull, nFull, nFull, 1.0, B, dwork.data(), 0.0, A, desc); + timer( + myid, + "U^-T^(A*U^-1)", + "3", + t); + } + ScalapackConnector::gemm( + 'T', + 'N', + nFull, + nFull, + nFull, + 1.0, + B, + dwork.data(), + 0.0, + A, + desc); if (loglevel > 1) { - timer(myid, "U^-T^(A*U^-1)", "3", t); + timer( + myid, + "U^-T^(A*U^-1)", + "3", + t); } } else @@ -101,57 +177,135 @@ int ELPA_Solver::generalized_eigenvector(double* A, if (loglevel > 1) { t = -1; - timer(myid, "B*A^T", "2", t); - } - ScalapackConnector::gemm('N', 'T', nFull, nFull, nFull, 1.0, B, A, 0.0, dwork.data(), desc); + timer( + myid, + "B*A^T", + "2", + t); + } + ScalapackConnector::gemm( + 'N', + 'T', + nFull, + nFull, + nFull, + 1.0, + B, + A, + 0.0, + dwork.data(), + desc); if (loglevel > 1) { - timer(myid, "B*A^T", "2", t); + timer( + myid, + "B*A^T", + "2", + t); } // calculate B*work^T = B*(B*A^T)^T and put to A -- original A*x=v*B*x was transform to a*x'=v*x' if (loglevel > 1) { t = -1; - timer(myid, "B*work^T = B*(B*A^T)^T", "3", t); - } - ScalapackConnector::gemm('N', 'T', nFull, nFull, nFull, 1.0, B, dwork.data(), 0.0, A, desc); + timer( + myid, + "B*work^T = B*(B*A^T)^T", + "3", + t); + } + ScalapackConnector::gemm( + 'N', + 'T', + nFull, + nFull, + nFull, + 1.0, + B, + dwork.data(), + 0.0, + A, + desc); if (loglevel > 1) { - timer(myid, "B*work^T = B*(B*A^T)^T", "3", t); + timer( + myid, + "B*work^T = B*(B*A^T)^T", + "3", + t); } } if ((loglevel > 0 && myid == 0) || loglevel > 1) { - timer(myid, "transform A to A~", "2", t); + timer( + myid, + "transform A to A~", + "2", + t); } if ((loglevel > 0 && myid == 0) || loglevel > 1) { t = -1; - timer(myid, "elpa_eigenvectors", "2", t); + timer( + myid, + "elpa_eigenvectors", + "2", + t); } if (loglevel > 2) - saveMatrix("A_tilde.dat", nFull, A, desc, cblacs_ctxt); + { + saveMatrix( + "A_tilde.dat", + nFull, + A, + desc, + cblacs_ctxt); + } info = eigenvector(A, EigenValue, EigenVector); if ((loglevel > 0 && myid == 0) || loglevel > 1) { - timer(myid, "elpa_eigenvectors", "2", t); + timer( + myid, + "elpa_eigenvectors", + "2", + t); } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; if (loglevel > 2) - saveMatrix("EigenVector_tilde.dat", nFull, EigenVector, desc, cblacs_ctxt); + { + saveMatrix( + "EigenVector_tilde.dat", + nFull, + EigenVector, + desc, + cblacs_ctxt); + } if (loglevel > 0 && myid == 0) { t = -1; - timer(myid, "composeEigenVector", "3", t); + timer( + myid, + "composeEigenVector", + "3", + t); } allinfo = composeEigenVector(DecomposedState, B, EigenVector); if (loglevel > 0 && myid == 0) { - timer(myid, "composeEigenVector", "3", t); + timer( + myid, + "composeEigenVector", + "3", + t); } return allinfo; } @@ -168,7 +322,11 @@ int ELPA_Solver::generalized_eigenvector(double* A, // B: decomposed right matrix // when DecomposedState is 1 or 2, B is U^{-1} // when DecomposedState is 3, B is B^{-1/2} -int ELPA_Solver::decomposeRightMatrix(double* B, double* EigenValue, double* EigenVector, int& DecomposedState) +int ELPA_Solver::decomposeRightMatrix( + double *B, + double *EigenValue, + double *EigenVector, + int &DecomposedState) { int info = 0; int allinfo = 0; @@ -181,14 +339,32 @@ int ELPA_Solver::decomposeRightMatrix(double* B, double* EigenValue, double* Eig if (loglevel > 1) { t = -1; - timer(myid, "pdpotrf_", "1", t); - } - info = ScalapackConnector::potrf('U', nFull, B, desc); + timer( + myid, + "pdpotrf_", + "1", + t); + } + info = ScalapackConnector::potrf( + 'U', + nFull, + B, + desc); if (loglevel > 1) { - timer(myid, "pdpotrf_", "1", t); - } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + timer( + myid, + "pdpotrf_", + "1", + t); + } + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; if (allinfo != 0) // pdpotrf fail, try elpa_cholesky_real { @@ -196,14 +372,28 @@ int ELPA_Solver::decomposeRightMatrix(double* B, double* EigenValue, double* Eig if (loglevel > 1) { t = -1; - timer(myid, "elpa_cholesky_d", "2", t); + timer( + myid, + "elpa_cholesky_d", + "2", + t); } elpa_cholesky(NEW_ELPA_HANDLE_POOL[handle_id], B, &info); if (loglevel > 1) { - timer(myid, "elpa_cholesky_d", "2", t); + timer( + myid, + "elpa_cholesky_d", + "2", + t); } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; } } @@ -213,14 +403,28 @@ int ELPA_Solver::decomposeRightMatrix(double* B, double* EigenValue, double* Eig if (loglevel > 1) { t = -1; - timer(myid, "elpa_cholesky_d", "1", t); + timer( + myid, + "elpa_cholesky_d", + "1", + t); } elpa_cholesky(NEW_ELPA_HANDLE_POOL[handle_id], B, &info); if (loglevel > 1) { - timer(myid, "elpa_cholesky_d", "1", t); - } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + timer( + myid, + "elpa_cholesky_d", + "1", + t); + } + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; if (allinfo != 0) { @@ -228,14 +432,32 @@ int ELPA_Solver::decomposeRightMatrix(double* B, double* EigenValue, double* Eig if (loglevel > 1) { t = -1; - timer(myid, "pdpotrf_", "2", t); + timer( + myid, + "pdpotrf_", + "2", + t); } - info = ScalapackConnector::potrf('U', nFull, B, desc); + info = ScalapackConnector::potrf( + 'U', + nFull, + B, + desc); if (loglevel > 1) { - timer(myid, "pdpotrf_", "2", t); + timer( + myid, + "pdpotrf_", + "2", + t); } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; } } @@ -246,37 +468,77 @@ int ELPA_Solver::decomposeRightMatrix(double* B, double* EigenValue, double* Eig if (loglevel > 1) { t = -1; - timer(myid, "clear low triangle", "1", t); + timer( + myid, + "clear low triangle", + "1", + t); } for (int j = 0; j < nacols; ++j) { - int jGlobal = globalIndex(j, nblk, npcols, mypcol); + int jGlobal = globalIndex( + j, + nblk, + npcols, + mypcol); for (int i = 0; i < narows; ++i) { - int iGlobal = globalIndex(i, nblk, nprows, myprow); + int iGlobal = globalIndex( + i, + nblk, + nprows, + myprow); if (iGlobal > jGlobal) + { B[i + j * static_cast(narows)] = 0; + } } } if (loglevel > 1) { - timer(myid, "clear low triangle", "1", t); + timer( + myid, + "clear low triangle", + "1", + t); } if (loglevel > 2) - saveMatrix("U.dat", nFull, B, desc, cblacs_ctxt); + { + saveMatrix( + "U.dat", + nFull, + B, + desc, + cblacs_ctxt); + } // calculate the inverse U^{-1} if (loglevel > 1) { t = -1; - timer(myid, "invert U", "1", t); + timer( + myid, + "invert U", + "1", + t); } elpa_invert_triangular(NEW_ELPA_HANDLE_POOL[handle_id], B, &info); if (loglevel > 1) { - timer(myid, "invert U", "1", t); + timer( + myid, + "invert U", + "1", + t); } if (loglevel > 2) - saveMatrix("U_inv.dat", nFull, B, desc, cblacs_ctxt); + { + saveMatrix( + "U_inv.dat", + nFull, + B, + desc, + cblacs_ctxt); + } } else { @@ -286,52 +548,92 @@ int ELPA_Solver::decomposeRightMatrix(double* B, double* EigenValue, double* Eig if (loglevel > 1) { t = -1; - timer(myid, "calculate eigenvalue and eigenvector of B", "1", t); + timer( + myid, + "calculate eigenvalue and eigenvector of B", + "1", + t); } // elpa_eigenvectors_all_host_arrays_d(NEW_ELPA_HANDLE_POOL[handle_id], B, EigenValue, EigenVector, &info); info = eigenvector(B, EigenValue, EigenVector); if (loglevel > 1) { - timer(myid, "calculate eigenvalue and eigenvector of B", "1", t); - } - MPI_Allreduce(&info, &allinfo, 1, MPI_INT, MPI_MAX, comm); + timer( + myid, + "calculate eigenvalue and eigenvector of B", + "1", + t); + } + MPI_Allreduce( + &info, + &allinfo, + 1, + MPI_INT, + MPI_MAX, + comm); allinfo = info; // calculate q*ev^{-1/2} and put to work for (int i = 0; i < nacols; ++i) { - int eidx = globalIndex(i, nblk, npcols, mypcol); + int eidx = globalIndex( + i, + nblk, + npcols, + mypcol); // double ev_sqrt=1.0/sqrt(ev[eidx]); double ev_sqrt = EigenValue[eidx] > DBL_MIN ? 1.0 / sqrt(EigenValue[eidx]) : 0; const std::size_t index_i = i * static_cast(lda); for (int j = 0; j < narows; ++j) + { dwork[index_i + j] = EigenVector[index_i + j] * ev_sqrt; + } } // calculate work*q=q*ev^{-1/2}*q^T, put to b, which is B^{-1/2} if (loglevel > 1) { t = -1; - timer(myid, "qevq=qev*q^T", "2", t); - } - ScalapackConnector::gemm('N', 'T', nFull, nFull, nFull, 1.0, dwork.data(), EigenVector, 0.0, B, desc); + timer( + myid, + "qevq=qev*q^T", + "2", + t); + } + ScalapackConnector::gemm( + 'N', + 'T', + nFull, + nFull, + nFull, + 1.0, + dwork.data(), + EigenVector, + 0.0, + B, + desc); if (loglevel > 1) { - timer(myid, "qevq=qev*q^T", "2", t); + timer( + myid, + "qevq=qev*q^T", + "2", + t); } } // if allinfo is still not 0 anyway, report error and quit - if(allinfo != 0) + if (allinfo != 0) { - if(myid == 0){ - ModuleBase::WARNING_QUIT("decomposeRightMatrix", - "Failed to decompose right matrix!\n info = " + std::to_string(allinfo)); + if (myid == 0) + { + ModuleBase::WARNING_QUIT( + "decomposeRightMatrix", "Failed to decompose right matrix!\n info = " + std::to_string(allinfo)); } } return allinfo; } -int ELPA_Solver::composeEigenVector(int DecomposedState, double* B, double* EigenVector) +int ELPA_Solver::composeEigenVector(int DecomposedState, double *B, double *EigenVector) { double t; if (DecomposedState == 1 || DecomposedState == 2) @@ -340,13 +642,31 @@ int ELPA_Solver::composeEigenVector(int DecomposedState, double* B, double* Eige if (loglevel > 1) { t = -1; - timer(myid, "Cpdtrmm", "1", t); - } - ScalapackConnector::trmm('L', 'U', 'N', 'N', nFull, nev, 1.0, B, EigenVector, desc); - //Cpdtrmm('L', 'U', 'N', 'N', nFull, nev, 1.0, B, EigenVector, desc); + timer( + myid, + "Cpdtrmm", + "1", + t); + } + ScalapackConnector::trmm( + 'L', + 'U', + 'N', + 'N', + nFull, + nev, + 1.0, + B, + EigenVector, + desc); + // Cpdtrmm('L', 'U', 'N', 'N', nFull, nev, 1.0, B, EigenVector, desc); if (loglevel > 1) { - timer(myid, "Cpdtrmm", "1", t); + timer( + myid, + "Cpdtrmm", + "1", + t); } } else @@ -355,12 +675,31 @@ int ELPA_Solver::composeEigenVector(int DecomposedState, double* B, double* Eige if (loglevel > 1) { t = -1; - timer(myid, "Cpdgemm", "1", t); - } - ScalapackConnector::gemm('T', 'N', nFull, nFull, nFull, 1.0, B, dwork.data(), 0.0, EigenVector, desc); + timer( + myid, + "Cpdgemm", + "1", + t); + } + ScalapackConnector::gemm( + 'T', + 'N', + nFull, + nFull, + nFull, + 1.0, + B, + dwork.data(), + 0.0, + EigenVector, + desc); if (loglevel > 1) { - timer(myid, "Cpdgemm", "1", t); + timer( + myid, + "Cpdgemm", + "1", + t); } } return 0; @@ -371,25 +710,40 @@ int ELPA_Solver::composeEigenVector(int DecomposedState, double* B, double* Eige // D: Diaganal matrix of eigenvalue // maxError: maximum error // meanError: mean error -void ELPA_Solver::verify(double* A, double* EigenValue, double* EigenVector, double& maxError, double& meanError) +void ELPA_Solver::verify( + double *A, + double *EigenValue, + double *EigenVector, + double &maxError, + double &meanError) { - double* V = EigenVector; + double *V = EigenVector; const std::size_t naloc = static_cast(narows) * nacols; - double* D = new double[naloc]; - double* R = dwork.data(); + double *D = new double[naloc]; + double *R = dwork.data(); for (int i = 0; i < naloc; ++i) + { D[i] = 0; + } for (int i = 0; i < nFull; ++i) { int localRow, localCol; int localProcRow, localProcCol; - localRow = localIndex(i, nblk, nprows, localProcRow); + localRow = localIndex( + i, + nblk, + nprows, + localProcRow); if (myprow == localProcRow) { - localCol = localIndex(i, nblk, npcols, localProcCol); + localCol = localIndex( + i, + nblk, + npcols, + localProcCol); if (mypcol == localProcCol) { const std::size_t idx = localRow + localCol * static_cast(narows); @@ -399,16 +753,47 @@ void ELPA_Solver::verify(double* A, double* EigenValue, double* EigenVector, dou } // R=V*D - ScalapackConnector::symm('R', 'U', nFull, nev, 1.0, D, V, 0.0, R, desc); + ScalapackConnector::symm( + 'R', + 'U', + nFull, + nev, + 1.0, + D, + V, + 0.0, + R, + desc); // R=A*V-V*D=A*V-R - ScalapackConnector::symm('L', 'U', nFull, nev, 1.0, A, V, -1.0, R, desc); + ScalapackConnector::symm( + 'L', + 'U', + nFull, + nev, + 1.0, + A, + V, + -1.0, + R, + desc); // calculate the maximum and mean value of sum_i{R(:,i)*R(:,i)} double sumError = 0; maxError = 0; for (int i = 1; i <= nev; ++i) { double E = 0.0; - ScalapackConnector::dot(nFull, E, R, 1, i, 1, R, 1, i, 1, desc); + ScalapackConnector::dot( + nFull, + E, + R, + 1, + i, + 1, + R, + 1, + i, + 1, + desc); // printf("myid: %d, i: %d, E: %lf\n", myid, i, E); sumError += E; maxError = maxError > E ? maxError : E; @@ -423,30 +808,41 @@ void ELPA_Solver::verify(double* A, double* EigenValue, double* EigenVector, dou // D: Diaganal matrix of eigenvalue // maxError: maximum absolute value of error // meanError: mean absolute value of error -void ELPA_Solver::verify(double* A, - double* B, - double* EigenValue, - double* EigenVector, - double& maxError, - double& meanError) +void ELPA_Solver::verify( + double *A, + double *B, + double *EigenValue, + double *EigenVector, + double &maxError, + double &meanError) { - double* V = EigenVector; + double *V = EigenVector; const std::size_t naloc = static_cast(narows) * nacols; - double* D = new double[naloc]; - double* R = new double[naloc]; + double *D = new double[naloc]; + double *R = new double[naloc]; for (int i = 0; i < naloc; ++i) + { D[i] = 0; + } for (int i = 0; i < nFull; ++i) { int localRow, localCol; int localProcRow, localProcCol; - localRow = localIndex(i, nblk, nprows, localProcRow); + localRow = localIndex( + i, + nblk, + nprows, + localProcRow); if (myprow == localProcRow) { - localCol = localIndex(i, nblk, npcols, localProcCol); + localCol = localIndex( + i, + nblk, + npcols, + localProcCol); if (mypcol == localProcCol) { const std::size_t idx = localRow + localCol * static_cast(narows); @@ -456,18 +852,59 @@ void ELPA_Solver::verify(double* A, } // dwork=B*V - ScalapackConnector::symm('L', 'U', nFull, nFull, 1.0, B, V, 0.0, dwork.data(), desc); + ScalapackConnector::symm( + 'L', + 'U', + nFull, + nFull, + 1.0, + B, + V, + 0.0, + dwork.data(), + desc); // R=B*V*D=dwork*D - ScalapackConnector::symm('R', 'U', nFull, nFull, 1.0, D, dwork.data(), 0.0, R, desc); + ScalapackConnector::symm( + 'R', + 'U', + nFull, + nFull, + 1.0, + D, + dwork.data(), + 0.0, + R, + desc); // R=A*V-B*V*D=A*V-R - ScalapackConnector::symm('L', 'U', nFull, nFull, 1.0, A, V, -1.0, R, desc); + ScalapackConnector::symm( + 'L', + 'U', + nFull, + nFull, + 1.0, + A, + V, + -1.0, + R, + desc); // calculate the maximum and mean value of sum_i{R(:,i)*R(:,i)} double sumError = 0; maxError = 0; for (int i = 1; i <= nev; ++i) { double E; - ScalapackConnector::dot(nFull, E, R, 1, i, 1, R, 1, i, 1, desc); + ScalapackConnector::dot( + nFull, + E, + R, + 1, + i, + 1, + R, + 1, + i, + 1, + desc); // printf("myid: %d, i: %d, E: %lf\n", myid, i, E); sumError += E; maxError = maxError > E ? maxError : E; diff --git a/source/source_hsolver/module_genelpa/elpa_runtime_check.cpp b/source/source_hsolver/module_genelpa/elpa_runtime_check.cpp index c2b94211eea..350d8170323 100644 --- a/source/source_hsolver/module_genelpa/elpa_runtime_check.cpp +++ b/source/source_hsolver/module_genelpa/elpa_runtime_check.cpp @@ -34,9 +34,17 @@ std::size_t index(const int row, const int col, const int leading_dimension) return static_cast(row) + static_cast(col) * leading_dimension; } -bool set_integer(elpa_t handle, const char* name, const int value, int& status) +bool set_integer( + elpa_t handle, + const char *name, + const int value, + int &status) { - elpa_set_integer(handle, name, value, &status); + elpa_set_integer( + handle, + name, + value, + &status); return status == ELPA_OK; } @@ -57,8 +65,7 @@ std::pair, std::vector> generate_hs() const double h_imag = distribution(generator); h[static_cast(i) * kN + j] = {h_real, h_imag}; - h[static_cast(j) * kN + i] - = i == j ? Complex{h_real, 0.0} : Complex{h_real, -h_imag}; + h[static_cast(j) * kN + i] = i == j ? Complex{h_real, 0.0} : Complex{h_real, -h_imag}; const double s_real = distribution(generator); const double s_imag = distribution(generator); @@ -73,8 +80,8 @@ std::pair, std::vector> generate_hs() Complex value{0.0, 0.0}; for (int k = 0; k < kN; ++k) { - value += s_tmp[static_cast(i) * kN + k] - * std::conj(s_tmp[static_cast(j) * kN + k]); + value += s_tmp[static_cast(i) * kN + k] * + std::conj(s_tmp[static_cast(j) * kN + k]); } if (i == j) { @@ -87,7 +94,7 @@ std::pair, std::vector> generate_hs() return {std::move(h), std::move(s)}; } -bool upper_cholesky(const std::vector& matrix, std::vector& upper) +bool upper_cholesky(const std::vector &matrix, std::vector &upper) { upper.assign(kN * kN, Complex{0.0, 0.0}); @@ -120,7 +127,7 @@ bool upper_cholesky(const std::vector& matrix, std::vector& up return true; } -std::vector invert_upper_triangular(const std::vector& upper) +std::vector invert_upper_triangular(const std::vector &upper) { std::vector inverse(kN * kN, Complex{0.0, 0.0}); @@ -140,7 +147,7 @@ std::vector invert_upper_triangular(const std::vector& upper) return inverse; } -std::vector multiply(const std::vector& left, const std::vector& right) +std::vector multiply(const std::vector &left, const std::vector &right) { std::vector result(kN * kN, Complex{0.0, 0.0}); for (int col = 0; col < kN; ++col) @@ -158,7 +165,7 @@ std::vector multiply(const std::vector& left, const std::vecto return result; } -std::vector conjugate_transpose(const std::vector& matrix) +std::vector conjugate_transpose(const std::vector &matrix) { std::vector result(kN * kN); for (int col = 0; col < kN; ++col) @@ -171,11 +178,11 @@ std::vector conjugate_transpose(const std::vector& matrix) return result; } -bool generate_a_tilde(std::vector& result) +bool generate_a_tilde(std::vector &result) { const std::pair, std::vector> hs = generate_hs(); - const std::vector& h_row_major = hs.first; - const std::vector& s_row_major = hs.second; + const std::vector &h_row_major = hs.first; + const std::vector &s_row_major = hs.second; std::vector a(kN * kN); std::vector s(kN * kN); @@ -200,7 +207,7 @@ bool generate_a_tilde(std::vector& result) return true; } -double frobenius_norm(const std::vector& matrix) +double frobenius_norm(const std::vector &matrix) { double norm_squared = 0.0; for (const Complex value : matrix) @@ -210,7 +217,7 @@ double frobenius_norm(const std::vector& matrix) return std::sqrt(norm_squared); } -double vector_norm(const Complex* vector, const int size) +double vector_norm(const Complex *vector, const int size) { double norm_squared = 0.0; for (int i = 0; i < size; ++i) @@ -220,10 +227,11 @@ double vector_norm(const Complex* vector, const int size) return std::sqrt(norm_squared); } -double relative_residual(const std::vector& matrix, - const Complex* eigenvector, - const double eigenvalue, - const double matrix_norm) +double relative_residual( + const std::vector &matrix, + const Complex *eigenvector, + const double eigenvalue, + const double matrix_norm) { std::vector residual(kN, Complex{0.0, 0.0}); @@ -239,11 +247,10 @@ double relative_residual(const std::vector& matrix, const double eigenvector_norm = vector_norm(eigenvector, kN); const double denominator = (matrix_norm + std::abs(eigenvalue)) * eigenvector_norm; - return denominator > 0.0 ? vector_norm(residual.data(), kN) / denominator - : vector_norm(residual.data(), kN); + return denominator > 0.0 ? vector_norm(residual.data(), kN) / denominator : vector_norm(residual.data(), kN); } -double orthogonality_error(const std::vector& eigenvectors) +double orthogonality_error(const std::vector &eigenvectors) { double error_squared = 0.0; @@ -281,9 +288,8 @@ ElpaRuntimeCheckResult failed_result() bool needs_runtime_check(const int kernel) { - return kernel == ELPA_2STAGE_COMPLEX_AVX_BLOCK2 - || kernel == ELPA_2STAGE_COMPLEX_AVX2_BLOCK2 - || kernel == ELPA_2STAGE_COMPLEX_AVX512_BLOCK2; + return kernel == ELPA_2STAGE_COMPLEX_AVX_BLOCK2 || kernel == ELPA_2STAGE_COMPLEX_AVX2_BLOCK2 || + kernel == ELPA_2STAGE_COMPLEX_AVX512_BLOCK2; } ElpaRuntimeCheckResult check_complex_block2_kernel(const int kernel) @@ -305,7 +311,8 @@ ElpaRuntimeCheckResult check_complex_block2_kernel(const int kernel) return failed_result(); } - const auto deallocate = [&handle](int& current_status) { + const auto deallocate = [&handle](int ¤t_status) + { int deallocate_status = ELPA_OK; elpa_deallocate(handle, &deallocate_status); if (current_status == ELPA_OK && deallocate_status != ELPA_OK) @@ -315,36 +322,85 @@ ElpaRuntimeCheckResult check_complex_block2_kernel(const int kernel) }; #ifdef _OPENMP - if (!set_integer(handle, "omp_threads", 1, status)) + if (!set_integer( + handle, + "omp_threads", + 1, + status)) { deallocate(status); return failed_result(); } #endif - if (!set_integer(handle, "na", kN, status) - || !set_integer(handle, "nev", kNev, status) - || !set_integer(handle, "local_nrows", kN, status) - || !set_integer(handle, "local_ncols", kN, status) - || !set_integer(handle, "nblk", kNblk, status) - || !set_integer(handle, "mpi_comm_parent", MPI_Comm_c2f(MPI_COMM_SELF), status) - || !set_integer(handle, "process_row", 0, status) - || !set_integer(handle, "process_col", 0, status)) + if (!set_integer( + handle, + "na", + kN, + status) || + !set_integer( + handle, + "nev", + kNev, + status) || + !set_integer( + handle, + "local_nrows", + kN, + status) || + !set_integer( + handle, + "local_ncols", + kN, + status) || + !set_integer( + handle, + "nblk", + kNblk, + status) || + !set_integer( + handle, + "mpi_comm_parent", + MPI_Comm_c2f(MPI_COMM_SELF), + status) || + !set_integer( + handle, + "process_row", + 0, + status) || + !set_integer( + handle, + "process_col", + 0, + status)) { deallocate(status); return failed_result(); } status = elpa_setup(handle); - if (status != ELPA_OK - || !set_integer(handle, "solver", ELPA_SOLVER_2STAGE, status) - || !set_integer(handle, "complex_kernel", kernel, status)) + if (status != ELPA_OK || + !set_integer( + handle, + "solver", + ELPA_SOLVER_2STAGE, + status) || + !set_integer( + handle, + "complex_kernel", + kernel, + status)) { deallocate(status); return failed_result(); } - elpa_eigenvectors(handle, matrix.data(), eigenvalues.data(), eigenvectors.data(), &status); + elpa_eigenvectors( + handle, + matrix.data(), + eigenvalues.data(), + eigenvectors.data(), + &status); if (status != ELPA_OK) { deallocate(status); @@ -355,12 +411,12 @@ ElpaRuntimeCheckResult check_complex_block2_kernel(const int kernel) double maximum_residual = 0.0; for (int band = 0; band < kNev; ++band) { - maximum_residual - = std::max(maximum_residual, - relative_residual(original_matrix, - &eigenvectors[index(0, band, kN)], - eigenvalues[band], - matrix_norm)); + maximum_residual = std::max( + maximum_residual, relative_residual( + original_matrix, + &eigenvectors[index(0, band, kN)], + eigenvalues[band], + matrix_norm)); } const double orthogonality = orthogonality_error(eigenvectors); @@ -376,7 +432,13 @@ void validate_elpa_complex_kernel(const int kernel, const int set_status, const { const int local_check_required = set_status == ELPA_OK && needs_runtime_check(kernel) ? 1 : 0; int any_check_required = 0; - MPI_Allreduce(&local_check_required, &any_check_required, 1, MPI_INT, MPI_MAX, comm); + MPI_Allreduce( + &local_check_required, + &any_check_required, + 1, + MPI_INT, + MPI_MAX, + comm); if (any_check_required == 0) { return; @@ -396,7 +458,13 @@ void validate_elpa_complex_kernel(const int kernel, const int set_status, const const int local_failed = result.passed ? 0 : 1; int any_failed = 0; - MPI_Allreduce(&local_failed, &any_failed, 1, MPI_INT, MPI_MAX, comm); + MPI_Allreduce( + &local_failed, + &any_failed, + 1, + MPI_INT, + MPI_MAX, + comm); if (any_failed == 0) { return; @@ -404,8 +472,20 @@ void validate_elpa_complex_kernel(const int kernel, const int set_status, const double max_residual = result.max_residual; double max_orthogonality = result.orthogonality; - MPI_Allreduce(MPI_IN_PLACE, &max_residual, 1, MPI_DOUBLE, MPI_MAX, comm); - MPI_Allreduce(MPI_IN_PLACE, &max_orthogonality, 1, MPI_DOUBLE, MPI_MAX, comm); + MPI_Allreduce( + MPI_IN_PLACE, + &max_residual, + 1, + MPI_DOUBLE, + MPI_MAX, + comm); + MPI_Allreduce( + MPI_IN_PLACE, + &max_orthogonality, + 1, + MPI_DOUBLE, + MPI_MAX, + comm); std::ostringstream message; message << "The selected ELPA complex 2-stage BLOCK2 kernel failed a runtime correctness check:\n\n " diff --git a/source/source_hsolver/module_genelpa/elpa_runtime_check.h b/source/source_hsolver/module_genelpa/elpa_runtime_check.h index 9ad6434bfac..48052d6d35b 100644 --- a/source/source_hsolver/module_genelpa/elpa_runtime_check.h +++ b/source/source_hsolver/module_genelpa/elpa_runtime_check.h @@ -1,7 +1,6 @@ #ifndef ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_ELPA_RUNTIME_CHECK_H #define ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_ELPA_RUNTIME_CHECK_H - #include void validate_elpa_complex_kernel(int kernel, int set_status, MPI_Comm comm); diff --git a/source/source_hsolver/module_genelpa/elpa_solver.h b/source/source_hsolver/module_genelpa/elpa_solver.h index 53a7c197710..7328cd62799 100644 --- a/source/source_hsolver/module_genelpa/elpa_solver.h +++ b/source/source_hsolver/module_genelpa/elpa_solver.h @@ -23,46 +23,67 @@ class ELPA_Solver * The context must remain valid for the lifetime of this object. * This class does NOT take ownership of the BLACS context. */ - ELPA_Solver(const bool isReal, - const MPI_Comm comm, - const int nev, - const int narows, - const int nacols, - const int* desc, - const bool reuse_handle_0 = true); - ELPA_Solver(const bool isReal, - const MPI_Comm comm, - const int nev, - const int narows, - const int nacols, - const int* desc, - const int* otherParameter); + ELPA_Solver( + const bool isReal, + const MPI_Comm comm, + const int nev, + const int narows, + const int nacols, + const int *desc, + const bool reuse_handle_0 = true); + ELPA_Solver( + const bool isReal, + const MPI_Comm comm, + const int nev, + const int narows, + const int nacols, + const int *desc, + const int *otherParameter); - int eigenvector(double* A, double* EigenValue, double* EigenVector); - int generalized_eigenvector(double* A, double* B, int& DecomposedState, double* EigenValue, double* EigenVector); - int eigenvector(std::complex* A, double* EigenValue, std::complex* EigenVector); - int generalized_eigenvector(std::complex* A, - std::complex* B, - int& DecomposedState, - double* EigenValue, - std::complex* EigenVector); + int eigenvector(double *A, double *EigenValue, double *EigenVector); + int generalized_eigenvector( + double *A, + double *B, + int &DecomposedState, + double *EigenValue, + double *EigenVector); + int eigenvector(std::complex *A, double *EigenValue, std::complex *EigenVector); + int generalized_eigenvector( + std::complex *A, + std::complex *B, + int &DecomposedState, + double *EigenValue, + std::complex *EigenVector); void setLoglevel(int loglevel); void setKernel(bool isReal, int Kernel); void setQR(int useQR); void outputParameters(); - void verify(double* A, double* EigenValue, double* EigenVector, double& maxRemain, double& meanRemain); - void verify(double* A, double* B, double* EigenValue, double* EigenVector, double& maxRemain, double& meanRemain); - void verify(std::complex* A, - double* EigenValue, - std::complex* EigenVector, - double& maxError, - double& meanError); - void verify(std::complex* A, - std::complex* B, - double* EigenValue, - std::complex* EigenVector, - double& maxError, - double& meanError); + void verify( + double *A, + double *EigenValue, + double *EigenVector, + double &maxRemain, + double &meanRemain); + void verify( + double *A, + double *B, + double *EigenValue, + double *EigenVector, + double &maxRemain, + double &meanRemain); + void verify( + std::complex *A, + double *EigenValue, + std::complex *EigenVector, + double &maxError, + double &meanError); + void verify( + std::complex *A, + std::complex *B, + double *EigenValue, + std::complex *EigenVector, + double &maxError, + double &meanError); void exit(); private: @@ -102,15 +123,24 @@ class ELPA_Solver int read_real_kernel(); int read_complex_kernel(); int allocate_work(); - int decomposeRightMatrix(double* B, double* EigenValue, double* EigenVector, int& DecomposedState); - int decomposeRightMatrix(std::complex* B, - double* EigenValue, - std::complex* EigenVector, - int& DecomposedState); - int composeEigenVector(int DecomposedState, double* B, double* EigenVector); - int composeEigenVector(int DecomposedState, std::complex* B, std::complex* EigenVector); + int decomposeRightMatrix( + double *B, + double *EigenValue, + double *EigenVector, + int &DecomposedState); + int decomposeRightMatrix( + std::complex *B, + double *EigenValue, + std::complex *EigenVector, + int &DecomposedState); + int composeEigenVector(int DecomposedState, double *B, double *EigenVector); + int composeEigenVector(int DecomposedState, std::complex *B, std::complex *EigenVector); // debug tool - void timer(int myid, const char function[], const char step[], double& t0); + void timer( + int myid, + const char function[], + const char step[], + double &t0); }; #endif // ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_ELPA_SOLVER_H diff --git a/source/source_hsolver/module_genelpa/pblas.h b/source/source_hsolver/module_genelpa/pblas.h index 5efca578455..4247cfd304a 100644 --- a/source/source_hsolver/module_genelpa/pblas.h +++ b/source/source_hsolver/module_genelpa/pblas.h @@ -1,44 +1,194 @@ #ifndef ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_PBLAS_H #define ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_PBLAS_H -void pddot_(int* n, double* dot, double* x, int* ix, int* jx, int* descx, int* incx, - double* y, int* iy, int* jy, int* descy, int* incy); - -void pzdotc_(int* n, double _Complex* dot, double _Complex* x, int* ix, int* jx, int* descx, int* incx, - double _Complex* y, int* iy, int* jy, int* descy, int* incy); -void pdsymv_(char* uplo, int* n, - double* alpha, double* a, int* ia, int* ja, int* desca, - double* x, int* ix, int* jx, int* descx, int* incx, - double* beta, double* y, int* iy, int* jy, int* descy, int* incy); -void pdtran_(int* m , int* n , - double* alpha , double* a , int* ia , int* ja , int* desca , - double* beta , double* c , int* ic , int* jc , int* descc ); +void pddot_( + int *n, + double *dot, + double *x, + int *ix, + int *jx, + int *descx, + int *incx, + double *y, + int *iy, + int *jy, + int *descy, + int *incy); -void pdgemm_(char* transa , char* transb , int* m , int* n , int* k , - double* alpha , double* a , int* ia , int* ja , int* desca , - double* b , int* ib , int* jb , int* descb , - double* beta , double* c , int* ic , int* jc , int* descc ); -void pzgemm_(char* transa , char* transb , int* m , int* n , int* k , - double _Complex* alpha , double _Complex* a , int* ia , int* ja , int* desca , - double _Complex* b , int* ib , int* jb , int* descb , - double _Complex* beta , double _Complex* c , int* ic , int* jc , int* descc ); -void pdsymm_(char* side , char* uplo , int* m , int* n , - double* alpha , double* a , int* ia , int* ja , int* desca , - double* b , int* ib , int* jb , int* descb , - double* beta , double* c , int* ic , int* jc , int* descc ); -void pzsymm_(char* side , char* uplo , int* m , int* n , - double _Complex* alpha , double _Complex* a , int* ia , int* ja , int* desca , - double _Complex* b , int* ib , int* jb , int* descb , - double _Complex* beta , double _Complex* c , int* ic , int* jc , int* descc ); -void pzhemm_(char* side , char* uplo , int* m , int* n , - double _Complex* alpha , double _Complex* a , int* ia , int* ja , int* desca , - double _Complex* b , int* ib , int* jb , int* descb , - double _Complex* beta , double _Complex* c , int* ic , int* jc , int* descc ); -void pdtrmm_(char* side , char* uplo , char* transa , char* diag , int* m , int* n , - double* alpha , double* a , int* ia , int* ja , int* desca , - double* b , int* ib , int* jb , int* descb ); -void pztrmm_(char* side , char* uplo , char* transa , char* diag , int* m , int* n , - double _Complex* alpha , double _Complex* a , int* ia , int* ja , int* desca , - double _Complex* b , int* ib , int* jb , int* descb ); +void pzdotc_( + int *n, + double _Complex *dot, + double _Complex *x, + int *ix, + int *jx, + int *descx, + int *incx, + double _Complex *y, + int *iy, + int *jy, + int *descy, + int *incy); +void pdsymv_( + char *uplo, + int *n, + double *alpha, + double *a, + int *ia, + int *ja, + int *desca, + double *x, + int *ix, + int *jx, + int *descx, + int *incx, + double *beta, + double *y, + int *iy, + int *jy, + int *descy, + int *incy); +void pdtran_( + int *m, + int *n, + double *alpha, + double *a, + int *ia, + int *ja, + int *desca, + double *beta, + double *c, + int *ic, + int *jc, + int *descc); + +void pdgemm_( + char *transa, + char *transb, + int *m, + int *n, + int *k, + double *alpha, + double *a, + int *ia, + int *ja, + int *desca, + double *b, + int *ib, + int *jb, + int *descb, + double *beta, + double *c, + int *ic, + int *jc, + int *descc); +void pzgemm_( + char *transa, + char *transb, + int *m, + int *n, + int *k, + double _Complex *alpha, + double _Complex *a, + int *ia, + int *ja, + int *desca, + double _Complex *b, + int *ib, + int *jb, + int *descb, + double _Complex *beta, + double _Complex *c, + int *ic, + int *jc, + int *descc); +void pdsymm_( + char *side, + char *uplo, + int *m, + int *n, + double *alpha, + double *a, + int *ia, + int *ja, + int *desca, + double *b, + int *ib, + int *jb, + int *descb, + double *beta, + double *c, + int *ic, + int *jc, + int *descc); +void pzsymm_( + char *side, + char *uplo, + int *m, + int *n, + double _Complex *alpha, + double _Complex *a, + int *ia, + int *ja, + int *desca, + double _Complex *b, + int *ib, + int *jb, + int *descb, + double _Complex *beta, + double _Complex *c, + int *ic, + int *jc, + int *descc); +void pzhemm_( + char *side, + char *uplo, + int *m, + int *n, + double _Complex *alpha, + double _Complex *a, + int *ia, + int *ja, + int *desca, + double _Complex *b, + int *ib, + int *jb, + int *descb, + double _Complex *beta, + double _Complex *c, + int *ic, + int *jc, + int *descc); +void pdtrmm_( + char *side, + char *uplo, + char *transa, + char *diag, + int *m, + int *n, + double *alpha, + double *a, + int *ia, + int *ja, + int *desca, + double *b, + int *ib, + int *jb, + int *descb); +void pztrmm_( + char *side, + char *uplo, + char *transa, + char *diag, + int *m, + int *n, + double _Complex *alpha, + double _Complex *a, + int *ia, + int *ja, + int *desca, + double _Complex *b, + int *ib, + int *jb, + int *descb); #endif // ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_PBLAS_H diff --git a/source/source_hsolver/module_genelpa/scalapack.h b/source/source_hsolver/module_genelpa/scalapack.h index 03a2b1a321b..ad3e0389b0f 100644 --- a/source/source_hsolver/module_genelpa/scalapack.h +++ b/source/source_hsolver/module_genelpa/scalapack.h @@ -2,15 +2,78 @@ #define ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_SCALAPACK_H // scalapack -int numroc_(const int *N, const int *NB, const int *IPROC, const int *ISRCPROC, const int *NPROCS); -void descinit_(int *DESC, const int *M, const int *N, const int *MB, const int *NB, const int *IRSRC, const int *ICSRC, const int *ICTXT, const int *LLD, int *INFO); -void pdpotrf_(const char *UPLO, const int *N, double *A, const int *IA, const int *JA, const int *DESCA, int *INFO); -void pzpotrf_(const char *UPLO, const int *N, double _Complex *A, const int *IA, const int *JA, const int *DESCA, int *INFO); -void pdsyev_(const char *JOBZ, const char *UPLO, int *N, double *A, int *IA, int *JA, int *DESCA, - double *W, double *Z, int *IZ, int *JZ, int *DESCZ, double *WORK, int *LWORK, int *INFO); -void pdgemr2d_(int *M, int *N, double *A, int *IA, int *JA, int *DESCA, - double *B, int *IB, int *JB, int *DESCB, int *ICTXT); -void pzgemr2d_(int *M, int *N, double _Complex *A, int *IA, int *JA, int *DESCA, - double _Complex *B, int *IB, int *JB, int *DESCB, int *ICTXT); +int numroc_( + const int *N, + const int *NB, + const int *IPROC, + const int *ISRCPROC, + const int *NPROCS); +void descinit_( + int *DESC, + const int *M, + const int *N, + const int *MB, + const int *NB, + const int *IRSRC, + const int *ICSRC, + const int *ICTXT, + const int *LLD, + int *INFO); +void pdpotrf_( + const char *UPLO, + const int *N, + double *A, + const int *IA, + const int *JA, + const int *DESCA, + int *INFO); +void pzpotrf_( + const char *UPLO, + const int *N, + double _Complex *A, + const int *IA, + const int *JA, + const int *DESCA, + int *INFO); +void pdsyev_( + const char *JOBZ, + const char *UPLO, + int *N, + double *A, + int *IA, + int *JA, + int *DESCA, + double *W, + double *Z, + int *IZ, + int *JZ, + int *DESCZ, + double *WORK, + int *LWORK, + int *INFO); +void pdgemr2d_( + int *M, + int *N, + double *A, + int *IA, + int *JA, + int *DESCA, + double *B, + int *IB, + int *JB, + int *DESCB, + int *ICTXT); +void pzgemr2d_( + int *M, + int *N, + double _Complex *A, + int *IA, + int *JA, + int *DESCA, + double _Complex *B, + int *IB, + int *JB, + int *DESCB, + int *ICTXT); #endif // ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_SCALAPACK_H diff --git a/source/source_hsolver/module_genelpa/utils.cpp b/source/source_hsolver/module_genelpa/utils.cpp index a334627c3a4..c37e54368aa 100644 --- a/source/source_hsolver/module_genelpa/utils.cpp +++ b/source/source_hsolver/module_genelpa/utils.cpp @@ -1,8 +1,8 @@ #include "utils.h" #include "source_base/module_external/blacs_connector.h" -#include "source_base/tool_quit.h" #include "source_base/module_external/scalapack_connector.h" +#include "source_base/tool_quit.h" #include #include @@ -12,10 +12,20 @@ #include #include // load matrix from the file -void loadMatrix(const char FileName[], int nFull, double* a, int* desca, int blacs_ctxt) +void loadMatrix( + const char FileName[], + int nFull, + double *a, + int *desca, + int blacs_ctxt) { int nprows, npcols, myprow, mypcol; - Cblacs_gridinfo(blacs_ctxt, &nprows, &npcols, &myprow, &mypcol); + Cblacs_gridinfo( + blacs_ctxt, + &nprows, + &npcols, + &myprow, + &mypcol); int myid = Cblacs_pnum(blacs_ctxt, myprow, mypcol); const int ROOT_PROC = 0; @@ -25,25 +35,37 @@ void loadMatrix(const char FileName[], int nFull, double* a, int* desca, int bla matrixFile.open(FileName); if (!matrixFile.is_open()) { - ModuleBase::WARNING_QUIT("module_genelpa::loadMatrix", - std::string("Failed to open matrix file: ") + FileName); + ModuleBase::WARNING_QUIT( + "module_genelpa::loadMatrix", std::string("Failed to open matrix file: ") + FileName); } } - double* b = nullptr; // buffer + double *b = nullptr; // buffer const int MAX_BUFFER_SIZE = 1e9; // max buffer size is 1GB int N = nFull; - int M - = std::max(1, std::min(nFull, (int)(MAX_BUFFER_SIZE / nFull / sizeof(double)))); // at lease 1 row, max size 1GB + int M = + std::max(1, std::min(nFull, (int)(MAX_BUFFER_SIZE / nFull / sizeof(double)))); // at lease 1 row, max size 1GB if (myid == ROOT_PROC) + { b = new double[M * N]; + } else + { b = new double[1]; + } // set descb, which has all elements in the only block in the root process // block size is M x N, so all elements are in the first process - int descb[9] = {1, blacs_ctxt, M, N, M, N, 0, 0, M}; + int descb[9] = {1, + blacs_ctxt, + M, + N, + M, + N, + 0, + 0, + M}; int ja = 1, ib = 1, jb = 1; for (int ia = 1; ia < nFull; ia += M) @@ -61,16 +83,33 @@ void loadMatrix(const char FileName[], int nFull, double* a, int* desca, int bla } } // gather data rows by rows from all processes - Cpdgemr2d(thisM, N, b, ib, jb, descb, a, ia, ja, desca, blacs_ctxt); + Cpdgemr2d( + thisM, + N, + b, + ib, + jb, + descb, + a, + ia, + ja, + desca, + blacs_ctxt); } if (myid == ROOT_PROC) + { matrixFile.close(); + } delete[] b; } -void saveLocalMatrix(const char filePrefix[], int narows, int nacols, double* a) +void saveLocalMatrix( + const char filePrefix[], + int narows, + int nacols, + double *a) { char FileName[80]; int myid = 0; @@ -81,12 +120,16 @@ void saveLocalMatrix(const char filePrefix[], int narows, int nacols, double* a) myid = 0; #endif - sprintf(FileName, "%s_%3.3d.dat", filePrefix, myid); + sprintf( + FileName, + "%s_%3.3d.dat", + filePrefix, + myid); matrixFile.open(FileName); if (!matrixFile.is_open()) { - ModuleBase::WARNING_QUIT("module_genelpa::saveLocalMatrix", - std::string("Failed to open matrix file for write: ") + FileName); + ModuleBase::WARNING_QUIT( + "module_genelpa::saveLocalMatrix", std::string("Failed to open matrix file for write: ") + FileName); } matrixFile.flags(std::ios_base::scientific); matrixFile.precision(17); @@ -104,10 +147,20 @@ void saveLocalMatrix(const char filePrefix[], int narows, int nacols, double* a) // use pdgemr2d to collect matrix from all processes to root process // and save to one completed matrix file -void saveMatrix(const char FileName[], int nFull, double* a, int* desca, int blacs_ctxt) +void saveMatrix( + const char FileName[], + int nFull, + double *a, + int *desca, + int blacs_ctxt) { int nprows, npcols, myprow, mypcol; - Cblacs_gridinfo(blacs_ctxt, &nprows, &npcols, &myprow, &mypcol); + Cblacs_gridinfo( + blacs_ctxt, + &nprows, + &npcols, + &myprow, + &mypcol); int myid = Cblacs_pnum(blacs_ctxt, myprow, mypcol); const int ROOT_PROC = 0; @@ -117,34 +170,57 @@ void saveMatrix(const char FileName[], int nFull, double* a, int* desca, int bla matrixFile.open(FileName); if (!matrixFile.is_open()) { - ModuleBase::WARNING_QUIT("module_genelpa::saveMatrix", - std::string("Failed to open matrix file for write: ") + FileName); + ModuleBase::WARNING_QUIT( + "module_genelpa::saveMatrix", std::string("Failed to open matrix file for write: ") + FileName); } matrixFile.flags(std::ios_base::scientific); matrixFile.precision(17); matrixFile.width(24); } - double* b = nullptr; // buffer + double *b = nullptr; // buffer const int MAX_BUFFER_SIZE = 1e9; // max buffer size is 1GB int N = nFull; - int M - = std::max(1, std::min(nFull, (int)(MAX_BUFFER_SIZE / nFull / sizeof(double)))); // at lease 1 row, max size 1GB + int M = + std::max(1, std::min(nFull, (int)(MAX_BUFFER_SIZE / nFull / sizeof(double)))); // at lease 1 row, max size 1GB if (myid == ROOT_PROC) + { b = new double[M * N]; + } else + { b = new double[1]; + } // set descb, which has all elements in the only block in the root process - int descb[9] = {1, blacs_ctxt, M, N, M, N, 0, 0, M}; + int descb[9] = {1, + blacs_ctxt, + M, + N, + M, + N, + 0, + 0, + M}; int ja = 1, ib = 1, jb = 1; for (int ia = 1; ia < nFull; ia += M) { int thisM = std::min(M, nFull - ia + 1); // nFull-ia+1 is the last few row to be saved // gather data rows by rows from all processes - Cpdgemr2d(thisM, N, a, ia, ja, desca, b, ib, jb, descb, blacs_ctxt); + Cpdgemr2d( + thisM, + N, + a, + ia, + ja, + desca, + b, + ib, + jb, + descb, + blacs_ctxt); // write to the file if (myid == ROOT_PROC) { @@ -160,16 +236,28 @@ void saveMatrix(const char FileName[], int nFull, double* a, int* desca, int bla } if (myid == ROOT_PROC) + { matrixFile.close(); + } delete[] b; } // load matrix from the file -void loadMatrix(const char FileName[], int nFull, std::complex* a, int* desca, int blacs_ctxt) +void loadMatrix( + const char FileName[], + int nFull, + std::complex *a, + int *desca, + int blacs_ctxt) { int nprows, npcols, myprow, mypcol; - Cblacs_gridinfo(blacs_ctxt, &nprows, &npcols, &myprow, &mypcol); + Cblacs_gridinfo( + blacs_ctxt, + &nprows, + &npcols, + &myprow, + &mypcol); int myid = Cblacs_pnum(blacs_ctxt, myprow, mypcol); const int ROOT_PROC = 0; @@ -179,26 +267,37 @@ void loadMatrix(const char FileName[], int nFull, std::complex* a, int* matrixFile.open(FileName); if (!matrixFile.is_open()) { - ModuleBase::WARNING_QUIT("module_genelpa::loadMatrix", - std::string("Failed to open matrix file: ") + FileName); + ModuleBase::WARNING_QUIT( + "module_genelpa::loadMatrix", std::string("Failed to open matrix file: ") + FileName); } } - std::complex* b; // buffer + std::complex *b; // buffer const int MAX_BUFFER_SIZE = 1e9; // max buffer size is 1GB int N = nFull; int M = std::max( - 1, - std::min(nFull, (int)(MAX_BUFFER_SIZE / nFull / (2 * sizeof(double))))); // at lease 1 row, max size 1GB + 1, std::min(nFull, (int)(MAX_BUFFER_SIZE / nFull / (2 * sizeof(double))))); // at lease 1 row, max size 1GB if (myid == ROOT_PROC) + { b = new std::complex[M * N]; + } else + { b = new std::complex[1]; + } // set descb, which has all elements in the only block in the root process // block size is M x N, so all elements are in the first process - int descb[9] = {1, blacs_ctxt, M, N, M, N, 0, 0, M}; + int descb[9] = {1, + blacs_ctxt, + M, + N, + M, + N, + 0, + 0, + M}; int ja = 1, ib = 1, jb = 1; for (int ia = 1; ia < nFull; ia += M) @@ -216,16 +315,33 @@ void loadMatrix(const char FileName[], int nFull, std::complex* a, int* } } // gather data rows by rows from all processes - Cpzgemr2d(thisM, N, b, ib, jb, descb, a, ia, ja, desca, blacs_ctxt); + Cpzgemr2d( + thisM, + N, + b, + ib, + jb, + descb, + a, + ia, + ja, + desca, + blacs_ctxt); } if (myid == ROOT_PROC) + { matrixFile.close(); + } delete[] b; } -void saveLocalMatrix(const char filePrefix[], int narows, int nacols, std::complex* a) +void saveLocalMatrix( + const char filePrefix[], + int narows, + int nacols, + std::complex *a) { char FileName[80]; int myid; @@ -236,12 +352,16 @@ void saveLocalMatrix(const char filePrefix[], int narows, int nacols, std::compl myid = 0; #endif - sprintf(FileName, "%s_%3.3d.dat", filePrefix, myid); + sprintf( + FileName, + "%s_%3.3d.dat", + filePrefix, + myid); matrixFile.open(FileName); if (!matrixFile.is_open()) { - ModuleBase::WARNING_QUIT("module_genelpa::saveLocalMatrix", - std::string("Failed to open matrix file for write: ") + FileName); + ModuleBase::WARNING_QUIT( + "module_genelpa::saveLocalMatrix", std::string("Failed to open matrix file for write: ") + FileName); } matrixFile.flags(std::ios_base::scientific); matrixFile.precision(17); @@ -259,10 +379,20 @@ void saveLocalMatrix(const char filePrefix[], int narows, int nacols, std::compl // use pzgemr2d to collect matrix from all processes to root process // and save to one completed matrix file -void saveMatrix(const char FileName[], int nFull, std::complex* a, int* desca, int blacs_ctxt) +void saveMatrix( + const char FileName[], + int nFull, + std::complex *a, + int *desca, + int blacs_ctxt) { int nprows, npcols, myprow, mypcol; - Cblacs_gridinfo(blacs_ctxt, &nprows, &npcols, &myprow, &mypcol); + Cblacs_gridinfo( + blacs_ctxt, + &nprows, + &npcols, + &myprow, + &mypcol); int myid = Cblacs_pnum(blacs_ctxt, myprow, mypcol); const int ROOT_PROC = 0; @@ -272,34 +402,57 @@ void saveMatrix(const char FileName[], int nFull, std::complex* a, int* matrixFile.open(FileName); if (!matrixFile.is_open()) { - ModuleBase::WARNING_QUIT("module_genelpa::saveMatrix", - std::string("Failed to open matrix file for write: ") + FileName); + ModuleBase::WARNING_QUIT( + "module_genelpa::saveMatrix", std::string("Failed to open matrix file for write: ") + FileName); } matrixFile.flags(std::ios_base::scientific); matrixFile.precision(17); matrixFile.width(24); } - std::complex* b; // buffer + std::complex *b; // buffer const int MAX_BUFFER_SIZE = 1e9; // max buffer size is 1GB int N = nFull; - int M - = std::max(1, std::min(nFull, (int)(MAX_BUFFER_SIZE / nFull / sizeof(double)))); // at lease 1 row, max size 1GB + int M = + std::max(1, std::min(nFull, (int)(MAX_BUFFER_SIZE / nFull / sizeof(double)))); // at lease 1 row, max size 1GB if (myid == ROOT_PROC) + { b = new std::complex[M * N]; + } else + { b = new std::complex[1]; + } // set descb, which has all elements in the only block in the root process - int descb[9] = {1, blacs_ctxt, M, N, M, N, 0, 0, M}; + int descb[9] = {1, + blacs_ctxt, + M, + N, + M, + N, + 0, + 0, + M}; int ja = 1, ib = 1, jb = 1; for (int ia = 1; ia < nFull; ia += M) { int transM = std::min(M, nFull - ia + 1); // nFull-ia+1 is the last few row to be saved // gather data rows by rows from all processes - Cpzgemr2d(transM, N, a, ia, ja, desca, b, ib, jb, descb, blacs_ctxt); + Cpzgemr2d( + transM, + N, + a, + ia, + ja, + desca, + b, + ib, + jb, + descb, + blacs_ctxt); // write to the file if (myid == ROOT_PROC) { @@ -315,7 +468,9 @@ void saveMatrix(const char FileName[], int nFull, std::complex* a, int* } if (myid == ROOT_PROC) + { matrixFile.close(); + } delete[] b; } diff --git a/source/source_hsolver/module_genelpa/utils.h b/source/source_hsolver/module_genelpa/utils.h index 73174b595a0..559e553fa45 100644 --- a/source/source_hsolver/module_genelpa/utils.h +++ b/source/source_hsolver/module_genelpa/utils.h @@ -4,7 +4,11 @@ #include #include -static inline int globalIndex(int localIndex, int nblk, int nprocs, int myproc) +static inline int globalIndex( + int localIndex, + int nblk, + int nprocs, + int myproc) { int iblock, gIndex; iblock = localIndex / nblk; @@ -12,27 +16,59 @@ static inline int globalIndex(int localIndex, int nblk, int nprocs, int myproc) return gIndex; } -static inline int localIndex(int globalIndex, int nblk, int nprocs, int& lcoalProc) +static inline int localIndex( + int globalIndex, + int nblk, + int nprocs, + int &lcoalProc) { lcoalProc = int((globalIndex % (nblk * nprocs)) / nblk); return int(globalIndex / (nblk * nprocs)) * nblk + globalIndex % nblk; } // load matrix from the file -void loadMatrix(const char FileName[], int nFull, double* a, int* desca, int blacs_ctxt); +void loadMatrix( + const char FileName[], + int nFull, + double *a, + int *desca, + int blacs_ctxt); -void saveLocalMatrix(const char filePrefix[], int narows, int nacols, double* a); +void saveLocalMatrix( + const char filePrefix[], + int narows, + int nacols, + double *a); // use pdgemr2d to collect matrix from all processes to root process // and save to one completed matrix file -void saveMatrix(const char FileName[], int nFull, double* a, int* desca, int blacs_ctxt); +void saveMatrix( + const char FileName[], + int nFull, + double *a, + int *desca, + int blacs_ctxt); -void loadMatrix(const char FileName[], int nFull, std::complex* a, int* desca, int blacs_ctxt); +void loadMatrix( + const char FileName[], + int nFull, + std::complex *a, + int *desca, + int blacs_ctxt); -void saveLocalMatrix(const char filePrefix[], int narows, int nacols, std::complex* a); +void saveLocalMatrix( + const char filePrefix[], + int narows, + int nacols, + std::complex *a); // use pzgemr2d to collect matrix from all processes to root process // and save to one completed matrix file -void saveMatrix(const char FileName[], int nFull, std::complex* a, int* desca, int blacs_ctxt); +void saveMatrix( + const char FileName[], + int nFull, + std::complex *a, + int *desca, + int blacs_ctxt); #endif // ABACUS_SOURCE_HSOLVER_MODULE_GENELPA_UTILS_H diff --git a/source/source_hsolver/module_linalg/cn_subspace.cpp b/source/source_hsolver/module_linalg/cn_subspace.cpp index 640f0b9dc77..abf62f88cb2 100644 --- a/source/source_hsolver/module_linalg/cn_subspace.cpp +++ b/source/source_hsolver/module_linalg/cn_subspace.cpp @@ -6,32 +6,95 @@ namespace hsolver { template -bool CNSubspace::prepare(LinearAlgebra& algebra, int ld, int dim, int bands, const T* u, const T* b) +bool CNSubspace::prepare( + LinearAlgebra &algebra, + int ld, + int dim, + int bands, + const T *u, + const T *b) { const int64_t elements = static_cast(ld) * bands; linear_buffer(&image_, elements); linear_buffer(&seed_, elements); linear_buffer(&residual_, elements); - linear_op().batch(ld, dim, bands, image(), u, b, T(2), T(-1), nullptr, nullptr, nullptr); - const std::vector> projected = algebra.projection_cross(ld, dim, bands, bands, u, image()); - std::vector> c = algebra.projection_cross(ld, dim, bands, bands, u, b); + linear_op().batch( + ld, + dim, + bands, + image(), + u, + b, + T(2), + T(-1), + nullptr, + nullptr, + nullptr); + const std::vector> projected = algebra.projection_cross( + ld, + dim, + bands, + bands, + u, + image()); + std::vector> c = algebra.projection_cross( + ld, + dim, + bands, + bands, + u, + b); if (!factor_.factor(projected, bands) || !factor_.solve(&c, bands)) { return false; } - for (const std::complex& value: c) + for (const std::complex &value : c) { if (!std::isfinite(std::abs(T(value)))) { return false; } } - algebra.expand(ld, dim, bands, bands, u, c, seed(), T(0)); - algebra.expand(ld, dim, bands, bands, image(), c, residual(), T(0)); - linear_op().batch(ld, dim, bands, residual(), b, residual(), T(1), T(-1), nullptr, nullptr, nullptr); + algebra.expand( + ld, + dim, + bands, + bands, + u, + c, + seed(), + T(0)); + algebra.expand( + ld, + dim, + bands, + bands, + image(), + c, + residual(), + T(0)); + linear_op().batch( + ld, + dim, + bands, + residual(), + b, + residual(), + T(1), + T(-1), + nullptr, + nullptr, + nullptr); const int stride = ld * bands; - const std::vector> norms = algebra.dots(ld, dim, bands, 1, stride, residual(), residual()); - for (const std::complex& value: norms) + const std::vector> norms = algebra.dots( + ld, + dim, + bands, + 1, + stride, + residual(), + residual()); + for (const std::complex &value : norms) { if (!std::isfinite(linear_norm(value))) { diff --git a/source/source_hsolver/module_linalg/cn_subspace.h b/source/source_hsolver/module_linalg/cn_subspace.h index a7492d30163..eb38e2536a5 100644 --- a/source/source_hsolver/module_linalg/cn_subspace.h +++ b/source/source_hsolver/module_linalg/cn_subspace.h @@ -5,8 +5,7 @@ namespace hsolver { /** @brief CN projection derived from A*U = 2*U - B without another Hamiltonian application. */ -template -class CNSubspace +template class CNSubspace { private: ct::Tensor image_; @@ -15,23 +14,17 @@ class CNSubspace LinearSmallLU factor_; public: - bool prepare(LinearAlgebra& algebra, int ld, int dim, int bands, const T* u, const T* b); - T* image() - { - return image_.template data(); - } - T* seed() - { - return seed_.template data(); - } - T* residual() - { - return residual_.template data(); - } - const LinearSmallLU& factor() const - { - return factor_; - } + bool prepare( + LinearAlgebra &algebra, + int ld, + int dim, + int bands, + const T *u, + const T *b); + T *image() { return image_.template data(); } + T *seed() { return seed_.template data(); } + T *residual() { return residual_.template data(); } + const LinearSmallLU &factor() const { return factor_; } }; } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/exp_action.cpp b/source/source_hsolver/module_linalg/exp_action.cpp index eb34865d733..ffa10432d87 100644 --- a/source/source_hsolver/module_linalg/exp_action.cpp +++ b/source/source_hsolver/module_linalg/exp_action.cpp @@ -9,21 +9,23 @@ namespace hsolver { template -ExpAction::ExpAction(const ExpControls& exp_ctrl, const diag_comm_info& comm) : exp_ctrl_(exp_ctrl), algebra_(comm) +ExpAction::ExpAction(const ExpControls &exp_ctrl, const diag_comm_info &comm) + : exp_ctrl_(exp_ctrl), algebra_(comm) { using Real = typename T::value_type; if (exp_ctrl_.rel_tol == 0.0) { exp_ctrl_.rel_tol = std::max(1e-10, 100.0 * std::numeric_limits::epsilon()); } - if (exp_ctrl_.taylor_order < 2 || exp_ctrl_.krylov_max < 2 || !std::isfinite(exp_ctrl_.rel_tol) || exp_ctrl_.rel_tol <= 0) + if (exp_ctrl_.taylor_order < 2 || exp_ctrl_.krylov_max < 2 || !std::isfinite(exp_ctrl_.rel_tol) || + exp_ctrl_.rel_tol <= 0) { throw std::invalid_argument("Invalid exponential controls."); } } template -std::unique_ptr> make_exp_action(const ExpControls& exp_ctrl, const diag_comm_info& comm) +std::unique_ptr> make_exp_action(const ExpControls &exp_ctrl, const diag_comm_info &comm) { if (exp_ctrl.expansion == ExpMethod::taylor) { @@ -34,16 +36,16 @@ std::unique_ptr> make_exp_action(const ExpControls& exp_ctr template class ExpAction, base_device::DEVICE_CPU>; template class ExpAction, base_device::DEVICE_CPU>; -template std::unique_ptr, base_device::DEVICE_CPU>> make_exp_action(const ExpControls&, - const diag_comm_info&); -template std::unique_ptr, base_device::DEVICE_CPU>> make_exp_action(const ExpControls&, - const diag_comm_info&); +template std::unique_ptr, base_device::DEVICE_CPU>> make_exp_action( + const ExpControls &, const diag_comm_info &); +template std::unique_ptr, base_device::DEVICE_CPU>> make_exp_action( + const ExpControls &, const diag_comm_info &); #if defined(__CUDA) || defined(__ROCM) template class ExpAction, base_device::DEVICE_GPU>; template class ExpAction, base_device::DEVICE_GPU>; -template std::unique_ptr, base_device::DEVICE_GPU>> make_exp_action(const ExpControls&, - const diag_comm_info&); -template std::unique_ptr, base_device::DEVICE_GPU>> make_exp_action(const ExpControls&, - const diag_comm_info&); +template std::unique_ptr, base_device::DEVICE_GPU>> make_exp_action( + const ExpControls &, const diag_comm_info &); +template std::unique_ptr, base_device::DEVICE_GPU>> make_exp_action( + const ExpControls &, const diag_comm_info &); #endif } // namespace hsolver diff --git a/source/source_hsolver/module_linalg/exp_action.h b/source/source_hsolver/module_linalg/exp_action.h index 974deaba102..68338bd0daf 100644 --- a/source/source_hsolver/module_linalg/exp_action.h +++ b/source/source_hsolver/module_linalg/exp_action.h @@ -47,28 +47,34 @@ struct ExpResult }; /** @brief Apply exp(-i*alpha*H_Ry); caller selects k, buffers must not alias. */ -template -class ExpAction +template class ExpAction { protected: ExpControls exp_ctrl_; LinearAlgebra algebra_; public: - ExpAction(const ExpControls& exp_ctrl, const diag_comm_info& comm); + ExpAction(const ExpControls &exp_ctrl, const diag_comm_info &comm); virtual ~ExpAction() = default; /** @brief Effective relative tolerance after resolving the precision-dependent default. */ - double rel_tol() const - { - return exp_ctrl_.rel_tol; - } - virtual ExpResult apply(const HSOperator& op, const OrbLayout& layout, double alpha, const T* input, T* output) = 0; - virtual ExpResult apply_pair(const HSOperator& op, const OrbLayout& layout, double alpha, const T* input, T* half, T* full) - = 0; + double rel_tol() const { return exp_ctrl_.rel_tol; } + virtual ExpResult apply( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *output) = 0; + virtual ExpResult apply_pair( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *half, + T *full) = 0; }; /** @brief Construct a backend without reading global input parameters. */ template -std::unique_ptr> make_exp_action(const ExpControls& exp_ctrl, const diag_comm_info& comm); +std::unique_ptr> make_exp_action(const ExpControls &exp_ctrl, const diag_comm_info &comm); } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/exp_lanczos.cpp b/source/source_hsolver/module_linalg/exp_lanczos.cpp index 9c020595748..dacda2a1672 100644 --- a/source/source_hsolver/module_linalg/exp_lanczos.cpp +++ b/source/source_hsolver/module_linalg/exp_lanczos.cpp @@ -13,7 +13,7 @@ namespace { using Wide = std::complex; -void add_work(ExpResult& total, const ExpResult& part) +void add_work(ExpResult &total, const ExpResult &part) { total.hpsi_calls += part.hpsi_calls; total.hpsi_cols += part.hpsi_cols; @@ -32,13 +32,12 @@ ExpResult failed_band(ExpResult result, int band, bool retry) } // namespace template -ExpLanczos::ExpLanczos(const ExpControls& exp_ctrl, const diag_comm_info& comm) +ExpLanczos::ExpLanczos(const ExpControls &exp_ctrl, const diag_comm_info &comm) : ExpAction(exp_ctrl, comm), comm_(comm), work_(comm, 1) { } -template -int ExpLanczos::batch_size(const OrbLayout& layout) const +template int ExpLanczos::batch_size(const OrbLayout &layout) const { // The layout can differ between ranks, but operator column counts must agree. double rows = std::max(1, layout.ld); @@ -59,28 +58,49 @@ int ExpLanczos::batch_size(const OrbLayout& layout) const } template -void ExpLanczos::combine(const OrbLayout& layout, int count, const T* basis, const std::vector& coefficients, T* output) +void ExpLanczos::combine( + const OrbLayout &layout, + int count, + const T *basis, + const std::vector &coefficients, + T *output) { ModuleBase::timer::start("ExpLanczos", "combine"); linear_buffer(&coefficients_, coefficients.size()); - T* data = coefficients_.template data(); - base_device::memory::synchronize_memory_op()(data, coefficients.data(), coefficients.size()); + T *data = coefficients_.template data(); + base_device::memory::synchronize_memory_op()( + data, coefficients.data(), coefficients.size()); const int dim = layout.npol == 2 ? layout.ld : layout.npw; const int stride = layout.ld * layout.bands; - linear_op().combine(layout.ld, dim, layout.bands, count, stride, basis, data, output); + linear_op().combine( + layout.ld, + dim, + layout.bands, + count, + stride, + basis, + data, + output); ModuleBase::timer::end("ExpLanczos", "combine"); } template -std::vector ExpLanczos::basis_gram(const OrbLayout& layout, int count, const T* basis) +std::vector ExpLanczos::basis_gram(const OrbLayout &layout, int count, const T *basis) { ModuleBase::timer::start("ExpLanczos", "basis_gram"); const int size = layout.bands * count * count; linear_buffer(&products_, size); - Wide* data = products_.template data(); + Wide *data = products_.template data(); const int dim = layout.npol == 2 ? layout.ld : layout.npw; const int stride = layout.ld * layout.bands; - linear_op().block_gram(layout.ld, dim, layout.bands, count, stride, basis, data); + linear_op().block_gram( + layout.ld, + dim, + layout.bands, + count, + stride, + basis, + data); std::vector gram(size); base_device::memory::synchronize_memory_op()(gram.data(), data, size); #ifdef __MPI @@ -94,11 +114,12 @@ std::vector ExpLanczos::basis_gram(const OrbLayout& layout, int } template -bool ExpLanczos::diagonalize(const std::vector& diagonal, - const std::vector& off_diagonal, - int band, - int bands, - int count) +bool ExpLanczos::diagonalize( + const std::vector &diagonal, + const std::vector &off_diagonal, + int band, + int bands, + int count) { ModuleBase::timer::start("ExpLanczos", "diagonalize"); small_vectors_.assign(count * count, 0.0); @@ -117,21 +138,31 @@ bool ExpLanczos::diagonalize(const std::vector& diagonal, const char job = 'V'; const char upper = 'U'; int info = 0; - ct::lapackConnector::heev(job, upper, count, small_vectors_.data(), count, small_values_.data(), small_work_.data(), work_size, info); + ct::lapackConnector::heev( + job, + upper, + count, + small_vectors_.data(), + count, + small_values_.data(), + small_work_.data(), + work_size, + info); ModuleBase::timer::end("ExpLanczos", "diagonalize"); return info == 0; } template -ExpResult ExpLanczos::attempt(const HSOperator& op, - const OrbLayout& layout, - double alpha, - double rel_tol, - const T* input, - T* output, - T* full, - bool reorthogonalize, - bool* lost_orthogonality) +ExpResult ExpLanczos::attempt( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + double rel_tol, + const T *input, + T *output, + T *full, + bool reorthogonalize, + bool *lost_orthogonality) { using Real = typename T::value_type; const int ld = layout.ld; @@ -156,11 +187,11 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, linear_buffer(&packed_, size); linear_buffer(&hpacked_, size); linear_buffer(&permutation_, bands); - T* q = basis_.template data(); - T* w = image_.template data(); - T* packed = packed_.template data(); - T* hpacked = hpacked_.template data(); - int* map = permutation_.template data(); + T *q = basis_.template data(); + T *w = image_.template data(); + T *packed = packed_.template data(); + T *hpacked = hpacked_.template data(); + int *map = permutation_.template data(); base_device::memory::set_memory_op()(output, 0, stride); if (full) { @@ -168,7 +199,14 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, } base_device::memory::synchronize_memory_op()(q, input, stride); zero_pad(q, layout); - const std::vector> norms = this->algebra_.dots(ld, dim, bands, 1, stride, q, q); + const std::vector> norms = this->algebra_.dots( + ld, + dim, + bands, + 1, + stride, + q, + q); std::vector scale(bands); std::vector factors(bands); std::vector done(bands, false); @@ -189,7 +227,18 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, done[b] = scale[b] == 0.0; factors[b] = done[b] ? T(0) : T(1.0 / scale[b]); } - this->work_.batch(ld, dim, bands, q, q, q, T(1), T(0), factors.data(), nullptr, nullptr); + this->work_.batch( + ld, + dim, + bands, + q, + q, + q, + T(1), + T(0), + factors.data(), + nullptr, + nullptr); for (int j = 0; j < limit; ++j) { active.clear(); @@ -205,29 +254,59 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, return result; } const int count = active.size(); - T* column = q + static_cast(j) * stride; + T *column = q + static_cast(j) * stride; if (count == bands) { - op.hpsi(column, w, ld, bands); + op.hpsi( + column, + w, + ld, + bands); } else { - base_device::memory::synchronize_memory_op()(map, active.data(), count); - linear_op().gather(ld, ld, count, 1, stride, column, packed, map); - op.hpsi(packed, hpacked, ld, count); + base_device::memory::synchronize_memory_op()( + map, active.data(), count); + linear_op().gather( + ld, + ld, + count, + 1, + stride, + column, + packed, + map); + op.hpsi( + packed, + hpacked, + ld, + count); base_device::memory::set_memory_op()(w, 0, stride); - linear_op().scatter(ld, ld, count, hpacked, w, map); + linear_op().scatter( + ld, + ld, + count, + hpacked, + w, + map); } zero_pad(w, layout); ++result.hpsi_calls; result.hpsi_cols += count; - const std::vector> projection = this->algebra_.dots(ld, dim, bands, 1, stride, column, w); + const std::vector> projection = this->algebra_.dots( + ld, + dim, + bands, + 1, + stride, + column, + w); const int m = j + 1; result.order_or_dim = m; const int first = std::max(0, j - 1); const int columns = j - first + 1; std::vector recurrence(columns * bands, T(0)); - for (int b: active) + for (int b : active) { if (!std::isfinite(std::abs(projection[b]))) { @@ -240,18 +319,30 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, recurrence[b] = T(-beta[(j - 1) * bands + b]); } } - const T* previous = q + static_cast(first) * stride; - combine(layout, columns, previous, recurrence, w); + const T *previous = q + static_cast(first) * stride; + combine( + layout, + columns, + previous, + recurrence, + w); if (reorthogonalize) { std::vector removed(bands, 0.0); for (int pass = 0; pass < 2; ++pass) { - const std::vector> projections = this->algebra_.dots(ld, dim, bands, m, stride, q, w); + const std::vector> projections = this->algebra_.dots( + ld, + dim, + bands, + m, + stride, + q, + w); std::vector correction(m * bands, T(0)); for (int i = 0; i < m; ++i) { - for (int b: active) + for (int b : active) { const Wide value = projections[i * bands + b]; if (!std::isfinite(std::abs(value))) @@ -262,18 +353,30 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, correction[i * bands + b] = T(-value); } } - combine(layout, m, q, correction, w); + combine( + layout, + m, + q, + correction, + w); } - for (int b: active) + for (int b : active) { // ||Q*C|| <= sum_i |C_i| for normalized columns. Accumulate // a Frobenius bound on the discarded recurrence corrections. correction_sq[b] += removed[b] * removed[b]; } } - const std::vector> residuals = this->algebra_.dots(ld, dim, bands, 1, stride, w, w); + const std::vector> residuals = this->algebra_.dots( + ld, + dim, + bands, + 1, + stride, + w, + w); std::vector candidates; - for (int b: active) + for (int b : active) { if (!std::isfinite(residuals[b].real()) || residuals[b].real() < 0.0) { @@ -310,7 +413,7 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, const double roundoff = std::numeric_limits::epsilon(); const double orth_tol = std::max(std::sqrt(roundoff), 0.1 * rel_tol); const double norm_tol = std::max(100.0 * m * std::numeric_limits::epsilon(), 0.1 * rel_tol); - for (int b: candidates) + for (int b : candidates) { double defect = 0.0; for (int row = 0; row < m; ++row) @@ -340,9 +443,14 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, { full_coefficients.assign(m * bands, T(0)); } - for (int b: candidates) + for (int b : candidates) { - if (!diagonalize(diagonal, beta, b, bands, m)) + if (!diagonalize( + diagonal, + beta, + b, + bands, + m)) { return failed_band(result, b, false); } @@ -376,18 +484,42 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, } } } - combine(layout, m, q, coefficients, output); + combine( + layout, + m, + q, + coefficients, + output); if (full) { - combine(layout, m, q, full_coefficients, full); + combine( + layout, + m, + q, + full_coefficients, + full); } - const std::vector> output_norms = this->algebra_.dots(ld, dim, bands, 1, stride, output, output); + const std::vector> output_norms = this->algebra_.dots( + ld, + dim, + bands, + 1, + stride, + output, + output); std::vector full_norms; if (full) { - full_norms = this->algebra_.dots(ld, dim, bands, 1, stride, full, full); + full_norms = this->algebra_.dots( + ld, + dim, + bands, + 1, + stride, + full, + full); } - for (int b: candidates) + for (int b : candidates) { const double norm_error = std::abs(std::sqrt(output_norms[b].real()) / scale[b] - 1.0); double full_error = 0.0; @@ -426,9 +558,20 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, } if (m < limit) { - T* next = q + static_cast(m) * stride; + T *next = q + static_cast(m) * stride; base_device::memory::set_memory_op()(next, 0, stride); - this->work_.batch(ld, dim, bands, next, w, w, T(1), T(0), factors.data(), nullptr, nullptr); + this->work_.batch( + ld, + dim, + bands, + next, + w, + w, + T(1), + T(0), + factors.data(), + nullptr, + nullptr); } } result.state = ExpState::retry; @@ -437,21 +580,40 @@ ExpResult ExpLanczos::attempt(const HSOperator& op, } template -ExpResult ExpLanczos::interval(const HSOperator& op, - const OrbLayout& layout, - double alpha, - double rel_tol, - const T* input, - T* output, - T* full, - int depth) +ExpResult ExpLanczos::interval( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + double rel_tol, + const T *input, + T *output, + T *full, + int depth) { bool lost_orthogonality = false; - ExpResult trial = attempt(op, layout, alpha, rel_tol, input, output, full, false, &lost_orthogonality); + ExpResult trial = attempt( + op, + layout, + alpha, + rel_tol, + input, + output, + full, + false, + &lost_orthogonality); if (lost_orthogonality) { const ExpResult previous = trial; - trial = attempt(op, layout, alpha, rel_tol, input, output, full, true, &lost_orthogonality); + trial = attempt( + op, + layout, + alpha, + rel_tol, + input, + output, + full, + true, + &lost_orthogonality); add_work(trial, previous); } if (trial.state != ExpState::retry) @@ -466,8 +628,8 @@ ExpResult ExpLanczos::interval(const HSOperator& op, ++trial.bisect_count; ct::Tensor midpoint; const int64_t size = static_cast(layout.ld) * layout.bands; - T* middle = output; - T* endpoint = full; + T *middle = output; + T *endpoint = full; double sub_alpha = alpha; if (!full) { @@ -478,13 +640,29 @@ ExpResult ExpLanczos::interval(const HSOperator& op, } const double sub_tol = rel_tol / 2.0; const int next_depth = depth + 1; - ExpResult left = interval(op, layout, sub_alpha, sub_tol, input, middle, nullptr, next_depth); + ExpResult left = interval( + op, + layout, + sub_alpha, + sub_tol, + input, + middle, + nullptr, + next_depth); add_work(left, trial); if (left.state != ExpState::success) { return left; } - ExpResult right = interval(op, layout, sub_alpha, sub_tol, middle, endpoint, nullptr, next_depth); + ExpResult right = interval( + op, + layout, + sub_alpha, + sub_tol, + middle, + endpoint, + nullptr, + next_depth); add_work(right, left); if (right.state == ExpState::success) { @@ -494,12 +672,13 @@ ExpResult ExpLanczos::interval(const HSOperator& op, } template -ExpResult ExpLanczos::evaluate(const HSOperator& op, - const OrbLayout& layout, - double alpha, - const T* input, - T* output, - T* full) +ExpResult ExpLanczos::evaluate( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *output, + T *full) { ModuleBase::timer::start("ExpLanczos", "evaluate"); ExpResult result; @@ -508,12 +687,23 @@ ExpResult ExpLanczos::evaluate(const HSOperator& op, for (int first = 0; first < layout.bands; first += batch_limit) { const int bands = std::min(batch_limit, layout.bands - first); - const OrbLayout batch{layout.ld, layout.npw, layout.npol, bands}; + const OrbLayout batch{layout.ld, + layout.npw, + layout.npol, + bands}; const int64_t offset = static_cast(first) * layout.ld; - const T* src = input + offset; - T* dst = output + offset; - T* full_dst = full ? full + offset : nullptr; - const ExpResult local = interval(op, batch, alpha, this->exp_ctrl_.rel_tol, src, dst, full_dst, 0); + const T *src = input + offset; + T *dst = output + offset; + T *full_dst = full ? full + offset : nullptr; + const ExpResult local = interval( + op, + batch, + alpha, + this->exp_ctrl_.rel_tol, + src, + dst, + full_dst, + 0); result.has_err_est = result.has_err_est && local.has_err_est; add_work(result, local); result.rel_err_est = std::max(result.rel_err_est, local.rel_err_est); @@ -529,20 +719,38 @@ ExpResult ExpLanczos::evaluate(const HSOperator& op, } template -ExpResult ExpLanczos::apply(const HSOperator& op, const OrbLayout& layout, double alpha, const T* input, T* output) +ExpResult ExpLanczos::apply( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *output) { - return evaluate(op, layout, alpha, input, output, nullptr); + return evaluate( + op, + layout, + alpha, + input, + output, + nullptr); } template -ExpResult ExpLanczos::apply_pair(const HSOperator& op, - const OrbLayout& layout, - double alpha, - const T* input, - T* half, - T* full) +ExpResult ExpLanczos::apply_pair( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *half, + T *full) { - return evaluate(op, layout, alpha, input, half, full); + return evaluate( + op, + layout, + alpha, + input, + half, + full); } template class ExpLanczos, base_device::DEVICE_CPU>; diff --git a/source/source_hsolver/module_linalg/exp_lanczos.h b/source/source_hsolver/module_linalg/exp_lanczos.h index e1a451f4258..fc3c192c0f1 100644 --- a/source/source_hsolver/module_linalg/exp_lanczos.h +++ b/source/source_hsolver/module_linalg/exp_lanczos.h @@ -5,8 +5,7 @@ namespace hsolver { -template -class ExpLanczos final : public ExpAction +template class ExpLanczos final : public ExpAction { private: const diag_comm_info comm_; @@ -21,34 +20,63 @@ class ExpLanczos final : public ExpAction std::vector small_vectors_; std::vector small_values_; std::vector small_work_; - int batch_size(const OrbLayout& layout) const; - void combine(const OrbLayout& layout, int count, const T* basis, const std::vector& coefficients, T* output); - std::vector> basis_gram(const OrbLayout& layout, int count, const T* basis); - bool diagonalize(const std::vector& diagonal, const std::vector& off_diagonal, int band, int bands, int count); - ExpResult attempt(const HSOperator& op, - const OrbLayout& layout, - double alpha, - double rel_tol, - const T* input, - T* output, - T* full, - bool reorthogonalize, - bool* lost_orthogonality); - ExpResult interval(const HSOperator& op, - const OrbLayout& layout, - double alpha, - double rel_tol, - const T* input, - T* output, - T* full, - int depth); - ExpResult evaluate(const HSOperator& op, const OrbLayout& layout, double alpha, const T* input, T* output, T* full); + int batch_size(const OrbLayout &layout) const; + void combine( + const OrbLayout &layout, + int count, + const T *basis, + const std::vector &coefficients, + T *output); + std::vector> basis_gram(const OrbLayout &layout, int count, const T *basis); + bool diagonalize( + const std::vector &diagonal, + const std::vector &off_diagonal, + int band, + int bands, + int count); + ExpResult attempt( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + double rel_tol, + const T *input, + T *output, + T *full, + bool reorthogonalize, + bool *lost_orthogonality); + ExpResult interval( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + double rel_tol, + const T *input, + T *output, + T *full, + int depth); + ExpResult evaluate( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *output, + T *full); public: - ExpLanczos(const ExpControls& exp_ctrl, const diag_comm_info& comm); - ExpResult apply(const HSOperator& op, const OrbLayout& layout, double alpha, const T* input, T* output) override; + ExpLanczos(const ExpControls &exp_ctrl, const diag_comm_info &comm); + ExpResult apply( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *output) override; /** @brief Share one Krylov construction for alpha and twice alpha. */ - ExpResult apply_pair(const HSOperator& op, const OrbLayout& layout, double alpha, const T* input, T* half, T* full) override; + ExpResult apply_pair( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *half, + T *full) override; }; } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/exp_taylor.cpp b/source/source_hsolver/module_linalg/exp_taylor.cpp index 4acb3108419..c074fff89f5 100644 --- a/source/source_hsolver/module_linalg/exp_taylor.cpp +++ b/source/source_hsolver/module_linalg/exp_taylor.cpp @@ -6,34 +6,54 @@ namespace hsolver { template -ExpTaylor::ExpTaylor(const ExpControls& exp_ctrl, const diag_comm_info& comm) : ExpAction(exp_ctrl, comm) +ExpTaylor::ExpTaylor(const ExpControls &exp_ctrl, const diag_comm_info &comm) + : ExpAction(exp_ctrl, comm) { } template -ExpResult ExpTaylor::apply(const HSOperator& op, const OrbLayout& layout, double alpha, const T* input, T* output) +ExpResult ExpTaylor::apply( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *output) { - return evaluate(op, layout, alpha, input, output, nullptr); + return evaluate( + op, + layout, + alpha, + input, + output, + nullptr); } template -ExpResult ExpTaylor::apply_pair(const HSOperator& op, - const OrbLayout& layout, - double alpha, - const T* input, - T* half, - T* full) +ExpResult ExpTaylor::apply_pair( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *half, + T *full) { - return evaluate(op, layout, alpha, input, half, full); + return evaluate( + op, + layout, + alpha, + input, + half, + full); } template -ExpResult ExpTaylor::evaluate(const HSOperator& op, - const OrbLayout& layout, - double alpha, - const T* input, - T* half, - T* full) +ExpResult ExpTaylor::evaluate( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *half, + T *full) { ModuleBase::timer::start("ExpTaylor", "evaluate"); const int ld = layout.ld; @@ -43,8 +63,8 @@ ExpResult ExpTaylor::evaluate(const HSOperator& op, const int64_t count = static_cast(ld) * bands; linear_buffer(&term_, count); linear_buffer(&image_, count); - T* term = term_.template data(); - T* image = image_.template data(); + T *term = term_.template data(); + T *image = image_.template data(); base_device::memory::synchronize_memory_op()(term, input, count); zero_pad(term, layout); base_device::memory::synchronize_memory_op()(half, term, count); @@ -58,24 +78,75 @@ ExpResult ExpTaylor::evaluate(const HSOperator& op, const double series_alpha = full ? 2.0 * alpha : alpha; for (int k = 1; k <= this->exp_ctrl_.taylor_order; ++k) { - op.hpsi(term, image, ld, bands); + op.hpsi( + term, + image, + ld, + bands); zero_pad(image, layout); const T coefficient(0, -series_alpha / k); - linear_op().batch(ld, dim, bands, term, image, image, coefficient, T(0), nullptr, nullptr, nullptr); + linear_op().batch( + ld, + dim, + bands, + term, + image, + image, + coefficient, + T(0), + nullptr, + nullptr, + nullptr); const T half_weight = full ? T(std::ldexp(1.0, -k)) : T(1); - linear_op().batch(ld, dim, bands, half, half, term, T(1), half_weight, nullptr, nullptr, nullptr); + linear_op().batch( + ld, + dim, + bands, + half, + half, + term, + T(1), + half_weight, + nullptr, + nullptr, + nullptr); if (full) { - linear_op().batch(ld, dim, bands, full, full, term, T(1), T(1), nullptr, nullptr, nullptr); + linear_op().batch( + ld, + dim, + bands, + full, + full, + term, + T(1), + T(1), + nullptr, + nullptr, + nullptr); } ++result.hpsi_calls; result.hpsi_cols += bands; } - const std::vector> norms = this->algebra_.dots(ld, dim, bands, 1, stride, half, half); + const std::vector> norms = this->algebra_.dots( + ld, + dim, + bands, + 1, + stride, + half, + half); std::vector> full_norms; if (full) { - full_norms = this->algebra_.dots(ld, dim, bands, 1, stride, full, full); + full_norms = this->algebra_.dots( + ld, + dim, + bands, + 1, + stride, + full, + full); } for (int b = 0; b < bands; ++b) { diff --git a/source/source_hsolver/module_linalg/exp_taylor.h b/source/source_hsolver/module_linalg/exp_taylor.h index dc0f1014ce1..c231dfc6dd2 100644 --- a/source/source_hsolver/module_linalg/exp_taylor.h +++ b/source/source_hsolver/module_linalg/exp_taylor.h @@ -4,18 +4,34 @@ namespace hsolver { -template -class ExpTaylor final : public ExpAction +template class ExpTaylor final : public ExpAction { private: ct::Tensor term_; ct::Tensor image_; - ExpResult evaluate(const HSOperator& op, const OrbLayout& layout, double alpha, const T* input, T* half, T* full); + ExpResult evaluate( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *half, + T *full); public: - ExpTaylor(const ExpControls& exp_ctrl, const diag_comm_info& comm); - ExpResult apply(const HSOperator& op, const OrbLayout& layout, double alpha, const T* input, T* output) override; - ExpResult apply_pair(const HSOperator& op, const OrbLayout& layout, double alpha, const T* input, T* half, T* full) override; + ExpTaylor(const ExpControls &exp_ctrl, const diag_comm_info &comm); + ExpResult apply( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *output) override; + ExpResult apply_pair( + const HSOperator &op, + const OrbLayout &layout, + double alpha, + const T *input, + T *half, + T *full) override; }; } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/hsolver_linear.cpp b/source/source_hsolver/module_linalg/hsolver_linear.cpp index 9a90963d534..717c6f62b46 100644 --- a/source/source_hsolver/module_linalg/hsolver_linear.cpp +++ b/source/source_hsolver/module_linalg/hsolver_linear.cpp @@ -4,7 +4,7 @@ namespace hsolver { -const char* linear_status_name(const LinearSolveStatus status) +const char *linear_status_name(const LinearSolveStatus status) { switch (status) { @@ -23,7 +23,7 @@ const char* linear_status_name(const LinearSolveStatus status) } template -HSolverLinear::HSolverLinear(const LinearControls& linear_ctrl, const diag_comm_info& comm) +HSolverLinear::HSolverLinear(const LinearControls &linear_ctrl, const diag_comm_info &comm) : max_iter_(linear_ctrl.max_iterations), reconstruct_(linear_ctrl.reconstruct) { using Real = typename GetTypeReal::type; @@ -39,7 +39,11 @@ HSolverLinear::HSolverLinear(const LinearControls& linear_ctrl, const } else if (linear_ctrl.solver == LinearMethod::gmres) { - gmres_.reset(new LinearGMRES(tolerance_, linear_ctrl.restart, linear_ctrl.max_iterations, comm)); + gmres_.reset(new LinearGMRES( + tolerance_, + linear_ctrl.restart, + linear_ctrl.max_iterations, + comm)); } else { @@ -48,43 +52,85 @@ HSolverLinear::HSolverLinear(const LinearControls& linear_ctrl, const } template -LinearSolveResult HSolverLinear::solve(const LinearOperator& op, - const LinearOperator& preconditioner, - int ld, - int nvec, - int dim, - T* x, - const T* b, - const T* initial_residual, - bool force_check) +LinearSolveResult HSolverLinear::solve( + const LinearOperator &op, + const LinearOperator &preconditioner, + int ld, + int nvec, + int dim, + T *x, + const T *b, + const T *initial_residual, + bool force_check) { - return solve(op, preconditioner, ld, nvec, dim, x, b, initial_residual, force_check, max_iter_, reconstruct_); + return solve( + op, + preconditioner, + ld, + nvec, + dim, + x, + b, + initial_residual, + force_check, + max_iter_, + reconstruct_); } template -LinearSolveResult HSolverLinear::solve(const LinearOperator& op, - const LinearOperator& preconditioner, - int ld, - int nvec, - int dim, - T* x, - const T* b, - const T* initial_residual, - bool force_check, - int max_iter, - bool reconstruct) +LinearSolveResult HSolverLinear::solve( + const LinearOperator &op, + const LinearOperator &preconditioner, + int ld, + int nvec, + int dim, + T *x, + const T *b, + const T *initial_residual, + bool force_check, + int max_iter, + bool reconstruct) { if (bicgstab_) { - return bicgstab_->solve(op, preconditioner, ld, nvec, dim, x, b, initial_residual, max_iter); + return bicgstab_->solve( + op, + preconditioner, + ld, + nvec, + dim, + x, + b, + initial_residual, + max_iter); } else if (cgs_) { - return cgs_->solve(op, preconditioner, ld, nvec, dim, x, b, initial_residual, max_iter); + return cgs_->solve( + op, + preconditioner, + ld, + nvec, + dim, + x, + b, + initial_residual, + max_iter); } else if (gmres_) { - return gmres_->solve(op, preconditioner, ld, nvec, dim, x, b, initial_residual, force_check, max_iter, reconstruct); + return gmres_->solve( + op, + preconditioner, + ld, + nvec, + dim, + x, + b, + initial_residual, + force_check, + max_iter, + reconstruct); } else { diff --git a/source/source_hsolver/module_linalg/hsolver_linear.h b/source/source_hsolver/module_linalg/hsolver_linear.h index 47c43057788..e27326d6d84 100644 --- a/source/source_hsolver/module_linalg/hsolver_linear.h +++ b/source/source_hsolver/module_linalg/hsolver_linear.h @@ -9,8 +9,7 @@ namespace hsolver { /** @brief Method selection and reusable numerical workspace, independent of physics. */ -template -class HSolverLinear +template class HSolverLinear { private: std::unique_ptr> bicgstab_; @@ -21,34 +20,33 @@ class HSolverLinear const bool reconstruct_; public: - HSolverLinear(const LinearControls& linear_ctrl, const diag_comm_info& comm); + HSolverLinear(const LinearControls &linear_ctrl, const diag_comm_info &comm); /** @brief Return the effective tolerance after resolving the precision-dependent default. */ - double tolerance() const - { - return tolerance_; - } + double tolerance() const { return tolerance_; } /** @brief Reuse an optional CN residual and control independent GMRES verification. */ - LinearSolveResult solve(const LinearOperator& op, - const LinearOperator& preconditioner, - int ld, - int nvec, - int dim, - T* x, - const T* b, - const T* initial_residual, - bool force_check); + LinearSolveResult solve( + const LinearOperator &op, + const LinearOperator &preconditioner, + int ld, + int nvec, + int dim, + T *x, + const T *b, + const T *initial_residual, + bool force_check); /** @brief Reuse solver storage with an explicit per-call budget and reconstruction policy. */ - LinearSolveResult solve(const LinearOperator& op, - const LinearOperator& preconditioner, - int ld, - int nvec, - int dim, - T* x, - const T* b, - const T* initial_residual, - bool force_check, - int max_iter, - bool reconstruct); + LinearSolveResult solve( + const LinearOperator &op, + const LinearOperator &preconditioner, + int ld, + int nvec, + int dim, + T *x, + const T *b, + const T *initial_residual, + bool force_check, + int max_iter, + bool reconstruct); }; } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/kernels/cuda/linear_op.cu b/source/source_hsolver/module_linalg/kernels/cuda/linear_op.cu index 9a984234f79..5560594e472 100644 --- a/source/source_hsolver/module_linalg/kernels/cuda/linear_op.cu +++ b/source/source_hsolver/module_linalg/kernels/cuda/linear_op.cu @@ -1,10 +1,10 @@ #include "source_hsolver/module_linalg/kernels/linear_op.h" -#include -#include #include #include +#include #include +#include namespace hsolver { @@ -22,8 +22,12 @@ void check_launch() } template -__global__ void dot_kernel(const int ld, const int dim, const thrust::complex* x, - const thrust::complex* y, thrust::complex* out) +__global__ void dot_kernel( + const int ld, + const int dim, + const thrust::complex *x, + const thrust::complex *y, + thrust::complex *out) { __shared__ Real re[linear_threads]; __shared__ Real im[linear_threads]; @@ -54,36 +58,58 @@ __global__ void dot_kernel(const int ld, const int dim, const thrust::complex -void linear_op::dot(const int ld, const int dim, const int nvec, - const T* x, const T* y, T* out) const -{ - if (nvec <= 0) { return; } +void linear_op::dot( + const int ld, + const int dim, + const int nvec, + const T *x, + const T *y, + T *out) const +{ + if (nvec <= 0) + { + return; + } using Complex = thrust::complex; - dot_kernel<<>>(ld, dim, reinterpret_cast(x), - reinterpret_cast(y), reinterpret_cast(out)); + dot_kernel<<>>( + ld, + dim, + reinterpret_cast(x), + reinterpret_cast(y), + reinterpret_cast(out)); check_launch(); } - template -__global__ void batch_kernel(int ld,int dim,C* out,const C* x,const C* y,C a,C b,const C* ca,const C* cb,const int* skip) +__global__ void batch_kernel( + int ld, + int dim, + C *out, + const C *x, + const C *y, + C a, + C b, + const C *ca, + const C *cb, + const int *skip) { - const int i=blockIdx.x*blockDim.x+threadIdx.x; - const int band=blockIdx.y; - if(i -__global__ void gmres_update_kernel(int ld, - int dim, - C* solution, - C* residual, - const C* direction, - const C* image, - const C* coefficients) +__global__ void gmres_update_kernel( + int ld, + int dim, + C *solution, + C *residual, + const C *direction, + const C *image, + const C *coefficients) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int band = blockIdx.y; @@ -105,34 +131,74 @@ __global__ void gmres_update_kernel(int ld, } template -__global__ void dots_kernel(int ld,int dim,int nvec,int tiles, - const thrust::complex* x,const thrust::complex* y, - const thrust::complex* z,const thrust::complex* w,thrust::complex* partial) +__global__ void dots_kernel( + int ld, + int dim, + int nvec, + int tiles, + const thrust::complex *x, + const thrust::complex *y, + const thrust::complex *z, + const thrust::complex *w, + thrust::complex *partial) { __shared__ Real re[256]; __shared__ Real im[256]; - const int tid=threadIdx.x; - const int band=blockIdx.x; - const int pair=blockIdx.z; - const thrust::complex* a=pair ? z : x; - const thrust::complex* b=pair ? w : y; - thrust::complex sum(0,0); - for(int i=blockIdx.y*256+tid;i *a = pair ? z : x; + const thrust::complex *b = pair ? w : y; + thrust::complex sum(0, 0); + for (int i = blockIdx.y * 256 + tid; i < dim; i += tiles * 256) + { + sum += thrust::conj(a[band * ld + i]) * b[band * ld + i]; + } + re[tid] = sum.real(); + im[tid] = sum.imag(); __syncthreads(); - for(int n=128;n>0;n/=2) { if(tid(re[0],im[0]); + for (int n = 128; n > 0; n /= 2) + { + if (tid < n) + { + re[tid] += re[tid + n]; + im[tid] += im[tid + n]; + } + __syncthreads(); + } + if (tid == 0) + { + partial[(pair * nvec + band) * tiles + blockIdx.y] = thrust::complex(re[0], im[0]); + } } template -__global__ void sum_tiles(int n,int tiles,const C* partial,C* out) +__global__ void sum_tiles( + int n, + int tiles, + const C *partial, + C *out) { - const int i=blockIdx.x*blockDim.x+threadIdx.x; - if(i -__global__ void gather_kernel(int ld,int dim,int stride,const C* in,C* out,const int* map) +__global__ void gather_kernel( + int ld, + int dim, + int stride, + const C *in, + C *out, + const int *map) { - const int i=blockIdx.x*blockDim.x+threadIdx.x; + const int i = blockIdx.x * blockDim.x + threadIdx.x; if (i < dim) { const std::int64_t slot_offset = static_cast(blockIdx.z) * stride; @@ -142,25 +208,47 @@ __global__ void gather_kernel(int ld,int dim,int stride,const C* in,C* out,const } } template -void linear_op::batch(int ld,int dim,int nvec,T* out,const T* x,const T* y, - T a,T b,const T* ca,const T* cb,const int* skip) const -{ - if(dim<=0 || nvec<=0)return; - using C=thrust::complex; - batch_kernel<<>>(ld,dim,reinterpret_cast(out), - reinterpret_cast(x),reinterpret_cast(y),C(a.real(),a.imag()),C(b.real(),b.imag()), - reinterpret_cast(ca),reinterpret_cast(cb),skip); +void linear_op::batch( + int ld, + int dim, + int nvec, + T *out, + const T *x, + const T *y, + T a, + T b, + const T *ca, + const T *cb, + const int *skip) const +{ + if (dim <= 0 || nvec <= 0) + { + return; + } + using C = thrust::complex; + batch_kernel<<>>( + ld, + dim, + reinterpret_cast(out), + reinterpret_cast(x), + reinterpret_cast(y), + C(a.real(), a.imag()), + C(b.real(), b.imag()), + reinterpret_cast(ca), + reinterpret_cast(cb), + skip); check_launch(); } template -void linear_op::gmres_update(int ld, - int dim, - int nvec, - T* solution, - T* residual, - const T* direction, - const T* image, - const T* coefficients) const +void linear_op::gmres_update( + int ld, + int dim, + int nvec, + T *solution, + T *residual, + const T *direction, + const T *image, + const T *coefficients) const { if (dim <= 0 || nvec <= 0) { @@ -168,101 +256,212 @@ void linear_op::gmres_update(int ld, } using C = thrust::complex; const dim3 grid((dim + linear_threads - 1) / linear_threads, nvec); - C* current_solution = reinterpret_cast(solution); - C* current_residual = reinterpret_cast(residual); - const C* current_direction = reinterpret_cast(direction); - const C* current_image = reinterpret_cast(image); - const C* coeff = reinterpret_cast(coefficients); - gmres_update_kernel<<>>(ld, dim, current_solution, current_residual, current_direction, current_image, coeff); + C *current_solution = reinterpret_cast(solution); + C *current_residual = reinterpret_cast(residual); + const C *current_direction = reinterpret_cast(direction); + const C *current_image = reinterpret_cast(image); + const C *coeff = reinterpret_cast(coefficients); + gmres_update_kernel<<>>( + ld, + dim, + current_solution, + current_residual, + current_direction, + current_image, + coeff); check_launch(); } template -void linear_op::dots(int ld,int dim,int nvec,const T* x,const T* y,const T* z,const T* w, - T* out,T* partial,int tiles) const -{ - if(nvec<=0)return; - using C=thrust::complex; - const int pairs=z ? 2 : 1; - C* target=reinterpret_cast(tiles==1 ? out : partial); - dots_kernel<<>>(ld,dim,nvec,tiles,reinterpret_cast(x), - reinterpret_cast(y),reinterpret_cast(z),reinterpret_cast(w),target); +void linear_op::dots( + int ld, + int dim, + int nvec, + const T *x, + const T *y, + const T *z, + const T *w, + T *out, + T *partial, + int tiles) const +{ + if (nvec <= 0) + { + return; + } + using C = thrust::complex; + const int pairs = z ? 2 : 1; + C *target = reinterpret_cast(tiles == 1 ? out : partial); + dots_kernel<<>>( + ld, + dim, + nvec, + tiles, + reinterpret_cast(x), + reinterpret_cast(y), + reinterpret_cast(z), + reinterpret_cast(w), + target); check_launch(); - if(tiles>1) + if (tiles > 1) { - sum_tiles<<<(pairs*nvec+255)/256,256>>>(pairs*nvec,tiles,target,reinterpret_cast(out)); + sum_tiles<<<(pairs * nvec + 255) / 256, 256>>>( + pairs * nvec, + tiles, + target, + reinterpret_cast(out)); check_launch(); } } template -void linear_op::gather(int ld,int dim,int nvec,int slots,int stride,const T* in,T* out,const int* map) const +void linear_op::gather( + int ld, + int dim, + int nvec, + int slots, + int stride, + const T *in, + T *out, + const int *map) const { - if(dim<=0 || nvec<=0)return; - using C=thrust::complex; - gather_kernel<<>>(ld,dim,stride,reinterpret_cast(in),reinterpret_cast(out),map); + if (dim <= 0 || nvec <= 0) + { + return; + } + using C = thrust::complex; + gather_kernel<<>>( + ld, + dim, + stride, + reinterpret_cast(in), + reinterpret_cast(out), + map); check_launch(); } - template -__global__ void swaps_kernel(int ld,int dim,int stride,int count,C* vectors,const int* pairs) +__global__ void swaps_kernel( + int ld, + int dim, + int stride, + int count, + C *vectors, + const int *pairs) { - const int i=blockIdx.x*blockDim.x+threadIdx.x; + const int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= dim) { return; } const std::int64_t offset = static_cast(blockIdx.y) * stride + i; - for(int k=0;k(pairs[2 * k]) * ld; const std::int64_t second = offset + static_cast(pairs[2 * k + 1]) * ld; - C& a = vectors[first]; - C& b = vectors[second]; - const C value=a; a=b; b=value; + C &a = vectors[first]; + C &b = vectors[second]; + const C value = a; + a = b; + b = value; } } template -void linear_op::swaps(int ld,int dim,int slots,int stride,int count,T* vectors,const int* pairs) const -{ - if(dim==0 || count==0)return; - using C=thrust::complex; - swaps_kernel<<>>(ld,dim,stride,count,reinterpret_cast(vectors),pairs); +void linear_op::swaps( + int ld, + int dim, + int slots, + int stride, + int count, + T *vectors, + const int *pairs) const +{ + if (dim == 0 || count == 0) + { + return; + } + using C = thrust::complex; + swaps_kernel<<>>( + ld, + dim, + stride, + count, + reinterpret_cast(vectors), + pairs); check_launch(); } - template -__global__ void bicg_update_kernel(int ld,int dim,bool direction,C* out,const C* x,const C* y,const C* a,const C* b) +__global__ void bicg_update_kernel( + int ld, + int dim, + bool direction, + C *out, + const C *x, + const C *y, + const C *a, + const C *b) { - const int i=blockIdx.x*blockDim.x+threadIdx.x; - const int band=blockIdx.y; - if(i -void linear_op::bicg_update(int ld,int dim,int nvec,bool direction,T* out,const T* x,const T* y,const T* a,const T* b) const +void linear_op::bicg_update( + int ld, + int dim, + int nvec, + bool direction, + T *out, + const T *x, + const T *y, + const T *a, + const T *b) const { - if(dim==0 || nvec==0)return; - using C=thrust::complex; - bicg_update_kernel<<>>(ld,dim,direction,reinterpret_cast(out), - reinterpret_cast(x),reinterpret_cast(y),reinterpret_cast(a),reinterpret_cast(b)); + if (dim == 0 || nvec == 0) + { + return; + } + using C = thrust::complex; + bicg_update_kernel<<>>( + ld, + dim, + direction, + reinterpret_cast(out), + reinterpret_cast(x), + reinterpret_cast(y), + reinterpret_cast(a), + reinterpret_cast(b)); check_launch(); } - template -__global__ void diagonal_kernel(int ld, int dim, const C* inverse, const C* x, C* y) +__global__ void diagonal_kernel( + int ld, + int dim, + const C *inverse, + const C *x, + C *y) { const int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i < dim) y[blockIdx.y * ld + i] = inverse[i] * x[blockIdx.y * ld + i]; + if (i < dim) + { + y[blockIdx.y * ld + i] = inverse[i] * x[blockIdx.y * ld + i]; + } } template -__global__ void cgs_direction_kernel(int ld, int dim, C* p, C* u, const C* r, const C* q, const C* beta) +__global__ void cgs_direction_kernel( + int ld, + int dim, + C *p, + C *u, + const C *r, + const C *q, + const C *beta) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int band = blockIdx.y; @@ -275,7 +474,14 @@ __global__ void cgs_direction_kernel(int ld, int dim, C* p, C* u, const C* r, co } template -__global__ void cgs_alpha_kernel(int ld, int dim, C* q, C* direction, const C* u, const C* v, const C* alpha) +__global__ void cgs_alpha_kernel( + int ld, + int dim, + C *q, + C *direction, + const C *u, + const C *v, + const C *alpha) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int band = blockIdx.y; @@ -288,7 +494,14 @@ __global__ void cgs_alpha_kernel(int ld, int dim, C* q, C* direction, const C* u } template -__global__ void cgs_finish_kernel(int ld, int dim, C* x, C* r, const C* direction, const C* ad, const C* alpha) +__global__ void cgs_finish_kernel( + int ld, + int dim, + C *x, + C *r, + const C *direction, + const C *ad, + const C *alpha) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int band = blockIdx.y; @@ -301,64 +514,119 @@ __global__ void cgs_finish_kernel(int ld, int dim, C* x, C* r, const C* directio } template -void linear_op::diagonal(int ld, int dim, int nvec, const T* inverse, const T* x, T* y) const +void linear_op::diagonal( + int ld, + int dim, + int nvec, + const T *inverse, + const T *x, + T *y) const { - if (dim == 0 || nvec == 0) return; + if (dim == 0 || nvec == 0) + { + return; + } using C = thrust::complex; - diagonal_kernel<<>>(ld, dim, - reinterpret_cast(inverse), - reinterpret_cast(x), - reinterpret_cast(y)); + diagonal_kernel<<>>( + ld, + dim, + reinterpret_cast(inverse), + reinterpret_cast(x), + reinterpret_cast(y)); check_launch(); } template -void linear_op::cgs_direction(int ld, int dim, int nvec, T* p, T* u, const T* r, const T* q, const T* beta) const +void linear_op::cgs_direction( + int ld, + int dim, + int nvec, + T *p, + T *u, + const T *r, + const T *q, + const T *beta) const { - if (dim == 0 || nvec == 0) return; + if (dim == 0 || nvec == 0) + { + return; + } using C = thrust::complex; - cgs_direction_kernel<<>>(ld, dim, - reinterpret_cast(p), - reinterpret_cast(u), - reinterpret_cast(r), - reinterpret_cast(q), - reinterpret_cast(beta)); + cgs_direction_kernel<<>>( + ld, + dim, + reinterpret_cast(p), + reinterpret_cast(u), + reinterpret_cast(r), + reinterpret_cast(q), + reinterpret_cast(beta)); check_launch(); } template -void linear_op::cgs_alpha(int ld, int dim, int nvec, T* q, T* direction, const T* u, const T* v, const T* alpha) const +void linear_op::cgs_alpha( + int ld, + int dim, + int nvec, + T *q, + T *direction, + const T *u, + const T *v, + const T *alpha) const { - if (dim == 0 || nvec == 0) return; + if (dim == 0 || nvec == 0) + { + return; + } using C = thrust::complex; - cgs_alpha_kernel<<>>(ld, dim, - reinterpret_cast(q), - reinterpret_cast(direction), - reinterpret_cast(u), - reinterpret_cast(v), - reinterpret_cast(alpha)); + cgs_alpha_kernel<<>>( + ld, + dim, + reinterpret_cast(q), + reinterpret_cast(direction), + reinterpret_cast(u), + reinterpret_cast(v), + reinterpret_cast(alpha)); check_launch(); } template -void linear_op::cgs_finish(int ld, int dim, int nvec, T* x, T* r, const T* direction, const T* ad, const T* alpha) const +void linear_op::cgs_finish( + int ld, + int dim, + int nvec, + T *x, + T *r, + const T *direction, + const T *ad, + const T *alpha) const { - if (dim == 0 || nvec == 0) return; + if (dim == 0 || nvec == 0) + { + return; + } using C = thrust::complex; - cgs_finish_kernel<<>>(ld, dim, - reinterpret_cast(x), - reinterpret_cast(r), - reinterpret_cast(direction), - reinterpret_cast(ad), - reinterpret_cast(alpha)); + cgs_finish_kernel<<>>( + ld, + dim, + reinterpret_cast(x), + reinterpret_cast(r), + reinterpret_cast(direction), + reinterpret_cast(ad), + reinterpret_cast(alpha)); check_launch(); } namespace { template -__global__ void wide_dot_kernel(int ld, int dim, int nvec, int stride, - const thrust::complex* basis, const thrust::complex* x, - thrust::complex* out) +__global__ void wide_dot_kernel( + int ld, + int dim, + int nvec, + int stride, + const thrust::complex *basis, + const thrust::complex *x, + thrust::complex *out) { __shared__ Accumulator re[linear_threads]; __shared__ Accumulator im[linear_threads]; @@ -370,48 +638,93 @@ __global__ void wide_dot_kernel(int ld, int dim, int nvec, int stride, thrust::complex sum(0, 0); for (int i = tid; i < dim; i += blockDim.x) { - sum += thrust::conj(thrust::complex(basis[basis_offset + i])) - * thrust::complex(x[band_offset + i]); + sum += thrust::conj(thrust::complex(basis[basis_offset + i])) * + thrust::complex(x[band_offset + i]); } re[tid] = sum.real(); im[tid] = sum.imag(); __syncthreads(); for (int s = blockDim.x / 2; s > 0; s /= 2) { - if (tid < s) { re[tid] += re[tid + s]; im[tid] += im[tid + s]; } + if (tid < s) + { + re[tid] += re[tid + s]; + im[tid] += im[tid + s]; + } __syncthreads(); } - if (tid == 0) out[blockIdx.x] = thrust::complex(re[0], im[0]); -} + if (tid == 0) + { + out[blockIdx.x] = thrust::complex(re[0], im[0]); + } } +} // namespace template -void linear_op::wide_dots(int ld, int dim, int nvec, int count, int stride, - const T* basis, const T* x, std::complex* out) const -{ - if (nvec * count == 0) return; +void linear_op::wide_dots( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *x, + std::complex *out) const +{ + if (nvec * count == 0) + { + return; + } using C = thrust::complex; - wide_dot_kernel<<>>(ld, dim, nvec, stride, - reinterpret_cast(basis), reinterpret_cast(x), - reinterpret_cast*>(out)); + wide_dot_kernel<<>>( + ld, + dim, + nvec, + stride, + reinterpret_cast(basis), + reinterpret_cast(x), + reinterpret_cast *>(out)); check_launch(); } template -void linear_op::native_dots(int ld, int dim, int nvec, int count, int stride, - const T* basis, const T* x, T* out) const -{ - if (nvec * count == 0) return; +void linear_op::native_dots( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *x, + T *out) const +{ + if (nvec * count == 0) + { + return; + } using C = thrust::complex; - wide_dot_kernel<<>>(ld, dim, nvec, stride, - reinterpret_cast(basis), reinterpret_cast(x), reinterpret_cast(out)); + wide_dot_kernel<<>>( + ld, + dim, + nvec, + stride, + reinterpret_cast(basis), + reinterpret_cast(x), + reinterpret_cast(out)); check_launch(); } namespace { template -__global__ void combine_kernel(int ld, int dim, int count, int stride, int nvec, - const C* basis, const C* coefficients, C* output) +__global__ void combine_kernel( + int ld, + int dim, + int count, + int stride, + int nvec, + const C *basis, + const C *coefficients, + C *output) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int b = blockIdx.y; @@ -428,7 +741,12 @@ __global__ void combine_kernel(int ld, int dim, int count, int stride, int nvec, } template -__global__ void scatter_kernel(int ld, int dim, const C* input, C* output, const int* map) +__global__ void scatter_kernel( + int ld, + int dim, + const C *input, + C *output, + const int *map) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int b = blockIdx.y; @@ -439,8 +757,14 @@ __global__ void scatter_kernel(int ld, int dim, const C* input, C* output, const } template -__global__ void block_gram_kernel(int ld, int dim, int nvec, int count, int stride, - const thrust::complex* basis, thrust::complex* output) +__global__ void block_gram_kernel( + int ld, + int dim, + int nvec, + int count, + int stride, + const thrust::complex *basis, + thrust::complex *output) { const int b = blockIdx.x % nvec; const int pair = blockIdx.x / nvec; @@ -484,8 +808,15 @@ __global__ void block_gram_kernel(int ld, int dim, int nvec, int count, int stri } // namespace template -void linear_op::combine(int ld, int dim, int nvec, int count, int stride, - const T* basis, const T* coefficients, T* output) const +void linear_op::combine( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *coefficients, + T *output) const { if (dim == 0 || nvec == 0) { @@ -493,14 +824,26 @@ void linear_op::combine(int ld, int dim, int nvec, i } using C = thrust::complex; const dim3 grid((dim + linear_threads - 1) / linear_threads, nvec); - combine_kernel<<>>(ld, dim, count, stride, nvec, - reinterpret_cast(basis), reinterpret_cast(coefficients), reinterpret_cast(output)); + combine_kernel<<>>( + ld, + dim, + count, + stride, + nvec, + reinterpret_cast(basis), + reinterpret_cast(coefficients), + reinterpret_cast(output)); check_launch(); } template -void linear_op::scatter(int ld, int dim, int nvec, - const T* input, T* output, const int* map) const +void linear_op::scatter( + int ld, + int dim, + int nvec, + const T *input, + T *output, + const int *map) const { if (dim == 0 || nvec == 0) { @@ -508,13 +851,24 @@ void linear_op::scatter(int ld, int dim, int nvec, } using C = thrust::complex; const dim3 grid((dim + linear_threads - 1) / linear_threads, nvec); - scatter_kernel<<>>(ld, dim, reinterpret_cast(input), reinterpret_cast(output), map); + scatter_kernel<<>>( + ld, + dim, + reinterpret_cast(input), + reinterpret_cast(output), + map); check_launch(); } template -void linear_op::block_gram(int ld, int dim, int nvec, int count, int stride, - const T* basis, std::complex* output) const +void linear_op::block_gram( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + std::complex *output) const { if (nvec == 0 || count == 0) { @@ -522,8 +876,14 @@ void linear_op::block_gram(int ld, int dim, int nvec } using C = thrust::complex; const int blocks = nvec * count * count; - block_gram_kernel<<>>(ld, dim, nvec, count, stride, reinterpret_cast(basis), - reinterpret_cast*>(output)); + block_gram_kernel<<>>( + ld, + dim, + nvec, + count, + stride, + reinterpret_cast(basis), + reinterpret_cast *>(output)); check_launch(); } diff --git a/source/source_hsolver/module_linalg/kernels/linear_op.cpp b/source/source_hsolver/module_linalg/kernels/linear_op.cpp index c5b194f1c22..7e6743308e1 100644 --- a/source/source_hsolver/module_linalg/kernels/linear_op.cpp +++ b/source/source_hsolver/module_linalg/kernels/linear_op.cpp @@ -20,7 +20,13 @@ bool parallel_work(const int dim, const int nvec) } // namespace template -void linear_op::dot(const int ld, const int dim, const int nvec, const T* x, const T* y, T* out) const +void linear_op::dot( + const int ld, + const int dim, + const int nvec, + const T *x, + const T *y, + T *out) const { for (int band = 0; band < nvec; ++band) { @@ -35,17 +41,18 @@ void linear_op::dot(const int ld, const int dim, const int nvec, cons } template -void linear_op::batch(int ld, - int dim, - int nvec, - T* out, - const T* x, - const T* y, - T a, - T b, - const T* ca, - const T* cb, - const int* skip) const +void linear_op::batch( + int ld, + int dim, + int nvec, + T *out, + const T *x, + const T *y, + T a, + T b, + const T *ca, + const T *cb, + const int *skip) const { #pragma omp parallel for schedule(static) if (parallel_work(dim, nvec)) for (int band = 0; band < nvec; ++band) @@ -64,14 +71,15 @@ void linear_op::batch(int ld, } } template -void linear_op::gmres_update(int ld, - int dim, - int nvec, - T* solution, - T* residual, - const T* direction, - const T* image, - const T* coefficients) const +void linear_op::gmres_update( + int ld, + int dim, + int nvec, + T *solution, + T *residual, + const T *direction, + const T *image, + const T *coefficients) const { #pragma omp parallel for schedule(static) if (parallel_work(dim, nvec)) for (int band = 0; band < nvec; ++band) @@ -94,8 +102,17 @@ void linear_op::gmres_update(int ld, } template -void linear_op::dots(int ld, int dim, int nvec, const T* x, const T* y, const T* z, const T* w, T* out, T* partial, int tiles) - const +void linear_op::dots( + int ld, + int dim, + int nvec, + const T *x, + const T *y, + const T *z, + const T *w, + T *out, + T *partial, + int tiles) const { // Independent tiles also expose parallelism when there are few bands. #pragma omp parallel for collapse(2) schedule(static) if (parallel_work(dim, nvec)) @@ -135,7 +152,15 @@ void linear_op::dots(int ld, int dim, int nvec, const T* x, const T* } } template -void linear_op::gather(int ld, int dim, int nvec, int slots, int stride, const T* in, T* out, const int* map) const +void linear_op::gather( + int ld, + int dim, + int nvec, + int slots, + int stride, + const T *in, + T *out, + const int *map) const { for (int slot = 0; slot < slots; ++slot) { @@ -153,7 +178,14 @@ void linear_op::gather(int ld, int dim, int nvec, int slots, int stri } template -void linear_op::swaps(int ld, int dim, int slots, int stride, int count, T* vectors, const int* pairs) const +void linear_op::swaps( + int ld, + int dim, + int slots, + int stride, + int count, + T *vectors, + const int *pairs) const { for (int slot = 0; slot < slots; ++slot) { @@ -171,8 +203,16 @@ void linear_op::swaps(int ld, int dim, int slots, int stride, int cou } template -void linear_op::bicg_update(int ld, int dim, int nvec, bool direction, T* out, const T* x, const T* y, const T* a, const T* b) - const +void linear_op::bicg_update( + int ld, + int dim, + int nvec, + bool direction, + T *out, + const T *x, + const T *y, + const T *a, + const T *b) const { #pragma omp parallel for collapse(2) schedule(static) if (parallel_work(dim, nvec)) for (int band = 0; band < nvec; ++band) @@ -180,13 +220,20 @@ void linear_op::bicg_update(int ld, int dim, int nvec, bool direction for (int i = 0; i < dim; ++i) { const int j = band * ld + i; - out[j] = direction ? x[j] + a[band] * (out[j] - b[band] * y[j]) : (out[j] + a[band] * x[j]) + b[band] * y[j]; + out[j] = + direction ? x[j] + a[band] * (out[j] - b[band] * y[j]) : (out[j] + a[band] * x[j]) + b[band] * y[j]; } } } template -void linear_op::diagonal(int ld, int dim, int nvec, const T* inverse, const T* x, T* y) const +void linear_op::diagonal( + int ld, + int dim, + int nvec, + const T *inverse, + const T *x, + T *y) const { #pragma omp parallel for collapse(2) schedule(static) if (parallel_work(dim, nvec)) for (int band = 0; band < nvec; ++band) @@ -199,7 +246,15 @@ void linear_op::diagonal(int ld, int dim, int nvec, const T* inverse, } template -void linear_op::cgs_direction(int ld, int dim, int nvec, T* p, T* u, const T* r, const T* q, const T* beta) const +void linear_op::cgs_direction( + int ld, + int dim, + int nvec, + T *p, + T *u, + const T *r, + const T *q, + const T *beta) const { #pragma omp parallel for collapse(2) schedule(static) if (parallel_work(dim, nvec)) for (int band = 0; band < nvec; ++band) @@ -214,7 +269,15 @@ void linear_op::cgs_direction(int ld, int dim, int nvec, T* p, T* u, } template -void linear_op::cgs_alpha(int ld, int dim, int nvec, T* q, T* direction, const T* u, const T* v, const T* alpha) const +void linear_op::cgs_alpha( + int ld, + int dim, + int nvec, + T *q, + T *direction, + const T *u, + const T *v, + const T *alpha) const { #pragma omp parallel for collapse(2) schedule(static) if (parallel_work(dim, nvec)) for (int band = 0; band < nvec; ++band) @@ -229,7 +292,15 @@ void linear_op::cgs_alpha(int ld, int dim, int nvec, T* q, T* directi } template -void linear_op::cgs_finish(int ld, int dim, int nvec, T* x, T* r, const T* direction, const T* ad, const T* alpha) const +void linear_op::cgs_finish( + int ld, + int dim, + int nvec, + T *x, + T *r, + const T *direction, + const T *ad, + const T *alpha) const { #pragma omp parallel for collapse(2) schedule(static) if (parallel_work(dim, nvec)) for (int band = 0; band < nvec; ++band) @@ -243,14 +314,15 @@ void linear_op::cgs_finish(int ld, int dim, int nvec, T* x, T* r, con } } template -void linear_op::wide_dots(int ld, - int dim, - int nvec, - int count, - int stride, - const T* basis, - const T* x, - std::complex* out) const +void linear_op::wide_dots( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *x, + std::complex *out) const { #ifdef _OPENMP #pragma omp parallel for collapse(2) if (parallel_work(dim, nvec * count)) @@ -264,14 +336,23 @@ void linear_op::wide_dots(int ld, std::complex sum = 0.0; for (int i = 0; i < dim; ++i) { - sum += std::conj(std::complex(basis[basis_offset + i])) * std::complex(x[band_offset + i]); + sum += + std::conj(std::complex(basis[basis_offset + i])) * std::complex(x[band_offset + i]); } out[j * nvec + b] = sum; } } } template -void linear_op::native_dots(int ld, int dim, int nvec, int count, int stride, const T* basis, const T* x, T* out) const +void linear_op::native_dots( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *x, + T *out) const { #ifdef _OPENMP #pragma omp parallel for collapse(2) if (parallel_work(dim, nvec * count)) @@ -293,7 +374,15 @@ void linear_op::native_dots(int ld, int dim, int nvec, int count, int } template -void linear_op::combine(int ld, int dim, int nvec, int count, int stride, const T* basis, const T* coefficients, T* output) const +void linear_op::combine( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *coefficients, + T *output) const { #pragma omp parallel for schedule(static) if (parallel_work(dim, nvec)) for (int b = 0; b < nvec; ++b) @@ -312,7 +401,13 @@ void linear_op::combine(int ld, int dim, int nvec, int count, int str } template -void linear_op::scatter(int ld, int dim, int nvec, const T* input, T* output, const int* map) const +void linear_op::scatter( + int ld, + int dim, + int nvec, + const T *input, + T *output, + const int *map) const { #pragma omp parallel for schedule(static) if (parallel_work(dim, nvec)) for (int b = 0; b < nvec; ++b) @@ -324,7 +419,14 @@ void linear_op::scatter(int ld, int dim, int nvec, const T* input, T* } template -void linear_op::block_gram(int ld, int dim, int nvec, int count, int stride, const T* basis, std::complex* output) const +void linear_op::block_gram( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + std::complex *output) const { #pragma omp parallel for collapse(2) schedule(static) if (parallel_work(dim, nvec * count)) for (int col = 0; col < count; ++col) diff --git a/source/source_hsolver/module_linalg/kernels/linear_op.h b/source/source_hsolver/module_linalg/kernels/linear_op.h index 7f59d8c418d..6d44e938d6e 100644 --- a/source/source_hsolver/module_linalg/kernels/linear_op.h +++ b/source/source_hsolver/module_linalg/kernels/linear_op.h @@ -9,79 +9,329 @@ namespace hsolver { /** @brief Elementwise operations; pointers and dot outputs belong to Device. */ -template -class linear_op +template class linear_op { public: /** @brief Add per-band linear combinations of Krylov columns; coefficients are column-major blocks. */ - void combine(int ld, int dim, int nvec, int count, int stride, const T* basis, const T* coefficients, T* output) const; + void combine( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *coefficients, + T *output) const; /** @brief Scatter packed columns to distinct original-band destinations. */ - void scatter(int ld, int dim, int nvec, const T* input, T* output, const int* map) const; + void scatter( + int ld, + int dim, + int nvec, + const T *input, + T *output, + const int *map) const; /** @brief Per-band Krylov Gram matrices in double precision, indexed as (column * count + row) * nvec + band. */ - void block_gram(int ld, int dim, int nvec, int count, int stride, const T* basis, std::complex* output) const; + void block_gram( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + std::complex *output) const; /** @brief Skip zero-coefficient columns; a null residual disables the paired residual update. */ - void gmres_update(int ld, int dim, int nvec, T* solution, T* residual, const T* direction, const T* image, const T* coefficients) const; + void gmres_update( + int ld, + int dim, + int nvec, + T *solution, + T *residual, + const T *direction, + const T *image, + const T *coefficients) const; /** @brief Batched products accumulated in wavefunction precision. */ - void native_dots(int ld, int dim, int nvec, int count, int stride, const T* basis, const T* x, T* out) const; + void native_dots( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *x, + T *out) const; /** @brief Batched products accumulated in double precision. */ - void wide_dots(int ld, int dim, int nvec, int count, int stride, const T* basis, const T* x, std::complex* out) const; + void wide_dots( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *x, + std::complex *out) const; /** @brief Apply the same inverse diagonal to every column. */ - void diagonal(int ld, int dim, int nvec, const T* inverse, const T* x, T* y) const; + void diagonal( + int ld, + int dim, + int nvec, + const T *inverse, + const T *x, + T *y) const; /** @brief Fused CGS search direction, alpha step, and final updates. */ - void cgs_direction(int ld, int dim, int nvec, T* p, T* u, const T* r, const T* q, const T* beta) const; - void cgs_alpha(int ld, int dim, int nvec, T* q, T* direction, const T* u, const T* v, const T* alpha) const; - void cgs_finish(int ld, int dim, int nvec, T* x, T* r, const T* direction, const T* ad, const T* alpha) const; + void cgs_direction( + int ld, + int dim, + int nvec, + T *p, + T *u, + const T *r, + const T *q, + const T *beta) const; + void cgs_alpha( + int ld, + int dim, + int nvec, + T *q, + T *direction, + const T *u, + const T *v, + const T *alpha) const; + void cgs_finish( + int ld, + int dim, + int nvec, + T *x, + T *r, + const T *direction, + const T *ad, + const T *alpha) const; /** @brief Batched updates with optional device coefficients and skip mask. */ - void batch(int ld, int dim, int nvec, T* out, const T* x, const T* y, T a, T b, const T* ca, const T* cb, const int* skip) const; + void batch( + int ld, + int dim, + int nvec, + T *out, + const T *x, + const T *y, + T a, + T b, + const T *ca, + const T *cb, + const int *skip) const; /** @brief One or two inner products using caller-owned partial sums. */ - void dots(int ld, int dim, int nvec, const T* x, const T* y, const T* z, const T* w, T* out, T* partial, int tiles) const; + void dots( + int ld, + int dim, + int nvec, + const T *x, + const T *y, + const T *z, + const T *w, + T *out, + T *partial, + int tiles) const; /** @brief Apply an ordered list of column swaps to all vector slots. */ - void swaps(int ld, int dim, int slots, int stride, int count, T* vectors, const int* pairs) const; + void swaps( + int ld, + int dim, + int slots, + int stride, + int count, + T *vectors, + const int *pairs) const; /** @brief Fuse the two BiCGSTAB direction or solution updates. */ - void bicg_update(int ld, int dim, int nvec, bool direction, T* out, const T* x, const T* y, const T* a, const T* b) const; + void bicg_update( + int ld, + int dim, + int nvec, + bool direction, + T *out, + const T *x, + const T *y, + const T *a, + const T *b) const; /** @brief Gather columns using a device destination-to-source map. */ - void gather(int ld, int dim, int nvec, int slots, int stride, const T* in, T* out, const int* map) const; - void dot(const int ld, const int dim, const int nvec, const T* x, const T* y, T* out) const; + void gather( + int ld, + int dim, + int nvec, + int slots, + int stride, + const T *in, + T *out, + const int *map) const; + void dot( + const int ld, + const int dim, + const int nvec, + const T *x, + const T *y, + T *out) const; }; #if defined(__CUDA) || defined(__ROCM) -template -class linear_op +template class linear_op { public: /** @brief Add per-band linear combinations of Krylov columns; coefficients are column-major blocks. */ - void combine(int ld, int dim, int nvec, int count, int stride, const T* basis, const T* coefficients, T* output) const; + void combine( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *coefficients, + T *output) const; /** @brief Scatter packed columns to distinct original-band destinations. */ - void scatter(int ld, int dim, int nvec, const T* input, T* output, const int* map) const; + void scatter( + int ld, + int dim, + int nvec, + const T *input, + T *output, + const int *map) const; /** @brief Per-band Krylov Gram matrices in double precision, indexed as (column * count + row) * nvec + band. */ - void block_gram(int ld, int dim, int nvec, int count, int stride, const T* basis, std::complex* output) const; + void block_gram( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + std::complex *output) const; /** @brief Skip zero-coefficient columns; a null residual disables the paired residual update. */ - void gmres_update(int ld, int dim, int nvec, T* solution, T* residual, const T* direction, const T* image, const T* coefficients) const; + void gmres_update( + int ld, + int dim, + int nvec, + T *solution, + T *residual, + const T *direction, + const T *image, + const T *coefficients) const; /** @brief Batched products accumulated in wavefunction precision. */ - void native_dots(int ld, int dim, int nvec, int count, int stride, const T* basis, const T* x, T* out) const; + void native_dots( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *x, + T *out) const; /** @brief Batched products accumulated in double precision. */ - void wide_dots(int ld, int dim, int nvec, int count, int stride, const T* basis, const T* x, std::complex* out) const; + void wide_dots( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *x, + std::complex *out) const; /** @brief Apply the same inverse diagonal to every column. */ - void diagonal(int ld, int dim, int nvec, const T* inverse, const T* x, T* y) const; + void diagonal( + int ld, + int dim, + int nvec, + const T *inverse, + const T *x, + T *y) const; /** @brief Fused CGS search direction, alpha step, and final updates. */ - void cgs_direction(int ld, int dim, int nvec, T* p, T* u, const T* r, const T* q, const T* beta) const; - void cgs_alpha(int ld, int dim, int nvec, T* q, T* direction, const T* u, const T* v, const T* alpha) const; - void cgs_finish(int ld, int dim, int nvec, T* x, T* r, const T* direction, const T* ad, const T* alpha) const; + void cgs_direction( + int ld, + int dim, + int nvec, + T *p, + T *u, + const T *r, + const T *q, + const T *beta) const; + void cgs_alpha( + int ld, + int dim, + int nvec, + T *q, + T *direction, + const T *u, + const T *v, + const T *alpha) const; + void cgs_finish( + int ld, + int dim, + int nvec, + T *x, + T *r, + const T *direction, + const T *ad, + const T *alpha) const; /** @brief Batched updates with optional device coefficients and skip mask. */ - void batch(int ld, int dim, int nvec, T* out, const T* x, const T* y, T a, T b, const T* ca, const T* cb, const int* skip) const; + void batch( + int ld, + int dim, + int nvec, + T *out, + const T *x, + const T *y, + T a, + T b, + const T *ca, + const T *cb, + const int *skip) const; /** @brief One or two inner products using caller-owned partial sums. */ - void dots(int ld, int dim, int nvec, const T* x, const T* y, const T* z, const T* w, T* out, T* partial, int tiles) const; + void dots( + int ld, + int dim, + int nvec, + const T *x, + const T *y, + const T *z, + const T *w, + T *out, + T *partial, + int tiles) const; /** @brief Apply an ordered list of column swaps to all vector slots. */ - void swaps(int ld, int dim, int slots, int stride, int count, T* vectors, const int* pairs) const; + void swaps( + int ld, + int dim, + int slots, + int stride, + int count, + T *vectors, + const int *pairs) const; /** @brief Fuse the two BiCGSTAB direction or solution updates. */ - void bicg_update(int ld, int dim, int nvec, bool direction, T* out, const T* x, const T* y, const T* a, const T* b) const; + void bicg_update( + int ld, + int dim, + int nvec, + bool direction, + T *out, + const T *x, + const T *y, + const T *a, + const T *b) const; /** @brief Gather columns using a device destination-to-source map. */ - void gather(int ld, int dim, int nvec, int slots, int stride, const T* in, T* out, const int* map) const; - void dot(const int ld, const int dim, const int nvec, const T* x, const T* y, T* out) const; + void gather( + int ld, + int dim, + int nvec, + int slots, + int stride, + const T *in, + T *out, + const int *map) const; + void dot( + const int ld, + const int dim, + const int nvec, + const T *x, + const T *y, + T *out) const; }; #endif diff --git a/source/source_hsolver/module_linalg/kernels/rocm/linear_op.hip.cu b/source/source_hsolver/module_linalg/kernels/rocm/linear_op.hip.cu index 27d5ca079bf..5186f852759 100644 --- a/source/source_hsolver/module_linalg/kernels/rocm/linear_op.hip.cu +++ b/source/source_hsolver/module_linalg/kernels/rocm/linear_op.hip.cu @@ -1,10 +1,10 @@ #include "source_hsolver/module_linalg/kernels/linear_op.h" -#include -#include #include #include +#include #include +#include namespace hsolver { @@ -22,8 +22,12 @@ void check_launch() } template -__global__ void dot_kernel(const int ld, const int dim, const thrust::complex* x, - const thrust::complex* y, thrust::complex* out) +__global__ void dot_kernel( + const int ld, + const int dim, + const thrust::complex *x, + const thrust::complex *y, + thrust::complex *out) { __shared__ Real re[linear_threads]; __shared__ Real im[linear_threads]; @@ -54,36 +58,58 @@ __global__ void dot_kernel(const int ld, const int dim, const thrust::complex -void linear_op::dot(const int ld, const int dim, const int nvec, - const T* x, const T* y, T* out) const -{ - if (nvec <= 0) { return; } +void linear_op::dot( + const int ld, + const int dim, + const int nvec, + const T *x, + const T *y, + T *out) const +{ + if (nvec <= 0) + { + return; + } using Complex = thrust::complex; - dot_kernel<<>>(ld, dim, reinterpret_cast(x), - reinterpret_cast(y), reinterpret_cast(out)); + dot_kernel<<>>( + ld, + dim, + reinterpret_cast(x), + reinterpret_cast(y), + reinterpret_cast(out)); check_launch(); } - template -__global__ void batch_kernel(int ld,int dim,C* out,const C* x,const C* y,C a,C b,const C* ca,const C* cb,const int* skip) +__global__ void batch_kernel( + int ld, + int dim, + C *out, + const C *x, + const C *y, + C a, + C b, + const C *ca, + const C *cb, + const int *skip) { - const int i=blockIdx.x*blockDim.x+threadIdx.x; - const int band=blockIdx.y; - if(i -__global__ void gmres_update_kernel(int ld, - int dim, - C* solution, - C* residual, - const C* direction, - const C* image, - const C* coefficients) +__global__ void gmres_update_kernel( + int ld, + int dim, + C *solution, + C *residual, + const C *direction, + const C *image, + const C *coefficients) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int band = blockIdx.y; @@ -105,34 +131,74 @@ __global__ void gmres_update_kernel(int ld, } template -__global__ void dots_kernel(int ld,int dim,int nvec,int tiles, - const thrust::complex* x,const thrust::complex* y, - const thrust::complex* z,const thrust::complex* w,thrust::complex* partial) +__global__ void dots_kernel( + int ld, + int dim, + int nvec, + int tiles, + const thrust::complex *x, + const thrust::complex *y, + const thrust::complex *z, + const thrust::complex *w, + thrust::complex *partial) { __shared__ Real re[256]; __shared__ Real im[256]; - const int tid=threadIdx.x; - const int band=blockIdx.x; - const int pair=blockIdx.z; - const thrust::complex* a=pair ? z : x; - const thrust::complex* b=pair ? w : y; - thrust::complex sum(0,0); - for(int i=blockIdx.y*256+tid;i *a = pair ? z : x; + const thrust::complex *b = pair ? w : y; + thrust::complex sum(0, 0); + for (int i = blockIdx.y * 256 + tid; i < dim; i += tiles * 256) + { + sum += thrust::conj(a[band * ld + i]) * b[band * ld + i]; + } + re[tid] = sum.real(); + im[tid] = sum.imag(); __syncthreads(); - for(int n=128;n>0;n/=2) { if(tid(re[0],im[0]); + for (int n = 128; n > 0; n /= 2) + { + if (tid < n) + { + re[tid] += re[tid + n]; + im[tid] += im[tid + n]; + } + __syncthreads(); + } + if (tid == 0) + { + partial[(pair * nvec + band) * tiles + blockIdx.y] = thrust::complex(re[0], im[0]); + } } template -__global__ void sum_tiles(int n,int tiles,const C* partial,C* out) +__global__ void sum_tiles( + int n, + int tiles, + const C *partial, + C *out) { - const int i=blockIdx.x*blockDim.x+threadIdx.x; - if(i -__global__ void gather_kernel(int ld,int dim,int stride,const C* in,C* out,const int* map) +__global__ void gather_kernel( + int ld, + int dim, + int stride, + const C *in, + C *out, + const int *map) { - const int i=blockIdx.x*blockDim.x+threadIdx.x; + const int i = blockIdx.x * blockDim.x + threadIdx.x; if (i < dim) { const std::int64_t slot_offset = static_cast(blockIdx.z) * stride; @@ -142,25 +208,47 @@ __global__ void gather_kernel(int ld,int dim,int stride,const C* in,C* out,const } } template -void linear_op::batch(int ld,int dim,int nvec,T* out,const T* x,const T* y, - T a,T b,const T* ca,const T* cb,const int* skip) const -{ - if(dim<=0 || nvec<=0)return; - using C=thrust::complex; - batch_kernel<<>>(ld,dim,reinterpret_cast(out), - reinterpret_cast(x),reinterpret_cast(y),C(a.real(),a.imag()),C(b.real(),b.imag()), - reinterpret_cast(ca),reinterpret_cast(cb),skip); +void linear_op::batch( + int ld, + int dim, + int nvec, + T *out, + const T *x, + const T *y, + T a, + T b, + const T *ca, + const T *cb, + const int *skip) const +{ + if (dim <= 0 || nvec <= 0) + { + return; + } + using C = thrust::complex; + batch_kernel<<>>( + ld, + dim, + reinterpret_cast(out), + reinterpret_cast(x), + reinterpret_cast(y), + C(a.real(), a.imag()), + C(b.real(), b.imag()), + reinterpret_cast(ca), + reinterpret_cast(cb), + skip); check_launch(); } template -void linear_op::gmres_update(int ld, - int dim, - int nvec, - T* solution, - T* residual, - const T* direction, - const T* image, - const T* coefficients) const +void linear_op::gmres_update( + int ld, + int dim, + int nvec, + T *solution, + T *residual, + const T *direction, + const T *image, + const T *coefficients) const { if (dim <= 0 || nvec <= 0) { @@ -168,101 +256,212 @@ void linear_op::gmres_update(int ld, } using C = thrust::complex; const dim3 grid((dim + linear_threads - 1) / linear_threads, nvec); - C* current_solution = reinterpret_cast(solution); - C* current_residual = reinterpret_cast(residual); - const C* current_direction = reinterpret_cast(direction); - const C* current_image = reinterpret_cast(image); - const C* coeff = reinterpret_cast(coefficients); - gmres_update_kernel<<>>(ld, dim, current_solution, current_residual, current_direction, current_image, coeff); + C *current_solution = reinterpret_cast(solution); + C *current_residual = reinterpret_cast(residual); + const C *current_direction = reinterpret_cast(direction); + const C *current_image = reinterpret_cast(image); + const C *coeff = reinterpret_cast(coefficients); + gmres_update_kernel<<>>( + ld, + dim, + current_solution, + current_residual, + current_direction, + current_image, + coeff); check_launch(); } template -void linear_op::dots(int ld,int dim,int nvec,const T* x,const T* y,const T* z,const T* w, - T* out,T* partial,int tiles) const -{ - if(nvec<=0)return; - using C=thrust::complex; - const int pairs=z ? 2 : 1; - C* target=reinterpret_cast(tiles==1 ? out : partial); - dots_kernel<<>>(ld,dim,nvec,tiles,reinterpret_cast(x), - reinterpret_cast(y),reinterpret_cast(z),reinterpret_cast(w),target); +void linear_op::dots( + int ld, + int dim, + int nvec, + const T *x, + const T *y, + const T *z, + const T *w, + T *out, + T *partial, + int tiles) const +{ + if (nvec <= 0) + { + return; + } + using C = thrust::complex; + const int pairs = z ? 2 : 1; + C *target = reinterpret_cast(tiles == 1 ? out : partial); + dots_kernel<<>>( + ld, + dim, + nvec, + tiles, + reinterpret_cast(x), + reinterpret_cast(y), + reinterpret_cast(z), + reinterpret_cast(w), + target); check_launch(); - if(tiles>1) + if (tiles > 1) { - sum_tiles<<<(pairs*nvec+255)/256,256>>>(pairs*nvec,tiles,target,reinterpret_cast(out)); + sum_tiles<<<(pairs * nvec + 255) / 256, 256>>>( + pairs * nvec, + tiles, + target, + reinterpret_cast(out)); check_launch(); } } template -void linear_op::gather(int ld,int dim,int nvec,int slots,int stride,const T* in,T* out,const int* map) const +void linear_op::gather( + int ld, + int dim, + int nvec, + int slots, + int stride, + const T *in, + T *out, + const int *map) const { - if(dim<=0 || nvec<=0)return; - using C=thrust::complex; - gather_kernel<<>>(ld,dim,stride,reinterpret_cast(in),reinterpret_cast(out),map); + if (dim <= 0 || nvec <= 0) + { + return; + } + using C = thrust::complex; + gather_kernel<<>>( + ld, + dim, + stride, + reinterpret_cast(in), + reinterpret_cast(out), + map); check_launch(); } - template -__global__ void swaps_kernel(int ld,int dim,int stride,int count,C* vectors,const int* pairs) +__global__ void swaps_kernel( + int ld, + int dim, + int stride, + int count, + C *vectors, + const int *pairs) { - const int i=blockIdx.x*blockDim.x+threadIdx.x; + const int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= dim) { return; } const std::int64_t offset = static_cast(blockIdx.y) * stride + i; - for(int k=0;k(pairs[2 * k]) * ld; const std::int64_t second = offset + static_cast(pairs[2 * k + 1]) * ld; - C& a = vectors[first]; - C& b = vectors[second]; - const C value=a; a=b; b=value; + C &a = vectors[first]; + C &b = vectors[second]; + const C value = a; + a = b; + b = value; } } template -void linear_op::swaps(int ld,int dim,int slots,int stride,int count,T* vectors,const int* pairs) const -{ - if(dim==0 || count==0)return; - using C=thrust::complex; - swaps_kernel<<>>(ld,dim,stride,count,reinterpret_cast(vectors),pairs); +void linear_op::swaps( + int ld, + int dim, + int slots, + int stride, + int count, + T *vectors, + const int *pairs) const +{ + if (dim == 0 || count == 0) + { + return; + } + using C = thrust::complex; + swaps_kernel<<>>( + ld, + dim, + stride, + count, + reinterpret_cast(vectors), + pairs); check_launch(); } - template -__global__ void bicg_update_kernel(int ld,int dim,bool direction,C* out,const C* x,const C* y,const C* a,const C* b) +__global__ void bicg_update_kernel( + int ld, + int dim, + bool direction, + C *out, + const C *x, + const C *y, + const C *a, + const C *b) { - const int i=blockIdx.x*blockDim.x+threadIdx.x; - const int band=blockIdx.y; - if(i -void linear_op::bicg_update(int ld,int dim,int nvec,bool direction,T* out,const T* x,const T* y,const T* a,const T* b) const +void linear_op::bicg_update( + int ld, + int dim, + int nvec, + bool direction, + T *out, + const T *x, + const T *y, + const T *a, + const T *b) const { - if(dim==0 || nvec==0)return; - using C=thrust::complex; - bicg_update_kernel<<>>(ld,dim,direction,reinterpret_cast(out), - reinterpret_cast(x),reinterpret_cast(y),reinterpret_cast(a),reinterpret_cast(b)); + if (dim == 0 || nvec == 0) + { + return; + } + using C = thrust::complex; + bicg_update_kernel<<>>( + ld, + dim, + direction, + reinterpret_cast(out), + reinterpret_cast(x), + reinterpret_cast(y), + reinterpret_cast(a), + reinterpret_cast(b)); check_launch(); } - template -__global__ void diagonal_kernel(int ld, int dim, const C* inverse, const C* x, C* y) +__global__ void diagonal_kernel( + int ld, + int dim, + const C *inverse, + const C *x, + C *y) { const int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i < dim) y[blockIdx.y * ld + i] = inverse[i] * x[blockIdx.y * ld + i]; + if (i < dim) + { + y[blockIdx.y * ld + i] = inverse[i] * x[blockIdx.y * ld + i]; + } } template -__global__ void cgs_direction_kernel(int ld, int dim, C* p, C* u, const C* r, const C* q, const C* beta) +__global__ void cgs_direction_kernel( + int ld, + int dim, + C *p, + C *u, + const C *r, + const C *q, + const C *beta) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int band = blockIdx.y; @@ -275,7 +474,14 @@ __global__ void cgs_direction_kernel(int ld, int dim, C* p, C* u, const C* r, co } template -__global__ void cgs_alpha_kernel(int ld, int dim, C* q, C* direction, const C* u, const C* v, const C* alpha) +__global__ void cgs_alpha_kernel( + int ld, + int dim, + C *q, + C *direction, + const C *u, + const C *v, + const C *alpha) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int band = blockIdx.y; @@ -288,7 +494,14 @@ __global__ void cgs_alpha_kernel(int ld, int dim, C* q, C* direction, const C* u } template -__global__ void cgs_finish_kernel(int ld, int dim, C* x, C* r, const C* direction, const C* ad, const C* alpha) +__global__ void cgs_finish_kernel( + int ld, + int dim, + C *x, + C *r, + const C *direction, + const C *ad, + const C *alpha) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int band = blockIdx.y; @@ -301,64 +514,119 @@ __global__ void cgs_finish_kernel(int ld, int dim, C* x, C* r, const C* directio } template -void linear_op::diagonal(int ld, int dim, int nvec, const T* inverse, const T* x, T* y) const +void linear_op::diagonal( + int ld, + int dim, + int nvec, + const T *inverse, + const T *x, + T *y) const { - if (dim == 0 || nvec == 0) return; + if (dim == 0 || nvec == 0) + { + return; + } using C = thrust::complex; - diagonal_kernel<<>>(ld, dim, - reinterpret_cast(inverse), - reinterpret_cast(x), - reinterpret_cast(y)); + diagonal_kernel<<>>( + ld, + dim, + reinterpret_cast(inverse), + reinterpret_cast(x), + reinterpret_cast(y)); check_launch(); } template -void linear_op::cgs_direction(int ld, int dim, int nvec, T* p, T* u, const T* r, const T* q, const T* beta) const +void linear_op::cgs_direction( + int ld, + int dim, + int nvec, + T *p, + T *u, + const T *r, + const T *q, + const T *beta) const { - if (dim == 0 || nvec == 0) return; + if (dim == 0 || nvec == 0) + { + return; + } using C = thrust::complex; - cgs_direction_kernel<<>>(ld, dim, - reinterpret_cast(p), - reinterpret_cast(u), - reinterpret_cast(r), - reinterpret_cast(q), - reinterpret_cast(beta)); + cgs_direction_kernel<<>>( + ld, + dim, + reinterpret_cast(p), + reinterpret_cast(u), + reinterpret_cast(r), + reinterpret_cast(q), + reinterpret_cast(beta)); check_launch(); } template -void linear_op::cgs_alpha(int ld, int dim, int nvec, T* q, T* direction, const T* u, const T* v, const T* alpha) const +void linear_op::cgs_alpha( + int ld, + int dim, + int nvec, + T *q, + T *direction, + const T *u, + const T *v, + const T *alpha) const { - if (dim == 0 || nvec == 0) return; + if (dim == 0 || nvec == 0) + { + return; + } using C = thrust::complex; - cgs_alpha_kernel<<>>(ld, dim, - reinterpret_cast(q), - reinterpret_cast(direction), - reinterpret_cast(u), - reinterpret_cast(v), - reinterpret_cast(alpha)); + cgs_alpha_kernel<<>>( + ld, + dim, + reinterpret_cast(q), + reinterpret_cast(direction), + reinterpret_cast(u), + reinterpret_cast(v), + reinterpret_cast(alpha)); check_launch(); } template -void linear_op::cgs_finish(int ld, int dim, int nvec, T* x, T* r, const T* direction, const T* ad, const T* alpha) const +void linear_op::cgs_finish( + int ld, + int dim, + int nvec, + T *x, + T *r, + const T *direction, + const T *ad, + const T *alpha) const { - if (dim == 0 || nvec == 0) return; + if (dim == 0 || nvec == 0) + { + return; + } using C = thrust::complex; - cgs_finish_kernel<<>>(ld, dim, - reinterpret_cast(x), - reinterpret_cast(r), - reinterpret_cast(direction), - reinterpret_cast(ad), - reinterpret_cast(alpha)); + cgs_finish_kernel<<>>( + ld, + dim, + reinterpret_cast(x), + reinterpret_cast(r), + reinterpret_cast(direction), + reinterpret_cast(ad), + reinterpret_cast(alpha)); check_launch(); } namespace { template -__global__ void wide_dot_kernel(int ld, int dim, int nvec, int stride, - const thrust::complex* basis, const thrust::complex* x, - thrust::complex* out) +__global__ void wide_dot_kernel( + int ld, + int dim, + int nvec, + int stride, + const thrust::complex *basis, + const thrust::complex *x, + thrust::complex *out) { __shared__ Accumulator re[linear_threads]; __shared__ Accumulator im[linear_threads]; @@ -370,48 +638,93 @@ __global__ void wide_dot_kernel(int ld, int dim, int nvec, int stride, thrust::complex sum(0, 0); for (int i = tid; i < dim; i += blockDim.x) { - sum += thrust::conj(thrust::complex(basis[basis_offset + i])) - * thrust::complex(x[band_offset + i]); + sum += thrust::conj(thrust::complex(basis[basis_offset + i])) * + thrust::complex(x[band_offset + i]); } re[tid] = sum.real(); im[tid] = sum.imag(); __syncthreads(); for (int s = blockDim.x / 2; s > 0; s /= 2) { - if (tid < s) { re[tid] += re[tid + s]; im[tid] += im[tid + s]; } + if (tid < s) + { + re[tid] += re[tid + s]; + im[tid] += im[tid + s]; + } __syncthreads(); } - if (tid == 0) out[blockIdx.x] = thrust::complex(re[0], im[0]); -} + if (tid == 0) + { + out[blockIdx.x] = thrust::complex(re[0], im[0]); + } } +} // namespace template -void linear_op::wide_dots(int ld, int dim, int nvec, int count, int stride, - const T* basis, const T* x, std::complex* out) const -{ - if (nvec * count == 0) return; +void linear_op::wide_dots( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *x, + std::complex *out) const +{ + if (nvec * count == 0) + { + return; + } using C = thrust::complex; - wide_dot_kernel<<>>(ld, dim, nvec, stride, - reinterpret_cast(basis), reinterpret_cast(x), - reinterpret_cast*>(out)); + wide_dot_kernel<<>>( + ld, + dim, + nvec, + stride, + reinterpret_cast(basis), + reinterpret_cast(x), + reinterpret_cast *>(out)); check_launch(); } template -void linear_op::native_dots(int ld, int dim, int nvec, int count, int stride, - const T* basis, const T* x, T* out) const -{ - if (nvec * count == 0) return; +void linear_op::native_dots( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *x, + T *out) const +{ + if (nvec * count == 0) + { + return; + } using C = thrust::complex; - wide_dot_kernel<<>>(ld, dim, nvec, stride, - reinterpret_cast(basis), reinterpret_cast(x), reinterpret_cast(out)); + wide_dot_kernel<<>>( + ld, + dim, + nvec, + stride, + reinterpret_cast(basis), + reinterpret_cast(x), + reinterpret_cast(out)); check_launch(); } namespace { template -__global__ void combine_kernel(int ld, int dim, int count, int stride, int nvec, - const C* basis, const C* coefficients, C* output) +__global__ void combine_kernel( + int ld, + int dim, + int count, + int stride, + int nvec, + const C *basis, + const C *coefficients, + C *output) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int b = blockIdx.y; @@ -428,7 +741,12 @@ __global__ void combine_kernel(int ld, int dim, int count, int stride, int nvec, } template -__global__ void scatter_kernel(int ld, int dim, const C* input, C* output, const int* map) +__global__ void scatter_kernel( + int ld, + int dim, + const C *input, + C *output, + const int *map) { const int i = blockIdx.x * blockDim.x + threadIdx.x; const int b = blockIdx.y; @@ -439,8 +757,14 @@ __global__ void scatter_kernel(int ld, int dim, const C* input, C* output, const } template -__global__ void block_gram_kernel(int ld, int dim, int nvec, int count, int stride, - const thrust::complex* basis, thrust::complex* output) +__global__ void block_gram_kernel( + int ld, + int dim, + int nvec, + int count, + int stride, + const thrust::complex *basis, + thrust::complex *output) { const int b = blockIdx.x % nvec; const int pair = blockIdx.x / nvec; @@ -484,8 +808,15 @@ __global__ void block_gram_kernel(int ld, int dim, int nvec, int count, int stri } // namespace template -void linear_op::combine(int ld, int dim, int nvec, int count, int stride, - const T* basis, const T* coefficients, T* output) const +void linear_op::combine( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + const T *coefficients, + T *output) const { if (dim == 0 || nvec == 0) { @@ -493,14 +824,26 @@ void linear_op::combine(int ld, int dim, int nvec, i } using C = thrust::complex; const dim3 grid((dim + linear_threads - 1) / linear_threads, nvec); - combine_kernel<<>>(ld, dim, count, stride, nvec, - reinterpret_cast(basis), reinterpret_cast(coefficients), reinterpret_cast(output)); + combine_kernel<<>>( + ld, + dim, + count, + stride, + nvec, + reinterpret_cast(basis), + reinterpret_cast(coefficients), + reinterpret_cast(output)); check_launch(); } template -void linear_op::scatter(int ld, int dim, int nvec, - const T* input, T* output, const int* map) const +void linear_op::scatter( + int ld, + int dim, + int nvec, + const T *input, + T *output, + const int *map) const { if (dim == 0 || nvec == 0) { @@ -508,13 +851,24 @@ void linear_op::scatter(int ld, int dim, int nvec, } using C = thrust::complex; const dim3 grid((dim + linear_threads - 1) / linear_threads, nvec); - scatter_kernel<<>>(ld, dim, reinterpret_cast(input), reinterpret_cast(output), map); + scatter_kernel<<>>( + ld, + dim, + reinterpret_cast(input), + reinterpret_cast(output), + map); check_launch(); } template -void linear_op::block_gram(int ld, int dim, int nvec, int count, int stride, - const T* basis, std::complex* output) const +void linear_op::block_gram( + int ld, + int dim, + int nvec, + int count, + int stride, + const T *basis, + std::complex *output) const { if (nvec == 0 || count == 0) { @@ -522,8 +876,14 @@ void linear_op::block_gram(int ld, int dim, int nvec } using C = thrust::complex; const int blocks = nvec * count * count; - block_gram_kernel<<>>(ld, dim, nvec, count, stride, reinterpret_cast(basis), - reinterpret_cast*>(output)); + block_gram_kernel<<>>( + ld, + dim, + nvec, + count, + stride, + reinterpret_cast(basis), + reinterpret_cast *>(output)); check_launch(); } diff --git a/source/source_hsolver/module_linalg/linear_algebra.cpp b/source/source_hsolver/module_linalg/linear_algebra.cpp index 5f35a19a2be..196f0fdaeac 100644 --- a/source/source_hsolver/module_linalg/linear_algebra.cpp +++ b/source/source_hsolver/module_linalg/linear_algebra.cpp @@ -11,7 +11,11 @@ namespace hsolver { -std::vector> linear_gram_basis(const std::vector>& gram, int n, double cutoff, int* rank) +std::vector> linear_gram_basis( + const std::vector> &gram, + int n, + double cutoff, + int *rank) { using Wide = std::complex; *rank = 0; @@ -89,7 +93,7 @@ std::vector> linear_gram_basis(const std::vector>& matrix, int n) +bool LinearSmallLU::factor(const std::vector> &matrix, int n) { size_ = 0; if (n <= 0 || matrix.size() != static_cast(n) * n) @@ -99,7 +103,7 @@ bool LinearSmallLU::factor(const std::vector>& matrix, int lu_ = matrix; pivots_.resize(n); double scale = 0.0; - for (const std::complex& value: matrix) + for (const std::complex &value : matrix) { const double magnitude = std::abs(value); if (!std::isfinite(magnitude)) @@ -141,7 +145,7 @@ bool LinearSmallLU::factor(const std::vector>& matrix, int return true; } -bool LinearSmallLU::solve(std::vector>* rhs, int columns) const +bool LinearSmallLU::solve(std::vector> *rhs, int columns) const { if (size_ == 0 || rhs->size() != static_cast(size_) * columns) { @@ -149,7 +153,7 @@ bool LinearSmallLU::solve(std::vector>* rhs, int columns) c } for (int j = 0; j < columns; ++j) { - std::complex* b = rhs->data() + j * size_; + std::complex *b = rhs->data() + j * size_; for (int k = 0; k < size_; ++k) { std::swap(b[k], b[pivots_[k]]); @@ -176,8 +180,7 @@ bool LinearSmallLU::solve(std::vector>* rhs, int columns) c } return true; } -template -void LinearAlgebra::reduce(std::vector* values) const +template void LinearAlgebra::reduce(std::vector *values) const { #ifdef __MPI if (comm_.nproc > 1) @@ -188,11 +191,23 @@ void LinearAlgebra::reduce(std::vector* values) const } template -std::vector> LinearAlgebra::projection_cross(int ld, int dim, int nx, int ny, const T* x, const T* y) +std::vector> LinearAlgebra::projection_cross( + int ld, + int dim, + int nx, + int ny, + const T *x, + const T *y) { if (std::is_same::value) { - return cross(ld, dim, nx, ny, x, y); + return cross( + ld, + dim, + nx, + ny, + x, + y); } const int64_t elements = static_cast(nx) * ny; std::vector native(elements, T(0)); @@ -201,9 +216,23 @@ std::vector> LinearAlgebra::projection_cross(int linear_buffer(&native_products_, elements); const T one(1); const T zero(0); - T* products = native_products_.template data(); - ModuleBase::gemm_op()('C', 'N', nx, ny, dim, &one, x, ld, y, ld, &zero, products, nx); - base_device::memory::synchronize_memory_op()(native.data(), products, native.size()); + T *products = native_products_.template data(); + ModuleBase::gemm_op()( + 'C', + 'N', + nx, + ny, + dim, + &one, + x, + ld, + y, + ld, + &zero, + products, + nx); + base_device::memory::synchronize_memory_op()( + native.data(), products, native.size()); } #ifdef __MPI if (comm_.nproc > 1) @@ -215,22 +244,38 @@ std::vector> LinearAlgebra::projection_cross(int } template -std::vector> LinearAlgebra::arnoldi_dots(int ld, - int dim, - int bands, - int count, - int stride, - const T* basis, - const T* x) +std::vector> LinearAlgebra::arnoldi_dots( + int ld, + int dim, + int bands, + int count, + int stride, + const T *basis, + const T *x) { if (std::is_same::value) { - return dots(ld, dim, bands, count, stride, basis, x); + return dots( + ld, + dim, + bands, + count, + stride, + basis, + x); } const int size = bands * count; linear_buffer(&native_products_, size); - T* products = native_products_.template data(); - linear_op().native_dots(ld, dim, bands, count, stride, basis, x, products); + T *products = native_products_.template data(); + linear_op().native_dots( + ld, + dim, + bands, + count, + stride, + basis, + x, + products); std::vector native(size); base_device::memory::synchronize_memory_op()(native.data(), products, size); #ifdef __MPI @@ -243,34 +288,48 @@ std::vector> LinearAlgebra::arnoldi_dots(int ld, } template -std::vector> LinearAlgebra::dots(int ld, - int dim, - int bands, - int count, - int stride, - const T* basis, - const T* x) +std::vector> LinearAlgebra::dots( + int ld, + int dim, + int bands, + int count, + int stride, + const T *basis, + const T *x) { const int size = bands * count; linear_buffer(&products_, size); - linear_op().wide_dots(ld, dim, bands, count, stride, basis, x, products_.template data()); + linear_op().wide_dots( + ld, + dim, + bands, + count, + stride, + basis, + x, + products_.template data()); std::vector result(size); - base_device::memory::synchronize_memory_op()(result.data(), - products_.template data(), - size); + base_device::memory::synchronize_memory_op()( + result.data(), products_.template data(), size); reduce(&result); return result; } template -std::vector> LinearAlgebra::cross(int ld, int dim, int nx, int ny, const T* x, const T* y) +std::vector> LinearAlgebra::cross( + int ld, + int dim, + int nx, + int ny, + const T *x, + const T *y) { const int64_t elements = static_cast(nx) * ny; std::vector result(elements, Wide(0)); if (dim > 0) { - const Wide* a = reinterpret_cast(x); - const Wide* b = reinterpret_cast(y); + const Wide *a = reinterpret_cast(x); + const Wide *b = reinterpret_cast(y); if (!std::is_same::value) { const int64_t left_elements = static_cast(ld) * nx; @@ -281,15 +340,15 @@ std::vector> LinearAlgebra::cross(int ld, int di for (int j = 0; j < nx; ++j) { const int64_t offset = static_cast(j) * ld; - Wide* destination = left_.template data() + offset; - const T* source = x + offset; + Wide *destination = left_.template data() + offset; + const T *source = x + offset; base_device::memory::cast_memory_op()(destination, source, dim); } for (int j = 0; j < ny; ++j) { const int64_t offset = static_cast(j) * ld; - Wide* destination = right_.template data() + offset; - const T* source = y + offset; + Wide *destination = right_.template data() + offset; + const T *source = y + offset; base_device::memory::cast_memory_op()(destination, source, dim); } a = left_.template data(); @@ -298,29 +357,45 @@ std::vector> LinearAlgebra::cross(int ld, int di linear_buffer(&products_, elements); const Wide one(1); const Wide zero(0); - ModuleBase::gemm_op()('C', 'N', nx, ny, dim, &one, a, ld, b, ld, &zero, products_.template data(), nx); - base_device::memory::synchronize_memory_op()(result.data(), - products_.template data(), - result.size()); + ModuleBase::gemm_op()( + 'C', + 'N', + nx, + ny, + dim, + &one, + a, + ld, + b, + ld, + &zero, + products_.template data(), + nx); + base_device::memory::synchronize_memory_op()( + result.data(), products_.template data(), result.size()); } reduce(&result); return result; } template -std::vector> LinearAlgebra::gram(int ld, int dim, int bands, const T* input) +std::vector> LinearAlgebra::gram( + int ld, + int dim, + int bands, + const T *input) { ModuleBase::timer::start("LinearAlgebra", "gram"); const int64_t elements = static_cast(bands) * bands; std::vector result(elements, Wide(0)); if (dim > 0 && bands > 0) { - const Wide* wide = reinterpret_cast(input); + const Wide *wide = reinterpret_cast(input); if (!std::is_same::value) { const int64_t input_elements = static_cast(ld) * bands; linear_buffer(&left_, input_elements); - Wide* converted = left_.template data(); + Wide *converted = left_.template data(); // The existing conversion kernels use int indices, even though their wrappers accept size_t. if (ld == dim && input_elements <= std::numeric_limits::max()) { @@ -332,8 +407,8 @@ std::vector> LinearAlgebra::gram(int ld, int dim for (int band = 0; band < bands; ++band) { const int64_t offset = static_cast(band) * ld; - Wide* destination = converted + offset; - const T* source = input + offset; + Wide *destination = converted + offset; + const T *source = input + offset; base_device::memory::cast_memory_op()(destination, source, dim); } } @@ -342,9 +417,23 @@ std::vector> LinearAlgebra::gram(int ld, int dim linear_buffer(&products_, elements); const Wide one(1); const Wide zero(0); - Wide* products = products_.template data(); - ModuleBase::gemm_op()('C', 'N', bands, bands, dim, &one, wide, ld, wide, ld, &zero, products, bands); - base_device::memory::synchronize_memory_op()(result.data(), products, elements); + Wide *products = products_.template data(); + ModuleBase::gemm_op()( + 'C', + 'N', + bands, + bands, + dim, + &one, + wide, + ld, + wide, + ld, + &zero, + products, + bands); + base_device::memory::synchronize_memory_op()( + result.data(), products, elements); } reduce(&result); ModuleBase::timer::end("LinearAlgebra", "gram"); @@ -352,7 +441,15 @@ std::vector> LinearAlgebra::gram(int ld, int dim } template -void LinearAlgebra::expand(int ld, int dim, int nx, int ny, const T* x, const std::vector& c, T* y, T beta) +void LinearAlgebra::expand( + int ld, + int dim, + int nx, + int ny, + const T *x, + const std::vector &c, + T *y, + T beta) { if (dim == 0 || nx == 0 || ny == 0) { @@ -360,10 +457,24 @@ void LinearAlgebra::expand(int ld, int dim, int nx, int ny, const T* } std::vector narrow(c.begin(), c.end()); linear_buffer(&coefficients_, narrow.size()); - T* coeff = coefficients_.template data(); - base_device::memory::synchronize_memory_op()(coeff, narrow.data(), narrow.size()); + T *coeff = coefficients_.template data(); + base_device::memory::synchronize_memory_op()( + coeff, narrow.data(), narrow.size()); const T one(1); - ModuleBase::gemm_op()('N', 'N', dim, ny, nx, &one, x, ld, coeff, nx, &beta, y, ld); + ModuleBase::gemm_op()( + 'N', + 'N', + dim, + ny, + nx, + &one, + x, + ld, + coeff, + nx, + &beta, + y, + ld); } template class LinearAlgebra, base_device::DEVICE_CPU>; diff --git a/source/source_hsolver/module_linalg/linear_algebra.h b/source/source_hsolver/module_linalg/linear_algebra.h index 9d6620d91b7..042236adcbb 100644 --- a/source/source_hsolver/module_linalg/linear_algebra.h +++ b/source/source_hsolver/module_linalg/linear_algebra.h @@ -14,13 +14,13 @@ namespace hsolver { /** @brief Device allocation helper, including type and empty-partition checks. */ -template -void linear_buffer(ct::Tensor* buffer, const int64_t size) +template void linear_buffer(ct::Tensor *buffer, const int64_t size) { using CtDevice = typename ct::PsiToContainer::type; const ct::DeviceType device = ct::DeviceTypeToEnum::value; const int64_t elements = std::max(1, size); - if (buffer->NumElements() < elements || buffer->data_type() != ct::DataTypeToEnum::value || buffer->device_type() != device) + if (buffer->NumElements() < elements || buffer->data_type() != ct::DataTypeToEnum::value || + buffer->device_type() != device) { *buffer = ct::Tensor(ct::DataTypeToEnum::value, device, {elements}); } @@ -35,16 +35,19 @@ class LinearSmallLU std::vector pivots_; public: - bool factor(const std::vector>& matrix, int n); - bool solve(std::vector>* rhs, int columns) const; + bool factor(const std::vector> &matrix, int n); + bool solve(std::vector> *rhs, int columns) const; }; /** @brief Rank-revealing coefficients for a small Gram matrix, with a norm cutoff. */ -std::vector> linear_gram_basis(const std::vector>& gram, int n, double cutoff, int* rank); +std::vector> linear_gram_basis( + const std::vector> &gram, + int n, + double cutoff, + int *rank); /** @brief Collective block algebra; only small coefficients cross the device boundary. */ -template -class LinearAlgebra +template class LinearAlgebra { private: using Wide = std::complex; @@ -55,30 +58,66 @@ class LinearAlgebra ct::Tensor right_; ct::Tensor native_products_; - void reduce(std::vector* values) const; + void reduce(std::vector *values) const; public: - explicit LinearAlgebra(const diag_comm_info& comm) : comm_(comm) - { - } + explicit LinearAlgebra(const diag_comm_info &comm) : comm_(comm) {} /** @brief Projection products and reductions in wavefunction precision; return double coefficients. */ - std::vector projection_cross(int ld, int dim, int nx, int ny, const T* x, const T* y); + std::vector projection_cross( + int ld, + int dim, + int nx, + int ny, + const T *x, + const T *y); /** @brief Arnoldi products in wavefunction precision; use dots() for reliable norms. */ - std::vector arnoldi_dots(int ld, int dim, int bands, int count, int stride, const T* basis, const T* x); + std::vector arnoldi_dots( + int ld, + int dim, + int bands, + int count, + int stride, + const T *basis, + const T *x); /** @brief Products between corresponding bands and several Arnoldi blocks. */ - std::vector dots(int ld, int dim, int bands, int count, int stride, const T* basis, const T* x); + std::vector dots( + int ld, + int dim, + int bands, + int count, + int stride, + const T *basis, + const T *x); /** @brief Global X-adjoint times Y, with double accumulation for both precisions. */ - std::vector cross(int ld, int dim, int nx, int ny, const T* x, const T* y); + std::vector cross( + int ld, + int dim, + int nx, + int ny, + const T *x, + const T *y); /** @brief Global X-adjoint times X, reusing one FP64 conversion of the valid rows. */ - std::vector gram(int ld, int dim, int bands, const T* input); + std::vector gram( + int ld, + int dim, + int bands, + const T *input); /** @brief Y = X*C + beta*Y, using a small host coefficient matrix. */ - void expand(int ld, int dim, int nx, int ny, const T* x, const std::vector& c, T* y, T beta); + void expand( + int ld, + int dim, + int nx, + int ny, + const T *x, + const std::vector &c, + T *y, + T beta); }; } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/linear_bicgstab.cpp b/source/source_hsolver/module_linalg/linear_bicgstab.cpp index 5ba240e92f0..af557ab1f09 100644 --- a/source/source_hsolver/module_linalg/linear_bicgstab.cpp +++ b/source/source_hsolver/module_linalg/linear_bicgstab.cpp @@ -7,12 +7,13 @@ namespace hsolver { template -LinearBiCGSTAB::LinearBiCGSTAB(const double tolerance, const diag_comm_info& comm) : tolerance_(tolerance), work_(comm, 9) +LinearBiCGSTAB::LinearBiCGSTAB(const double tolerance, const diag_comm_info &comm) + : tolerance_(tolerance), work_(comm, 9) { } template -bool LinearBiCGSTAB::breakdown(const T divisor, const int band, LinearSolveResult* result) const +bool LinearBiCGSTAB::breakdown(const T divisor, const int band, LinearSolveResult *result) const { if (!linear_bad_divisor(divisor)) { @@ -23,8 +24,7 @@ bool LinearBiCGSTAB::breakdown(const T divisor, const int band, Linea return true; } -template -void LinearBiCGSTAB::retire(const int band) +template void LinearBiCGSTAB::retire(const int band) { const int last = active_ - 1; if (band != last) @@ -44,14 +44,29 @@ void LinearBiCGSTAB::retire(const int band) template void LinearBiCGSTAB::retire_converged(const bool alpha_step, const bool identity) { - const T* residual = work_.data(alpha_step ? intermediate_slot : residual_slot); + const T *residual = work_.data(alpha_step ? intermediate_slot : residual_slot); if (alpha_step) { - work_.dot(ld_, dim_, active_, residual, residual, norm_.data()); + work_.dot( + ld_, + dim_, + active_, + residual, + residual, + norm_.data()); } else { - work_.dot_pair(ld_, dim_, active_, residual, residual, work_.data(shadow_slot), residual, norm_.data(), rho_.data()); + work_.dot_pair( + ld_, + dim_, + active_, + residual, + residual, + work_.data(shadow_slot), + residual, + norm_.data(), + rho_.data()); } swaps_.clear(); skip_.assign(active_, 1); @@ -72,17 +87,18 @@ void LinearBiCGSTAB::retire_converged(const bool alpha_step, const bo if (alpha_step) { const int update_slot = identity ? direction_slot : precond_direction_slot; - work_.batch(ld_, - dim_, - active_, - work_.data(solution_slot), - work_.data(solution_slot), - work_.data(update_slot), - T(1), - T(1), - nullptr, - alpha_.data(), - skip_.data()); + work_.batch( + ld_, + dim_, + active_, + work_.data(solution_slot), + work_.data(solution_slot), + work_.data(update_slot), + T(1), + T(1), + nullptr, + alpha_.data(), + skip_.data()); } for (int band = active_ - 1; band >= 0; --band) { @@ -96,36 +112,60 @@ void LinearBiCGSTAB::retire_converged(const bool alpha_step, const bo } template -bool LinearBiCGSTAB::iterate(const LinearOperator& op, - const LinearOperator& preconditioner, - LinearSolveResult* result) +bool LinearBiCGSTAB::iterate( + const LinearOperator &op, const LinearOperator &preconditioner, LinearSolveResult *result) { - T* r = work_.data(residual_slot); + T *r = work_.data(residual_slot); const bool identity = preconditioner.is_identity(); - T* shadow = work_.data(shadow_slot); - T* p = work_.data(direction_slot); - T* v = work_.data(image_slot); - T* s = work_.data(intermediate_slot); - T* y = work_.data(identity ? direction_slot : precond_direction_slot); - T* z = work_.data(identity ? intermediate_slot : precond_intermediate_slot); - T* t = work_.data(intermediate_image_slot); - T* solution = work_.data(solution_slot); + T *shadow = work_.data(shadow_slot); + T *p = work_.data(direction_slot); + T *v = work_.data(image_slot); + T *s = work_.data(intermediate_slot); + T *y = work_.data(identity ? direction_slot : precond_direction_slot); + T *z = work_.data(identity ? intermediate_slot : precond_intermediate_slot); + T *t = work_.data(intermediate_image_slot); + T *solution = work_.data(solution_slot); for (int band = 0; band < active_; ++band) { - if (breakdown(rho_[band], band, result) || breakdown(rho_prev_[band], band, result) || breakdown(omega_[band], band, result)) + if (breakdown(rho_[band], band, result) || breakdown(rho_prev_[band], band, result) || + breakdown(omega_[band], band, result)) { return false; } beta_[band] = (rho_[band] / rho_prev_[band]) * (alpha_[band] / omega_[band]); } - work_.bicg_update(ld_, dim_, active_, true, p, r, v, beta_.data(), omega_.data()); + work_.bicg_update( + ld_, + dim_, + active_, + true, + p, + r, + v, + beta_.data(), + omega_.data()); if (!identity) { - preconditioner.apply(p, y, ld_, active_); + preconditioner.apply( + p, + y, + ld_, + active_); } - work_.apply(op, y, v, ld_, active_); - work_.dot(ld_, dim_, active_, shadow, v, denominator_.data()); + work_.apply( + op, + y, + v, + ld_, + active_); + work_.dot( + ld_, + dim_, + active_, + shadow, + v, + denominator_.data()); for (int band = 0; band < active_; ++band) { if (breakdown(denominator_[band], band, result)) @@ -134,7 +174,18 @@ bool LinearBiCGSTAB::iterate(const LinearOperator& op, } alpha_[band] = rho_[band] / denominator_[band]; } - work_.batch(ld_, dim_, active_, s, r, v, T(1), T(-1), nullptr, alpha_.data(), nullptr); + work_.batch( + ld_, + dim_, + active_, + s, + r, + v, + T(1), + T(-1), + nullptr, + alpha_.data(), + nullptr); // Retire exact alpha steps before evaluating omega = /. retire_converged(true, identity); if (active_ == 0) @@ -143,10 +194,28 @@ bool LinearBiCGSTAB::iterate(const LinearOperator& op, } if (!identity) { - preconditioner.apply(s, z, ld_, active_); + preconditioner.apply( + s, + z, + ld_, + active_); } - work_.apply(op, z, t, ld_, active_); - work_.dot_pair(ld_, dim_, active_, t, s, t, t, denominator_.data(), tt_.data()); + work_.apply( + op, + z, + t, + ld_, + active_); + work_.dot_pair( + ld_, + dim_, + active_, + t, + s, + t, + t, + denominator_.data(), + tt_.data()); for (int band = 0; band < active_; ++band) { if (breakdown(tt_[band], band, result)) @@ -161,25 +230,53 @@ bool LinearBiCGSTAB::iterate(const LinearOperator& op, rho_prev_[band] = rho_[band]; } - work_.bicg_update(ld_, dim_, active_, false, solution, y, z, alpha_.data(), omega_.data()); - work_.batch(ld_, dim_, active_, r, s, t, T(1), T(-1), nullptr, omega_.data(), nullptr); + work_.bicg_update( + ld_, + dim_, + active_, + false, + solution, + y, + z, + alpha_.data(), + omega_.data()); + work_.batch( + ld_, + dim_, + active_, + r, + s, + t, + T(1), + T(-1), + nullptr, + omega_.data(), + nullptr); retire_converged(false, identity); return true; } template -LinearSolveResult LinearBiCGSTAB::solve(const LinearOperator& op, - const LinearOperator& preconditioner, - int ld, - int nband, - int dim, - T* x, - const T* b, - const T* initial_residual, - const int max_iterations) +LinearSolveResult LinearBiCGSTAB::solve( + const LinearOperator &op, + const LinearOperator &preconditioner, + int ld, + int nband, + int dim, + T *x, + const T *b, + const T *initial_residual, + const int max_iterations) { const LinearSolveTimer timer("LinearBiCGSTAB"); - work_.prepare(ld, dim, nband, x, b, tolerance_, max_iterations); + work_.prepare( + ld, + dim, + nband, + x, + b, + tolerance_, + max_iterations); work_.reset_statistics(); ld_ = ld; dim_ = dim; @@ -195,7 +292,13 @@ LinearSolveResult LinearBiCGSTAB::solve(const LinearOperator::solve(const LinearOperator::solve(const LinearOperator= max_iterations || result.iterations == cycle_start - || !std::isfinite(result.max_residual)) + work_.verify( + op, + ld, + dim, + nband, + x, + b, + original_threshold, + work_.data(residual_slot), + &result); + if (result.status != LinearSolveStatus::residual_mismatch || result.iterations >= max_iterations || + result.iterations == cycle_start || !std::isfinite(result.max_residual)) { return result; } diff --git a/source/source_hsolver/module_linalg/linear_bicgstab.h b/source/source_hsolver/module_linalg/linear_bicgstab.h index 3131c7b10b5..17e1889a140 100644 --- a/source/source_hsolver/module_linalg/linear_bicgstab.h +++ b/source/source_hsolver/module_linalg/linear_bicgstab.h @@ -4,8 +4,7 @@ namespace hsolver { /** @brief Right-preconditioned BiCGSTAB for independent complex right-hand sides. */ -template -class LinearBiCGSTAB final +template class LinearBiCGSTAB final { private: enum WorkspaceSlot @@ -39,7 +38,7 @@ class LinearBiCGSTAB final std::vector skip_; public: - LinearBiCGSTAB(const double tolerance, const diag_comm_info& comm); + LinearBiCGSTAB(const double tolerance, const diag_comm_info &comm); /** * @brief Solve A*x=b, using x as initial guess and M as inverse preconditioner. * @param ld Column stride, at least dim. Padding is neither read nor overwritten. @@ -48,21 +47,24 @@ class LinearBiCGSTAB final * @param max_iterations Iteration budget for this call, shared by all restart cycles. * @note Accepts only true residuals below tolerance*max(1, norm(b)) per column. */ - LinearSolveResult solve(const LinearOperator& op, - const LinearOperator& preconditioner, - int ld, - int nband, - int dim, - T* x, - const T* b, - const T* initial_residual, - const int max_iterations); + LinearSolveResult solve( + const LinearOperator &op, + const LinearOperator &preconditioner, + int ld, + int nband, + int dim, + T *x, + const T *b, + const T *initial_residual, + const int max_iterations); private: - bool iterate(const LinearOperator& op, const LinearOperator& preconditioner, LinearSolveResult* result); + bool iterate( + const LinearOperator &op, const LinearOperator &preconditioner, + LinearSolveResult *result); void retire(const int band); void retire_converged(const bool alpha_step, const bool identity); - bool breakdown(const T divisor, const int band, LinearSolveResult* result) const; + bool breakdown(const T divisor, const int band, LinearSolveResult *result) const; }; } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/linear_cgs.cpp b/source/source_hsolver/module_linalg/linear_cgs.cpp index 1ec12b9047a..2ee15d6cb64 100644 --- a/source/source_hsolver/module_linalg/linear_cgs.cpp +++ b/source/source_hsolver/module_linalg/linear_cgs.cpp @@ -5,22 +5,23 @@ namespace hsolver { template -LinearCGS::LinearCGS(const double tolerance, const diag_comm_info& comm) : tolerance_(tolerance), work_(comm, 9) +LinearCGS::LinearCGS(const double tolerance, const diag_comm_info &comm) + : tolerance_(tolerance), work_(comm, 9) { } -template -void LinearCGS::retire_converged() +template void LinearCGS::retire_converged() { - work_.dot_pair(ld_, - dim_, - active_, - work_.data(residual_slot), - work_.data(residual_slot), - work_.data(shadow_slot), - work_.data(residual_slot), - norm_.data(), - rho_.data()); + work_.dot_pair( + ld_, + dim_, + active_, + work_.data(residual_slot), + work_.data(residual_slot), + work_.data(shadow_slot), + work_.data(residual_slot), + norm_.data(), + rho_.data()); swaps_.clear(); for (int band = active_ - 1; band >= 0; --band) { @@ -44,20 +45,21 @@ void LinearCGS::retire_converged() } template -bool LinearCGS::iterate(const LinearOperator& op, - const LinearOperator& preconditioner, - const bool first_iteration, - LinearSolveResult* result) +bool LinearCGS::iterate( + const LinearOperator &op, + const LinearOperator &preconditioner, + const bool first_iteration, + LinearSolveResult *result) { - T* r = work_.data(residual_slot); - T* shadow = work_.data(shadow_slot); - T* p = work_.data(direction_slot); - T* q = work_.data(q_slot); - T* u = work_.data(u_slot); - T* v = work_.data(image_slot); - T* scratch = work_.data(scratch_slot); - T* direction = work_.data(update_slot); - T* solution = work_.data(solution_slot); + T *r = work_.data(residual_slot); + T *shadow = work_.data(shadow_slot); + T *p = work_.data(direction_slot); + T *q = work_.data(q_slot); + T *u = work_.data(u_slot); + T *v = work_.data(image_slot); + T *scratch = work_.data(scratch_slot); + T *direction = work_.data(update_slot); + T *solution = work_.data(solution_slot); const bool identity = preconditioner.is_identity(); for (int band = 0; band < active_; ++band) { @@ -69,14 +71,37 @@ bool LinearCGS::iterate(const LinearOperator& op, } beta_[band] = first_iteration ? T(0) : rho_[band] / rho_prev_[band]; } - work_.cgs_direction(ld_, dim_, active_, p, u, r, q, beta_.data()); + work_.cgs_direction( + ld_, + dim_, + active_, + p, + u, + r, + q, + beta_.data()); if (!identity) { - preconditioner.apply(p, scratch, ld_, active_); + preconditioner.apply( + p, + scratch, + ld_, + active_); } - const T* preconditioned_direction = identity ? p : scratch; - work_.apply(op, preconditioned_direction, v, ld_, active_); - work_.dot(ld_, dim_, active_, shadow, v, denominator_.data()); + const T *preconditioned_direction = identity ? p : scratch; + work_.apply( + op, + preconditioned_direction, + v, + ld_, + active_); + work_.dot( + ld_, + dim_, + active_, + shadow, + v, + denominator_.data()); for (int band = 0; band < active_; ++band) { if (linear_bad_divisor(denominator_[band])) @@ -87,31 +112,63 @@ bool LinearCGS::iterate(const LinearOperator& op, } alpha_[band] = rho_[band] / denominator_[band]; } - work_.cgs_alpha(ld_, dim_, active_, q, direction, u, v, alpha_.data()); + work_.cgs_alpha( + ld_, + dim_, + active_, + q, + direction, + u, + v, + alpha_.data()); if (!identity) { - preconditioner.apply(direction, scratch, ld_, active_); + preconditioner.apply( + direction, + scratch, + ld_, + active_); } - const T* update = identity ? direction : scratch; - work_.apply(op, update, v, ld_, active_); - work_.cgs_finish(ld_, dim_, active_, solution, r, update, v); + const T *update = identity ? direction : scratch; + work_.apply( + op, + update, + v, + ld_, + active_); + work_.cgs_finish( + ld_, + dim_, + active_, + solution, + r, + update, + v); rho_prev_ = rho_; return true; } template -LinearSolveResult LinearCGS::solve(const LinearOperator& op, - const LinearOperator& preconditioner, - int ld, - int nband, - int dim, - T* x, - const T* b, - const T* initial_residual, - const int max_iterations) +LinearSolveResult LinearCGS::solve( + const LinearOperator &op, + const LinearOperator &preconditioner, + int ld, + int nband, + int dim, + T *x, + const T *b, + const T *initial_residual, + const int max_iterations) { const LinearSolveTimer timer("LinearCGS"); - work_.prepare(ld, dim, nband, x, b, tolerance_, max_iterations); + work_.prepare( + ld, + dim, + nband, + x, + b, + tolerance_, + max_iterations); work_.reset_statistics(); ld_ = ld; dim_ = dim; @@ -123,7 +180,13 @@ LinearSolveResult LinearCGS::solve(const LinearOperator& o norm_.resize(nband); threshold_.resize(nband); original_.resize(nband); - work_.dot(ld, dim, nband, b, b, norm_.data()); + work_.dot( + ld, + dim, + nband, + b, + b, + norm_.data()); for (int band = 0; band < nband; ++band) { const double rhs_norm = linear_norm(norm_[band]); @@ -138,16 +201,38 @@ LinearSolveResult LinearCGS::solve(const LinearOperator& o threshold_ = original_threshold; std::iota(original_.begin(), original_.end(), 0); rho_prev_.assign(nband, T(1)); - work_.copy(ld, dim, nband, x, work_.data(solution_slot)); + work_.copy( + ld, + dim, + nband, + x, + work_.data(solution_slot)); if (initial_residual && result.restarts == 0) { - work_.copy(ld, dim, nband, initial_residual, work_.data(residual_slot)); + work_.copy( + ld, + dim, + nband, + initial_residual, + work_.data(residual_slot)); } else { - work_.residual(op, ld, dim, nband, work_.data(solution_slot), b, work_.data(residual_slot)); + work_.residual( + op, + ld, + dim, + nband, + work_.data(solution_slot), + b, + work_.data(residual_slot)); } - work_.copy(ld, dim, nband, work_.data(residual_slot), work_.data(shadow_slot)); + work_.copy( + ld, + dim, + nband, + work_.data(residual_slot), + work_.data(shadow_slot)); retire_converged(); result.status = LinearSolveStatus::max_iterations; const int cycle_start = result.iterations; @@ -157,25 +242,44 @@ LinearSolveResult LinearCGS::solve(const LinearOperator& o { const bool first_iteration = result.iterations == cycle_start; ++result.iterations; - if (!iterate(op, preconditioner, first_iteration, &result)) + if (!iterate( + op, + preconditioner, + first_iteration, + &result)) { break; } retire_converged(); } } - catch (const LinearPreconditionerError&) + catch (const LinearPreconditionerError &) { result.status = LinearSolveStatus::preconditioner_failure; } - work_.restore(ld, dim, nband, original_, work_.data(solution_slot), x); + work_.restore( + ld, + dim, + nband, + original_, + work_.data(solution_slot), + x); if (active_ == 0) { result.status = LinearSolveStatus::residual_mismatch; } - work_.verify(op, ld, dim, nband, x, b, original_threshold, work_.data(residual_slot), &result); - if (result.status != LinearSolveStatus::residual_mismatch || result.iterations >= max_iterations || result.iterations == cycle_start - || !std::isfinite(result.max_residual)) + work_.verify( + op, + ld, + dim, + nband, + x, + b, + original_threshold, + work_.data(residual_slot), + &result); + if (result.status != LinearSolveStatus::residual_mismatch || result.iterations >= max_iterations || + result.iterations == cycle_start || !std::isfinite(result.max_residual)) { return result; } diff --git a/source/source_hsolver/module_linalg/linear_cgs.h b/source/source_hsolver/module_linalg/linear_cgs.h index dc2f96c92ba..1bc2d68c518 100644 --- a/source/source_hsolver/module_linalg/linear_cgs.h +++ b/source/source_hsolver/module_linalg/linear_cgs.h @@ -5,8 +5,7 @@ namespace hsolver { /** @brief Right-preconditioned CGS with a compact active set of columns. */ -template -class LinearCGS final +template class LinearCGS final { private: enum WorkspaceSlot @@ -37,28 +36,30 @@ class LinearCGS final std::vector swaps_; public: - LinearCGS(const double tolerance, const diag_comm_info& comm); + LinearCGS(const double tolerance, const diag_comm_info &comm); /** * @brief Solve A*x=b in Device memory, preserving padding and caller column order. * @param initial_residual Optional initial residual, reused on the first cycle only; otherwise nullptr. * @param max_iterations Iteration budget for this call, shared by all restart cycles. * @note Uses tolerance*norm(b), or tolerance for a zero right-hand side. */ - LinearSolveResult solve(const LinearOperator& op, - const LinearOperator& preconditioner, - int ld, - int nband, - int dim, - T* x, - const T* b, - const T* initial_residual, - const int max_iterations); + LinearSolveResult solve( + const LinearOperator &op, + const LinearOperator &preconditioner, + int ld, + int nband, + int dim, + T *x, + const T *b, + const T *initial_residual, + const int max_iterations); private: - bool iterate(const LinearOperator& op, - const LinearOperator& preconditioner, - const bool first_iteration, - LinearSolveResult* result); + bool iterate( + const LinearOperator &op, + const LinearOperator &preconditioner, + const bool first_iteration, + LinearSolveResult *result); void retire_converged(); }; } // namespace hsolver diff --git a/source/source_hsolver/module_linalg/linear_gmres.cpp b/source/source_hsolver/module_linalg/linear_gmres.cpp index a5772b16505..91e1f68423f 100644 --- a/source/source_hsolver/module_linalg/linear_gmres.cpp +++ b/source/source_hsolver/module_linalg/linear_gmres.cpp @@ -6,7 +6,11 @@ namespace hsolver { template -LinearGMRES::LinearGMRES(double tolerance, int restart, int max_iter, const diag_comm_info& comm) +LinearGMRES::LinearGMRES( + double tolerance, + int restart, + int max_iter, + const diag_comm_info &comm) : tolerance_(tolerance), restart_(0), work_(comm, 2), algebra_(comm) { const int iter_limit = std::max(1, max_iter); @@ -17,14 +21,12 @@ LinearGMRES::LinearGMRES(double tolerance, int restart, int max_iter, } } -template -T* LinearGMRES::vector(int slot) +template T *LinearGMRES::vector(int slot) { return krylov_.template data() + static_cast(slot) * stride_; } -template -void LinearGMRES::retire(int band) +template void LinearGMRES::retire(int band) { const int last = --active_; if (band == last) @@ -40,62 +42,72 @@ void LinearGMRES::retire(int band) std::swap(cosine_[band], cosine_[last]); } -template -void LinearGMRES::apply_swaps(int last) +template void LinearGMRES::apply_swaps(int last) { work_.swap_columns(ld_, dim_, swaps_); const int basis_count = last + 2; - work_.swap_vectors(ld_, dim_, basis_count, stride_, basis(0), swaps_); + work_.swap_vectors( + ld_, + dim_, + basis_count, + stride_, + basis(0), + swaps_); if (last >= 0) { const int direction_count = last + 1; - work_.swap_vectors(ld_, dim_, direction_count, stride_, direction(0), swaps_); - work_.swap_vectors(ld_, dim_, direction_count, stride_, image(0), swaps_); + work_.swap_vectors( + ld_, + dim_, + direction_count, + stride_, + direction(0), + swaps_); + work_.swap_vectors( + ld_, + dim_, + direction_count, + stride_, + image(0), + swaps_); } swaps_.clear(); } -template -T* LinearGMRES::residual() -{ - return work_.data(0); -} +template T *LinearGMRES::residual() { return work_.data(0); } -template -T* LinearGMRES::solution() -{ - return work_.data(1); -} +template T *LinearGMRES::solution() { return work_.data(1); } -template -T* LinearGMRES::basis(int index) -{ - return vector(index); -} +template T *LinearGMRES::basis(int index) { return vector(index); } -template -T* LinearGMRES::direction(int index) +template T *LinearGMRES::direction(int index) { const int slot = restart_ + 1 + index; return vector(slot); } -template -T* LinearGMRES::image(int index) +template T *LinearGMRES::image(int index) { const int slot = 2 * restart_ + 1 + index; return vector(slot); } template -void LinearGMRES::orthogonalize(int j, T* next, std::vector* coefficients) +void LinearGMRES::orthogonalize(int j, T *next, std::vector *coefficients) { - std::vector& coeff = *coefficients; + std::vector &coeff = *coefficients; const int basis_count = j + 1; // Two-pass classical Gram-Schmidt batches the global reductions. for (int pass = 0; pass < 2; ++pass) { - const std::vector dots = algebra_.arnoldi_dots(ld_, dim_, active_, basis_count, stride_, basis(0), next); + const std::vector dots = algebra_.arnoldi_dots( + ld_, + dim_, + active_, + basis_count, + stride_, + basis(0), + next); for (int i = 0; i <= j; ++i) { for (int b = 0; b < active_; ++b) @@ -103,15 +115,25 @@ void LinearGMRES::orthogonalize(int j, T* next, std::vector* coeff h_[b][i + j * (restart_ + 1)] += dots[i * active_ + b]; coeff[b] = T(dots[i * active_ + b]); } - work_.batch(ld_, dim_, active_, next, next, basis(i), T(1), T(-1), nullptr, coeff.data(), nullptr); + work_.batch( + ld_, + dim_, + active_, + next, + next, + basis(i), + T(1), + T(-1), + nullptr, + coeff.data(), + nullptr); } } } -template -bool LinearGMRES::update_qr(int b, int j, double norm) +template bool LinearGMRES::update_qr(int b, int j, double norm) { - Wide* column = h_[b].data() + j * (restart_ + 1); + Wide *column = h_[b].data() + j * (restart_ + 1); column[j + 1] = norm; for (int i = 0; i < j; ++i) { @@ -136,7 +158,7 @@ bool LinearGMRES::update_qr(int b, int j, double norm) } template -bool LinearGMRES::back_substitute(int b, int j, std::vector* weights) +bool LinearGMRES::back_substitute(int b, int j, std::vector *weights) { const typename std::vector::const_iterator weights_end = g_[b].cbegin() + j + 1; weights->assign(g_[b].cbegin(), weights_end); @@ -156,16 +178,17 @@ bool LinearGMRES::back_substitute(int b, int j, std::vector* we } template -void LinearGMRES::update_solution(int j, - const std::vector& done, - const std::vector>& weights, - bool reconstruct, - std::vector* coefficients) +void LinearGMRES::update_solution( + int j, + const std::vector &done, + const std::vector> &weights, + bool reconstruct, + std::vector *coefficients) { - std::vector& coeff = *coefficients; + std::vector &coeff = *coefficients; const bool any_done = std::find(done.begin(), done.end(), 1) != done.end(); - T* current_solution = solution(); - T* current_residual = reconstruct ? residual() : nullptr; + T *current_solution = solution(); + T *current_residual = reconstruct ? residual() : nullptr; for (int i = 0; any_done && i <= j; ++i) { bool has_zero_coefficients = false; @@ -174,22 +197,23 @@ void LinearGMRES::update_solution(int j, coeff[b] = done[b] ? T(weights[b][i]) : T(0); has_zero_coefficients = has_zero_coefficients || coeff[b] == T(0); } - const T* current_direction = direction(i); - const T* current_image = reconstruct ? image(i) : nullptr; - work_.gmres_update(ld_, - dim_, - active_, - current_solution, - current_residual, - current_direction, - current_image, - coeff.data(), - has_zero_coefficients); + const T *current_direction = direction(i); + const T *current_image = reconstruct ? image(i) : nullptr; + work_.gmres_update( + ld_, + dim_, + active_, + current_solution, + current_residual, + current_direction, + current_image, + coeff.data(), + has_zero_coefficients); } } template -bool LinearGMRES::start_cycle(const std::vector& threshold, std::vector* coefficients) +bool LinearGMRES::start_cycle(const std::vector &threshold, std::vector *coefficients) { const int projected_rows = restart_ + 1; const int64_t projected_elements = static_cast(projected_rows) * restart_; @@ -197,8 +221,15 @@ bool LinearGMRES::start_cycle(const std::vector& threshold, s g_.assign(bands_, std::vector(projected_rows, 0)); sine_.assign(bands_, std::vector(restart_, 0)); cosine_.assign(bands_, std::vector(restart_, 0)); - std::vector norms = algebra_.dots(ld_, dim_, bands_, 1, stride_, residual(), residual()); - std::vector& coeff = *coefficients; + std::vector norms = algebra_.dots( + ld_, + dim_, + bands_, + 1, + stride_, + residual(), + residual()); + std::vector &coeff = *coefficients; active_ = bands_; swaps_.clear(); std::iota(order_.begin(), order_.end(), 0); @@ -212,7 +243,18 @@ bool LinearGMRES::start_cycle(const std::vector& threshold, s g_[b][0] = norm; coeff[b] = norm == 0 ? T(0) : T(1.0 / norm); } - work_.batch(ld_, dim_, bands_, basis(0), residual(), residual(), T(1), T(0), coeff.data(), nullptr, nullptr); + work_.batch( + ld_, + dim_, + bands_, + basis(0), + residual(), + residual(), + T(1), + T(0), + coeff.data(), + nullptr, + nullptr); for (int b = bands_ - 1; b >= 0; --b) { if (std::abs(g_[b][0]) <= 0.8 * threshold[order_[b]]) @@ -225,12 +267,13 @@ bool LinearGMRES::start_cycle(const std::vector& threshold, s } template -bool LinearGMRES::cycle(const LinearOperator& op, - const LinearOperator& preconditioner, - const std::vector& threshold, - bool reconstruct, - int max_iter, - LinearSolveResult* result) +bool LinearGMRES::cycle( + const LinearOperator &op, + const LinearOperator &preconditioner, + const std::vector &threshold, + bool reconstruct, + int max_iter, + LinearSolveResult *result) { std::vector coeff(bands_); if (!start_cycle(threshold, &coeff)) @@ -239,15 +282,36 @@ bool LinearGMRES::cycle(const LinearOperator& op, } for (int j = 0; j < restart_ && active_ > 0 && result->iterations < max_iter; ++j) { - T* z = direction(j); - T* raw = image(j); + T *z = direction(j); + T *raw = image(j); const int next_index = j + 1; - T* next = basis(next_index); - preconditioner.apply(basis(j), z, ld_, active_); - work_.apply(op, z, raw, ld_, active_); - work_.copy(ld_, dim_, active_, raw, next); + T *next = basis(next_index); + preconditioner.apply( + basis(j), + z, + ld_, + active_); + work_.apply( + op, + z, + raw, + ld_, + active_); + work_.copy( + ld_, + dim_, + active_, + raw, + next); orthogonalize(j, next, &coeff); - const std::vector norms = algebra_.dots(ld_, dim_, active_, 1, stride_, next, next); + const std::vector norms = algebra_.dots( + ld_, + dim_, + active_, + 1, + stride_, + next, + next); ++result->iterations; std::vector done(active_, 0); std::vector> weights(active_); @@ -262,8 +326,8 @@ bool LinearGMRES::cycle(const LinearOperator& op, { return false; } - done[b] = std::abs(g_[b][j + 1]) <= 0.8 * threshold[order_[b]] || norm <= std::numeric_limits::min() || j + 1 == restart_ - || result->iterations == max_iter; + done[b] = std::abs(g_[b][j + 1]) <= 0.8 * threshold[order_[b]] || + norm <= std::numeric_limits::min() || j + 1 == restart_ || result->iterations == max_iter; coeff[b] = done[b] ? T(0) : T(1.0 / norm); if (!done[b]) { @@ -274,8 +338,24 @@ bool LinearGMRES::cycle(const LinearOperator& op, return false; } } - work_.batch(ld_, dim_, active_, next, next, next, T(1), T(0), coeff.data(), nullptr, nullptr); - update_solution(j, done, weights, reconstruct, &coeff); + work_.batch( + ld_, + dim_, + active_, + next, + next, + next, + T(1), + T(0), + coeff.data(), + nullptr, + nullptr); + update_solution( + j, + done, + weights, + reconstruct, + &coeff); for (int b = active_ - 1; b >= 0; --b) { if (done[b]) @@ -289,20 +369,28 @@ bool LinearGMRES::cycle(const LinearOperator& op, } template -LinearSolveResult LinearGMRES::solve(const LinearOperator& op, - const LinearOperator& preconditioner, - int ld, - int nvec, - int dim, - T* x, - const T* b, - const T* initial_residual, - bool force_check, - int max_iter, - bool reconstruct) +LinearSolveResult LinearGMRES::solve( + const LinearOperator &op, + const LinearOperator &preconditioner, + int ld, + int nvec, + int dim, + T *x, + const T *b, + const T *initial_residual, + bool force_check, + int max_iter, + bool reconstruct) { const LinearSolveTimer timer("LinearGMRES"); - work_.prepare(ld, dim, nvec, x, b, tolerance_, max_iter); + work_.prepare( + ld, + dim, + nvec, + x, + b, + tolerance_, + max_iter); work_.reset_statistics(); ld_ = ld; dim_ = dim; @@ -312,7 +400,14 @@ LinearSolveResult LinearGMRES::solve(const LinearOperator& const int64_t krylov_elements = (3 * static_cast(restart_) + 1) * stride_; linear_buffer(&krylov_, krylov_elements); order_.resize(nvec); - const std::vector rhs_norms = algebra_.dots(ld, dim, nvec, 1, stride_, b, b); + const std::vector rhs_norms = algebra_.dots( + ld, + dim, + nvec, + 1, + stride_, + b, + b); std::vector threshold(nvec); for (int i = 0; i < nvec; ++i) { @@ -324,35 +419,71 @@ LinearSolveResult LinearGMRES::solve(const LinearOperator& work_.clear(); if (initial_residual) { - work_.copy(ld, dim, nvec, initial_residual, residual()); + work_.copy( + ld, + dim, + nvec, + initial_residual, + residual()); } else { - work_.residual(op, ld, dim, nvec, x, b, residual()); + work_.residual( + op, + ld, + dim, + nvec, + x, + b, + residual()); } while (true) { // Restarts retain the independent residual computed below in the original band order. - work_.copy(ld, dim, nvec, x, solution()); + work_.copy( + ld, + dim, + nvec, + x, + solution()); const int start = result.iterations; bool regular = false; try { - regular = cycle(op, preconditioner, threshold, reconstruct, max_iter, &result); + regular = cycle( + op, + preconditioner, + threshold, + reconstruct, + max_iter, + &result); result.status = regular ? LinearSolveStatus::max_iterations : LinearSolveStatus::breakdown; } - catch (const LinearPreconditionerError&) + catch (const LinearPreconditionerError &) { // Unlike completed Arnoldi steps, this failed attempt has not yet been counted. ++result.iterations; result.status = LinearSolveStatus::preconditioner_failure; } - work_.restore(ld, dim, nvec, order_, solution(), x); + work_.restore( + ld, + dim, + nvec, + order_, + solution(), + x); bool accepted = regular && reconstruct; bool invalid_reconstruction = false; if (accepted) { - const std::vector norms = algebra_.dots(ld, dim, nvec, 1, stride_, residual(), residual()); + const std::vector norms = algebra_.dots( + ld, + dim, + nvec, + 1, + stride_, + residual(), + residual()); result.max_residual = 0; for (int i = 0; i < nvec; ++i) { @@ -373,10 +504,24 @@ LinearSolveResult LinearGMRES::solve(const LinearOperator& return result; } } - work_.residual(op, ld, dim, nvec, x, b, residual()); + work_.residual( + op, + ld, + dim, + nvec, + x, + b, + residual()); ++result.true_checks; work_.statistics(&result); - const std::vector true_norms = algebra_.dots(ld, dim, nvec, 1, stride_, residual(), residual()); + const std::vector true_norms = algebra_.dots( + ld, + dim, + nvec, + 1, + stride_, + residual(), + residual()); result.max_residual = 0; result.failed_band = -1; for (int i = 0; i < nvec; ++i) @@ -413,7 +558,8 @@ LinearSolveResult LinearGMRES::solve(const LinearOperator& return result; } const bool retry_initial = initial_residual && result.restarts == 0; - if (result.iterations >= max_iter || (result.iterations == start && !retry_initial) || !std::isfinite(result.max_residual)) + if (result.iterations >= max_iter || (result.iterations == start && !retry_initial) || + !std::isfinite(result.max_residual)) { return result; } diff --git a/source/source_hsolver/module_linalg/linear_gmres.h b/source/source_hsolver/module_linalg/linear_gmres.h index 241e06e4a5b..b1a7795031f 100644 --- a/source/source_hsolver/module_linalg/linear_gmres.h +++ b/source/source_hsolver/module_linalg/linear_gmres.h @@ -6,8 +6,7 @@ namespace hsolver { /** @brief Restarted right-preconditioned GMRES with independent batched columns. */ -template -class LinearGMRES +template class LinearGMRES { private: using Wide = std::complex; @@ -29,44 +28,52 @@ class LinearGMRES int stride_ = 0; int active_ = 0; - T* vector(int slot); - T* residual(); - T* solution(); - T* basis(int index); - T* direction(int index); - T* image(int index); - bool start_cycle(const std::vector& threshold, std::vector* coefficients); - void orthogonalize(int j, T* next, std::vector* coefficients); + T *vector(int slot); + T *residual(); + T *solution(); + T *basis(int index); + T *direction(int index); + T *image(int index); + bool start_cycle(const std::vector &threshold, std::vector *coefficients); + void orthogonalize(int j, T *next, std::vector *coefficients); bool update_qr(int b, int j, double norm); - bool back_substitute(int b, int j, std::vector* weights); - void update_solution(int j, - const std::vector& done, - const std::vector>& weights, - bool reconstruct, - std::vector* coefficients); + bool back_substitute(int b, int j, std::vector *weights); + void update_solution( + int j, + const std::vector &done, + const std::vector> &weights, + bool reconstruct, + std::vector *coefficients); void retire(int band); void apply_swaps(int last); - bool cycle(const LinearOperator& op, - const LinearOperator& preconditioner, - const std::vector& threshold, - bool reconstruct, - int max_iter, - LinearSolveResult* result); + bool cycle( + const LinearOperator &op, + const LinearOperator &preconditioner, + const std::vector &threshold, + bool reconstruct, + int max_iter, + LinearSolveResult *result); public: - LinearGMRES(double tolerance, int restart, int max_iter, const diag_comm_info& comm); - /** @brief Use the per-call budget and reconstruction policy; force_check requests an independent final application. */ - LinearSolveResult solve(const LinearOperator& op, - const LinearOperator& preconditioner, - int ld, - int nvec, - int dim, - T* x, - const T* b, - const T* initial_residual, - bool force_check, - int max_iter, - bool reconstruct); + LinearGMRES( + double tolerance, + int restart, + int max_iter, + const diag_comm_info &comm); + /** @brief Use the per-call budget and reconstruction policy; force_check requests an independent final application. + */ + LinearSolveResult solve( + const LinearOperator &op, + const LinearOperator &preconditioner, + int ld, + int nvec, + int dim, + T *x, + const T *b, + const T *initial_residual, + bool force_check, + int max_iter, + bool reconstruct); }; } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/linear_low_rank.cpp b/source/source_hsolver/module_linalg/linear_low_rank.cpp index cbbed22817c..27239c06f57 100644 --- a/source/source_hsolver/module_linalg/linear_low_rank.cpp +++ b/source/source_hsolver/module_linalg/linear_low_rank.cpp @@ -6,15 +6,16 @@ namespace hsolver { template -void LinearResponse::update(LinearAlgebra& algebra, - int ld, - int dim, - int bands, - const T* solution, - const T* seed, - const T* seed_residual, - double tolerance, - ct::Tensor* workspace) +void LinearResponse::update( + LinearAlgebra &algebra, + int ld, + int dim, + int bands, + const T *solution, + const T *seed, + const T *seed_residual, + double tolerance, + ct::Tensor *workspace) { rank_ = 0; const int64_t elements = static_cast(ld) * bands; @@ -23,49 +24,125 @@ void LinearResponse::update(LinearAlgebra& algebra, linear_buffer(&images_, size); const int64_t workspace_elements = 4 * size; linear_buffer(workspace, workspace_elements); - T* z = workspace->template data(); - T* w = z + size; + T *z = workspace->template data(); + T *w = z + size; using Real = typename GetTypeReal::type; const double tolerance_cutoff = 100 * tolerance; const double roundoff_cutoff = 100 * static_cast(std::numeric_limits::epsilon()); const double cutoff = std::max(tolerance_cutoff, roundoff_cutoff); - linear_op().batch(ld, dim, bands, z, solution, seed, T(1), T(-1), nullptr, nullptr, nullptr); + linear_op().batch( + ld, + dim, + bands, + z, + solution, + seed, + T(1), + T(-1), + nullptr, + nullptr, + nullptr); if (dim > 0) { - base_device::memory::synchronize_memory_2d_op()(w, ld, seed_residual, ld, dim, bands); + base_device::memory::synchronize_memory_2d_op()( + w, + ld, + seed_residual, + ld, + dim, + bands); } // Form a rank-revealing block basis, then repair its Gram matrix once. // This avoids a device transfer and MPI reduction for every history column. - std::vector> gram = algebra.cross(ld, dim, bands, bands, w, w); - std::vector> c = linear_gram_basis(gram, bands, cutoff, &rank_); + std::vector> gram = algebra.cross( + ld, + dim, + bands, + bands, + w, + w); + std::vector> c = linear_gram_basis( + gram, + bands, + cutoff, + &rank_); if (rank_ == 0) { return; } - T* rz = z + 2 * size; - T* rw = rz + size; - algebra.expand(ld, dim, bands, rank_, z, c, rz, T(0)); - algebra.expand(ld, dim, bands, rank_, w, c, rw, T(0)); - gram = algebra.cross(ld, dim, rank_, rank_, rw, rw); + T *rz = z + 2 * size; + T *rw = rz + size; + algebra.expand( + ld, + dim, + bands, + rank_, + z, + c, + rz, + T(0)); + algebra.expand( + ld, + dim, + bands, + rank_, + w, + c, + rw, + T(0)); + gram = algebra.cross( + ld, + dim, + rank_, + rank_, + rw, + rw); int refined_rank = 0; - c = linear_gram_basis(gram, rank_, 1e-6, &refined_rank); + c = linear_gram_basis( + gram, + rank_, + 1e-6, + &refined_rank); if (refined_rank > 0) { - algebra.expand(ld, dim, rank_, refined_rank, rz, c, directions_.template data(), T(0)); - algebra.expand(ld, dim, rank_, refined_rank, rw, c, images_.template data(), T(0)); + algebra.expand( + ld, + dim, + rank_, + refined_rank, + rz, + c, + directions_.template data(), + T(0)); + algebra.expand( + ld, + dim, + rank_, + refined_rank, + rw, + c, + images_.template data(), + T(0)); } rank_ = refined_rank; if (rank_ > 0) { - gram = algebra.cross(ld, dim, rank_, rank_, images(), images()); + gram = algebra.cross( + ld, + dim, + rank_, + rank_, + images(), + images()); const double roundoff_limit = 100.0 * std::numeric_limits::epsilon(); const double limit = std::max(1e-8, roundoff_limit); for (int j = 0; j < rank_; ++j) { for (int i = 0; i < rank_; ++i) { - if (!std::isfinite(std::abs(gram[i + j * rank_])) - || std::abs(gram[i + j * rank_] - (i == j ? std::complex(1) : std::complex(0))) > limit) + if (!std::isfinite(std::abs(gram[i + j * rank_])) || + std::abs(gram[i + j * rank_] - (i == j ? std::complex(1) : std::complex(0))) > + limit) { rank_ = 0; return; @@ -76,30 +153,51 @@ void LinearResponse::update(LinearAlgebra& algebra, } template -void LinearLowRank::prepare_response(int ld, int rank, const T* directions, const T* images, ct::Tensor* workspace) +void LinearLowRank::prepare_response( + int ld, + int rank, + const T *directions, + const T *images, + ct::Tensor *workspace) { - prepare(ld, rank, directions, images, images, nullptr, workspace); + prepare( + ld, + rank, + directions, + images, + images, + nullptr, + workspace); } template -void LinearLowRank::prepare_subspace(int ld, - int rank, - const T* basis, - const T* images, - const LinearSmallLU& factor, - ct::Tensor* workspace) +void LinearLowRank::prepare_subspace( + int ld, + int rank, + const T *basis, + const T *images, + const LinearSmallLU &factor, + ct::Tensor *workspace) { - prepare(ld, rank, basis, images, basis, &factor, workspace); + prepare( + ld, + rank, + basis, + images, + basis, + &factor, + workspace); } template -void LinearLowRank::prepare(int ld, - int rank, - const T* z, - const T* w, - const T* test, - const LinearSmallLU* factor, - ct::Tensor* workspace) +void LinearLowRank::prepare( + int ld, + int rank, + const T *z, + const T *w, + const T *test, + const LinearSmallLU *factor, + ct::Tensor *workspace) { rank_ = rank; test_ = rank > 0 ? test : nullptr; @@ -111,41 +209,94 @@ void LinearLowRank::prepare(int ld, } const int64_t size = static_cast(ld) * rank; linear_buffer(workspace, size); - T* correction = workspace->template data(); + T *correction = workspace->template data(); if (diagonal_) { - linear_op().diagonal(ld, dim_, rank, diagonal_, w, correction); + linear_op().diagonal( + ld, + dim_, + rank, + diagonal_, + w, + correction); } else if (dim_ > 0) { - base_device::memory::synchronize_memory_2d_op()(correction, ld, w, ld, dim_, rank); + base_device::memory::synchronize_memory_2d_op()( + correction, + ld, + w, + ld, + dim_, + rank); } - linear_op().batch(ld, dim_, rank, correction, z, correction, T(1), T(-1), nullptr, nullptr, nullptr); + linear_op().batch( + ld, + dim_, + rank, + correction, + z, + correction, + T(1), + T(-1), + nullptr, + nullptr, + nullptr); correction_ = correction; } template -void LinearLowRank::apply(const T* x, T* y, int ld, int nvec) const +void LinearLowRank::apply( + const T *x, + T *y, + int ld, + int nvec) const { if (diagonal_) { - linear_op().diagonal(ld, dim_, nvec, diagonal_, x, y); + linear_op().diagonal( + ld, + dim_, + nvec, + diagonal_, + x, + y); } else if (dim_ > 0) { - base_device::memory::synchronize_memory_2d_op()(y, ld, x, ld, dim_, nvec); + base_device::memory::synchronize_memory_2d_op()( + y, + ld, + x, + ld, + dim_, + nvec); } if (rank_ == 0) { return; } - std::vector> coefficients = algebra_.projection_cross(ld, dim_, rank_, nvec, test_, x); + std::vector> coefficients = algebra_.projection_cross( + ld, + dim_, + rank_, + nvec, + test_, + x); if (factor_ && !factor_->solve(&coefficients, nvec)) { // Changing the preconditioner inside a recurrence invalidates BiCGSTAB and CGS. throw LinearPreconditionerError(); } - algebra_.expand(ld, dim_, rank_, nvec, correction_, coefficients, y, T(1)); + algebra_.expand( + ld, + dim_, + rank_, + nvec, + correction_, + coefficients, + y, + T(1)); } template class LinearResponse, base_device::DEVICE_CPU>; diff --git a/source/source_hsolver/module_linalg/linear_low_rank.h b/source/source_hsolver/module_linalg/linear_low_rank.h index 7c3cc483a4f..b59521b9cac 100644 --- a/source/source_hsolver/module_linalg/linear_low_rank.h +++ b/source/source_hsolver/module_linalg/linear_low_rank.h @@ -6,8 +6,7 @@ namespace hsolver { /** @brief A bounded, paired-orthogonalized response history for one k point. */ -template -class LinearResponse +template class LinearResponse { private: ct::Tensor directions_; @@ -15,78 +14,83 @@ class LinearResponse int rank_ = 0; public: - int rank() const - { - return rank_; - } - void clear() - { - rank_ = 0; - } - const T* directions() const - { - return directions_.template data(); - } - const T* images() const - { - return images_.template data(); - } + int rank() const { return rank_; } + void clear() { rank_ = 0; } + const T *directions() const { return directions_.template data(); } + const T *images() const { return images_.template data(); } /** @brief Replace the history only after a successful solve; discard unresolved directions. */ - void update(LinearAlgebra& algebra, - int ld, - int dim, - int bands, - const T* solution, - const T* seed, - const T* seed_residual, - double tolerance, - ct::Tensor* workspace); + void update( + LinearAlgebra &algebra, + int ld, + int dim, + int bands, + const T *solution, + const T *seed, + const T *seed_residual, + double tolerance, + ct::Tensor *workspace); }; /** @brief Diagonal inverse with an optional response or Galerkin coarse correction. */ -template -class LinearLowRank final : public LinearOperator +template class LinearLowRank final : public LinearOperator { private: - LinearAlgebra& algebra_; - const T* diagonal_; + LinearAlgebra &algebra_; + const T *diagonal_; const int dim_; - const T* test_ = nullptr; + const T *test_ = nullptr; int rank_ = 0; - const LinearSmallLU* factor_ = nullptr; - const T* correction_ = nullptr; + const LinearSmallLU *factor_ = nullptr; + const T *correction_ = nullptr; public: - LinearLowRank(LinearAlgebra& algebra, const T* diagonal, int dim) : algebra_(algebra), diagonal_(diagonal), dim_(dim) - { - } - bool is_identity() const override - { - return diagonal_ == nullptr && rank_ == 0; - } - int rank() const + LinearLowRank(LinearAlgebra &algebra, const T *diagonal, int dim) + : algebra_(algebra), diagonal_(diagonal), dim_(dim) { - return rank_; } + bool is_identity() const override { return diagonal_ == nullptr && rank_ == 0; } + int rank() const { return rank_; } /** @brief Prepare a response correction with orthonormal images. * @param images Borrowed projection vectors; keep them valid and unchanged until the last apply. * @param workspace Caller-owned correction storage; do not modify or resize it until the last apply. */ - void prepare_response(int ld, int rank, const T* directions, const T* images, ct::Tensor* workspace); + void prepare_response( + int ld, + int rank, + const T *directions, + const T *images, + ct::Tensor *workspace); /** @brief Prepare a Galerkin coarse correction. * @param basis Borrowed projection vectors; keep them valid and unchanged until the last apply. * @param factor Borrowed factorization of basis^H*images; keep it valid and unchanged until the last apply. * @param workspace Caller-owned correction storage; do not modify or resize it until the last apply. */ - void prepare_subspace(int ld, int rank, const T* basis, const T* images, const LinearSmallLU& factor, ct::Tensor* workspace); + void prepare_subspace( + int ld, + int rank, + const T *basis, + const T *images, + const LinearSmallLU &factor, + ct::Tensor *workspace); /** @brief Apply the fixed correction; throw LinearPreconditionerError if the coarse solve fails. */ - void apply(const T* x, T* y, int ld, int nvec) const override; + void apply( + const T *x, + T *y, + int ld, + int nvec) const override; private: /** @brief Form Z-D*W and retain the projection data for subsequent applications. */ - void prepare(int ld, int rank, const T* z, const T* w, const T* test, const LinearSmallLU* factor, ct::Tensor* workspace); + void prepare( + int ld, + int rank, + const T *z, + const T *w, + const T *test, + const LinearSmallLU *factor, + ct::Tensor *workspace); }; } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/linear_operator.h b/source/source_hsolver/module_linalg/linear_operator.h index f41ee21677f..c550deebb9a 100644 --- a/source/source_hsolver/module_linalg/linear_operator.h +++ b/source/source_hsolver/module_linalg/linear_operator.h @@ -12,10 +12,7 @@ namespace hsolver class LinearPreconditionerError final : public std::exception { public: - const char* what() const noexcept override - { - return "Preconditioner application failed."; - } + const char *what() const noexcept override { return "Preconditioner application failed."; } }; /** @@ -26,12 +23,15 @@ class LinearPreconditionerError final : public std::exception * operator acts on every column, including reordered or compacted blocks. * A zero local row count must still participate in operator collectives. */ -template -class LinearOperator +template class LinearOperator { public: virtual ~LinearOperator() = default; - virtual void apply(const T* x, T* y, const int ld, const int nvec) const = 0; + virtual void apply( + const T *x, + T *y, + const int ld, + const int nvec) const = 0; /** @brief Allow solvers to bypass identity preconditioner copies. */ virtual bool is_identity() const { return false; } }; diff --git a/source/source_hsolver/module_linalg/linear_solver_types.h b/source/source_hsolver/module_linalg/linear_solver_types.h index 2d26f6400f8..de6a76be0b5 100644 --- a/source/source_hsolver/module_linalg/linear_solver_types.h +++ b/source/source_hsolver/module_linalg/linear_solver_types.h @@ -47,7 +47,7 @@ struct LinearControls bool reconstruct = false; }; -const char* linear_status_name(const LinearSolveStatus status); +const char *linear_status_name(const LinearSolveStatus status); } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/linear_workspace.cpp b/source/source_hsolver/module_linalg/linear_workspace.cpp index e4fe906459e..017eda99b87 100644 --- a/source/source_hsolver/module_linalg/linear_workspace.cpp +++ b/source/source_hsolver/module_linalg/linear_workspace.cpp @@ -14,18 +14,12 @@ namespace hsolver { -LinearSolveTimer::LinearSolveTimer(const char* solver) : solver_(solver) -{ - ModuleBase::timer::start(solver_, "solve"); -} +LinearSolveTimer::LinearSolveTimer(const char *solver) : solver_(solver) { ModuleBase::timer::start(solver_, "solve"); } -LinearSolveTimer::~LinearSolveTimer() -{ - ModuleBase::timer::end(solver_, "solve"); -} +LinearSolveTimer::~LinearSolveTimer() { ModuleBase::timer::end(solver_, "solve"); } template -LinearWorkspace::LinearWorkspace(const diag_comm_info& comm, int slots) : slots_(slots), comm_(comm) +LinearWorkspace::LinearWorkspace(const diag_comm_info &comm, int slots) : slots_(slots), comm_(comm) { if (slots_ <= 0) { @@ -42,17 +36,18 @@ LinearWorkspace::LinearWorkspace(const diag_comm_info& comm, int slot } template -void LinearWorkspace::prepare(const int ld, - const int dim, - const int nvec, - const T* x, - const T* b, - const double tolerance, - const int max_iter) +void LinearWorkspace::prepare( + const int ld, + const int dim, + const int nvec, + const T *x, + const T *b, + const double tolerance, + const int max_iter) { - if (dim < 0 || ld < dim || nvec <= 0 || !std::isfinite(tolerance) || tolerance <= 0.0 || max_iter < 0 - || ld > std::numeric_limits::max() / nvec || (ld > 0 && (x == nullptr || b == nullptr)) || (ld > 0 && x == b) - || comm_.nproc < 1) + if (dim < 0 || ld < dim || nvec <= 0 || !std::isfinite(tolerance) || tolerance <= 0.0 || max_iter < 0 || + ld > std::numeric_limits::max() / nvec || (ld > 0 && (x == nullptr || b == nullptr)) || + (ld > 0 && x == b) || comm_.nproc < 1) { throw std::invalid_argument("Invalid linear-solver layout, buffers, or convergence controls."); } @@ -62,101 +57,164 @@ void LinearWorkspace::prepare(const int ld, { using CtDevice = typename ct::PsiToContainer::type; const int64_t slot_elements = size; - vectors_ = ct::Tensor(ct::DataTypeToEnum::value, ct::DeviceTypeToEnum::value, {slots_, slot_elements}); + vectors_ = + ct::Tensor(ct::DataTypeToEnum::value, ct::DeviceTypeToEnum::value, {slots_, slot_elements}); capacity_ = size; } } -template -void LinearWorkspace::clear() +template void LinearWorkspace::clear() { const size_t elements = static_cast(slots_) * capacity_; base_device::memory::set_memory_op()(vectors_.template data(), 0, elements); } -template -void LinearWorkspace::reset_statistics() +template void LinearWorkspace::reset_statistics() { operator_calls_ = 0; operator_columns_ = 0; } -template -void LinearWorkspace::statistics(LinearSolveResult* result) const +template void LinearWorkspace::statistics(LinearSolveResult *result) const { result->operator_calls = operator_calls_; result->operator_columns = operator_columns_; } template -void LinearWorkspace::apply(const LinearOperator& op, const T* x, T* y, int ld, int nvec) +void LinearWorkspace::apply( + const LinearOperator &op, + const T *x, + T *y, + int ld, + int nvec) { ModuleBase::timer::start("LinearWorkspace", "apply"); ++operator_calls_; operator_columns_ += nvec; - op.apply(x, y, ld, nvec); + op.apply( + x, + y, + ld, + nvec); ModuleBase::timer::end("LinearWorkspace", "apply"); } -template -T* LinearWorkspace::data(const int slot) +template T *LinearWorkspace::data(const int slot) { return vectors_.data() + static_cast(slot) * capacity_; } template -void LinearWorkspace::copy(const int ld, const int dim, const int nvec, const T* x, T* y) const +void LinearWorkspace::copy( + const int ld, + const int dim, + const int nvec, + const T *x, + T *y) const { if (dim > 0) { - base_device::memory::synchronize_memory_2d_op()(y, ld, x, ld, dim, nvec); + base_device::memory::synchronize_memory_2d_op()( + y, + ld, + x, + ld, + dim, + nvec); } } template -void LinearWorkspace::dot(const int ld, const int dim, const int nvec, const T* x, const T* y, T* out) +void LinearWorkspace::dot( + const int ld, + const int dim, + const int nvec, + const T *x, + const T *y, + T *out) { - dot_pair(ld, dim, nvec, x, y, nullptr, nullptr, out, nullptr); + dot_pair( + ld, + dim, + nvec, + x, + y, + nullptr, + nullptr, + out, + nullptr); } template -void LinearWorkspace::residual(const LinearOperator& op, - const int ld, - const int dim, - const int nvec, - const T* x, - const T* b, - T* r) +void LinearWorkspace::residual( + const LinearOperator &op, + const int ld, + const int dim, + const int nvec, + const T *x, + const T *b, + T *r) { - apply(op, x, r, ld, nvec); - linear_op().batch(ld, dim, nvec, r, b, r, T(1), T(-1), nullptr, nullptr, nullptr); + apply( + op, + x, + r, + ld, + nvec); + linear_op().batch( + ld, + dim, + nvec, + r, + b, + r, + T(1), + T(-1), + nullptr, + nullptr, + nullptr); } template -void LinearWorkspace::verify(const LinearOperator& op, - const int ld, - const int dim, - const int nvec, - const T* x, - const T* b, - const std::vector& threshold, - T* scratch, - LinearSolveResult* result) +void LinearWorkspace::verify( + const LinearOperator &op, + const int ld, + const int dim, + const int nvec, + const T *x, + const T *b, + const std::vector &threshold, + T *scratch, + LinearSolveResult *result) { - residual(op, ld, dim, nvec, x, b, scratch); + residual( + op, + ld, + dim, + nvec, + x, + b, + scratch); ++result->true_checks; result->operator_calls = operator_calls_; result->operator_columns = operator_columns_; std::vector norms(nvec); - dot(ld, dim, nvec, scratch, scratch, norms.data()); + dot(ld, + dim, + nvec, + scratch, + scratch, + norms.data()); result->max_residual = 0.0; result->failed_band = -1; bool converged = true; for (int band = 0; band < nvec; ++band) { const double error = linear_norm(norms[band]); - result->max_residual = std::isfinite(error) ? std::max(result->max_residual, error) : std::numeric_limits::infinity(); + result->max_residual = + std::isfinite(error) ? std::max(result->max_residual, error) : std::numeric_limits::infinity(); if (!std::isfinite(error) || !std::isfinite(threshold[band]) || error > threshold[band]) { if (result->failed_band < 0) @@ -174,7 +232,16 @@ void LinearWorkspace::verify(const LinearOperator& op, } template -void LinearWorkspace::dot_pair(int ld, int dim, int nvec, const T* x, const T* y, const T* z, const T* w, T* first, T* second) +void LinearWorkspace::dot_pair( + int ld, + int dim, + int nvec, + const T *x, + const T *y, + const T *z, + const T *w, + T *first, + T *second) { ModuleBase::timer::start("LinearWorkspace", "dot_pair"); if (nvec == 0) @@ -198,8 +265,19 @@ void LinearWorkspace::dot_pair(int ld, int dim, int nvec, const T* x, partial_ = ct::Tensor(ct::DataTypeToEnum::value, device, {partial_elements}); } host_dots_.resize(count); - linear_op().dots(ld, dim, nvec, x, y, z, w, dots_.template data(), partial_.template data(), tiles); - base_device::memory::synchronize_memory_op()(host_dots_.data(), dots_.template data(), count); + linear_op().dots( + ld, + dim, + nvec, + x, + y, + z, + w, + dots_.template data(), + partial_.template data(), + tiles); + base_device::memory::synchronize_memory_op()( + host_dots_.data(), dots_.template data(), count); #ifdef __MPI if (comm_.nproc > 1) { @@ -216,17 +294,18 @@ void LinearWorkspace::dot_pair(int ld, int dim, int nvec, const T* x, } template -void LinearWorkspace::batch(int ld, - int dim, - int nvec, - T* out, - const T* x, - const T* y, - T a, - T b, - const T* ca, - const T* cb, - const int* skip) +void LinearWorkspace::batch( + int ld, + int dim, + int nvec, + T *out, + const T *x, + const T *y, + T a, + T b, + const T *ca, + const T *cb, + const int *skip) { ModuleBase::timer::start("LinearWorkspace", "batch"); if (dim == 0 || nvec == 0) @@ -241,42 +320,55 @@ void LinearWorkspace::batch(int ld, { coefficients_ = ct::Tensor(ct::DataTypeToEnum::value, device, {coefficient_elements}); } - T* coeff = coefficients_.template data(); + T *coeff = coefficients_.template data(); if (ca) { base_device::memory::synchronize_memory_op()(coeff, ca, nvec); } if (cb) { - T* second_coefficients = coeff + nvec; + T *second_coefficients = coeff + nvec; base_device::memory::synchronize_memory_op()(second_coefficients, cb, nvec); } - const int* mask = nullptr; + const int *mask = nullptr; if (skip) { if (mask_.NumElements() < nvec) { mask_ = ct::Tensor(ct::DataTypeToEnum::value, device, {nvec}); } - base_device::memory::synchronize_memory_op()(mask_.template data(), skip, nvec); + base_device::memory::synchronize_memory_op()( + mask_.template data(), skip, nvec); mask = mask_.template data(); } - const T* first_coefficients = ca ? coeff : nullptr; - const T* second_coefficients = cb ? coeff + nvec : nullptr; - linear_op().batch(ld, dim, nvec, out, x, y, a, b, first_coefficients, second_coefficients, mask); + const T *first_coefficients = ca ? coeff : nullptr; + const T *second_coefficients = cb ? coeff + nvec : nullptr; + linear_op().batch( + ld, + dim, + nvec, + out, + x, + y, + a, + b, + first_coefficients, + second_coefficients, + mask); ModuleBase::timer::end("LinearWorkspace", "batch"); } template -void LinearWorkspace::gmres_update(int ld, - int dim, - int nvec, - T* solution, - T* residual, - const T* direction, - const T* image, - const T* coefficients, - bool has_zero_coefficients) +void LinearWorkspace::gmres_update( + int ld, + int dim, + int nvec, + T *solution, + T *residual, + const T *direction, + const T *image, + const T *coefficients, + bool has_zero_coefficients) { ModuleBase::timer::start("LinearWorkspace", "gmres_update"); if (dim == 0 || nvec == 0) @@ -290,28 +382,59 @@ void LinearWorkspace::gmres_update(int ld, { coefficients_ = ct::Tensor(ct::DataTypeToEnum::value, device, {nvec}); } - T* coeff = coefficients_.template data(); + T *coeff = coefficients_.template data(); base_device::memory::synchronize_memory_op()(coeff, coefficients, nvec); if (std::is_same::value && residual == nullptr && !has_zero_coefficients) { // Keep the existing GPU path when neither zero-column skipping nor a paired update is needed. - linear_op().batch(ld, dim, nvec, solution, solution, direction, T(1), T(1), nullptr, coeff, nullptr); + linear_op().batch( + ld, + dim, + nvec, + solution, + solution, + direction, + T(1), + T(1), + nullptr, + coeff, + nullptr); } else { - linear_op().gmres_update(ld, dim, nvec, solution, residual, direction, image, coeff); + linear_op().gmres_update( + ld, + dim, + nvec, + solution, + residual, + direction, + image, + coeff); } ModuleBase::timer::end("LinearWorkspace", "gmres_update"); } template -void LinearWorkspace::swap_columns(int ld, int dim, const std::vector& pairs) +void LinearWorkspace::swap_columns(int ld, int dim, const std::vector &pairs) { - swap_vectors(ld, dim, slots_, capacity_, vectors_.template data(), pairs); + swap_vectors( + ld, + dim, + slots_, + capacity_, + vectors_.template data(), + pairs); } template -void LinearWorkspace::swap_vectors(int ld, int dim, int slots, int stride, T* vectors, const std::vector& pairs) +void LinearWorkspace::swap_vectors( + int ld, + int dim, + int slots, + int stride, + T *vectors, + const std::vector &pairs) { if (pairs.empty() || dim == 0) { @@ -321,15 +444,29 @@ void LinearWorkspace::swap_vectors(int ld, int dim, int slots, int st const int64_t permutation_elements = pairs.size(); if (permutation_.NumElements() < permutation_elements) { - permutation_ = ct::Tensor(ct::DataTypeToEnum::value, ct::DeviceTypeToEnum::value, {permutation_elements}); + permutation_ = + ct::Tensor(ct::DataTypeToEnum::value, ct::DeviceTypeToEnum::value, {permutation_elements}); } - int* map = permutation_.template data(); + int *map = permutation_.template data(); base_device::memory::synchronize_memory_op()(map, pairs.data(), pairs.size()); const int pair_count = pairs.size() / 2; - linear_op().swaps(ld, dim, slots, stride, pair_count, vectors, map); + linear_op().swaps( + ld, + dim, + slots, + stride, + pair_count, + vectors, + map); } template -void LinearWorkspace::restore(int ld, int dim, int nvec, const std::vector& order, const T* source, T* destination) +void LinearWorkspace::restore( + int ld, + int dim, + int nvec, + const std::vector &order, + const T *source, + T *destination) { if (dim == 0) { @@ -345,15 +482,31 @@ void LinearWorkspace::restore(int ld, int dim, int nvec, const std::v { inverse[order[band]] = band; } - int* map = permutation_.template data(); + int *map = permutation_.template data(); base_device::memory::synchronize_memory_op()(map, inverse.data(), nvec); const int stride = ld * nvec; - linear_op().gather(ld, dim, nvec, 1, stride, source, destination, map); + linear_op().gather( + ld, + dim, + nvec, + 1, + stride, + source, + destination, + map); } template -void LinearWorkspace::bicg_update(int ld, int dim, int nvec, bool direction, T* out, const T* x, const T* y, const T* a, const T* b) +void LinearWorkspace::bicg_update( + int ld, + int dim, + int nvec, + bool direction, + T *out, + const T *x, + const T *y, + const T *a, + const T *b) { ModuleBase::timer::start("LinearWorkspace", "bicg_update"); if (dim == 0 || nvec == 0) @@ -365,25 +518,42 @@ void LinearWorkspace(nvec); if (coefficients_.NumElements() < coefficient_elements) { - coefficients_ = ct::Tensor(ct::DataTypeToEnum::value, ct::DeviceTypeToEnum::value, {coefficient_elements}); + coefficients_ = + ct::Tensor(ct::DataTypeToEnum::value, ct::DeviceTypeToEnum::value, {coefficient_elements}); } host_coefficients_.resize(coefficient_elements); - const T* a_end = a + nvec; - const T* b_end = b + nvec; + const T *a_end = a + nvec; + const T *b_end = b + nvec; const typename std::vector::iterator second_begin = host_coefficients_.begin() + nvec; std::copy(a, a_end, host_coefficients_.begin()); std::copy(b, b_end, second_begin); - T* coeff = coefficients_.template data(); - base_device::memory::synchronize_memory_op()(coeff, - host_coefficients_.data(), - coefficient_elements); - const T* second_coefficients = coeff + nvec; - linear_op().bicg_update(ld, dim, nvec, direction, out, x, y, coeff, second_coefficients); + T *coeff = coefficients_.template data(); + base_device::memory::synchronize_memory_op()( + coeff, host_coefficients_.data(), coefficient_elements); + const T *second_coefficients = coeff + nvec; + linear_op().bicg_update( + ld, + dim, + nvec, + direction, + out, + x, + y, + coeff, + second_coefficients); ModuleBase::timer::end("LinearWorkspace", "bicg_update"); } template -void LinearWorkspace::cgs_direction(int ld, int dim, int nvec, T* p, T* u, const T* r, const T* q, const T* beta) +void LinearWorkspace::cgs_direction( + int ld, + int dim, + int nvec, + T *p, + T *u, + const T *r, + const T *q, + const T *beta) { ModuleBase::timer::start("LinearWorkspace", "cgs_direction"); if (dim == 0 || nvec == 0) @@ -396,14 +566,30 @@ void LinearWorkspace::cgs_direction(int ld, int dim, int nvec, T* p, { coefficients_ = ct::Tensor(ct::DataTypeToEnum::value, ct::DeviceTypeToEnum::value, {nvec}); } - T* coeff = coefficients_.template data(); + T *coeff = coefficients_.template data(); base_device::memory::synchronize_memory_op()(coeff, beta, nvec); - linear_op().cgs_direction(ld, dim, nvec, p, u, r, q, coeff); + linear_op().cgs_direction( + ld, + dim, + nvec, + p, + u, + r, + q, + coeff); ModuleBase::timer::end("LinearWorkspace", "cgs_direction"); } template -void LinearWorkspace::cgs_alpha(int ld, int dim, int nvec, T* q, T* direction, const T* u, const T* v, const T* alpha) +void LinearWorkspace::cgs_alpha( + int ld, + int dim, + int nvec, + T *q, + T *direction, + const T *u, + const T *v, + const T *alpha) { ModuleBase::timer::start("LinearWorkspace", "cgs_alpha"); if (dim == 0 || nvec == 0) @@ -411,14 +597,29 @@ void LinearWorkspace::cgs_alpha(int ld, int dim, int nvec, T* q, T* d ModuleBase::timer::end("LinearWorkspace", "cgs_alpha"); return; } - T* coeff = coefficients_.template data(); + T *coeff = coefficients_.template data(); base_device::memory::synchronize_memory_op()(coeff, alpha, nvec); - linear_op().cgs_alpha(ld, dim, nvec, q, direction, u, v, coeff); + linear_op().cgs_alpha( + ld, + dim, + nvec, + q, + direction, + u, + v, + coeff); ModuleBase::timer::end("LinearWorkspace", "cgs_alpha"); } template -void LinearWorkspace::cgs_finish(int ld, int dim, int nvec, T* x, T* r, const T* direction, const T* ad) +void LinearWorkspace::cgs_finish( + int ld, + int dim, + int nvec, + T *x, + T *r, + const T *direction, + const T *ad) { ModuleBase::timer::start("LinearWorkspace", "cgs_finish"); if (dim == 0 || nvec == 0) @@ -426,7 +627,15 @@ void LinearWorkspace::cgs_finish(int ld, int dim, int nvec, T* x, T* ModuleBase::timer::end("LinearWorkspace", "cgs_finish"); return; } - linear_op().cgs_finish(ld, dim, nvec, x, r, direction, ad, coefficients_.template data()); + linear_op().cgs_finish( + ld, + dim, + nvec, + x, + r, + direction, + ad, + coefficients_.template data()); ModuleBase::timer::end("LinearWorkspace", "cgs_finish"); } template class LinearWorkspace, base_device::DEVICE_CPU>; diff --git a/source/source_hsolver/module_linalg/linear_workspace.h b/source/source_hsolver/module_linalg/linear_workspace.h index 7b955f7e484..8bba4f9fe24 100644 --- a/source/source_hsolver/module_linalg/linear_workspace.h +++ b/source/source_hsolver/module_linalg/linear_workspace.h @@ -19,18 +19,17 @@ namespace hsolver class LinearSolveTimer { private: - const char* solver_; + const char *solver_; public: - explicit LinearSolveTimer(const char* solver); + explicit LinearSolveTimer(const char *solver); ~LinearSolveTimer(); - LinearSolveTimer(const LinearSolveTimer&) = delete; - LinearSolveTimer& operator=(const LinearSolveTimer&) = delete; + LinearSolveTimer(const LinearSolveTimer &) = delete; + LinearSolveTimer &operator=(const LinearSolveTimer &) = delete; }; /** @brief Reusable device arrays and collective block algebra for linear solvers. */ -template -class LinearWorkspace +template class LinearWorkspace { private: ct::Tensor vectors_; @@ -49,62 +48,156 @@ class LinearWorkspace public: /** @brief Allocate only the recurrence slots required by the calling solver. */ - LinearWorkspace(const diag_comm_info& comm, int slots); - void prepare(const int ld, const int dim, const int nvec, const T* x, const T* b, const double tolerance, const int max_iter); + LinearWorkspace(const diag_comm_info &comm, int slots); + void prepare( + const int ld, + const int dim, + const int nvec, + const T *x, + const T *b, + const double tolerance, + const int max_iter); /** @brief Initialize recurrence storage once per cycle, including padding. */ void clear(); void reset_statistics(); - void statistics(LinearSolveResult* result) const; - void swap_vectors(int ld, int dim, int slots, int stride, T* vectors, const std::vector& pairs); - void apply(const LinearOperator& op, const T* x, T* y, int ld, int nvec); - void cgs_direction(int ld, int dim, int nvec, T* p, T* u, const T* r, const T* q, const T* beta); - void cgs_alpha(int ld, int dim, int nvec, T* q, T* direction, const T* u, const T* v, const T* alpha); + void statistics(LinearSolveResult *result) const; + void swap_vectors( + int ld, + int dim, + int slots, + int stride, + T *vectors, + const std::vector &pairs); + void apply( + const LinearOperator &op, + const T *x, + T *y, + int ld, + int nvec); + void cgs_direction( + int ld, + int dim, + int nvec, + T *p, + T *u, + const T *r, + const T *q, + const T *beta); + void cgs_alpha( + int ld, + int dim, + int nvec, + T *q, + T *direction, + const T *u, + const T *v, + const T *alpha); /** @brief Finish the CGS step using the coefficients uploaded by cgs_alpha. */ - void cgs_finish(int ld, int dim, int nvec, T* x, T* r, const T* direction, const T* ad); - T* data(const int slot); - void dot(const int ld, const int dim, const int nvec, const T* x, const T* y, T* out); + void cgs_finish( + int ld, + int dim, + int nvec, + T *x, + T *r, + const T *direction, + const T *ad); + T *data(const int slot); + void dot( + const int ld, + const int dim, + const int nvec, + const T *x, + const T *y, + T *out); /** @brief Return two packed global inner products in a single collective. */ - void dot_pair(int ld, int dim, int nvec, const T* x, const T* y, const T* z, const T* w, T* first, T* second); + void dot_pair( + int ld, + int dim, + int nvec, + const T *x, + const T *y, + const T *z, + const T *w, + T *first, + T *second); /** @brief Upload per-column coefficients and update all unmasked columns. */ - void batch(int ld, int dim, int nvec, T* out, const T* x, const T* y, T a, T b, const T* ca, const T* cb, const int* skip); + void batch( + int ld, + int dim, + int nvec, + T *out, + const T *x, + const T *y, + T a, + T b, + const T *ca, + const T *cb, + const int *skip); /** @brief Upload coefficients once; the caller reports zeros to select the GPU update path. */ - void gmres_update(int ld, - int dim, - int nvec, - T* solution, - T* residual, - const T* direction, - const T* image, - const T* coefficients, - bool has_zero_coefficients); + void gmres_update( + int ld, + int dim, + int nvec, + T *solution, + T *residual, + const T *direction, + const T *image, + const T *coefficients, + bool has_zero_coefficients); /** @brief Reorder solver columns after host convergence decisions. */ - void swap_columns(int ld, int dim, const std::vector& pairs); + void swap_columns(int ld, int dim, const std::vector &pairs); /** @brief Restore original column order into caller storage. */ - void restore(int ld, int dim, int nvec, const std::vector& order, const T* source, T* destination); + void restore( + int ld, + int dim, + int nvec, + const std::vector &order, + const T *source, + T *destination); /** @brief Upload two coefficient arrays and execute a fused recurrence. */ - void bicg_update(int ld, int dim, int nvec, bool direction, T* out, const T* x, const T* y, const T* a, const T* b); - void copy(const int ld, const int dim, const int nvec, const T* x, T* y) const; - void residual(const LinearOperator& op, const int ld, const int dim, const int nvec, const T* x, const T* b, T* r); - void verify(const LinearOperator& op, - const int ld, - const int dim, - const int nvec, - const T* x, - const T* b, - const std::vector& threshold, - T* scratch, - LinearSolveResult* result); + void bicg_update( + int ld, + int dim, + int nvec, + bool direction, + T *out, + const T *x, + const T *y, + const T *a, + const T *b); + void copy( + const int ld, + const int dim, + const int nvec, + const T *x, + T *y) const; + void residual( + const LinearOperator &op, + const int ld, + const int dim, + const int nvec, + const T *x, + const T *b, + T *r); + void verify( + const LinearOperator &op, + const int ld, + const int dim, + const int nvec, + const T *x, + const T *b, + const std::vector &threshold, + T *scratch, + LinearSolveResult *result); }; -template -bool linear_bad_divisor(const T value) +template bool linear_bad_divisor(const T value) { using Real = typename GetTypeReal::type; return !std::isfinite(std::abs(value)) || std::abs(value) <= std::numeric_limits::min(); } -template -double linear_norm(const T value) +template double linear_norm(const T value) { return std::isfinite(std::abs(value)) ? std::sqrt(std::max(0.0, static_cast(std::real(value)))) : std::numeric_limits::infinity(); diff --git a/source/source_hsolver/module_linalg/orb_buffer.cpp b/source/source_hsolver/module_linalg/orb_buffer.cpp index 4edc782e9ed..1b836500122 100644 --- a/source/source_hsolver/module_linalg/orb_buffer.cpp +++ b/source/source_hsolver/module_linalg/orb_buffer.cpp @@ -4,27 +4,30 @@ namespace hsolver { -template -void zero_pad(T* data, const OrbLayout& layout) +template void zero_pad(T *data, const OrbLayout &layout) { const int stride = layout.ld / layout.npol; if (stride > layout.npw && layout.bands > 0) { - T* tail = data + layout.npw; + T *tail = data + layout.npw; const size_t width = stride - layout.npw; const size_t columns = static_cast(layout.bands) * layout.npol; - base_device::memory::set_memory_2d_op()(tail, stride, 0, width, columns); + base_device::memory::set_memory_2d_op()( + tail, + stride, + 0, + width, + columns); } } -template -const T* clean_orbs(const T* input, const OrbLayout& layout, ct::Tensor* scratch) +template const T *clean_orbs(const T *input, const OrbLayout &layout, ct::Tensor *scratch) { if (layout.npol == 1) { return input; } - T* data = scratch->template data(); + T *data = scratch->template data(); const size_t count = static_cast(layout.ld) * layout.bands; if (count > 0) { @@ -34,22 +37,18 @@ const T* clean_orbs(const T* input, const OrbLayout& layout, ct::Tensor* scratch return data; } -template void zero_pad, base_device::DEVICE_CPU>(std::complex*, const OrbLayout&); -template void zero_pad, base_device::DEVICE_CPU>(std::complex*, const OrbLayout&); -template const std::complex* clean_orbs, base_device::DEVICE_CPU>(const std::complex*, - const OrbLayout&, - ct::Tensor*); -template const std::complex* clean_orbs, base_device::DEVICE_CPU>(const std::complex*, - const OrbLayout&, - ct::Tensor*); +template void zero_pad, base_device::DEVICE_CPU>(std::complex *, const OrbLayout &); +template void zero_pad, base_device::DEVICE_CPU>(std::complex *, const OrbLayout &); +template const std::complex *clean_orbs, base_device::DEVICE_CPU>( + const std::complex *, const OrbLayout &, ct::Tensor *); +template const std::complex *clean_orbs, base_device::DEVICE_CPU>( + const std::complex *, const OrbLayout &, ct::Tensor *); #if defined(__CUDA) || defined(__ROCM) -template void zero_pad, base_device::DEVICE_GPU>(std::complex*, const OrbLayout&); -template void zero_pad, base_device::DEVICE_GPU>(std::complex*, const OrbLayout&); -template const std::complex* clean_orbs, base_device::DEVICE_GPU>(const std::complex*, - const OrbLayout&, - ct::Tensor*); -template const std::complex* clean_orbs, base_device::DEVICE_GPU>(const std::complex*, - const OrbLayout&, - ct::Tensor*); +template void zero_pad, base_device::DEVICE_GPU>(std::complex *, const OrbLayout &); +template void zero_pad, base_device::DEVICE_GPU>(std::complex *, const OrbLayout &); +template const std::complex *clean_orbs, base_device::DEVICE_GPU>( + const std::complex *, const OrbLayout &, ct::Tensor *); +template const std::complex *clean_orbs, base_device::DEVICE_GPU>( + const std::complex *, const OrbLayout &, ct::Tensor *); #endif } // namespace hsolver diff --git a/source/source_hsolver/module_linalg/orb_buffer.h b/source/source_hsolver/module_linalg/orb_buffer.h index 1f6960d832d..7969ee419ed 100644 --- a/source/source_hsolver/module_linalg/orb_buffer.h +++ b/source/source_hsolver/module_linalg/orb_buffer.h @@ -16,13 +16,12 @@ struct OrbLayout }; /** @brief Clear all spin-component tails, including scalar-orbital padding. */ -template -void zero_pad(T* data, const OrbLayout& layout); +template void zero_pad(T *data, const OrbLayout &layout); /** @brief Borrow scalar input, or copy spinors into scratch and clear their padding. * Scratch must have room for ld * bands coefficients; input remains unchanged. */ template -const T* clean_orbs(const T* input, const OrbLayout& layout, ct::Tensor* scratch); +const T *clean_orbs(const T *input, const OrbLayout &layout, ct::Tensor *scratch); } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/orthonormal.cpp b/source/source_hsolver/module_linalg/orthonormal.cpp index 942056a1000..78fbe1f7cb0 100644 --- a/source/source_hsolver/module_linalg/orthonormal.cpp +++ b/source/source_hsolver/module_linalg/orthonormal.cpp @@ -9,7 +9,7 @@ namespace hsolver { namespace { -bool positive_norms(const std::vector>& gram, int bands) +bool positive_norms(const std::vector> &gram, int bands) { for (int band = 0; band < bands; ++band) { @@ -23,36 +23,54 @@ bool positive_norms(const std::vector>& gram, int bands) } // namespace template -Orthonormal::Orthonormal(const diag_comm_info& comm, LinearAlgebra& algebra) : comm_(comm), algebra_(algebra) +Orthonormal::Orthonormal(const diag_comm_info &comm, LinearAlgebra &algebra) + : comm_(comm), algebra_(algebra) { } template -std::vector> Orthonormal::gram(const T* input, int ld, int dim, int bands) +std::vector> Orthonormal::gram( + const T *input, + int ld, + int dim, + int bands) { ModuleBase::timer::start("Orthonormal", "gram"); - std::vector> result = algebra_.gram(ld, dim, bands, input); + std::vector> result = algebra_.gram( + ld, + dim, + bands, + input); ModuleBase::timer::end("Orthonormal", "gram"); return result; } template -bool Orthonormal::factor(const std::vector>& g, - int bands, - OrthMethod method, - std::vector>* transform) +bool Orthonormal::factor( + const std::vector> &g, + int bands, + OrthMethod method, + std::vector> *transform) { ModuleBase::timer::start("Orthonormal", "factor"); bool valid = false; if (comm_.rank == 0) { - valid = orth_transform(g, bands, method, transform); + valid = orth_transform( + g, + bands, + method, + transform); } double status = static_cast(valid); #ifdef __MPI if (comm_.nproc > 1) { - Parallel_Common::bcast_data(&status, 1, comm_.comm, 0); + Parallel_Common::bcast_data( + &status, + 1, + comm_.comm, + 0); } #endif valid = status != 0.0; @@ -64,7 +82,11 @@ bool Orthonormal::factor(const std::vector>& g, { // Every partition must apply the same coefficients and take the same recovery branch. const int elements = bands * bands; - Parallel_Common::bcast_data(transform->data(), elements, comm_.comm, 0); + Parallel_Common::bcast_data( + transform->data(), + elements, + comm_.comm, + 0); } #endif } @@ -73,12 +95,13 @@ bool Orthonormal::factor(const std::vector>& g, } template -void Orthonormal::rotate(const T* input, - T* output, - int ld, - int dim, - int bands, - const std::vector>& transform) +void Orthonormal::rotate( + const T *input, + T *output, + int ld, + int dim, + int bands, + const std::vector> &transform) { ModuleBase::timer::start("Orthonormal", "rotate"); std::vector> delta(transform); @@ -88,26 +111,51 @@ void Orthonormal::rotate(const T* input, } if (dim > 0) { - base_device::memory::synchronize_memory_2d_op()(output, ld, input, ld, dim, bands); + base_device::memory::synchronize_memory_2d_op()( + output, + ld, + input, + ld, + dim, + bands); } - algebra_.expand(ld, dim, bands, bands, input, delta, output, T(1)); + algebra_.expand( + ld, + dim, + bands, + bands, + input, + delta, + output, + T(1)); ModuleBase::timer::end("Orthonormal", "rotate"); } template -bool Orthonormal::try_candidate(T* input, - int ld, - int dim, - int bands, - const std::vector>& transform, - std::vector>* g, - OrthResult* result) +bool Orthonormal::try_candidate( + T *input, + int ld, + int dim, + int bands, + const std::vector> &transform, + std::vector> *g, + OrthResult *result) { ModuleBase::timer::start("Orthonormal", "try_candidate"); - T* candidate = candidate_.template data(); - rotate(input, candidate, ld, dim, bands, transform); + T *candidate = candidate_.template data(); + rotate( + input, + candidate, + ld, + dim, + bands, + transform); ++result->passes; - const std::vector> check = gram(candidate, ld, dim, bands); + const std::vector> check = gram( + candidate, + ld, + dim, + bands); const double error = orth_error(check, bands); const bool valid = std::isfinite(error) && positive_norms(check, bands); const bool improved = valid && error < result->after; @@ -115,7 +163,13 @@ bool Orthonormal::try_candidate(T* input, { if (dim > 0) { - base_device::memory::synchronize_memory_2d_op()(input, ld, candidate, ld, dim, bands); + base_device::memory::synchronize_memory_2d_op()( + input, + ld, + candidate, + ld, + dim, + bands); } result->after = error; *g = check; @@ -124,7 +178,8 @@ bool Orthonormal::try_candidate(T* input, else if (!valid || result->after > orth_tolerance()) { result->failure = valid ? OrthFailure::no_improvement : OrthFailure::invalid_candidate; - result->reason += std::string(orth_method_name(result->actual)) + ": " + orth_failure_name(result->failure) + "; "; + result->reason += + std::string(orth_method_name(result->actual)) + ": " + orth_failure_name(result->failure) + "; "; ++result->rejected; } ModuleBase::timer::end("Orthonormal", "try_candidate"); @@ -132,13 +187,14 @@ bool Orthonormal::try_candidate(T* input, } template -void Orthonormal::correct(T* input, - int ld, - int dim, - int bands, - OrthMethod method, - std::vector>* g, - OrthResult* result) +void Orthonormal::correct( + T *input, + int ld, + int dim, + int bands, + OrthMethod method, + std::vector> *g, + OrthResult *result) { ModuleBase::timer::start("Orthonormal", "correct"); std::vector methods{method}; @@ -164,15 +220,27 @@ void Orthonormal::correct(T* input, { ++result->fallbacks; } - const bool factored = factor(*g, bands, result->actual, &transform); + const bool factored = factor( + *g, + bands, + result->actual, + &transform); if (factored) { - improved = try_candidate(input, ld, dim, bands, transform, g, result); + improved = try_candidate( + input, + ld, + dim, + bands, + transform, + g, + result); } else { result->failure = OrthFailure::factorization_failed; - result->reason += std::string(orth_method_name(result->actual)) + ": " + orth_failure_name(result->failure) + "; "; + result->reason += + std::string(orth_method_name(result->actual)) + ": " + orth_failure_name(result->failure) + "; "; } if (improved || result->after <= orth_tolerance()) { @@ -196,14 +264,24 @@ void Orthonormal::correct(T* input, } template -OrthResult Orthonormal::inspect(const T* input, int ld, int dim, int bands, bool full_gram, bool collect_norms) +OrthResult Orthonormal::inspect( + const T *input, + int ld, + int dim, + int bands, + bool full_gram, + bool collect_norms) { ModuleBase::timer::start("Orthonormal", "inspect"); OrthResult result; result.gram_checked = full_gram; if (full_gram) { - const std::vector> g = gram(input, ld, dim, bands); + const std::vector> g = gram( + input, + ld, + dim, + bands); result.before = orth_error(g, bands); result.after = result.before; for (int band = 0; collect_norms && band < bands; ++band) @@ -229,7 +307,14 @@ OrthResult Orthonormal::inspect(const T* input, int ld, int dim, int std::vector> norms; if (bands > 0) { - norms = algebra_.dots(ld, dim, bands, 1, 0, input, input); + norms = algebra_.dots( + ld, + dim, + bands, + 1, + 0, + input, + input); } result.status = OrthStatus::disabled; for (int band = 0; band < bands; ++band) @@ -263,18 +348,34 @@ OrthResult Orthonormal::inspect(const T* input, int ld, int dim, int } template -OrthResult Orthonormal::apply(T* input, int ld, int dim, int bands, OrthMethod method, bool collect_norms) +OrthResult Orthonormal::apply( + T *input, + int ld, + int dim, + int bands, + OrthMethod method, + bool collect_norms) { ModuleBase::timer::start("Orthonormal", "apply"); if (method == OrthMethod::none) { - const OrthResult result = inspect(input, ld, dim, bands, false, collect_norms); + const OrthResult result = inspect( + input, + ld, + dim, + bands, + false, + collect_norms); ModuleBase::timer::end("Orthonormal", "apply"); return result; } OrthResult result; result.gram_checked = true; - std::vector> g = gram(input, ld, dim, bands); + std::vector> g = gram( + input, + ld, + dim, + bands); result.before = orth_error(g, bands); result.after = result.before; if (!std::isfinite(result.before)) @@ -291,7 +392,14 @@ OrthResult Orthonormal::apply(T* input, int ld, int dim, int bands, O } else { - correct(input, ld, dim, bands, method, &g, &result); + correct( + input, + ld, + dim, + bands, + method, + &g, + &result); } for (int i = 0; collect_norms && i < bands; ++i) { diff --git a/source/source_hsolver/module_linalg/orthonormal.h b/source/source_hsolver/module_linalg/orthonormal.h index aea343b2d91..cfb78ad49fc 100644 --- a/source/source_hsolver/module_linalg/orthonormal.h +++ b/source/source_hsolver/module_linalg/orthonormal.h @@ -16,8 +16,8 @@ enum class OrthMethod lowdin, newton_schulz }; -OrthMethod parse_orth_method(const std::string& name); -const char* orth_method_name(OrthMethod method); +OrthMethod parse_orth_method(const std::string &name); +const char *orth_method_name(OrthMethod method); enum class OrthStatus { @@ -39,11 +39,10 @@ enum class OrthFailure no_improvement, tolerance_not_met }; -const char* orth_failure_name(OrthFailure failure); +const char *orth_failure_name(OrthFailure failure); /** @brief Acceptance tolerance for the native wavefunction precision. */ -template -constexpr double orth_tolerance() +template constexpr double orth_tolerance() { return std::is_same>::value ? 1e-6 : 1e-12; } @@ -65,40 +64,74 @@ struct OrthResult }; /** @brief Maximum elementwise distance from the identity, including nonfinite detection. */ -double orth_error(const std::vector>& gram, int bands); +double orth_error(const std::vector> &gram, int bands); /** @brief Construct one correction; the caller validates candidates and chooses fallbacks. */ -bool orth_transform(const std::vector>& gram, - int bands, - OrthMethod method, - std::vector>* transform); +bool orth_transform( + const std::vector> &gram, + int bands, + OrthMethod method, + std::vector> *transform); /** @brief Pool-local orthonormalization with FP64 products and native-precision updates. */ -template -class Orthonormal +template class Orthonormal { private: const diag_comm_info comm_; - LinearAlgebra& algebra_; + LinearAlgebra &algebra_; ct::Tensor candidate_; - bool factor(const std::vector>& gram, int bands, OrthMethod method, std::vector>* transform); - bool try_candidate(T* input, - int ld, - int dim, - int bands, - const std::vector>& transform, - std::vector>* gram, - OrthResult* result); - void correct(T* input, int ld, int dim, int bands, OrthMethod method, std::vector>* gram, OrthResult* result); - void rotate(const T* input, T* output, int ld, int dim, int bands, const std::vector>& transform); - std::vector> gram(const T* input, int ld, int dim, int bands); + bool factor( + const std::vector> &gram, + int bands, + OrthMethod method, + std::vector> *transform); + bool try_candidate( + T *input, + int ld, + int dim, + int bands, + const std::vector> &transform, + std::vector> *gram, + OrthResult *result); + void correct( + T *input, + int ld, + int dim, + int bands, + OrthMethod method, + std::vector> *gram, + OrthResult *result); + void rotate( + const T *input, + T *output, + int ld, + int dim, + int bands, + const std::vector> &transform); + std::vector> gram( + const T *input, + int ld, + int dim, + int bands); public: /** @brief Borrow sequential-use workspace with the same communicator; it must outlive this object. */ - Orthonormal(const diag_comm_info& comm, LinearAlgebra& algebra); + Orthonormal(const diag_comm_info &comm, LinearAlgebra &algebra); /** @brief Inspect norms and optionally overlaps without changing orbitals or enforcing a correction tolerance. */ - OrthResult inspect(const T* input, int ld, int dim, int bands, bool full_gram, bool collect_norms); + OrthResult inspect( + const T *input, + int ld, + int dim, + int bands, + bool full_gram, + bool collect_norms); /** @brief Preserve input on rejected candidates; a failed overall result is not usable. */ - OrthResult apply(T* input, int ld, int dim, int bands, OrthMethod method, bool collect_norms); + OrthResult apply( + T *input, + int ld, + int dim, + int bands, + OrthMethod method, + bool collect_norms); }; } // namespace hsolver #endif diff --git a/source/source_hsolver/module_linalg/orthonormal_small.cpp b/source/source_hsolver/module_linalg/orthonormal_small.cpp index b982b347cba..aa6a63cc1cb 100644 --- a/source/source_hsolver/module_linalg/orthonormal_small.cpp +++ b/source/source_hsolver/module_linalg/orthonormal_small.cpp @@ -23,24 +23,52 @@ std::vector identity(int n) return result; } -void multiply(const std::vector& a, const std::vector& b, int n, std::vector* result) +void multiply( + const std::vector &a, + const std::vector &b, + int n, + std::vector *result) { const Wide one(1); const Wide zero(0); - ModuleBase::gemm_op()('N', 'N', n, n, n, &one, a.data(), n, b.data(), n, &zero, result->data(), n); + ModuleBase::gemm_op()( + 'N', + 'N', + n, + n, + n, + &one, + a.data(), + n, + b.data(), + n, + &zero, + result->data(), + n); } -bool cholesky(const std::vector& g, int n, std::vector* c) +bool cholesky(const std::vector &g, int n, std::vector *c) { ModuleBase::timer::start("Orthonormal", "cholesky"); *c = g; const char upper = 'U'; const char nonunit = 'N'; int info = 0; - ct::lapackConnector::potrf(upper, n, c->data(), n, info); + ct::lapackConnector::potrf( + upper, + n, + c->data(), + n, + info); if (info == 0) { - ct::lapackConnector::trtri(upper, nonunit, n, c->data(), n, info); + ct::lapackConnector::trtri( + upper, + nonunit, + n, + c->data(), + n, + info); } for (int j = 0; j < n; ++j) { @@ -53,7 +81,7 @@ bool cholesky(const std::vector& g, int n, std::vector* c) return info == 0; } -bool lowdin(const std::vector& g, int n, std::vector* c) +bool lowdin(const std::vector &g, int n, std::vector *c) { ModuleBase::timer::start("Orthonormal", "lowdin"); std::vector u(g); @@ -67,21 +95,22 @@ bool lowdin(const std::vector& g, int n, std::vector* c) std::vector rwork(lrwork); std::vector iwork(liwork); int info = 0; - ct::lapackConnector::heevd(vectors, - upper, - n, - u.data(), - n, - values.data(), - work.data(), - lwork, - rwork.data(), - lrwork, - iwork.data(), - liwork, - info); + ct::lapackConnector::heevd( + vectors, + upper, + n, + u.data(), + n, + values.data(), + work.data(), + lwork, + rwork.data(), + lrwork, + iwork.data(), + liwork, + info); bool valid = info == 0; - for (double value: values) + for (double value : values) { valid = valid && std::isfinite(value) && value > 0.0; } @@ -98,13 +127,26 @@ bool lowdin(const std::vector& g, int n, std::vector* c) c->resize(n * n); const Wide one(1); const Wide zero(0); - ModuleBase::gemm_op()('N', 'C', n, n, n, &one, scaled.data(), n, u.data(), n, &zero, c->data(), n); + ModuleBase::gemm_op()( + 'N', + 'C', + n, + n, + n, + &one, + scaled.data(), + n, + u.data(), + n, + &zero, + c->data(), + n); } ModuleBase::timer::end("Orthonormal", "lowdin"); return valid; } -bool newton_schulz(const std::vector& g, int n, std::vector* c) +bool newton_schulz(const std::vector &g, int n, std::vector *c) { ModuleBase::timer::start("Orthonormal", "newton_schulz"); constexpr double iteration_tol = 1e-12; @@ -138,8 +180,16 @@ bool newton_schulz(const std::vector& g, int n, std::vector* c) std::vector residual(n * n); for (int iteration = 1; iteration <= max_updates; ++iteration) { - multiply(*c, *c, n, &square); - multiply(g, square, n, &residual); + multiply( + *c, + *c, + n, + &square); + multiply( + g, + square, + n, + &residual); if (orth_error(residual, n) <= iteration_tol) { valid = true; @@ -154,7 +204,11 @@ bool newton_schulz(const std::vector& g, int n, std::vector* c) residual[i] = 0.5 * (3.0 * unit[i] - residual[i]); } // The square is no longer needed; reuse its storage without aliasing GEMM inputs. - multiply(*c, residual, n, &square); + multiply( + *c, + residual, + n, + &square); c->swap(square); } } @@ -163,7 +217,7 @@ bool newton_schulz(const std::vector& g, int n, std::vector* c) } } // namespace -const char* orth_method_name(OrthMethod method) +const char *orth_method_name(OrthMethod method) { switch (method) { @@ -178,9 +232,12 @@ const char* orth_method_name(OrthMethod method) } } -OrthMethod parse_orth_method(const std::string& name) +OrthMethod parse_orth_method(const std::string &name) { - for (OrthMethod method: {OrthMethod::none, OrthMethod::cholesky, OrthMethod::lowdin, OrthMethod::newton_schulz}) + for (OrthMethod method : {OrthMethod::none, + OrthMethod::cholesky, + OrthMethod::lowdin, + OrthMethod::newton_schulz}) { if (name == orth_method_name(method)) { @@ -190,7 +247,7 @@ OrthMethod parse_orth_method(const std::string& name) throw std::invalid_argument("Unsupported orthonormalization method: " + name); } -double orth_error(const std::vector& g, int n) +double orth_error(const std::vector &g, int n) { double error = 0.0; for (int j = 0; j < n; ++j) @@ -209,7 +266,7 @@ double orth_error(const std::vector& g, int n) return error; } -const char* orth_failure_name(OrthFailure failure) +const char *orth_failure_name(OrthFailure failure) { switch (failure) { @@ -232,7 +289,11 @@ const char* orth_failure_name(OrthFailure failure) } } -bool orth_transform(const std::vector& g, int n, OrthMethod method, std::vector* c) +bool orth_transform( + const std::vector &g, + int n, + OrthMethod method, + std::vector *c) { ModuleBase::timer::start("Orthonormal", "orth_transform"); bool valid = false; @@ -250,7 +311,7 @@ bool orth_transform(const std::vector& g, int n, OrthMethod method, std::v default: valid = false; } - for (const Wide& value: *c) + for (const Wide &value : *c) { valid = valid && std::isfinite(std::abs(value)); } diff --git a/source/source_hsolver/module_linalg/unittests/linear_test_utils.h b/source/source_hsolver/module_linalg/unittests/linear_test_utils.h index b3c49647677..e04a4725cfa 100644 --- a/source/source_hsolver/module_linalg/unittests/linear_test_utils.h +++ b/source/source_hsolver/module_linalg/unittests/linear_test_utils.h @@ -24,8 +24,7 @@ inline hsolver::diag_comm_info world_comm() } // Column-major matrices with a decoupled first eigenvector and a dense complex block. -template -std::vector hermitian(const int n) +template std::vector hermitian(const int n) { std::vector h(n * n, T(0)); for (int j = 0; j < n; ++j) @@ -42,7 +41,11 @@ std::vector hermitian(const int n) } template -std::vector multiply(const std::vector& a, const std::vector& x, const int n, const int nvec) +std::vector multiply( + const std::vector &a, + const std::vector &x, + const int n, + const int nvec) { std::vector b(n * nvec, T(0)); for (int band = 0; band < nvec; ++band) @@ -60,7 +63,13 @@ std::vector multiply(const std::vector& a, const std::vector& x, const // Extract the local rows of column-major reference data, leaving padding zero. template -std::vector local_columns(const std::vector& global, int n, int columns, int start, int dim, int ld) +std::vector local_columns( + const std::vector &global, + int n, + int columns, + int start, + int dim, + int ld) { std::vector local(ld * columns, T(0)); for (int j = 0; j < columns; ++j) @@ -75,18 +84,37 @@ std::vector local_columns(const std::vector& global, int n, int columns, i // Solve in double precision even when the iterative input was rounded to float. template -std::vector lapack_solve(const std::vector& a, const std::vector& b, const int n, const int nvec) +std::vector lapack_solve( + const std::vector &a, + const std::vector &b, + const int n, + const int nvec) { std::vector lu(a.begin(), a.end()); std::vector x(b.begin(), b.end()); std::vector pivots(n); int info = 0; - container::lapackConnector::getrf(n, n, lu.data(), n, pivots.data(), info); + container::lapackConnector::getrf( + n, + n, + lu.data(), + n, + pivots.data(), + info); if (info != 0) { throw std::runtime_error("LAPACK reference factorization failed"); } - container::lapackConnector::getrs('N', n, nvec, lu.data(), n, pivots.data(), x.data(), n, info); + container::lapackConnector::getrs( + 'N', + n, + nvec, + lu.data(), + n, + pivots.data(), + x.data(), + n, + info); if (info != 0) { throw std::runtime_error("LAPACK reference solve failed"); diff --git a/source/source_hsolver/module_linalg/unittests/test_cn_subspace.cpp b/source/source_hsolver/module_linalg/unittests/test_cn_subspace.cpp index db79d3be884..d4a0a0b5e76 100644 --- a/source/source_hsolver/module_linalg/unittests/test_cn_subspace.cpp +++ b/source/source_hsolver/module_linalg/unittests/test_cn_subspace.cpp @@ -8,8 +8,7 @@ namespace { -template -class CNSubspaceTest : public testing::Test +template class CNSubspaceTest : public testing::Test { }; using Precisions = testing::Types; @@ -40,7 +39,7 @@ TYPED_TEST(CNSubspaceTest, ProjectedSeedAndResidualOrRankDeficientFallback) } hsolver::LinearAlgebra algebra(comm); hsolver::CNSubspace projection; - for (const bool singular: {false, true}) + for (const bool singular : {false, true}) { SCOPED_TRACE(singular); std::vector u(n * bands); @@ -52,11 +51,37 @@ TYPED_TEST(CNSubspaceTest, ProjectedSeedAndResidualOrRankDeficientFallback) u[j * n + i] = std::polar(TypeParam(1.0 / std::sqrt(n)), TypeParam(phase)); } } - const std::vector b = linear_test::multiply(right, u, n, bands); - const std::vector au = linear_test::multiply(a, u, n, bands); - const std::vector local_u = linear_test::local_columns(u, n, bands, start, dim, ld); - const std::vector local_b = linear_test::local_columns(b, n, bands, start, dim, ld); - const bool prepared = projection.prepare(algebra, ld, dim, bands, local_u.data(), local_b.data()); + const std::vector b = linear_test::multiply( + right, + u, + n, + bands); + const std::vector au = linear_test::multiply( + a, + u, + n, + bands); + const std::vector local_u = linear_test::local_columns( + u, + n, + bands, + start, + dim, + ld); + const std::vector local_b = linear_test::local_columns( + b, + n, + bands, + start, + dim, + ld); + const bool prepared = projection.prepare( + algebra, + ld, + dim, + bands, + local_u.data(), + local_b.data()); if (singular) { EXPECT_FALSE(prepared); @@ -76,7 +101,11 @@ TYPED_TEST(CNSubspaceTest, ProjectedSeedAndResidualOrRankDeficientFallback) } } } - const std::vector coefficients = linear_test::lapack_solve(projected, projected_rhs, bands, bands); + const std::vector coefficients = linear_test::lapack_solve( + projected, + projected_rhs, + bands, + bands); std::vector actual(n * bands, Wide(0)); for (int j = 0; j < bands; ++j) { @@ -97,7 +126,11 @@ TYPED_TEST(CNSubspaceTest, ProjectedSeedAndResidualOrRankDeficientFallback) Parallel_Common::reduce_data(actual.data(), count, comm.comm); #endif const std::vector wide_a(a.begin(), a.end()); - const std::vector ax = linear_test::multiply(wide_a, actual, n, bands); + const std::vector ax = linear_test::multiply( + wide_a, + actual, + n, + bands); for (int j = 0; j < bands; ++j) { for (int i = 0; i < dim; ++i) diff --git a/source/source_hsolver/module_linalg/unittests/test_exp_action.cpp b/source/source_hsolver/module_linalg/unittests/test_exp_action.cpp index af15cdabca0..db1ddb43932 100644 --- a/source/source_hsolver/module_linalg/unittests/test_exp_action.cpp +++ b/source/source_hsolver/module_linalg/unittests/test_exp_action.cpp @@ -14,8 +14,7 @@ using Wide = std::complex; using Device = base_device::DEVICE_CPU; // A full complex Hermitian matrix, with spin components separated by padding. -template -class DenseExp final : public hsolver::HSOperator +template class DenseExp final : public hsolver::HSOperator { public: std::vector matrix; @@ -23,18 +22,14 @@ class DenseExp final : public hsolver::HSOperator mutable int calls = 0; mutable int cols = 0; - explicit DenseExp(int spins) : npol(spins) - { - matrix = linear_test::hermitian(6); - } - int row(int i, int ld) const - { - return i / (6 / npol) * (ld / npol) + i % (6 / npol); - } - void update_k(int) override - { - } - void hpsi(const T* x, T* y, int ld, int bands) const override + explicit DenseExp(int spins) : npol(spins) { matrix = linear_test::hermitian(6); } + int row(int i, int ld) const { return i / (6 / npol) * (ld / npol) + i % (6 / npol); } + void update_k(int) override {} + void hpsi( + const T *x, + T *y, + int ld, + int bands) const override { ++calls; cols += bands; @@ -50,14 +45,17 @@ class DenseExp final : public hsolver::HSOperator } } } - void spsi(const T*, T*, int, int) const override + void spsi( + const T *, + T *, + int, + int) const override { FAIL() << "PW exponential must not request an overlap operator"; } }; -template -std::vector pack_input(const DenseExp& op, const hsolver::OrbLayout& layout) +template std::vector pack_input(const DenseExp &op, const hsolver::OrbLayout &layout) { std::vector input(layout.ld * layout.bands, T(30, -20)); for (int b = 0; b < layout.bands; ++b) @@ -72,7 +70,7 @@ std::vector pack_input(const DenseExp& op, const hsolver::OrbLayout& layou // Diagonalize the entire matrix, independently of the Krylov approximation. template -std::vector exact_exp(const std::vector& matrix, const std::vector& input, double alpha) +std::vector exact_exp(const std::vector &matrix, const std::vector &input, double alpha) { const int n = 6; const int lwork = 64; @@ -83,7 +81,17 @@ std::vector exact_exp(const std::vector& matrix, const std::vector rwork(3 * n); std::vector values(n); int status = 0; - ct::lapackConnector::heev(job, upper, n, vectors.data(), n, values.data(), work.data(), lwork, rwork.data(), status); + ct::lapackConnector::heev( + job, + upper, + n, + vectors.data(), + n, + values.data(), + work.data(), + lwork, + rwork.data(), + status); EXPECT_EQ(status, 0); std::vector output(n, Wide(0)); for (int k = 0; k < n; ++k) @@ -103,13 +111,14 @@ std::vector exact_exp(const std::vector& matrix, const std::vector -void check_output(const DenseExp& op, - const hsolver::OrbLayout& layout, - const std::vector& input, - const std::vector& output, - double alpha, - int order, - double tol) +void check_output( + const DenseExp &op, + const hsolver::OrbLayout &layout, + const std::vector &input, + const std::vector &output, + double alpha, + int order, + double tol) { for (int b = 0; b < layout.bands; ++b) { @@ -131,7 +140,11 @@ void check_output(const DenseExp& op, const std::vector matrix(op.matrix.begin(), op.matrix.end()); for (int k = 1; k <= order; ++k) { - term = linear_test::multiply(matrix, term, 6, 1); + term = linear_test::multiply( + matrix, + term, + 6, + 1); for (int i = 0; i < 6; ++i) { term[i] *= Wide(0, -alpha / k); @@ -162,8 +175,7 @@ void check_output(const DenseExp& op, } } -template -class ExpTest : public testing::Test +template class ExpTest : public testing::Test { }; using Precisions = testing::Types; @@ -173,22 +185,38 @@ TYPED_TEST(ExpTest, LanczosMatchesFullExponential) { using T = std::complex; const double tol = std::is_same::value ? 2e-5 : 2e-10; - for (int npol: {1, 2}) + for (int npol : {1, 2}) { SCOPED_TRACE(npol); DenseExp op(npol); - const hsolver::OrbLayout layout{10, 6 / npol, npol, 10}; + const hsolver::OrbLayout layout{10, + 6 / npol, + npol, + 10}; const std::vector input = pack_input(op, layout); std::vector output(input.size()); hsolver::ExpControls exp_ctrl; - std::unique_ptr> exp = hsolver::make_exp_action(exp_ctrl, linear_test::world_comm()); - const hsolver::ExpResult result = exp->apply(op, layout, 0.4, input.data(), output.data()); + std::unique_ptr> exp = + hsolver::make_exp_action(exp_ctrl, linear_test::world_comm()); + const hsolver::ExpResult result = exp->apply( + op, + layout, + 0.4, + input.data(), + output.data()); ASSERT_EQ(result.state, hsolver::ExpState::success); EXPECT_TRUE(result.has_err_est); EXPECT_LE(result.rel_err_est, exp->rel_tol()); EXPECT_EQ(result.hpsi_calls, op.calls); EXPECT_EQ(result.hpsi_cols, op.cols); - check_output(op, layout, input, output, 0.4, 0, tol); + check_output( + op, + layout, + input, + output, + 0.4, + 0, + tol); } } @@ -196,51 +224,96 @@ TYPED_TEST(ExpTest, TaylorPairMatchesBothPolynomials) { using T = std::complex; const double tol = std::is_same::value ? 2e-6 : 2e-14; - for (int npol: {1, 2}) + for (int npol : {1, 2}) { SCOPED_TRACE(npol); DenseExp op(npol); - const hsolver::OrbLayout layout{10, 6 / npol, npol, 3}; + const hsolver::OrbLayout layout{10, + 6 / npol, + npol, + 3}; const std::vector input = pack_input(op, layout); std::vector half(input.size()); std::vector full(input.size()); hsolver::ExpControls exp_ctrl; exp_ctrl.expansion = hsolver::ExpMethod::taylor; exp_ctrl.taylor_order = 6; - std::unique_ptr> exp = hsolver::make_exp_action(exp_ctrl, linear_test::world_comm()); - const hsolver::ExpResult result = exp->apply_pair(op, layout, 0.15, input.data(), half.data(), full.data()); + std::unique_ptr> exp = + hsolver::make_exp_action(exp_ctrl, linear_test::world_comm()); + const hsolver::ExpResult result = exp->apply_pair( + op, + layout, + 0.15, + input.data(), + half.data(), + full.data()); ASSERT_EQ(result.state, hsolver::ExpState::success); EXPECT_FALSE(result.has_err_est); EXPECT_EQ(op.calls, exp_ctrl.taylor_order); EXPECT_EQ(result.hpsi_calls, op.calls); EXPECT_EQ(result.hpsi_cols, exp_ctrl.taylor_order * layout.bands); - check_output(op, layout, input, half, 0.15, exp_ctrl.taylor_order, tol); - check_output(op, layout, input, full, 0.3, exp_ctrl.taylor_order, tol); + check_output( + op, + layout, + input, + half, + 0.15, + exp_ctrl.taylor_order, + tol); + check_output( + op, + layout, + input, + full, + 0.3, + exp_ctrl.taylor_order, + tol); } } TEST(ExpAction, LanczosSubdividesOrReportsExhaustion) { DenseExp op(1); - const hsolver::OrbLayout layout{8, 6, 1, 2}; + const hsolver::OrbLayout layout{8, + 6, + 1, + 2}; const std::vector input = pack_input(op, layout); std::vector output(input.size()); hsolver::ExpControls exp_ctrl; exp_ctrl.krylov_max = 4; exp_ctrl.rel_tol = 1e-8; - std::unique_ptr> exp = hsolver::make_exp_action(exp_ctrl, linear_test::world_comm()); - const hsolver::ExpResult result = exp->apply(op, layout, 1.0, input.data(), output.data()); + std::unique_ptr> exp = + hsolver::make_exp_action(exp_ctrl, linear_test::world_comm()); + const hsolver::ExpResult result = exp->apply( + op, + layout, + 1.0, + input.data(), + output.data()); ASSERT_EQ(result.state, hsolver::ExpState::success); EXPECT_GT(result.bisect_count, 0); EXPECT_LE(result.rel_err_est, exp_ctrl.rel_tol); EXPECT_EQ(result.hpsi_calls, op.calls); EXPECT_EQ(result.hpsi_cols, op.cols); - check_output(op, layout, input, output, 1.0, 0, 5e-8); + check_output( + op, + layout, + input, + output, + 1.0, + 0, + 5e-8); exp_ctrl.krylov_max = 2; exp_ctrl.rel_tol = 1e-14; exp = hsolver::make_exp_action(exp_ctrl, linear_test::world_comm()); - const hsolver::ExpResult failed = exp->apply(op, layout, 1.0, input.data(), output.data()); + const hsolver::ExpResult failed = exp->apply( + op, + layout, + 1.0, + input.data(), + output.data()); EXPECT_EQ(failed.state, hsolver::ExpState::failed); EXPECT_GT(failed.bisect_count, 0); EXPECT_GE(failed.failed_band, 0); @@ -250,16 +323,25 @@ TEST(ExpAction, LanczosSubdividesOrReportsExhaustion) TEST(ExpAction, NonfiniteInputReportsOriginalBand) { DenseExp op(2); - const hsolver::OrbLayout layout{10, 3, 2, 10}; + const hsolver::OrbLayout layout{10, + 3, + 2, + 10}; std::vector input = pack_input(op, layout); input[9 * layout.ld] = Wide(std::numeric_limits::quiet_NaN(), 0); std::vector output(input.size()); - for (hsolver::ExpMethod method: {hsolver::ExpMethod::lanczos, hsolver::ExpMethod::taylor}) + for (hsolver::ExpMethod method : {hsolver::ExpMethod::lanczos, hsolver::ExpMethod::taylor}) { hsolver::ExpControls exp_ctrl; exp_ctrl.expansion = method; - std::unique_ptr> exp = hsolver::make_exp_action(exp_ctrl, linear_test::world_comm()); - const hsolver::ExpResult result = exp->apply(op, layout, 0.1, input.data(), output.data()); + std::unique_ptr> exp = + hsolver::make_exp_action(exp_ctrl, linear_test::world_comm()); + const hsolver::ExpResult result = exp->apply( + op, + layout, + 0.1, + input.data(), + output.data()); EXPECT_EQ(result.state, hsolver::ExpState::failed); EXPECT_EQ(result.failed_band, 9); } diff --git a/source/source_hsolver/module_linalg/unittests/test_hsolver_linear.cpp b/source/source_hsolver/module_linalg/unittests/test_hsolver_linear.cpp index f8009bd9ef8..4146ce9cde8 100644 --- a/source/source_hsolver/module_linalg/unittests/test_hsolver_linear.cpp +++ b/source/source_hsolver/module_linalg/unittests/test_hsolver_linear.cpp @@ -10,21 +10,19 @@ namespace { -template -class IdentityOperator final : public hsolver::LinearOperator +template class IdentityOperator final : public hsolver::LinearOperator { private: const int dim_; public: - explicit IdentityOperator(const int dim) : dim_(dim) - { - } - bool is_identity() const override - { - return true; - } - void apply(const T* x, T* y, const int ld, const int nvec) const override + explicit IdentityOperator(const int dim) : dim_(dim) {} + bool is_identity() const override { return true; } + void apply( + const T *x, + T *y, + const int ld, + const int nvec) const override { for (int band = 0; band < nvec; ++band) { @@ -36,16 +34,24 @@ class IdentityOperator final : public hsolver::LinearOperator } }; -template -class DenseOperator final : public hsolver::LinearOperator +template class DenseOperator final : public hsolver::LinearOperator { public: - DenseOperator(const std::vector& matrix, const int n, const int start, const int dim, const hsolver::diag_comm_info& comm) + DenseOperator( + const std::vector &matrix, + const int n, + const int start, + const int dim, + const hsolver::diag_comm_info &comm) : matrix_(matrix), n_(n), start_(start), dim_(dim), comm_(comm) { } - void apply(const T* x, T* y, const int ld, const int nvec) const override + void apply( + const T *x, + T *y, + const int ld, + const int nvec) const override { std::vector full(n_ * nvec, T(0)); for (int band = 0; band < nvec; ++band) @@ -59,7 +65,11 @@ class DenseOperator final : public hsolver::LinearOperator const int count = full.size(); Parallel_Common::reduce_data(full.data(), count, comm_.comm); #endif - const std::vector product = linear_test::multiply(matrix_, full, n_, nvec); + const std::vector product = linear_test::multiply( + matrix_, + full, + n_, + nvec); for (int band = 0; band < nvec; ++band) { for (int i = 0; i < dim_; ++i) @@ -70,22 +80,29 @@ class DenseOperator final : public hsolver::LinearOperator } private: - const std::vector& matrix_; + const std::vector &matrix_; const int n_; const int start_; const int dim_; const hsolver::diag_comm_info comm_; }; -template -class RightPreconditioner final : public hsolver::LinearOperator +template class RightPreconditioner final : public hsolver::LinearOperator { public: - RightPreconditioner(const std::vector& matrix, const int n, const int start, const int dim) + RightPreconditioner( + const std::vector &matrix, + const int n, + const int start, + const int dim) : matrix_(matrix), n_(n), start_(start), dim_(dim) { } - void apply(const T* x, T* y, const int ld, const int nvec) const override + void apply( + const T *x, + T *y, + const int ld, + const int nvec) const override { for (int band = 0; band < nvec; ++band) { @@ -98,14 +115,13 @@ class RightPreconditioner final : public hsolver::LinearOperator } private: - const std::vector& matrix_; + const std::vector &matrix_; const int n_; const int start_; const int dim_; }; -template -class LinearSolveTest : public testing::Test +template class LinearSolveTest : public testing::Test { }; using Precisions = testing::Types; @@ -117,9 +133,10 @@ TYPED_TEST(LinearSolveTest, DenseReferenceAndReusedBatches) const hsolver::diag_comm_info comm = linear_test::world_comm(); const double tolerance = std::is_same::value ? 1e-12 : 2e-6; const double solution_tol = std::is_same::value ? 1e-10 : 1e-4; - for (const hsolver::LinearMethod method: {hsolver::LinearMethod::bicgstab, hsolver::LinearMethod::cgs, hsolver::LinearMethod::gmres}) + for (const hsolver::LinearMethod method : + {hsolver::LinearMethod::bicgstab, hsolver::LinearMethod::cgs, hsolver::LinearMethod::gmres}) { - for (const bool preconditioned: {false, true}) + for (const bool preconditioned : {false, true}) { hsolver::LinearControls linear_ctrl; linear_ctrl.solver = method; @@ -127,17 +144,21 @@ TYPED_TEST(LinearSolveTest, DenseReferenceAndReusedBatches) linear_ctrl.restart = 2; hsolver::HSolverLinear solver(linear_ctrl, comm); // Shrink, grow, and use zero local rows without replacing the solver. - for (const int n: {11, 5, 17, 1}) + for (const int n : {11, + 5, + 17, + 1}) { const int start = n * comm.rank / comm.nproc; const int end = n * (comm.rank + 1) / comm.nproc; const int dim = end - start; const int ld = dim + 3; const int nvec = n == 5 ? 3 : 5; - for (const bool cn_matrix: {true, false}) + for (const bool cn_matrix : {true, false}) { - SCOPED_TRACE(testing::Message() << "n=" << n << " method=" << static_cast(method) - << " preconditioned=" << preconditioned << " cn=" << cn_matrix); + SCOPED_TRACE( + testing::Message() << "n=" << n << " method=" << static_cast(method) + << " preconditioned=" << preconditioned << " cn=" << cn_matrix); std::vector a = linear_test::hermitian(n); for (int j = 0; j < n; ++j) { @@ -169,8 +190,16 @@ TYPED_TEST(LinearSolveTest, DenseReferenceAndReusedBatches) std::fill(exact.begin() + n, exact.begin() + 2 * n, T(0)); std::fill(exact.begin() + 2 * n, exact.begin() + 3 * n, T(0)); exact[2 * n] = T(0.7, -0.2); - const std::vector b = linear_test::multiply(a, exact, n, nvec); - const std::vector reference = linear_test::lapack_solve(a, b, n, nvec); + const std::vector b = linear_test::multiply( + a, + exact, + n, + nvec); + const std::vector reference = linear_test::lapack_solve( + a, + b, + n, + nvec); const T sentinel(123, -45); std::vector rhs(ld * nvec, sentinel); std::vector x(ld * nvec, sentinel); @@ -183,17 +212,44 @@ TYPED_TEST(LinearSolveTest, DenseReferenceAndReusedBatches) } } const std::vector rhs_before = rhs; - DenseOperator op(a, n, start, dim, comm); - RightPreconditioner precond(a, n, start, dim); + DenseOperator op( + a, + n, + start, + dim, + comm); + RightPreconditioner precond( + a, + n, + start, + dim); hsolver::LinearSolveResult result; if (preconditioned) { - result = solver.solve(op, precond, ld, nvec, dim, x.data(), rhs.data(), nullptr, true); + result = solver.solve( + op, + precond, + ld, + nvec, + dim, + x.data(), + rhs.data(), + nullptr, + true); } else { const IdentityOperator identity(dim); - result = solver.solve(op, identity, ld, nvec, dim, x.data(), rhs.data(), nullptr, true); + result = solver.solve( + op, + identity, + ld, + nvec, + dim, + x.data(), + rhs.data(), + nullptr, + true); } EXPECT_EQ(result.status, hsolver::LinearSolveStatus::converged); EXPECT_EQ(result.failed_band, -1); @@ -208,7 +264,8 @@ TYPED_TEST(LinearSolveTest, DenseReferenceAndReusedBatches) for (int i = 0; i < dim; ++i) { full[band * n + start + i] = x[band * ld + i]; - EXPECT_LT(std::abs(full[band * n + start + i] - reference[band * n + start + i]), solution_tol); + EXPECT_LT( + std::abs(full[band * n + start + i] - reference[band * n + start + i]), solution_tol); } for (int i = dim; i < ld; ++i) { @@ -220,7 +277,11 @@ TYPED_TEST(LinearSolveTest, DenseReferenceAndReusedBatches) Parallel_Common::reduce_data(full.data(), count, comm.comm); #endif const std::vector a_double(a.begin(), a.end()); - const std::vector ax = linear_test::multiply(a_double, full, n, nvec); + const std::vector ax = linear_test::multiply( + a_double, + full, + n, + nvec); double max_residual = 0.0; for (int band = 0; band < nvec; ++band) { @@ -261,16 +322,22 @@ TEST(LinearSolveFailure, ReportsOriginalColumnAndTrueResidual) const int start = n * comm.rank / comm.nproc; const int dim = n * (comm.rank + 1) / comm.nproc - start; const int ld = dim + 1; - for (const hsolver::LinearMethod method: {hsolver::LinearMethod::bicgstab, hsolver::LinearMethod::cgs, hsolver::LinearMethod::gmres}) + for (const hsolver::LinearMethod method : + {hsolver::LinearMethod::bicgstab, hsolver::LinearMethod::cgs, hsolver::LinearMethod::gmres}) { - for (const bool singular: {false, true}) + for (const bool singular : {false, true}) { std::vector a = linear_test::hermitian(n); if (singular) { std::fill(a.begin(), a.end(), T(0)); } - DenseOperator op(a, n, start, dim, comm); + DenseOperator op( + a, + n, + start, + dim, + comm); hsolver::LinearControls linear_ctrl; linear_ctrl.solver = method; linear_ctrl.tolerance = 1e-13; @@ -284,14 +351,27 @@ TEST(LinearSolveFailure, ReportsOriginalColumnAndTrueResidual) b[ld + i] = T(1, 0.1 * (start + i)); } const IdentityOperator identity(dim); - const hsolver::LinearSolveResult result = solver.solve(op, identity, ld, 3, dim, x.data(), b.data(), nullptr, true); - const hsolver::LinearSolveStatus expected - = singular ? hsolver::LinearSolveStatus::breakdown : hsolver::LinearSolveStatus::max_iterations; + const hsolver::LinearSolveResult result = solver.solve( + op, + identity, + ld, + 3, + dim, + x.data(), + b.data(), + nullptr, + true); + const hsolver::LinearSolveStatus expected = + singular ? hsolver::LinearSolveStatus::breakdown : hsolver::LinearSolveStatus::max_iterations; EXPECT_EQ(result.status, expected); EXPECT_EQ(result.failed_band, 1); EXPECT_LE(result.iterations, linear_ctrl.max_iterations); std::vector ax(3 * ld, T(0)); - op.apply(x.data(), ax.data(), ld, 3); + op.apply( + x.data(), + ax.data(), + ld, + 3); double residual2 = 0.0; for (int i = 0; i < dim; ++i) { @@ -305,8 +385,7 @@ TEST(LinearSolveFailure, ReportsOriginalColumnAndTrueResidual) } } } -template -void check_reconstruction(const bool corrupt_initial) +template void check_reconstruction(const bool corrupt_initial) { const hsolver::diag_comm_info comm = linear_test::world_comm(); const int n = 8; @@ -328,8 +407,16 @@ void check_reconstruction(const bool corrupt_initial) exact[n + i] = T(std::cos(0.3 * i), std::sin(0.4 * i)); } exact[2 * n] = T(1); - const std::vector rhs = linear_test::multiply(matrix, exact, n, bands); - const std::vector reference = linear_test::lapack_solve(matrix, rhs, n, bands); + const std::vector rhs = linear_test::multiply( + matrix, + exact, + n, + bands); + const std::vector reference = linear_test::lapack_solve( + matrix, + rhs, + n, + bands); std::vector b(ld * bands, T(0)); std::vector x(ld * bands, T(0)); std::vector bad_residual(ld * bands, T(0)); @@ -347,11 +434,25 @@ void check_reconstruction(const bool corrupt_initial) linear_ctrl.restart = 2; linear_ctrl.max_iterations = 100; linear_ctrl.reconstruct = true; - DenseOperator op(matrix, n, start, dim, comm); + DenseOperator op( + matrix, + n, + start, + dim, + comm); const IdentityOperator identity(dim); hsolver::HSolverLinear solver(linear_ctrl, comm); - const T* initial = corrupt_initial ? bad_residual.data() : nullptr; - const hsolver::LinearSolveResult result = solver.solve(op, identity, ld, bands, dim, x.data(), b.data(), initial, corrupt_initial); + const T *initial = corrupt_initial ? bad_residual.data() : nullptr; + const hsolver::LinearSolveResult result = solver.solve( + op, + identity, + ld, + bands, + dim, + x.data(), + b.data(), + initial, + corrupt_initial); ASSERT_EQ(result.status, hsolver::LinearSolveStatus::converged); EXPECT_EQ(result.failed_band, -1); EXPECT_GT(result.restarts, 0); @@ -360,7 +461,11 @@ void check_reconstruction(const bool corrupt_initial) EXPECT_EQ(result.reconstructed, !corrupt_initial); EXPECT_EQ(result.reconstruction_fallbacks, corrupt_initial ? 1 : 0); std::vector ax(ld * bands, T(0)); - op.apply(x.data(), ax.data(), ld, bands); + op.apply( + x.data(), + ax.data(), + ld, + bands); for (int band = 0; band < bands; ++band) { double error = 0.0; @@ -369,7 +474,9 @@ void check_reconstruction(const bool corrupt_initial) { error += std::norm(linear_test::Complex(ax[band * ld + i]) - linear_test::Complex(b[band * ld + i])); norm += std::norm(linear_test::Complex(b[band * ld + i])); - EXPECT_LT(std::abs(linear_test::Complex(x[band * ld + i]) - reference[band * n + start + i]), 10 * linear_ctrl.tolerance); + EXPECT_LT( + std::abs(linear_test::Complex(x[band * ld + i]) - reference[band * n + start + i]), + 10 * linear_ctrl.tolerance); } #ifdef __MPI Parallel_Common::reduce_data(&error, 1, comm.comm); @@ -383,7 +490,16 @@ void check_reconstruction(const bool corrupt_initial) linear_ctrl.max_iterations = 1; hsolver::HSolverLinear limited(linear_ctrl, comm); std::fill(x.begin(), x.end(), T(0)); - const hsolver::LinearSolveResult exhausted = limited.solve(op, identity, ld, bands, dim, x.data(), b.data(), initial, true); + const hsolver::LinearSolveResult exhausted = limited.solve( + op, + identity, + ld, + bands, + dim, + x.data(), + b.data(), + initial, + true); EXPECT_EQ(exhausted.status, hsolver::LinearSolveStatus::max_iterations); EXPECT_EQ(exhausted.failed_band, 1); EXPECT_EQ(exhausted.iterations, linear_ctrl.max_iterations); @@ -402,12 +518,17 @@ TYPED_TEST(LinearSolveTest, ReconstructionRecoversIncorrectInitialResidual) } } // namespace -int main(int argc, char** argv) +int main(int argc, char **argv) { int nproc = 1; int nthread = 1; int rank = 0; - Parallel_Global::read_pal_param(argc, argv, nproc, nthread, rank); + Parallel_Global::read_pal_param( + argc, + argv, + nproc, + nthread, + rank); #ifdef __MPI // This numerical test creates no application pools; the shared finalizer owns these handles. POOL_WORLD = MPI_COMM_NULL; diff --git a/source/source_hsolver/module_linalg/unittests/test_linear_low_rank.cpp b/source/source_hsolver/module_linalg/unittests/test_linear_low_rank.cpp index 4639d920a9f..26574f6632f 100644 --- a/source/source_hsolver/module_linalg/unittests/test_linear_low_rank.cpp +++ b/source/source_hsolver/module_linalg/unittests/test_linear_low_rank.cpp @@ -6,8 +6,7 @@ namespace { -template -class LinearLowRankTest : public testing::Test +template class LinearLowRankTest : public testing::Test { }; using Precisions = testing::Types; @@ -39,21 +38,54 @@ TYPED_TEST(LinearLowRankTest, CoarseAndHistoryCorrectionsMatchDenseReference) w[j * n + i] = T(1.0, 0.2 * (i + 1)) * z[j * n + i]; } } - const std::vector local_z = linear_test::local_columns(z, n, rank, start, dim, ld); - const std::vector local_w = linear_test::local_columns(w, n, rank, start, dim, ld); - const std::vector local_d = linear_test::local_columns(diagonal, n, 1, start, dim, ld); - const std::vector local_x = linear_test::local_columns(x, n, 1, start, dim, ld); + const std::vector local_z = linear_test::local_columns( + z, + n, + rank, + start, + dim, + ld); + const std::vector local_w = linear_test::local_columns( + w, + n, + rank, + start, + dim, + ld); + const std::vector local_d = linear_test::local_columns( + diagonal, + n, + 1, + start, + dim, + ld); + const std::vector local_x = linear_test::local_columns( + x, + n, + 1, + start, + dim, + ld); const std::vector zero(ld * rank, T(0)); hsolver::LinearAlgebra algebra(comm); hsolver::LinearResponse response; ct::Tensor workspace; - response.update(algebra, ld, dim, rank, local_z.data(), zero.data(), local_w.data(), 1e-8, &workspace); + response.update( + algebra, + ld, + dim, + rank, + local_z.data(), + zero.data(), + local_w.data(), + 1e-8, + &workspace); ASSERT_EQ(response.rank(), rank); ct::Tensor correction_workspace; - for (const bool history: {false, true}) + for (const bool history : {false, true}) { SCOPED_TRACE(history); - const std::vector& test = history ? w : z; + const std::vector &test = history ? w : z; std::vector gram(rank * rank, Wide(0)); std::vector projected(rank, Wide(0)); for (int i = 0; i < rank; ++i) @@ -67,22 +99,41 @@ TYPED_TEST(LinearLowRankTest, CoarseAndHistoryCorrectionsMatchDenseReference) } } } - const std::vector coefficients = linear_test::lapack_solve(gram, projected, rank, 1); + const std::vector coefficients = linear_test::lapack_solve( + gram, + projected, + rank, + 1); hsolver::LinearSmallLU factor; ASSERT_TRUE(factor.factor(gram, rank)); hsolver::LinearLowRank preconditioner(algebra, local_d.data(), dim); if (history) { - preconditioner.prepare_response(ld, response.rank(), response.directions(), response.images(), &correction_workspace); + preconditioner.prepare_response( + ld, + response.rank(), + response.directions(), + response.images(), + &correction_workspace); } else { - preconditioner.prepare_subspace(ld, rank, local_z.data(), local_w.data(), factor, &correction_workspace); + preconditioner.prepare_subspace( + ld, + rank, + local_z.data(), + local_w.data(), + factor, + &correction_workspace); } ASSERT_EQ(preconditioner.rank(), rank); std::vector actual(ld * rank, T(0)); // Both corrections must satisfy P*(A*Z) = Z on the represented space. - preconditioner.apply(local_w.data(), actual.data(), ld, rank); + preconditioner.apply( + local_w.data(), + actual.data(), + ld, + rank); for (int j = 0; j < rank; ++j) { for (int i = 0; i < dim; ++i) @@ -90,7 +141,11 @@ TYPED_TEST(LinearLowRankTest, CoarseAndHistoryCorrectionsMatchDenseReference) EXPECT_LT(std::abs(Wide(actual[j * ld + i]) - Wide(local_z[j * ld + i])), tolerance); } } - preconditioner.apply(local_x.data(), actual.data(), ld, 1); + preconditioner.apply( + local_x.data(), + actual.data(), + ld, + 1); for (int i = 0; i < dim; ++i) { const int row = start + i; @@ -103,21 +158,45 @@ TYPED_TEST(LinearLowRankTest, CoarseAndHistoryCorrectionsMatchDenseReference) } if (history) { - preconditioner.prepare_response(ld, 0, nullptr, nullptr, &correction_workspace); + preconditioner.prepare_response( + ld, + 0, + nullptr, + nullptr, + &correction_workspace); } else { - preconditioner.prepare_subspace(ld, 0, nullptr, nullptr, factor, &correction_workspace); + preconditioner.prepare_subspace( + ld, + 0, + nullptr, + nullptr, + factor, + &correction_workspace); } EXPECT_EQ(preconditioner.rank(), 0); - preconditioner.apply(local_x.data(), actual.data(), ld, 1); + preconditioner.apply( + local_x.data(), + actual.data(), + ld, + 1); for (int i = 0; i < dim; ++i) { EXPECT_LT(std::abs(actual[i] - local_d[i] * local_x[i]), tolerance); } } // A new unresolved response must discard an older usable history. - response.update(algebra, ld, dim, rank, zero.data(), zero.data(), zero.data(), 1e-8, &workspace); + response.update( + algebra, + ld, + dim, + rank, + zero.data(), + zero.data(), + zero.data(), + 1e-8, + &workspace); EXPECT_EQ(response.rank(), 0); } } // namespace diff --git a/source/source_hsolver/module_linalg/unittests/test_orthonormal.cpp b/source/source_hsolver/module_linalg/unittests/test_orthonormal.cpp index d664559b6d1..4ef8edd67cd 100644 --- a/source/source_hsolver/module_linalg/unittests/test_orthonormal.cpp +++ b/source/source_hsolver/module_linalg/unittests/test_orthonormal.cpp @@ -25,8 +25,7 @@ hsolver::diag_comm_info self_comm() #endif } -template -std::vector overlaps(const std::vector& psi) +template std::vector overlaps(const std::vector &psi) { std::vector gram(bands * bands, Wide(0)); for (int j = 0; j < bands; ++j) @@ -42,8 +41,7 @@ std::vector overlaps(const std::vector& psi) return gram; } -template -std::vector orbitals() +template std::vector orbitals() { const double nan = std::numeric_limits::quiet_NaN(); std::vector psi(ld * bands, T(nan, nan)); @@ -57,8 +55,7 @@ std::vector orbitals() return psi; } -template -void check_orthogonal(const std::vector& psi) +template void check_orthogonal(const std::vector &psi) { const std::vector gram = overlaps(psi); const double tolerance = hsolver::orth_tolerance(); @@ -72,8 +69,7 @@ void check_orthogonal(const std::vector& psi) } } -template -class OrthonormalTest : public testing::Test +template class OrthonormalTest : public testing::Test { }; using Precisions = testing::Types; @@ -85,12 +81,19 @@ TYPED_TEST(OrthonormalTest, CorrectsComplexOverlapsWithoutFallback) const hsolver::diag_comm_info comm = self_comm(); hsolver::LinearAlgebra algebra(comm); hsolver::Orthonormal orth(comm, algebra); - for (const hsolver::OrthMethod method: {hsolver::OrthMethod::cholesky, hsolver::OrthMethod::lowdin, hsolver::OrthMethod::newton_schulz}) + for (const hsolver::OrthMethod method : + {hsolver::OrthMethod::cholesky, hsolver::OrthMethod::lowdin, hsolver::OrthMethod::newton_schulz}) { SCOPED_TRACE(hsolver::orth_method_name(method)); std::vector psi = orbitals(); const std::vector original(psi); - const hsolver::OrthResult result = orth.apply(psi.data(), ld, dim, bands, method, false); + const hsolver::OrthResult result = orth.apply( + psi.data(), + ld, + dim, + bands, + method, + false); ASSERT_EQ(result.status, hsolver::OrthStatus::accepted); EXPECT_EQ(result.actual, method); EXPECT_EQ(result.fallbacks, 0); @@ -113,7 +116,13 @@ TYPED_TEST(OrthonormalTest, NewtonSchulzFallsBackOutsideItsRange) std::vector psi(ld * bands, T(0)); psi[0] = T(2); psi[ld + 1] = T(2); - const hsolver::OrthResult result = orth.apply(psi.data(), ld, dim, bands, hsolver::OrthMethod::newton_schulz, false); + const hsolver::OrthResult result = orth.apply( + psi.data(), + ld, + dim, + bands, + hsolver::OrthMethod::newton_schulz, + false); ASSERT_EQ(result.status, hsolver::OrthStatus::accepted); EXPECT_EQ(result.actual, hsolver::OrthMethod::cholesky); EXPECT_EQ(result.fallbacks, 1); @@ -129,9 +138,9 @@ TYPED_TEST(OrthonormalTest, RejectsInvalidOrbitalsWithoutOverwritingThem) hsolver::Orthonormal orth(comm, algebra); const double nan = std::numeric_limits::quiet_NaN(); const double inf = std::numeric_limits::infinity(); - for (const hsolver::OrthMethod method: {hsolver::OrthMethod::cholesky, hsolver::OrthMethod::none}) + for (const hsolver::OrthMethod method : {hsolver::OrthMethod::cholesky, hsolver::OrthMethod::none}) { - for (const double value: {0.0, nan, inf}) + for (const double value : {0.0, nan, inf}) { SCOPED_TRACE(hsolver::orth_method_name(method)); SCOPED_TRACE(value); @@ -139,10 +148,17 @@ TYPED_TEST(OrthonormalTest, RejectsInvalidOrbitalsWithoutOverwritingThem) psi[0] = T(value); psi[ld + 1] = T(1); const std::vector original(psi); - const hsolver::OrthResult result = orth.apply(psi.data(), ld, dim, bands, method, false); + const hsolver::OrthResult result = orth.apply( + psi.data(), + ld, + dim, + bands, + method, + false); EXPECT_EQ(result.status, hsolver::OrthStatus::failed); - const hsolver::OrthFailure nonfinite - = method == hsolver::OrthMethod::none ? hsolver::OrthFailure::nonfinite_norm : hsolver::OrthFailure::nonfinite_gram; + const hsolver::OrthFailure nonfinite = method == hsolver::OrthMethod::none + ? hsolver::OrthFailure::nonfinite_norm + : hsolver::OrthFailure::nonfinite_gram; const hsolver::OrthFailure expected = value == 0.0 ? hsolver::OrthFailure::nonpositive_norm : nonfinite; EXPECT_EQ(result.failure, expected); EXPECT_EQ(std::memcmp(psi.data(), original.data(), psi.size() * sizeof(T)), 0); @@ -152,7 +168,13 @@ TYPED_TEST(OrthonormalTest, RejectsInvalidOrbitalsWithoutOverwritingThem) duplicate[0] = T(1); duplicate[ld] = T(1); const std::vector original(duplicate); - const hsolver::OrthResult result = orth.apply(duplicate.data(), ld, dim, bands, hsolver::OrthMethod::cholesky, false); + const hsolver::OrthResult result = orth.apply( + duplicate.data(), + ld, + dim, + bands, + hsolver::OrthMethod::cholesky, + false); EXPECT_EQ(result.status, hsolver::OrthStatus::failed); EXPECT_EQ(result.failure, hsolver::OrthFailure::factorization_failed); EXPECT_EQ(std::memcmp(duplicate.data(), original.data(), duplicate.size() * sizeof(T)), 0); @@ -167,10 +189,22 @@ TYPED_TEST(OrthonormalTest, InitialInspectionAndNoneAreReadOnly) std::vector psi = orbitals(); const std::vector original(psi); const std::vector gram = overlaps(psi); - for (const bool collect: {false, true}) + for (const bool collect : {false, true}) { - const hsolver::OrthResult inspected = orth.inspect(psi.data(), ld, dim, bands, true, collect); - const hsolver::OrthResult disabled = orth.apply(psi.data(), ld, dim, bands, hsolver::OrthMethod::none, collect); + const hsolver::OrthResult inspected = orth.inspect( + psi.data(), + ld, + dim, + bands, + true, + collect); + const hsolver::OrthResult disabled = orth.apply( + psi.data(), + ld, + dim, + bands, + hsolver::OrthMethod::none, + collect); EXPECT_EQ(inspected.status, hsolver::OrthStatus::inspected); EXPECT_GT(inspected.after, hsolver::orth_tolerance()); EXPECT_EQ(disabled.status, hsolver::OrthStatus::disabled); diff --git a/source/source_hsolver/module_pexsi/dist_bcd_matrix.cpp b/source/source_hsolver/module_pexsi/dist_bcd_matrix.cpp index 93a6dcb5b0d..853e5e5bce0 100644 --- a/source/source_hsolver/module_pexsi/dist_bcd_matrix.cpp +++ b/source/source_hsolver/module_pexsi/dist_bcd_matrix.cpp @@ -4,20 +4,26 @@ #include extern "C" { - void Cblacs_gridinfo(int icontxt, int* nprow, int* npcol, int* myprow, int* mypcol); + void Cblacs_gridinfo( + int icontxt, + int *nprow, + int *npcol, + int *myprow, + int *mypcol); int Cblacs_pnum(int blacs_ctxt, int prow, int pcol); }; namespace pexsi { -DistBCDMatrix::DistBCDMatrix(MPI_Comm comm, - MPI_Group group, - int blacs_ctxt, - int size, - int nblk, - int nrow, - int ncol, - char layout) +DistBCDMatrix::DistBCDMatrix( + MPI_Comm comm, + MPI_Group group, + int blacs_ctxt, + int size, + int nblk, + int nrow, + int ncol, + char layout) { this->comm = comm; this->group = group; @@ -38,7 +44,12 @@ DistBCDMatrix::DistBCDMatrix(MPI_Comm comm, if (comm != MPI_COMM_NULL) { MPI_Comm_rank(comm, &this->myproc); - Cblacs_gridinfo(blacs_ctxt, &this->nprows, &this->npcols, &this->myprow, &this->mypcol); + Cblacs_gridinfo( + blacs_ctxt, + &this->nprows, + &this->npcols, + &this->myprow, + &this->mypcol); } else { @@ -53,13 +64,26 @@ DistBCDMatrix::DistBCDMatrix(MPI_Comm comm, if (myid_in_comm_world == 0) { MPI_Comm_size(comm, &this->nprocs); - int PARA_BCAST[4] = {this->nblk, this->nprocs, this->nprows, this->npcols}; - MPI_Bcast(&PARA_BCAST[0], 4, MPI_INT, 0, MPI_COMM_WORLD); + int PARA_BCAST[4] = {this->nblk, + this->nprocs, + this->nprows, + this->npcols}; + MPI_Bcast( + &PARA_BCAST[0], + 4, + MPI_INT, + 0, + MPI_COMM_WORLD); } else { int PARA_BCAST[4]; - MPI_Bcast(&PARA_BCAST[0], 4, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Bcast( + &PARA_BCAST[0], + 4, + MPI_INT, + 0, + MPI_COMM_WORLD); this->nblk = PARA_BCAST[0]; this->nprocs = PARA_BCAST[1]; this->nprows = PARA_BCAST[2]; @@ -76,13 +100,15 @@ DistBCDMatrix::DistBCDMatrix(MPI_Comm comm, } } } - MPI_Bcast(this->prowpcol2pnum, this->nprocs, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Bcast( + this->prowpcol2pnum, + this->nprocs, + MPI_INT, + 0, + MPI_COMM_WORLD); } -DistBCDMatrix::~DistBCDMatrix() -{ - delete[] prowpcol2pnum; -} +DistBCDMatrix::~DistBCDMatrix() { delete[] prowpcol2pnum; } int DistBCDMatrix::globalRow(const int localRow) { @@ -94,21 +120,18 @@ int DistBCDMatrix::globalCol(const int localCol) return (localCol / nblk * npcols + mypcol) * nblk + localCol % nblk; } -int DistBCDMatrix::localRow(const int globalRow, int& myprow) +int DistBCDMatrix::localRow(const int globalRow, int &myprow) { myprow = int((globalRow % (nblk * nprows)) / nblk); return int(globalRow / (nblk * nprows)) * nblk + globalRow % nblk; } -int DistBCDMatrix::localCol(const int globalCol, int& mypcol) +int DistBCDMatrix::localCol(const int globalCol, int &mypcol) { mypcol = int((globalCol % (nblk * npcols)) / nblk); return int(globalCol / (nblk * npcols)) * nblk + globalCol % nblk; } -int DistBCDMatrix::pnum(const int prow, const int pcol) -{ - return this->prowpcol2pnum[prow * this->npcols + pcol]; -} +int DistBCDMatrix::pnum(const int prow, const int pcol) { return this->prowpcol2pnum[prow * this->npcols + pcol]; } } // namespace pexsi #endif \ No newline at end of file diff --git a/source/source_hsolver/module_pexsi/dist_bcd_matrix.h b/source/source_hsolver/module_pexsi/dist_bcd_matrix.h index 6822efdccd4..1c36749657d 100644 --- a/source/source_hsolver/module_pexsi/dist_bcd_matrix.h +++ b/source/source_hsolver/module_pexsi/dist_bcd_matrix.h @@ -13,35 +13,28 @@ namespace pexsi class DistBCDMatrix { public: - DistBCDMatrix(MPI_Comm comm, MPI_Group group, int blacs_ctxt, int size, int nblk, int nrow, int ncol, char layout); + DistBCDMatrix( + MPI_Comm comm, + MPI_Group group, + int blacs_ctxt, + int size, + int nblk, + int nrow, + int ncol, + char layout); ~DistBCDMatrix(); int globalRow(const int localRow); int globalCol(const int localCol); - int localRow(const int globalRow, int& myprow); - int localCol(const int globalCol, int& mypcol); + int localRow(const int globalRow, int &myprow); + int localCol(const int globalCol, int &mypcol); int pnum(const int prow, const int pcol); - const MPI_Comm get_comm() const - { - return comm; - }; - const MPI_Group get_group() const - { - return group; - }; - const int get_nrow() const - { - return nrow; - }; - const int get_ncol() const - { - return ncol; - }; - const char get_layout() const - { - return layout; - }; + const MPI_Comm get_comm() const { return comm; }; + const MPI_Group get_group() const { return group; }; + const int get_nrow() const { return nrow; }; + const int get_ncol() const { return ncol; }; + const char get_layout() const { return layout; }; private: // MPI communicator @@ -75,7 +68,7 @@ class DistBCDMatrix // current process id int myproc; - int* prowpcol2pnum = nullptr; + int *prowpcol2pnum = nullptr; // the local data layout // 'R' or 'r' for row-major, which is used in C/C++ // 'C' or 'c' for column-major, which is used in Fortran diff --git a/source/source_hsolver/module_pexsi/dist_ccs_matrix.cpp b/source/source_hsolver/module_pexsi/dist_ccs_matrix.cpp index 0a73653c8ee..b8c40e24d15 100644 --- a/source/source_hsolver/module_pexsi/dist_ccs_matrix.cpp +++ b/source/source_hsolver/module_pexsi/dist_ccs_matrix.cpp @@ -33,7 +33,14 @@ DistCCSMatrix::DistCCSMatrix(int size_in, int nnzLocal_in) this->size = size_in; this->nnzLocal = nnzLocal_in; MPI_Request req; - MPI_Iallreduce(&nnzLocal, &this->nnz, 1, MPI_INT, MPI_SUM, this->comm, &req); + MPI_Iallreduce( + &nnzLocal, + &this->nnz, + 1, + MPI_INT, + MPI_SUM, + this->comm, + &req); this->numColLocal = 0; this->colptrLocal = new int[size]; this->rowindLocal = new int[nnzLocal]; @@ -49,7 +56,11 @@ DistCCSMatrix::DistCCSMatrix(MPI_Comm comm_in, int nproc_data_in, int size_in) int nproc_data_range[3] = {0, this->nproc_data - 1, 1}; // create processes group with data: this->group_data and associated communicator MPI_Comm_group(this->comm, &this->group); - MPI_Group_range_incl(this->group, 1, &nproc_data_range, &this->group_data); + MPI_Group_range_incl( + this->group, + 1, + &nproc_data_range, + &this->group_data); this->comm_data = MPI_COMM_NULL; MPI_Comm_create(this->comm, this->group_data, &this->comm_data); this->size = size_in; @@ -84,17 +95,16 @@ DistCCSMatrix::DistCCSMatrix(MPI_Comm comm_in, int nproc_data_in, int size_in) } } -int DistCCSMatrix::globalCol(int localCol) -{ - return this->firstCol + localCol; -} +int DistCCSMatrix::globalCol(int localCol) { return this->firstCol + localCol; } // NOTE: the process id is 0-based -int DistCCSMatrix::localCol(int globalCol, int& mypcol) +int DistCCSMatrix::localCol(int globalCol, int &mypcol) { mypcol = int(globalCol / int(this->size / this->nproc_data)); if (mypcol >= this->nproc_data) + { mypcol = this->nproc_data - 1; + } return mypcol > 0 ? globalCol - (this->size / this->nproc_data) * mypcol : globalCol; } @@ -103,7 +113,13 @@ void DistCCSMatrix::setnnz(int nnzLocal_in) { if (this->comm_data != MPI_COMM_NULL) { - MPI_Allreduce(&nnzLocal_in, &this->nnz, 1, MPI_INT, MPI_SUM, this->comm_data); + MPI_Allreduce( + &nnzLocal_in, + &this->nnz, + 1, + MPI_INT, + MPI_SUM, + this->comm_data); this->nnzLocal = nnzLocal_in; this->rowindLocal = new int[nnzLocal]; this->colptrLocal[this->numColLocal] = nnzLocal_in + 1; diff --git a/source/source_hsolver/module_pexsi/dist_ccs_matrix.h b/source/source_hsolver/module_pexsi/dist_ccs_matrix.h index b4a25fdd9ea..a2d73a648bc 100644 --- a/source/source_hsolver/module_pexsi/dist_ccs_matrix.h +++ b/source/source_hsolver/module_pexsi/dist_ccs_matrix.h @@ -14,48 +14,26 @@ class DistCCSMatrix DistCCSMatrix(MPI_Comm comm); DistCCSMatrix(int size, int nnzLocal); DistCCSMatrix(MPI_Comm comm, int size, int nnzLocal); - DistCCSMatrix(MPI_Comm comm, int size, int nnzLocal, double* valLocal, int* index); + DistCCSMatrix( + MPI_Comm comm, + int size, + int nnzLocal, + double *valLocal, + int *index); int globalCol(int localCol); - int localCol(int globalCol, int& mypcol); + int localCol(int globalCol, int &mypcol); void setnnz(int nnzLocal); - const MPI_Comm get_comm() const - { - return comm; - }; - const MPI_Group get_group() const - { - return group; - }; - const MPI_Group get_group_data() const - { - return group_data; - }; - const int get_size() const - { - return size; - }; - const int get_nnz() const - { - return nnz; - }; - const int get_nnzlocal() const - { - return nnzLocal; - }; - const int get_numcol_local() const - { - return numColLocal; - }; - int* get_colptr_local() const - { - return colptrLocal; - }; - int* get_rowind_local() const - { - return rowindLocal; - }; + const MPI_Comm get_comm() const { return comm; }; + const MPI_Group get_group() const { return group; }; + const MPI_Group get_group_data() const { return group_data; }; + const int get_size() const { return size; }; + const int get_nnz() const { return nnz; }; + const int get_nnzlocal() const { return nnzLocal; }; + const int get_numcol_local() const { return numColLocal; }; + int *get_colptr_local() const { return colptrLocal; }; + int *get_rowind_local() const { return rowindLocal; }; ~DistCCSMatrix(); @@ -66,7 +44,7 @@ class DistCCSMatrix // total number of processes and the processes with data in int nprocs = 0; - int nproc_data =0; + int nproc_data = 0; MPI_Group group_data; MPI_Comm comm_data; @@ -83,11 +61,11 @@ class DistCCSMatrix int numColLocal; // the first column index in current process - int firstCol=0; + int firstCol = 0; // Array stores the indices to the nonzero row indices in rowptrLocal and nzvalLocal - int* colptrLocal = nullptr; - int* rowindLocal = nullptr; + int *colptrLocal = nullptr; + int *rowindLocal = nullptr; // friend class DistMatrixTransformer; }; diff --git a/source/source_hsolver/module_pexsi/dist_mat_tf.cpp b/source/source_hsolver/module_pexsi/dist_mat_tf.cpp index 8af5d43ef1e..fd95ec3b3ab 100644 --- a/source/source_hsolver/module_pexsi/dist_mat_tf.cpp +++ b/source/source_hsolver/module_pexsi/dist_mat_tf.cpp @@ -20,19 +20,20 @@ namespace pexsi // wether this function is called for the first time for a index array; nprocs: total number of processes size_process: // the number of indices in each process displacement_process: the start position in each process index: the array // contains the indices -inline int DistMatrixTransformer::MinimumIndexPosition(const bool isFirst, - const int nprocs, - int* size_process, - int* displacement_process, - const int* index) +inline int DistMatrixTransformer::MinimumIndexPosition( + const bool isFirst, + const int nprocs, + int *size_process, + int *displacement_process, + const int *index) { // usually the minimum index is continuous, so it will be a good idea to // check the one next to the previous index first. static int pre_position = 0; // previous position in index array of minimum index, static int pre_process = 0; // the process contains previous index - int minimum_index - = INT_MAX; // the minimum index, initial value is a large number which is larger than any other index; + int minimum_index = + INT_MAX; // the minimum index, initial value is a large number which is larger than any other index; int minimum_position = -1; int minimum_process = -1; @@ -96,22 +97,30 @@ inline int DistMatrixTransformer::MinimumIndexPosition(const bool isFirst, } } -inline void DistMatrixTransformer::buildCCSParameter(const int size, - const int nprocs, - std::vector size_process, - std::vector displacement_process, - const int* position_index, - DistCCSMatrix& DST_Matrix, - int* buffer2ccsIndex) +inline void DistMatrixTransformer::buildCCSParameter( + const int size, + const int nprocs, + std::vector size_process, + std::vector displacement_process, + const int *position_index, + DistCCSMatrix &DST_Matrix, + int *buffer2ccsIndex) { // find the minimum one from left buffer index if (DST_Matrix.get_nnzlocal() <= 0) + { return; + } int pre_col = -1; int nnz_now = 0; int p_mini = 0; - p_mini = MinimumIndexPosition(true, nprocs, &size_process[0], &displacement_process[0], position_index); + p_mini = MinimumIndexPosition( + true, + nprocs, + &size_process[0], + &displacement_process[0], + position_index); while (p_mini >= 0) { int index_mini = position_index[p_mini]; @@ -126,45 +135,56 @@ inline void DistMatrixTransformer::buildCCSParameter(const int size, // copy data from buffer to M, be careful M is a 0-based array buffer2ccsIndex[nnz_now] = p_mini; ++nnz_now; - p_mini = MinimumIndexPosition(false, nprocs, &size_process[0], &displacement_process[0], position_index); + p_mini = MinimumIndexPosition( + false, + nprocs, + &size_process[0], + &displacement_process[0], + position_index); } // The last element of colptrLocal is nnzLocal+1 DST_Matrix.get_colptr_local()[DST_Matrix.get_numcol_local()] = nnz_now + 1; } -inline void DistMatrixTransformer::buffer2CCSvalue(int nnzLocal, - int* buffer2ccsIndex, - double* buffer, - double* nzvalLocal) +inline void DistMatrixTransformer::buffer2CCSvalue( + int nnzLocal, + int *buffer2ccsIndex, + double *buffer, + double *nzvalLocal) { for (int i = 0; i < nnzLocal; ++i) { nzvalLocal[i] = buffer[buffer2ccsIndex[i]]; } } -inline void DistMatrixTransformer::countMatrixDistribution(int N, double* A, std::map& P) +inline void DistMatrixTransformer::countMatrixDistribution(int N, double *A, std::map &P) { for (int i = 0; i < N; ++i) { int key = 0; if (fabs(A[i] < 1e-31)) + { key = -100; + } else + { key = floor(log10(fabs(A[i]))); + } ++P[key]; } } // find out the index of non-zero elements -inline int DistMatrixTransformer::getNonZeroIndex(char layout, - const int nrow, - const int ncol, - double* H_2d, - double* S_2d, - const double ZERO_Limit, - int& nnz, - std::vector& rowidx, - std::vector& colidx) +inline int DistMatrixTransformer::getNonZeroIndex( + char layout, + const int nrow, + const int ncol, + double *H_2d, + double *S_2d, + const double ZERO_Limit, + int &nnz, + std::vector &rowidx, + std::vector &colidx) { int idx = 0; nnz = 0; @@ -209,21 +229,22 @@ inline int DistMatrixTransformer::getNonZeroIndex(char layout, return 0; } -int DistMatrixTransformer::buildTransformParameter(DistBCDMatrix& SRC_Matrix, - DistCCSMatrix& DST_Matrix, - const int NPROC_TRANS, - MPI_Group& GROUP_TRANS, - MPI_Comm& COMM_TRANS, - const int nnz, - std::vector& rowidx, - std::vector& colidx, - int& sender_size, - std::vector& sender_size_process, - std::vector& sender_displacement_process, - int& receiver_size, - std::vector& receiver_size_process, - std::vector& receiver_displacement_process, - std::vector& buffer2ccsIndex) +int DistMatrixTransformer::buildTransformParameter( + DistBCDMatrix &SRC_Matrix, + DistCCSMatrix &DST_Matrix, + const int NPROC_TRANS, + MPI_Group &GROUP_TRANS, + MPI_Comm &COMM_TRANS, + const int nnz, + std::vector &rowidx, + std::vector &colidx, + int &sender_size, + std::vector &sender_size_process, + std::vector &sender_displacement_process, + int &receiver_size, + std::vector &receiver_size_process, + std::vector &receiver_displacement_process, + std::vector &buffer2ccsIndex) { int myproc; MPI_Comm_rank(MPI_COMM_WORLD, &myproc); @@ -235,19 +256,44 @@ int DistMatrixTransformer::buildTransformParameter(DistBCDMatrix& SRC_Matrix, if (myproc == 0) { MPI_Group_size(DST_Matrix.get_group_data(), &nproc_data); - MPI_Bcast(&nproc_data, 1, MPI_INT, 0, COMM_TRANS); + MPI_Bcast( + &nproc_data, + 1, + MPI_INT, + 0, + COMM_TRANS); proc_map_data_trans.resize(nproc_data, 0); for (int i = 0; i < nproc_data; ++i) { - MPI_Group_translate_ranks(DST_Matrix.get_group_data(), 1, &i, GROUP_TRANS, &proc_map_data_trans[i]); - } - MPI_Bcast(&proc_map_data_trans[0], nproc_data, MPI_INT, 0, COMM_TRANS); + MPI_Group_translate_ranks( + DST_Matrix.get_group_data(), + 1, + &i, + GROUP_TRANS, + &proc_map_data_trans[i]); + } + MPI_Bcast( + &proc_map_data_trans[0], + nproc_data, + MPI_INT, + 0, + COMM_TRANS); } else { - MPI_Bcast(&nproc_data, 1, MPI_INT, 0, COMM_TRANS); + MPI_Bcast( + &nproc_data, + 1, + MPI_INT, + 0, + COMM_TRANS); proc_map_data_trans.resize(nproc_data, 0); - MPI_Bcast(&proc_map_data_trans[0], nproc_data, MPI_INT, 0, COMM_TRANS); + MPI_Bcast( + &proc_map_data_trans[0], + nproc_data, + MPI_INT, + 0, + COMM_TRANS); } for (int i = 0; i < nnz; ++i) @@ -260,7 +306,14 @@ int DistMatrixTransformer::buildTransformParameter(DistBCDMatrix& SRC_Matrix, ++sender_size_process[dst_process_trans]; } // transfer sender index size to receiver index size - MPI_Alltoall(&sender_size_process[0], 1, MPI_INT, &receiver_size_process[0], 1, MPI_INT, COMM_TRANS); + MPI_Alltoall( + &sender_size_process[0], + 1, + MPI_INT, + &receiver_size_process[0], + 1, + MPI_INT, + COMM_TRANS); // setup all2all parameters sender_displacement_process[0] = 0; for (int i = 1; i < NPROC_TRANS; ++i) @@ -292,33 +345,36 @@ int DistMatrixTransformer::buildTransformParameter(DistBCDMatrix& SRC_Matrix, // transfer index to receiver std::vector receiver_index(receiver_size); - MPI_Alltoallv(&sender_index[0], - &sender_size_process[0], - &sender_displacement_process[0], - MPI_INT, - &receiver_index[0], - &receiver_size_process[0], - &receiver_displacement_process[0], - MPI_INT, - COMM_TRANS); + MPI_Alltoallv( + &sender_index[0], + &sender_size_process[0], + &sender_displacement_process[0], + MPI_INT, + &receiver_index[0], + &receiver_size_process[0], + &receiver_displacement_process[0], + MPI_INT, + COMM_TRANS); // setup buffer2ccsIndex based on receiver_index buffer2ccsIndex.resize(receiver_size); DST_Matrix.setnnz(receiver_size); - buildCCSParameter(receiver_size, - NPROC_TRANS, - receiver_size_process, - receiver_displacement_process, - &receiver_index[0], - DST_Matrix, - &buffer2ccsIndex[0]); + buildCCSParameter( + receiver_size, + NPROC_TRANS, + receiver_size_process, + receiver_displacement_process, + &receiver_index[0], + DST_Matrix, + &buffer2ccsIndex[0]); return 0; } -int DistMatrixTransformer::newGroupCommTrans(DistBCDMatrix& SRC_Matrix, - DistCCSMatrix& DST_Matrix, - MPI_Group& GROUP_TRANS, - MPI_Comm& COMM_TRANS) +int DistMatrixTransformer::newGroupCommTrans( + DistBCDMatrix &SRC_Matrix, + DistCCSMatrix &DST_Matrix, + MPI_Group &GROUP_TRANS, + MPI_Comm &COMM_TRANS) { // build transfortram communicator which contains both processes of BCD processors and // CCS processors with nonzero elements @@ -327,7 +383,7 @@ int DistMatrixTransformer::newGroupCommTrans(DistBCDMatrix& SRC_Matrix, return 0; } -int DistMatrixTransformer::deleteGroupCommTrans(MPI_Group& GROUP_TRANS, MPI_Comm& COMM_TRANS) +int DistMatrixTransformer::deleteGroupCommTrans(MPI_Group &GROUP_TRANS, MPI_Comm &COMM_TRANS) { MPI_Group_free(&GROUP_TRANS); if (COMM_TRANS != MPI_COMM_NULL) @@ -341,17 +397,22 @@ int DistMatrixTransformer::deleteGroupCommTrans(MPI_Group& GROUP_TRANS, MPI_Comm // two destination matrices share the same non-zero elements positions // if either of two elements in source matrices is non-zeros, the elements in the destination matrices are non-zero, // even if one of them is acturely zero All matrices must have same MPI communicator -int DistMatrixTransformer::transformBCDtoCCS(DistBCDMatrix& SRC_Matrix, - double* H_2d, - double* S_2d, - const double ZERO_Limit, - DistCCSMatrix& DST_Matrix, - double*& H_ccs, - double*& S_ccs) +int DistMatrixTransformer::transformBCDtoCCS( + DistBCDMatrix &SRC_Matrix, + double *H_2d, + double *S_2d, + const double ZERO_Limit, + DistCCSMatrix &DST_Matrix, + double *&H_ccs, + double *&S_ccs) { MPI_Group GROUP_TRANS; MPI_Comm COMM_TRANS = MPI_COMM_NULL; - newGroupCommTrans(SRC_Matrix, DST_Matrix, GROUP_TRANS, COMM_TRANS); + newGroupCommTrans( + SRC_Matrix, + DST_Matrix, + GROUP_TRANS, + COMM_TRANS); if (COMM_TRANS != MPI_COMM_NULL) { // set up sender and receiver= @@ -370,34 +431,36 @@ int DistMatrixTransformer::transformBCDtoCCS(DistBCDMatrix& SRC_Matrix, int nnz = 0; if (SRC_Matrix.get_comm() != MPI_COMM_NULL) { - getNonZeroIndex(SRC_Matrix.get_layout(), - SRC_Matrix.get_nrow(), - SRC_Matrix.get_ncol(), - H_2d, - S_2d, - ZERO_Limit, - nnz, - rowidx, - colidx); + getNonZeroIndex( + SRC_Matrix.get_layout(), + SRC_Matrix.get_nrow(), + SRC_Matrix.get_ncol(), + H_2d, + S_2d, + ZERO_Limit, + nnz, + rowidx, + colidx); } // build all2all transformation parameters and the map index of receiver buffer std::vector buffer2ccsIndex; - buildTransformParameter(SRC_Matrix, - DST_Matrix, - NPROC_TRANS, - GROUP_TRANS, - COMM_TRANS, - nnz, - rowidx, - colidx, - sender_size, - sender_size_process, - sender_displacement_process, - receiver_size, - receiver_size_process, - receiver_displacement_process, - buffer2ccsIndex); -// Do transformation + buildTransformParameter( + SRC_Matrix, + DST_Matrix, + NPROC_TRANS, + GROUP_TRANS, + COMM_TRANS, + nnz, + rowidx, + colidx, + sender_size, + sender_size_process, + sender_displacement_process, + receiver_size, + receiver_size_process, + receiver_displacement_process, + buffer2ccsIndex); + // Do transformation std::vector sender_buffer(sender_size); std::vector receiver_buffer(receiver_size); // put H to sender buffer @@ -416,19 +479,24 @@ int DistMatrixTransformer::transformBCDtoCCS(DistBCDMatrix& SRC_Matrix, } } // do all2all transformation - MPI_Alltoallv(&sender_buffer[0], - &sender_size_process[0], - &sender_displacement_process[0], - MPI_DOUBLE, - &receiver_buffer[0], - &receiver_size_process[0], - &receiver_displacement_process[0], - MPI_DOUBLE, - COMM_TRANS); -// collect H from receiver buffer + MPI_Alltoallv( + &sender_buffer[0], + &sender_size_process[0], + &sender_displacement_process[0], + MPI_DOUBLE, + &receiver_buffer[0], + &receiver_size_process[0], + &receiver_displacement_process[0], + MPI_DOUBLE, + COMM_TRANS); + // collect H from receiver buffer delete[] H_ccs; H_ccs = new double[receiver_size]; - buffer2CCSvalue(receiver_size, &buffer2ccsIndex[0], &receiver_buffer[0], H_ccs); + buffer2CCSvalue( + receiver_size, + &buffer2ccsIndex[0], + &receiver_buffer[0], + H_ccs); // put S to sender buffer if (SRC_Matrix.get_layout() == 'R' || SRC_Matrix.get_layout() == 'r') @@ -446,19 +514,24 @@ int DistMatrixTransformer::transformBCDtoCCS(DistBCDMatrix& SRC_Matrix, } } // do all2all transformation - MPI_Alltoallv(&sender_buffer[0], - &sender_size_process[0], - &sender_displacement_process[0], - MPI_DOUBLE, - &receiver_buffer[0], - &receiver_size_process[0], - &receiver_displacement_process[0], - MPI_DOUBLE, - COMM_TRANS); -// collect S from receiver buffer + MPI_Alltoallv( + &sender_buffer[0], + &sender_size_process[0], + &sender_displacement_process[0], + MPI_DOUBLE, + &receiver_buffer[0], + &receiver_size_process[0], + &receiver_displacement_process[0], + MPI_DOUBLE, + COMM_TRANS); + // collect S from receiver buffer delete[] S_ccs; S_ccs = new double[receiver_size]; - buffer2CCSvalue(receiver_size, &buffer2ccsIndex[0], &receiver_buffer[0], S_ccs); + buffer2CCSvalue( + receiver_size, + &buffer2ccsIndex[0], + &receiver_buffer[0], + S_ccs); } // clear and return deleteGroupCommTrans(GROUP_TRANS, COMM_TRANS); @@ -467,18 +540,23 @@ int DistMatrixTransformer::transformBCDtoCCS(DistBCDMatrix& SRC_Matrix, // transform two sparse matrices from Compressed Column Storage (CCS) to block cyclic distribution (BCD) distribution // two source matrices share the same non-zero elements positions -int DistMatrixTransformer::transformCCStoBCD(DistCCSMatrix& SRC_Matrix, - double* DMnzvalLocal, - double* EDMnzvalLocal, - DistBCDMatrix& DST_Matrix, - double* DM, - double* EDM) +int DistMatrixTransformer::transformCCStoBCD( + DistCCSMatrix &SRC_Matrix, + double *DMnzvalLocal, + double *EDMnzvalLocal, + DistBCDMatrix &DST_Matrix, + double *DM, + double *EDM) { int myproc; MPI_Comm_rank(MPI_COMM_WORLD, &myproc); MPI_Group GROUP_TRANS; MPI_Comm COMM_TRANS = MPI_COMM_NULL; - newGroupCommTrans(DST_Matrix, SRC_Matrix, GROUP_TRANS, COMM_TRANS); + newGroupCommTrans( + DST_Matrix, + SRC_Matrix, + GROUP_TRANS, + COMM_TRANS); if (COMM_TRANS != MPI_COMM_NULL) { // init DM and EDM with 0 @@ -501,31 +579,59 @@ int DistMatrixTransformer::transformCCStoBCD(DistCCSMatrix& SRC_Matrix, if (myproc_trans == 0) { MPI_Group_size(DST_Matrix.get_group(), &nproc_bcd); - MPI_Bcast(&nproc_bcd, 1, MPI_INT, 0, COMM_TRANS); + MPI_Bcast( + &nproc_bcd, + 1, + MPI_INT, + 0, + COMM_TRANS); proc_map_bcd_trans.resize(nproc_bcd, 0); for (int i = 0; i < nproc_bcd; ++i) { - MPI_Group_translate_ranks(DST_Matrix.get_group(), 1, &i, GROUP_TRANS, &proc_map_bcd_trans[i]); + MPI_Group_translate_ranks( + DST_Matrix.get_group(), + 1, + &i, + GROUP_TRANS, + &proc_map_bcd_trans[i]); } - MPI_Bcast(&proc_map_bcd_trans[0], nproc_bcd, MPI_INT, 0, COMM_TRANS); + MPI_Bcast( + &proc_map_bcd_trans[0], + nproc_bcd, + MPI_INT, + 0, + COMM_TRANS); } else { - MPI_Bcast(&nproc_bcd, 1, MPI_INT, 0, COMM_TRANS); + MPI_Bcast( + &nproc_bcd, + 1, + MPI_INT, + 0, + COMM_TRANS); proc_map_bcd_trans.resize(nproc_bcd, 0); - MPI_Bcast(&proc_map_bcd_trans[0], nproc_bcd, MPI_INT, 0, COMM_TRANS); + MPI_Bcast( + &proc_map_bcd_trans[0], + nproc_bcd, + MPI_INT, + 0, + COMM_TRANS); } // setup sender_size_process // std::fill(sender_size_process.begin(), sender_size_process.end(), 0); for (int i = 0; i < NPROC_TRANS; ++i) + { sender_size_process[i] = 0; + } for (int icol = 0; icol < SRC_Matrix.get_numcol_local(); ++icol) { int g_col = SRC_Matrix.globalCol(icol); int recv_pcol_bcd; int recv_col = DST_Matrix.localCol(g_col, recv_pcol_bcd); - - for (int rowidx = SRC_Matrix.get_colptr_local()[icol] - 1; rowidx < SRC_Matrix.get_colptr_local()[icol + 1] - 1; ++rowidx) + + for (int rowidx = SRC_Matrix.get_colptr_local()[icol] - 1; + rowidx < SRC_Matrix.get_colptr_local()[icol + 1] - 1; ++rowidx) { int g_row = SRC_Matrix.get_rowind_local()[rowidx] - 1; int recv_prow_bcd; @@ -539,8 +645,17 @@ int DistMatrixTransformer::transformCCStoBCD(DistCCSMatrix& SRC_Matrix, // setup receiver_size_process // std::fill(receiver_size_process.begin(), receiver_size_process.end(), 0); for (int i = 0; i < NPROC_TRANS; ++i) + { receiver_size_process[i] = 0; - MPI_Alltoall(&sender_size_process[0], 1, MPI_INT, &receiver_size_process[0], 1, MPI_INT, COMM_TRANS); + } + MPI_Alltoall( + &sender_size_process[0], + 1, + MPI_INT, + &receiver_size_process[0], + 1, + MPI_INT, + COMM_TRANS); // setup sender_displacement and receiver_displacement sender_displacement_process[0] = 0; receiver_displacement_process[0] = 0; @@ -554,11 +669,11 @@ int DistMatrixTransformer::transformCCStoBCD(DistCCSMatrix& SRC_Matrix, // setup up sender index and receiver index int sender_size = SRC_Matrix.get_nnzlocal(); - int* sender_index = nullptr; - double* sender_buffer = nullptr; - int* dst_index = nullptr; - int* receiver_index = nullptr; - double* receiver_buffer = nullptr; + int *sender_index = nullptr; + double *sender_buffer = nullptr; + int *dst_index = nullptr; + int *receiver_index = nullptr; + double *receiver_buffer = nullptr; if (sender_size > 0) { @@ -621,7 +736,8 @@ int DistMatrixTransformer::transformCCStoBCD(DistCCSMatrix& SRC_Matrix, int g_col = SRC_Matrix.globalCol(icol); int recv_pcol_bcd; int recv_col = DST_Matrix.localCol(g_col, recv_pcol_bcd); - for (int rowidx = SRC_Matrix.get_colptr_local()[icol] - 1; rowidx < SRC_Matrix.get_colptr_local()[icol + 1] - 1; ++rowidx) + for (int rowidx = SRC_Matrix.get_colptr_local()[icol] - 1; + rowidx < SRC_Matrix.get_colptr_local()[icol + 1] - 1; ++rowidx) { int g_row = SRC_Matrix.get_rowind_local()[rowidx] - 1; int recv_prow_bcd; @@ -648,15 +764,16 @@ int DistMatrixTransformer::transformCCStoBCD(DistCCSMatrix& SRC_Matrix, receiver_displacement_process[i] *= 2; } - MPI_Alltoallv(&dst_index[0], - &sender_size_process[0], - &sender_displacement_process[0], - MPI_INT, - &receiver_index[0], - &receiver_size_process[0], - &receiver_displacement_process[0], - MPI_INT, - COMM_TRANS); + MPI_Alltoallv( + &dst_index[0], + &sender_size_process[0], + &sender_displacement_process[0], + MPI_INT, + &receiver_index[0], + &receiver_size_process[0], + &receiver_displacement_process[0], + MPI_INT, + COMM_TRANS); // reset size and displacement for transfering matrix value by alltoall for (int i = 0; i < NPROC_TRANS; ++i) @@ -675,15 +792,16 @@ int DistMatrixTransformer::transformCCStoBCD(DistCCSMatrix& SRC_Matrix, } // transfer sender buffer to receiver buffer - MPI_Alltoallv(&sender_buffer[0], - &sender_size_process[0], - &sender_displacement_process[0], - MPI_DOUBLE, - &receiver_buffer[0], - &receiver_size_process[0], - &receiver_displacement_process[0], - MPI_DOUBLE, - COMM_TRANS); + MPI_Alltoallv( + &sender_buffer[0], + &sender_size_process[0], + &sender_displacement_process[0], + MPI_DOUBLE, + &receiver_buffer[0], + &receiver_size_process[0], + &receiver_displacement_process[0], + MPI_DOUBLE, + COMM_TRANS); // transform receiver_buffer to DM if (DST_Matrix.get_layout() == 'R' || DST_Matrix.get_layout() == 'r') @@ -716,15 +834,16 @@ int DistMatrixTransformer::transformCCStoBCD(DistCCSMatrix& SRC_Matrix, } // transfer sender buffer to receiver buffer - MPI_Alltoallv(&sender_buffer[0], - &sender_size_process[0], - &sender_displacement_process[0], - MPI_DOUBLE, - &receiver_buffer[0], - &receiver_size_process[0], - &receiver_displacement_process[0], - MPI_DOUBLE, - COMM_TRANS); + MPI_Alltoallv( + &sender_buffer[0], + &sender_size_process[0], + &sender_displacement_process[0], + MPI_DOUBLE, + &receiver_buffer[0], + &receiver_size_process[0], + &receiver_displacement_process[0], + MPI_DOUBLE, + COMM_TRANS); // transform receiver_buffer to EDM if (DST_Matrix.get_layout() == 'R' || DST_Matrix.get_layout() == 'r') @@ -755,7 +874,6 @@ int DistMatrixTransformer::transformCCStoBCD(DistCCSMatrix& SRC_Matrix, delete[] dst_index; delete[] receiver_index; delete[] receiver_buffer; - } deleteGroupCommTrans(GROUP_TRANS, COMM_TRANS); return 0; diff --git a/source/source_hsolver/module_pexsi/dist_mat_tf.h b/source/source_hsolver/module_pexsi/dist_mat_tf.h index 7d705a68540..58068cf4f02 100644 --- a/source/source_hsolver/module_pexsi/dist_mat_tf.h +++ b/source/source_hsolver/module_pexsi/dist_mat_tf.h @@ -1,8 +1,8 @@ #ifndef DIST_MAT_TF_H #define DIST_MAT_TF_H -#include #include +#include #include // transform a sparse matrix from block cyclic distribution (BCD) to Compressed Column Storage (CCS) distribution // they should have same MPI communicator @@ -21,71 +21,82 @@ class DistCCSMatrix; namespace DistMatrixTransformer { -int MinimumIndexPosition(const bool isFirst, - const int nprocs, - int* size_process, - int* displacement_process, - const int* index); +int MinimumIndexPosition( + const bool isFirst, + const int nprocs, + int *size_process, + int *displacement_process, + const int *index); -void buildCCSParameter(const int size, - const int nprocs, - std::vector size_process, - std::vector displacement_process, - const int* position_index, - DistCCSMatrix& DST_Matrix, - int* buffer2ccsIndex); +void buildCCSParameter( + const int size, + const int nprocs, + std::vector size_process, + std::vector displacement_process, + const int *position_index, + DistCCSMatrix &DST_Matrix, + int *buffer2ccsIndex); -void buffer2CCSvalue(int nnzLocal, int* buffer2ccsIndex, double* buffer, double* nzvalLocal); +void buffer2CCSvalue( + int nnzLocal, + int *buffer2ccsIndex, + double *buffer, + double *nzvalLocal); -void countMatrixDistribution(int N, double* A, std::map& P); +void countMatrixDistribution(int N, double *A, std::map &P); -int getNonZeroIndex(char layout, - const int nrow, - const int ncol, - double* H_2d, - double* S_2d, - const double ZERO_Limit, - int& nnz, - std::vector& rowidx, - std::vector& colidx); +int getNonZeroIndex( + char layout, + const int nrow, + const int ncol, + double *H_2d, + double *S_2d, + const double ZERO_Limit, + int &nnz, + std::vector &rowidx, + std::vector &colidx); -int buildTransformParameter(DistBCDMatrix& SRC_Matrix, - DistCCSMatrix& DST_Matrix, - const int NPROC_TRANS, - MPI_Group& GROUP_TRANS, - MPI_Comm& COMM_TRANS, - const int nnz, - std::vector& rowidx, - std::vector& colidx, - int& sender_size, - std::vector& sender_size_process, - std::vector& sender_displacement_process, - int& receiver_size, - std::vector& receiver_size_process, - std::vector& receiver_displacement_process, - std::vector& buffer2ccsIndex); +int buildTransformParameter( + DistBCDMatrix &SRC_Matrix, + DistCCSMatrix &DST_Matrix, + const int NPROC_TRANS, + MPI_Group &GROUP_TRANS, + MPI_Comm &COMM_TRANS, + const int nnz, + std::vector &rowidx, + std::vector &colidx, + int &sender_size, + std::vector &sender_size_process, + std::vector &sender_displacement_process, + int &receiver_size, + std::vector &receiver_size_process, + std::vector &receiver_displacement_process, + std::vector &buffer2ccsIndex); -int newGroupCommTrans(DistBCDMatrix& SRC_Matrix, - DistCCSMatrix& DST_Matrix, - MPI_Group& GROUP_TRANS, - MPI_Comm& COMM_TRANS); +int newGroupCommTrans( + DistBCDMatrix &SRC_Matrix, + DistCCSMatrix &DST_Matrix, + MPI_Group &GROUP_TRANS, + MPI_Comm &COMM_TRANS); -int deleteGroupCommTrans(MPI_Group& GROUP_TRANS, MPI_Comm& COMM_TRANS); +int deleteGroupCommTrans(MPI_Group &GROUP_TRANS, MPI_Comm &COMM_TRANS); -int transformBCDtoCCS(DistBCDMatrix& SRC_Matrix, - double* H_2d, - double* S_2d, - const double ZERO_Limit, - DistCCSMatrix& DST_Matrix, - double*& H_ccs, - double*& S_ccs); +int transformBCDtoCCS( + DistBCDMatrix &SRC_Matrix, + double *H_2d, + double *S_2d, + const double ZERO_Limit, + DistCCSMatrix &DST_Matrix, + double *&H_ccs, + double *&S_ccs); -int transformCCStoBCD(DistCCSMatrix& SRC_Matrix, - double* DMnzvalLocal, - double* ENDnzvalLocal, - DistBCDMatrix& DST_Matrix, - double* DM_2d, - double* ED_2d); +int transformCCStoBCD( + DistCCSMatrix &SRC_Matrix, + double *DMnzvalLocal, + double *ENDnzvalLocal, + DistBCDMatrix &DST_Matrix, + double *DM_2d, + double *ED_2d); }; // namespace DistMatrixTransformer } // namespace pexsi #endif // DIST_MAT_TF_H \ No newline at end of file diff --git a/source/source_hsolver/module_pexsi/pexsi_solver.cpp b/source/source_hsolver/module_pexsi/pexsi_solver.cpp index f58cd9bb9de..5202b664568 100644 --- a/source/source_hsolver/module_pexsi/pexsi_solver.cpp +++ b/source/source_hsolver/module_pexsi/pexsi_solver.cpp @@ -37,16 +37,17 @@ double PEXSI_Solver::pexsi_mu_guard = 0.0; double PEXSI_Solver::pexsi_elec_thr = 0.0; double PEXSI_Solver::pexsi_zero_thr = 0.0; -void PEXSI_Solver::prepare(const int blacs_text, - const int nb, - const int nrow, - const int ncol, - const int nlocal, - const double nelec, - const double* h, - const double* s, - double*& _DM, - double*& _EDM) +void PEXSI_Solver::prepare( + const int blacs_text, + const int nb, + const int nrow, + const int ncol, + const int nlocal, + const double nelec, + const double *h, + const double *s, + double *&_DM, + double *&_EDM) { this->blacs_text = blacs_text; this->nb = nb; @@ -54,8 +55,8 @@ void PEXSI_Solver::prepare(const int blacs_text, this->ncol = ncol; this->nlocal = nlocal; this->nelec = nelec; - this->h = const_cast(h); - this->s = const_cast(s); + this->h = const_cast(h); + this->s = const_cast(s); this->DM = _DM; this->EDM = _EDM; this->totalEnergyH = 0.0; @@ -73,50 +74,43 @@ int PEXSI_Solver::solve(double mu0, const int world_nproc) MPI_Comm_group(DIAG_WORLD, &world_group); int grid_proc_range[3] = {0, (world_nproc / grid_np) * grid_np - 1, world_nproc / grid_np}; - MPI_Group_range_incl(world_group, 1, &grid_proc_range, &grid_group); + MPI_Group_range_incl( + world_group, + 1, + &grid_proc_range, + &grid_group); - simplePEXSI(DIAG_WORLD, - DIAG_WORLD, - grid_group, - this->blacs_text, - this->nlocal, - this->nb, - this->nrow, - this->ncol, - 'c', - this->h, - this->s, - this->nelec, - "PEXSIOPTION", - this->DM, - this->EDM, - this->totalEnergyH, - this->totalEnergyS, - this->totalFreeEnergy, - mu, - mu0); + simplePEXSI( + DIAG_WORLD, + DIAG_WORLD, + grid_group, + this->blacs_text, + this->nlocal, + this->nb, + this->nrow, + this->ncol, + 'c', + this->h, + this->s, + this->nelec, + "PEXSIOPTION", + this->DM, + this->EDM, + this->totalEnergyH, + this->totalEnergyS, + this->totalFreeEnergy, + mu, + mu0); return 0; } -const double PEXSI_Solver::get_totalFreeEnergy() const -{ - return totalFreeEnergy; -} +const double PEXSI_Solver::get_totalFreeEnergy() const { return totalFreeEnergy; } -const double PEXSI_Solver::get_totalEnergyH() const -{ - return totalEnergyH; -} +const double PEXSI_Solver::get_totalEnergyH() const { return totalEnergyH; } -const double PEXSI_Solver::get_totalEnergyS() const -{ - return totalEnergyS; -} +const double PEXSI_Solver::get_totalEnergyS() const { return totalEnergyS; } -const double PEXSI_Solver::get_mu() const -{ - return mu; -} +const double PEXSI_Solver::get_mu() const { return mu; } } // namespace pexsi #endif diff --git a/source/source_hsolver/module_pexsi/pexsi_solver.h b/source/source_hsolver/module_pexsi/pexsi_solver.h index 71637e64ef9..7a74a2b7b4f 100644 --- a/source/source_hsolver/module_pexsi/pexsi_solver.h +++ b/source/source_hsolver/module_pexsi/pexsi_solver.h @@ -8,16 +8,17 @@ namespace pexsi class PEXSI_Solver { public: - void prepare(const int blacs_text, - const int nb, - const int nrow, - const int ncol, - const int nlocal, - const double nelec, - const double* h, - const double* s, - double*& DM, - double*& EDM); + void prepare( + const int blacs_text, + const int nb, + const int nrow, + const int ncol, + const int nlocal, + const double nelec, + const double *h, + const double *s, + double *&DM, + double *&EDM); int solve(double mu0, const int world_nproc); const double get_totalFreeEnergy() const; const double get_totalEnergyH() const; @@ -27,104 +28,105 @@ class PEXSI_Solver //========================================================== // PEXSI related variables //========================================================== - /** + /** * @brief Number of terms in the pole expansion. - */ + */ static int pexsi_npole; - /** + /** * @brief Whether inertia counting is used at the very beginning. - */ + */ static bool pexsi_inertia; - /** + /** * @brief Maximum number of PEXSI iterations after each inertia counting procedure. - */ + */ static int pexsi_nmax; - /** + /** * @brief Whether to construct PSelInv communication pattern. - */ + */ static bool pexsi_comm; - /** - * @brief Whether to use symmetric storage space used by the Selected Inversion algorithm for symmetric matrices. - */ + /** + * @brief Whether to use symmetric storage space used by the Selected Inversion algorithm for symmetric matrices. + */ static bool pexsi_storage; - /** - * @brief Ordering strategy for factorization and selected inversion. - */ + /** + * @brief Ordering strategy for factorization and selected inversion. + */ static int pexsi_ordering; - /** - * @brief row permutation strategy for factorization and selected inversion. - */ + /** + * @brief row permutation strategy for factorization and selected inversion. + */ static int pexsi_row_ordering; - /** + /** * @brief Number of processors for PARMETIS/PT-SCOTCH. Only used if the ordering == 0. - */ + */ static int pexsi_nproc; - /** + /** * @brief Matrix structure. * - = 0 : Unsymmetric matrix * - = 1 : Symmetric matrix (default). - */ + */ static bool pexsi_symm; - /** + /** * @brief Transpose. * - = 0 : Factor non transposed matrix (default). * - = 1 : Factor transposed matrix. - */ + */ static bool pexsi_trans; - /** + /** * @brief The pole expansion method to be used. * - = 1 : Cauchy Contour Integral method used. * - = 2 : Moussa optimized method. - */ + */ static int pexsi_method; - /** + /** * @brief The point parallelizaion of PEXSI. * - = 2 : Recommend two points parallelization - */ + */ static int pexsi_nproc_pole; - /** - * @brief Temperature, in the same unit as H - */ + /** + * @brief Temperature, in the same unit as H + */ static double pexsi_temp; - /** + /** * @brief Spectral gap. **Note** This can be set to be 0 in most cases. - */ + */ static double pexsi_gap; - /** + /** * @brief An upper bound for the spectral radius of \f$S^{-1} H\f$. - */ + */ static double pexsi_delta_e; - /** + /** * @brief Initial guess of lower bound for mu. - */ + */ static double pexsi_mu_lower; - /** + /** * @brief Initial guess of upper bound for mu. - */ + */ static double pexsi_mu_upper; - /** + /** * @brief Initial guess for mu (for the solver) (AG) - */ + */ static double pexsi_mu; - /** + /** * @brief Stopping criterion in terms of the chemical potential for the inertia counting procedure. - */ + */ static double pexsi_mu_thr; - /** + /** * @brief If the chemical potential is not in the initial interval, the interval is expanded by muInertiaExpansion. - */ + */ static double pexsi_mu_expand; - /** + /** * @brief Safe guard criterion in terms of the chemical potential to reinvoke the inertia counting procedure. - */ + */ static double pexsi_mu_guard; - /** - * @brief Stopping criterion of the %PEXSI iteration in terms of the number of electrons compared to numElectronExact. - */ + /** + * @brief Stopping criterion of the %PEXSI iteration in terms of the number of electrons compared to + * numElectronExact. + */ static double pexsi_elec_thr; - /** + /** * @brief If the absolute value of CCS matrix element is less than this value, it will be considered as zero. - */ + */ static double pexsi_zero_thr; private: @@ -134,10 +136,10 @@ class PEXSI_Solver int ncol; int nlocal; ///< global dimension of the NAO Hamiltonian double nelec; ///< total number of electrons - double* h = nullptr; - double* s = nullptr; - double* DM = nullptr; - double* EDM = nullptr; + double *h = nullptr; + double *s = nullptr; + double *DM = nullptr; + double *EDM = nullptr; double totalEnergyH; double totalEnergyS; double totalFreeEnergy; diff --git a/source/source_hsolver/module_pexsi/simple_pexsi.cpp b/source/source_hsolver/module_pexsi/simple_pexsi.cpp index d91a6ad8650..7d98e701ad4 100644 --- a/source/source_hsolver/module_pexsi/simple_pexsi.cpp +++ b/source/source_hsolver/module_pexsi/simple_pexsi.cpp @@ -23,7 +23,7 @@ namespace pexsi { -inline void strtolower(char* sa, char* sb) +inline void strtolower(char *sa, char *sb) { char c = '\0'; int len = strlen(sa); @@ -35,7 +35,7 @@ inline void strtolower(char* sa, char* sb) sb[len] = '\0'; } -inline void setDefaultOption(int* int_para, double* double_para) +inline void setDefaultOption(int *int_para, double *double_para) { double_para[0] = 2; double_para[2] = 0; @@ -50,11 +50,12 @@ inline void setDefaultOption(int* int_para, double* double_para) int_para[15] = 1; } -int loadPEXSIOption(MPI_Comm comm, - const std::string PexsiOptionFile, - PPEXSIOptions& options, - int& numProcessPerPole, - double& ZERO_Limit) +int loadPEXSIOption( + MPI_Comm comm, + const std::string PexsiOptionFile, + PPEXSIOptions &options, + int &numProcessPerPole, + double &ZERO_Limit) { // temp variable arrays read from conf file and will be bcast to all processors @@ -159,7 +160,7 @@ int loadPEXSIOption(MPI_Comm comm, return 0; } -void splitNProc2NProwNPcol(const int NPROC, int& nprow, int& npcol) +void splitNProc2NProwNPcol(const int NPROC, int &nprow, int &npcol) { int integral_part = (int)sqrt(NPROC); if (NPROC % integral_part == 0) @@ -184,37 +185,42 @@ void splitNProc2NProwNPcol(const int NPROC, int& nprow, int& npcol) for (i = flag; i > 0; ++i) { if (NPROC % i == 0) + { break; + } } nprow = i; npcol = NPROC / i; } } -int simplePEXSI(MPI_Comm comm_PEXSI, - MPI_Comm comm_2D, - MPI_Group group_2D, - const int blacs_ctxt, // communicator parameters - const int size, - const int nblk, - const int nrow, - const int ncol, - char layout, // matrix parameters - double* H, - double* S, // input matrices - const double numElectronExact, - const std::string PexsiOptionFile, // pexsi parameters file - double*& DM, - double*& EDM, // output matrices - double& totalEnergyH, - double& totalEnergyS, - double& totalFreeEnergy, // output energy - double& mu, - double mu0) +int simplePEXSI( + MPI_Comm comm_PEXSI, + MPI_Comm comm_2D, + MPI_Group group_2D, + const int blacs_ctxt, // communicator parameters + const int size, + const int nblk, + const int nrow, + const int ncol, + char layout, // matrix parameters + double *H, + double *S, // input matrices + const double numElectronExact, + const std::string PexsiOptionFile, // pexsi parameters file + double *&DM, + double *&EDM, // output matrices + double &totalEnergyH, + double &totalEnergyS, + double &totalFreeEnergy, // output energy + double &mu, + double mu0) { if (comm_2D == MPI_COMM_NULL && comm_PEXSI == MPI_COMM_NULL) + { return 0; + } int myid = 0; std::ofstream f_log; if (comm_PEXSI != MPI_COMM_NULL) @@ -227,7 +233,12 @@ int simplePEXSI(MPI_Comm comm_PEXSI, PPEXSISetDefaultOptions(&options); int numProcessPerPole = 0; double ZERO_Limit = 0.0; - loadPEXSIOption(comm_PEXSI, PexsiOptionFile, options, numProcessPerPole, ZERO_Limit); + loadPEXSIOption( + comm_PEXSI, + PexsiOptionFile, + options, + numProcessPerPole, + ZERO_Limit); options.mu0 = mu0; ModuleBase::timer::start("Diago_LCAO_Matrix", "setup_PEXSI_plan"); @@ -243,10 +254,15 @@ int simplePEXSI(MPI_Comm comm_PEXSI, ModuleBase::timer::start("Diago_LCAO_Matrix", "PEXSIPlanInit"); if (comm_PEXSI != MPI_COMM_NULL) { - plan = PPEXSIPlanInitialize(comm_PEXSI, pexsi_prow, pexsi_pcol, outputFileIndex, &info); + plan = PPEXSIPlanInitialize( + comm_PEXSI, + pexsi_prow, + pexsi_pcol, + outputFileIndex, + &info); } ModuleBase::timer::end("Diago_LCAO_Matrix", "PEXSIPlanInit"); - + ModuleBase::timer::end("Diago_LCAO_Matrix", "setup_PEXSI_plan"); // create compressed column storage distribution matrix parameter @@ -256,19 +272,33 @@ int simplePEXSI(MPI_Comm comm_PEXSI, // LiuXh modify 2021-03-30, add DONE(ofs_running,"xx") for test // DONE(ofs_running,"create compressed column storage distribution matrix parameter, finish"); - // create block cyclic distribution matrix parameter - DistBCDMatrix SRC_Matrix(comm_2D, group_2D, blacs_ctxt, size, nblk, nrow, ncol, layout); + DistBCDMatrix SRC_Matrix( + comm_2D, + group_2D, + blacs_ctxt, + size, + nblk, + nrow, + ncol, + layout); // LiuXh modify 2021-03-30, add DONE(ofs_running,"xx") for test // DONE(ofs_running,"create block cyclic distribution matrix parameter, finish"); - double* HnzvalLocal = nullptr; - double* SnzvalLocal = nullptr; - double* DMnzvalLocal = nullptr; - double* EDMnzvalLocal = nullptr; - double* FDMnzvalLocal = nullptr; + double *HnzvalLocal = nullptr; + double *SnzvalLocal = nullptr; + double *DMnzvalLocal = nullptr; + double *EDMnzvalLocal = nullptr; + double *FDMnzvalLocal = nullptr; // transform H and S from 2D block cyclic distribution to compressed column sparse matrix // LiuXh modify 2021-03-30, add DONE(ofs_running,"xx") for test - DistMatrixTransformer::transformBCDtoCCS(SRC_Matrix, H, S, ZERO_Limit, DST_Matrix, HnzvalLocal, SnzvalLocal); + DistMatrixTransformer::transformBCDtoCCS( + SRC_Matrix, + H, + S, + ZERO_Limit, + DST_Matrix, + HnzvalLocal, + SnzvalLocal); // MPI_Barrier(MPI_COMM_WORLD); // LiuXh modify 2021-03-30, add DONE(ofs_running,"xx") for test if (comm_PEXSI != MPI_COMM_NULL) @@ -276,18 +306,19 @@ int simplePEXSI(MPI_Comm comm_PEXSI, // Load H and S to PEXSI int isSIdentity = 0; - PPEXSILoadRealHSMatrix(plan, - options, - size, - DST_Matrix.get_nnz(), - DST_Matrix.get_nnzlocal(), - DST_Matrix.get_numcol_local(), - DST_Matrix.get_colptr_local(), - DST_Matrix.get_rowind_local(), - HnzvalLocal, - isSIdentity, - SnzvalLocal, - &info); + PPEXSILoadRealHSMatrix( + plan, + options, + size, + DST_Matrix.get_nnz(), + DST_Matrix.get_nnzlocal(), + DST_Matrix.get_numcol_local(), + DST_Matrix.get_colptr_local(), + DST_Matrix.get_rowind_local(), + HnzvalLocal, + isSIdentity, + SnzvalLocal, + &info); double nelec = 0.0; double muMinInertia = 0.0; @@ -296,39 +327,47 @@ int simplePEXSI(MPI_Comm comm_PEXSI, int numTotalInertiaIter = 0; // Number of total inertia[out] // LiuXh modify 2021-04-29, add DONE(ofs_running,"xx") for test ModuleBase::timer::start("Diago_LCAO_Matrix", "PEXSIDFT"); - PPEXSIDFTDriver2(plan, // PEXSI plan[in] - &options, // PEXSI Options[in] - numElectronExact, // exact electron number[in] - &mu, // chemical potential[out] - &nelec, // number of electrons[out] - // &muMinInertia, // Lower bound for mu after the last inertia[out] - // &muMaxInertia, // Upper bound for mu after the last inertia[out] - &numTotalInertiaIter, // Number of total inertia[out] - // &numTotalPEXSIIter, // number of total pexsi evaluation procedure[out] - &info); // 0: successful; otherwise: unsuccessful + PPEXSIDFTDriver2( + plan, // PEXSI plan[in] + &options, // PEXSI Options[in] + numElectronExact, // exact electron number[in] + &mu, // chemical potential[out] + &nelec, // number of electrons[out] + // &muMinInertia, // Lower bound for mu after the last inertia[out] + // &muMaxInertia, // Upper bound for mu after the last inertia[out] + &numTotalInertiaIter, // Number of total inertia[out] + // &numTotalPEXSIIter, // number of total pexsi evaluation procedure[out] + &info); // 0: successful; otherwise: unsuccessful // LiuXh modify 2021-04-29, add DONE(ofs_running,"xx") for test ModuleBase::timer::end("Diago_LCAO_Matrix", "PEXSIDFT"); // retrieve the results from the plan if (DMnzvalLocal != nullptr) + { delete[] DMnzvalLocal; + } if (EDMnzvalLocal != nullptr) + { delete[] EDMnzvalLocal; + } if (FDMnzvalLocal != nullptr) + { delete[] FDMnzvalLocal; + } DMnzvalLocal = new double[DST_Matrix.get_nnzlocal()]; EDMnzvalLocal = new double[DST_Matrix.get_nnzlocal()]; FDMnzvalLocal = new double[DST_Matrix.get_nnzlocal()]; if (myid < numProcessPerPole) { - PPEXSIRetrieveRealDFTMatrix(plan, - DMnzvalLocal, - EDMnzvalLocal, - FDMnzvalLocal, - &totalEnergyH, - &totalEnergyS, - &totalFreeEnergy, - &info); + PPEXSIRetrieveRealDFTMatrix( + plan, + DMnzvalLocal, + EDMnzvalLocal, + FDMnzvalLocal, + &totalEnergyH, + &totalEnergyS, + &totalFreeEnergy, + &info); } // clean PEXSI PPEXSIPlanFinalize(plan, &info); @@ -345,7 +384,13 @@ int simplePEXSI(MPI_Comm comm_PEXSI, } // LiuXh modify 2021-04-29, add DONE(ofs_running,"xx") for test ModuleBase::timer::start("Diago_LCAO_Matrix", "TransMAT22D"); - DistMatrixTransformer::transformCCStoBCD(DST_Matrix, DMnzvalLocal, EDMnzvalLocal, SRC_Matrix, DM, EDM); + DistMatrixTransformer::transformCCStoBCD( + DST_Matrix, + DMnzvalLocal, + EDMnzvalLocal, + SRC_Matrix, + DM, + EDM); ModuleBase::timer::end("Diago_LCAO_Matrix", "TransMAT22D"); // LiuXh modify 2021-04-29, add DONE(ofs_running,"xx") for test diff --git a/source/source_hsolver/module_pexsi/simple_pexsi.h b/source/source_hsolver/module_pexsi/simple_pexsi.h index 1aa2d632d8b..22fde0bdb46 100644 --- a/source/source_hsolver/module_pexsi/simple_pexsi.h +++ b/source/source_hsolver/module_pexsi/simple_pexsi.h @@ -6,25 +6,26 @@ // a simple interface for calling pexsi with 2D block cyclic distributed matrix namespace pexsi { -int simplePEXSI(MPI_Comm comm_PEXSI, - MPI_Comm comm_2D, - MPI_Group group_2D, - const int blacs_ctxt, // communicator parameters - const int size, - const int nblk, - const int nrow, - const int ncol, - char layout, // input matrix parameters - double* H, - double* S, // input matrices - const double nElectronExact, - const std::string PexsiOptionFile, // pexsi parameters file - double*& DM, - double*& EDM, // output matrices - double& totalEnergyH, - double& totalEnergyS, - double& totalFreeEnergy, - double& mu, - double mu0); +int simplePEXSI( + MPI_Comm comm_PEXSI, + MPI_Comm comm_2D, + MPI_Group group_2D, + const int blacs_ctxt, // communicator parameters + const int size, + const int nblk, + const int nrow, + const int ncol, + char layout, // input matrix parameters + double *H, + double *S, // input matrices + const double nElectronExact, + const std::string PexsiOptionFile, // pexsi parameters file + double *&DM, + double *&EDM, // output matrices + double &totalEnergyH, + double &totalEnergyS, + double &totalFreeEnergy, + double &mu, + double mu0); } #endif // SIMPLE_PEXSI_H \ No newline at end of file