Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion source/source_io/module_parameter/read_inp_sys.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -452,7 +452,8 @@ In socket_driver mode, this flag controls whether the returned frame advertises
item.description = R"(Controls k-point parallelism. The value must be positive and should not exceed either the number of k-points or the number of MPI processes.
* For PW calculations, divide all MPI processes into persistent k-point pools. Each pool stores and processes a subset of the k-points.
* For LCAO calculations with lapack, genelpa, elpa, or scalapack_gvx, divide the diagonalization work into temporary k-point pools. After diagonalization, the eigenvalues and distributed wavefunctions are restored for all k-points before occupations, density matrices, and output are evaluated.
* Multi-process LCAO cusolver uses its own active-GPU distribution and does not use this value to define its k-point layout. Other LCAO eigensolvers do not use the temporary k-point-pool implementation.)";
* Multi-process LCAO cusolver uses its own active-GPU distribution and does not use this value to define its k-point layout. Other LCAO eigensolvers do not use the temporary k-point-pool implementation.
* On DSP builds, the mtblas/C_PACK libraries require kpar (kpar_lcao for LCAO) times bndpar to equal the number of MPI processes exactly; ABACUS aborts at startup otherwise.)";
item.default_value = "1";
item.unit = "";
read_sync_int(input.kpar);
Expand All @@ -478,6 +479,39 @@ In socket_driver mode, this flag controls whether the returned frame advertises
}
#endif
};
#ifdef __DSP
item.check_value = [](const Input_Item& item, const Parameter& para) {
// The mtblas/C_PACK libraries on DSP hardware require exactly one
// process per k-point pool: the effective pool count (kpar for PW,
// kpar_lcao for LCAO, the other being 1) times bndpar must equal
// the number of MPI processes. Other configurations make the vendor
// routines return inconsistent results across ranks and the SCF
// silently diverges, so reject them at input time. See issue #8228.
// bndpar <= 0 is rejected by the bndpar input check; skip here so
// the diagnostic below never divides by zero.
if (para.input.bndpar <= 0)
{
return;
}
const int nproc = para.sys.nproc;
const bool layout_valid = dsp_kpar_layout_valid(nproc,
para.input.kpar,
para.sys.kpar_lcao,
para.input.bndpar);
if (!layout_valid)
{
const std::string msg =
"On DSP hardware, the mtblas/C_PACK libraries require kpar (kpar_lcao for LCAO) * bndpar"
" to equal the number of MPI processes, but got NPROC=" + std::to_string(nproc)
+ ", kpar=" + std::to_string(para.input.kpar)
+ ", kpar_lcao=" + std::to_string(para.sys.kpar_lcao)
+ ", bndpar=" + std::to_string(para.input.bndpar)
+ ". Please set kpar=" + std::to_string(nproc / para.input.bndpar)
Comment thread
DanielZhao0432 marked this conversation as resolved.
Resolved
+ " or adjust the process count.";
ModuleBase::WARNING_QUIT("ReadInput", msg);
}
};
#endif
this->add_item(item);
add_int_bcast(sys.kpar_lcao);
}
Expand Down
21 changes: 21 additions & 0 deletions source/source_io/module_parameter/read_input_tool.h
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#include <algorithm>
#include <sstream>
#include <stdexcept>
#include <string>
Expand Down Expand Up @@ -241,3 +242,23 @@ void reset_vector(std::vector<T>& vec, int size, T default_value)
vec.resize(size, default_value);
}
}

namespace ModuleIO
{
/**
* @brief Check whether a k-point/band parallel layout satisfies the DSP vendor
* library constraint.
*
* On DSP hardware, the mtblas/C_PACK libraries require exactly one MPI process
* per k-point pool: the effective pool count (kpar for PW, kpar_lcao for LCAO,
* the other being 1) times bndpar must equal nproc exactly. Defined inline here
* so that both the __DSP input check and non-DSP unit tests can use it.
* See issue #8228.
*/
inline bool dsp_kpar_layout_valid(const int nproc, const int kpar, const int kpar_lcao, const int bndpar)
{
const int kpar_eff = std::max(kpar, kpar_lcao);
const int expected_nproc = kpar_eff * bndpar;
return nproc == expected_nproc;
}
} // namespace ModuleIO
6 changes: 6 additions & 0 deletions source/source_io/unittests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,12 @@ AddTest(
SOURCES test_parse_args.cpp ../parse_args.cpp ../input_help.cpp
)

AddTest(
TARGET MODULE_IO_test_read_inp_sys
LIBS base device
SOURCES test_read_inp_sys.cpp
)

AddTest(
TARGET MODULE_IO_input_help_test
LIBS parameter base device io_input
Expand Down
50 changes: 50 additions & 0 deletions source/source_io/unittests/test_read_inp_sys.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#include "gtest/gtest.h"

#include "source_io/module_parameter/read_input_tool.h"

/************************************************
* unit tests of dsp_kpar_layout_valid
*
* The helper encodes the DSP vendor library (mtblas/C_PACK) constraint
* checked by the kpar input validation under __DSP: the effective k-point
* pool count (kpar for PW, kpar_lcao for LCAO, the other being 1) times
* bndpar must equal the number of MPI processes. See issue #8228.
***********************************************/

TEST(DspKparLayoutValidTest, PWKparPools)
{
// PW: kpar keeps the user value, kpar_lcao stays 1
EXPECT_TRUE(ModuleIO::dsp_kpar_layout_valid(4, 4, 1, 1));
// default kpar=1 on 4 processes: the silently diverging case of #8228
EXPECT_FALSE(ModuleIO::dsp_kpar_layout_valid(4, 1, 1, 1));
// serial run is always fine
EXPECT_TRUE(ModuleIO::dsp_kpar_layout_valid(1, 1, 1, 1));
// more processes than pools
EXPECT_FALSE(ModuleIO::dsp_kpar_layout_valid(8, 4, 1, 1));
}

TEST(DspKparLayoutValidTest, LCAOKparReset)
{
// LCAO moves the user kpar into kpar_lcao and forces kpar to 1
EXPECT_TRUE(ModuleIO::dsp_kpar_layout_valid(4, 1, 4, 1));
EXPECT_FALSE(ModuleIO::dsp_kpar_layout_valid(4, 1, 2, 1));
EXPECT_FALSE(ModuleIO::dsp_kpar_layout_valid(4, 1, 1, 1));
}

TEST(DspKparLayoutValidTest, BndparExemption)
{
// the bndpar+kpar layout that #7420 intended to allow
EXPECT_TRUE(ModuleIO::dsp_kpar_layout_valid(4, 2, 1, 2));
EXPECT_FALSE(ModuleIO::dsp_kpar_layout_valid(4, 1, 1, 2));
EXPECT_FALSE(ModuleIO::dsp_kpar_layout_valid(8, 2, 1, 2));
}

TEST(DspKparLayoutValidTest, DegenerateValues)
{
// non-positive factors are invalid layouts, and must be rejected by plain
// arithmetic without any division so no undefined behavior can occur
EXPECT_FALSE(ModuleIO::dsp_kpar_layout_valid(4, 0, 0, 1));
EXPECT_FALSE(ModuleIO::dsp_kpar_layout_valid(4, 1, 1, 0));
EXPECT_FALSE(ModuleIO::dsp_kpar_layout_valid(4, -1, -1, 1));
EXPECT_FALSE(ModuleIO::dsp_kpar_layout_valid(0, 1, 1, 1));
}
Loading