From 0c568aa187d20d7e74a8124ea62cb71f12266fda Mon Sep 17 00:00:00 2001 From: danieltowner Date: Thu, 8 Oct 2026 21:35:57 +0100 Subject: [PATCH] Rewrote `to_register` utility Previously `to_register` would return a builtin with enough elements to fill a complete register of the appropriate size, but this often wouldn't be allowed as a parameter to an intrinsic. The new version converts the register type to the one of the common intrinsic types (e.g., __m256i, __m512d, __m128h). This simplifies intrinsic calls by allowing `to_register` to handle the type conversion without also requiring an explicit cast at the intrinsic call site. This work is an important step in eliminating the use of `flax-vector-conversions` since it allows intrinsic calls without implicit conversions. --- include/xvec/detail/core.hpp | 56 ++++++++++++++---------- include/xvec/detail/target_config.hpp | 63 +++++++++++++++++++++++++++ include/xvec/simd | 21 ++++----- include/xvec/x86/permute.hpp | 8 ++-- include/xvec/x86/x86_impl.hpp | 42 +++++++++--------- 5 files changed, 131 insertions(+), 59 deletions(-) diff --git a/include/xvec/detail/core.hpp b/include/xvec/detail/core.hpp index ebba3f8..210adaf 100644 --- a/include/xvec/detail/core.hpp +++ b/include/xvec/detail/core.hpp @@ -437,30 +437,33 @@ struct vec_traits /// The compiler's representation of the simd::vec type. using builtin_type __attribute__((__vector_size__(sizeof(element_type) * _numStorageElements))) = element_type; - /// @brief The best register size to use for operations on this builtin type. - using register_type __attribute__((__vector_size__(sizeof(element_type) * register_size))) = element_type; - - /// @brief Allow the builtin data to be turned into the closest matching - /// target register type. This will return the smallest target register that - /// can contain all the elements of the builtin type. It is an error to try to - /// store the data in a register which is too small. In that case the user - /// must break the builtin type into pieces which do fit in a register (e.g., - /// using fit_to_size, or extract). If the builtin type contains less data - /// than the register then the unused register elements will have undefined - /// values. + /// @brief The register to use for intrinsic operations on this vec (e.g., for + /// vec it would be __m256d). Note that unininitialised elements + /// may be inserted to grow the vector to a size which can be represented in a + /// register. + using register_type = decltype(detail::get_register_type()); + + /// Convert the vector data into a native register value which can be passed + /// directly into an intrinsic. Firstly the data is expanded to full a + /// complete native register (e.g., 128-bit, 256-bit, etc) using uninitialised + /// bits. Secondly, the register is converted into the most appropriate native + /// register type which can be used directly by an intrinsic. For example, + /// vec would become __m128i, vec would become __m256, and + /// so on. This function must not be used on a vec which is too big to fit a + /// native register (use simd::chunk to break it into register-sized pieces). + /// Note that while many intrinsics accept a register which contains + /// uninitialised data, this data might lead to incorrect operation. The + /// caller is responsible for ensuring that the uninitialised bits are set to + /// a safe value before calling an intrinsic which requires it. static constexpr register_type to_register(builtin_type v) { - // Use a static assert rather than a requires to allow this error to be - // reported instead of rather than resulting in a confusing lookup failure. - constexpr auto _regSize = register_size<_Tp, _Np>; + constexpr auto _regSize = register_size<_Tp, _Np>; static_assert(_Np <= _regSize, "Converting to a smaller register loses data"); - if constexpr (_Np == _regSize) - return v; - else - { - return [=](std::index_sequence<_Idx...>) { + + auto r = [=](std::index_sequence<_Idx...>) { return __builtin_shufflevector(v, v, (_Idx < _Np ? int(_Idx) : -1)...); }(std::make_index_sequence<_regSize>()); - } + + return register_type(r); } /// @brief A named constructor for creating a builtin type from a register value. @@ -470,12 +473,17 @@ struct vec_traits constexpr auto _regSize = register_size<_Tp, _Np>; static_assert(_Np <= _regSize, "Converting from a smaller register creates undefined values"); if constexpr (_Np == _regSize) - return r; + return builtin_type(r); else { - return [=](std::index_sequence<_Idx...>) { - return __builtin_shufflevector(r, r, ((_Idx < _numDataElements) ? int(_Idx) : -1)...); - }(std::make_index_sequence<_numStorageElements>()); + // Convert to a full register of the builtin elements. + using FullRegBuiltin = typename vec::builtin_type; + auto b = FullRegBuiltin(r); + + // Then shuffle down to the correct number of elements. + return builtin_type([=](std::index_sequence<_Idx...>) { + return __builtin_shufflevector(b, b, _Idx...); + }(std::make_index_sequence<_numStorageElements>())); } } diff --git a/include/xvec/detail/target_config.hpp b/include/xvec/detail/target_config.hpp index 4daa0a0..5780b29 100644 --- a/include/xvec/detail/target_config.hpp +++ b/include/xvec/detail/target_config.hpp @@ -8,6 +8,8 @@ #pragma once +#include + #include #include @@ -20,6 +22,10 @@ namespace _XVEC_NAMESPACE::simd /// generic operations. struct generic_tag {}; +/// If a type supplied to get_register_type is invalid, mark that as such with +/// this special type. +struct invalid_register {}; + #if defined(__SSE__) and !defined(_XVEC_FORCE_SCALAR) // These target tags are specific to x86 and provide a hierarchy of @@ -78,6 +84,54 @@ inline constexpr int register_size = (hasAvx && (sizeof(_Tp) * _Np) > 16) ? (32 / sizeof(_Tp)) : 16 / sizeof(_Tp); +/// Generate the appropriate register type for the given vec type. For example, +/// a small vector of ints would be converted to the smallest native register +/// capable of storing all the bits, with padding added to bring it up to the +/// register size. the type of the element is also taken into account (e.g., +/// `double` elements would convert to a `d` suffix like __m256d). +template +consteval auto get_register_type() { + constexpr auto bytes = sizeof(T); + + using _Tp = std::remove_cvref_t()[0])>; + + constexpr bool isFloat = std::same_as; + constexpr bool isDouble = std::same_as; + + if constexpr (bytes > maxBytesInVec) + return invalid_register{}; + else +#if defined(__AVX512FP16__) + if constexpr (std::same_as<_Tp, _Float16>) + { + if constexpr (bytes > 32) return __m512h(); + else if constexpr (bytes > 16) return __m256h(); + else return __m128h(); + } else +#endif +#if defined(__AVX512F__) + if constexpr (bytes > 32) + { + if constexpr (isFloat) return __m512(); + else if constexpr (isDouble) return __m512d(); + else return __m512i(); + } else +#endif +#if defined(__AVX__) + if constexpr (bytes > 16) + { + if constexpr (isFloat) return __m256(); + else if constexpr (isDouble) return __m256d(); + else return __m256i(); + } else +#endif + { + if constexpr (isFloat) return __m128(); + else if constexpr (isDouble) return __m128d(); + else return __m128i(); + } +} + } // namespace detail #else @@ -93,6 +147,15 @@ namespace detail /// target-specific code. inline constexpr int maxBytesInVec = 16; template inline constexpr int register_size = maxBytesInVec / sizeof(_Tp); + +/// In generic targets, there is no way to know what the register type should +/// be, so return an invalid_register type to indicate that the target does not +/// support the requested type. This will cause a compile-time error if the user +/// tries to use a type which is not supported by the target. Targets which +/// support a specific register type should provide a specialization of this +/// function which returns the appropriate register type. +template consteval auto get_register_type() { return invalid_register{}; } + } struct compact_mask_tag : public generic_tag {}; diff --git a/include/xvec/simd b/include/xvec/simd index d56a36a..788f12e 100644 --- a/include/xvec/simd +++ b/include/xvec/simd @@ -153,15 +153,16 @@ public: constexpr basic_vec(const builtin_type& v) noexcept : data(v) {} /// Allow a vec to be constructed from a target register, such as the return - /// value of an intrinsic. If the vec object is smaller than the register - /// type then some values may be discarded, so the constructor is explicit to - /// make the user be clear about this. Note that in the special case that the - /// vec is already the exact size of the target register, this constructor - /// will be disabled because the register type will match the compiler's own - /// builtin type. + /// value of an intrinsic. If the vec object is smaller than the register type + /// then some values may be discarded, so the constructor is explicit to make + /// the user be clear about this. Note that there is also a constructor which + /// accepts a builtin type, so in the case where a register is directly + /// convertible to a builtin, this would introduce ambiguity and so is + /// disabled. /// @ingroup simd_constructor explicit constexpr basic_vec(register_type r) noexcept - requires (!std::same_as) : data(traits::from_register(r)) {} + requires (!std::is_convertible_v) + : data(traits::from_register(r)) {} /// @brief Constructs a basic_vec by reading the contents of an object with the same /// number of values (e.g., span, C-array, std::array). Both unmasked and masked variants are provided. @@ -266,9 +267,9 @@ public: /// This will return the smallest target register that can contain all the elements of the vec. explicit constexpr operator typename traits::register_type() const { return traits::to_register(data); } - /// @brief Convert the vec into the nearest register size which is large enough to store all the vec elements. - /// This is more convenient to use than the explicit register_type cast since there is no need to work out the register - /// type. + /// Convert the mask into a register which can be passed to an intrinsic + /// (e.g., on x86 `vec` would become `__m256d`). Vectors which + /// don't fit into a native register must be broken into smaller pieces first. constexpr typename traits::register_type to_register() const { return traits::to_register(data); } /// @brief Return the builtin value. diff --git a/include/xvec/x86/permute.hpp b/include/xvec/x86/permute.hpp index 1519da3..99f6e12 100644 --- a/include/xvec/x86/permute.hpp +++ b/include/xvec/x86/permute.hpp @@ -76,7 +76,7 @@ constexpr auto permute_byte_shuffle (const basic_vec<_Tp, _AbiT>& v, const basic static_assert(basic_vec<_Tp, _AbiT>::size <= 16 && basic_vec<_Ip, _AbiI>::size <= 16, "Data and indexes must fit first lane"); // :TODO: In future both indexes and values could extend beyond lanes through // duplication or multi-lane index merging, which will be faster than using permutex[2]var_epi8. - return vec<_Tp, basic_vec<_Ip, _AbiI>::size>(_mm_shuffle_epi8(v.to_register(), indexes.to_register() & 0xf)); + return vec<_Tp, basic_vec<_Ip, _AbiI>::size>(_mm_shuffle_epi8(v.to_register(), (indexes & uint8_t(0xf)).to_register())); } /// Permute one register of Intel AVX2 values. @@ -162,7 +162,7 @@ inline target_overloads permutexvar { [] _Vec>(_Vec v, _Vec i) { return _mm256_permutexvar_epi8(i.to_register(), v.to_register()); }, [] _Vec>(_Vec v, _Vec i) { return _mm512_permutexvar_epi8(i.to_register(), v.to_register()); }, #else - [] _Vec>(_Vec v, _Vec i) { return __m256i(_mm512_permutex2var_epi8_emulated(__m512i(v.to_register()), __m512i(i.to_register()), __m512i())); }, + [] _Vec>(_Vec v, _Vec i) { return __m256i(_mm512_permutex2var_epi8_emulated(v.to_register(), i.to_register(), __m512i())); }, [] _Vec>(_Vec v, _Vec i) { return _mm512_permutex2var_epi8_emulated(v.to_register(), i.to_register(), __m512i()); }, #endif @@ -180,7 +180,7 @@ inline target_overloads permutex2var { [] _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm256_permutex2var_epi8(v0.to_register(), i.to_register(), v1.to_register()); }, [] _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi8(v0.to_register(), i.to_register(), v1.to_register()); }, #else - [] _Vec>(_Vec v0, _Vec i, _Vec v1) { return __m256i(_mm512_permutex2var_epi8_emulated(__m512i(v0.to_register()), __m512i(i.to_register()), v1.to_register())); }, + [] _Vec>(_Vec v0, _Vec i, _Vec v1) { return __m256i(_mm512_permutex2var_epi8_emulated(v0.to_register(), i.to_register(), v1.to_register())); }, [] _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi8_emulated(v0.to_register(), i.to_register(), v1.to_register()); }, #endif @@ -394,7 +394,7 @@ inline target_overloads compress_one_register { [] _Vec>(_Vec v, auto m, auto f) { return _Vec(_mm512_mask_compress_epi8(f.to_register(), m.to_register(), v.to_register())); }, - // int 16 + // int 16 [] _Vec>(_Vec v, auto m, auto f) { return _Vec(_mm_mask_compress_epi16(f.to_register(), m.to_register(), v.to_register())); }, [] _Vec>(_Vec v, auto m, auto f) diff --git a/include/xvec/x86/x86_impl.hpp b/include/xvec/x86/x86_impl.hpp index e714360..ed6b7c3 100644 --- a/include/xvec/x86/x86_impl.hpp +++ b/include/xvec/x86/x86_impl.hpp @@ -440,7 +440,7 @@ constexpr void store_masked(x86_tag, const _Vp& value_original, std::span<_Up, _ constexpr auto numSseElements = 16 / sizeof(_Up); auto impl = [=](_Vec x, auto m, auto idx) { // Note that the mask register is zero extended to ensure no extra mask elements appear in it during `to_register'. - _mm_maskmoveu_si128(__m128i(x.to_register()), __m128i(grow(m).to_register()), (char*)(to.data() + idx)); + _mm_maskmoveu_si128(x.to_register(), grow(m).to_register(), (char*)(to.data() + idx)); }; // Use the mask to limit the bounds of the write if requested by the user. @@ -478,9 +478,9 @@ constexpr void store_masked(x86_avx2_tag, const _Vp& value_original, std::span<_ const auto rc = permute(s, perm_uninitResize); if constexpr (sizeof(_Up) == 4) - _mm256_maskstore_epi32(reinterpret_cast(to.data() + idx), rm.to_register(), __m256i(rc.to_register())); + _mm256_maskstore_epi32(reinterpret_cast(to.data() + idx), rm.to_register(), rc.to_register()); else if constexpr (sizeof(_Up) == 8) - _mm256_maskstore_epi64(reinterpret_cast(to.data() + idx), rm.to_register(), __m256i(rc.to_register())); + _mm256_maskstore_epi64(reinterpret_cast(to.data() + idx), rm.to_register(), rc.to_register()); else static_assert(dependent_false<_Vec>, "Unimplemented masked store"); }; @@ -505,24 +505,24 @@ constexpr void store_masked(x86_avx512_tag, const _Vp& value_original, std::span mandates_for_store(value_original, to, flags); target_overloads call_insn { - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi8(p, m, __m128i(v.to_register())); }, - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi16(p, m, __m128i(v.to_register())); }, - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi32(p, m, __m128i(v.to_register())); }, - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, m, __m128i(v.to_register())); }, - - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi8(p, m, __m256i(v.to_register())); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi16(p, m, __m256i(v.to_register())); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi32(p, m, __m256i(v.to_register())); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, m, __m256i(v.to_register())); }, - - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi8(p, m, __m512i(v.to_register())); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi16(p, m, __m512i(v.to_register())); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi32(p, m, __m512i(v.to_register())); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, m, __m512i(v.to_register())); }, - - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, dupBits(m), __m128i(v.to_register())); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, dupBits(m), __m256i(v.to_register())); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, dupBits(m), __m512i(v.to_register())); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi8(p, m, v.to_register()); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi16(p, m, v.to_register()); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi32(p, m, v.to_register()); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, m, v.to_register()); }, + + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi8(p, m, v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi16(p, m, v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi32(p, m, v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, m, v.to_register()); }, + + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi8(p, m, v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi16(p, m, v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi32(p, m, v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, m, v.to_register()); }, + + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, dupBits(m), v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, dupBits(m), v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, dupBits(m), v.to_register()); }, [=](auto unhandled, auto, auto) { static_assert(dependent_false, "Unimplemented partial masked store"); } };