diff --git a/include/xvec/detail/core.hpp b/include/xvec/detail/core.hpp index ebba3f8..210adaf 100644 --- a/include/xvec/detail/core.hpp +++ b/include/xvec/detail/core.hpp @@ -437,30 +437,33 @@ struct vec_traits /// The compiler's representation of the simd::vec type. using builtin_type __attribute__((__vector_size__(sizeof(element_type) * _numStorageElements))) = element_type; - /// @brief The best register size to use for operations on this builtin type. - using register_type __attribute__((__vector_size__(sizeof(element_type) * register_size))) = element_type; - - /// @brief Allow the builtin data to be turned into the closest matching - /// target register type. This will return the smallest target register that - /// can contain all the elements of the builtin type. It is an error to try to - /// store the data in a register which is too small. In that case the user - /// must break the builtin type into pieces which do fit in a register (e.g., - /// using fit_to_size, or extract). If the builtin type contains less data - /// than the register then the unused register elements will have undefined - /// values. + /// @brief The register to use for intrinsic operations on this vec (e.g., for + /// vec it would be __m256d). Note that unininitialised elements + /// may be inserted to grow the vector to a size which can be represented in a + /// register. + using register_type = decltype(detail::get_register_type()); + + /// Convert the vector data into a native register value which can be passed + /// directly into an intrinsic. Firstly the data is expanded to full a + /// complete native register (e.g., 128-bit, 256-bit, etc) using uninitialised + /// bits. Secondly, the register is converted into the most appropriate native + /// register type which can be used directly by an intrinsic. For example, + /// vec would become __m128i, vec would become __m256, and + /// so on. This function must not be used on a vec which is too big to fit a + /// native register (use simd::chunk to break it into register-sized pieces). + /// Note that while many intrinsics accept a register which contains + /// uninitialised data, this data might lead to incorrect operation. The + /// caller is responsible for ensuring that the uninitialised bits are set to + /// a safe value before calling an intrinsic which requires it. static constexpr register_type to_register(builtin_type v) { - // Use a static assert rather than a requires to allow this error to be - // reported instead of rather than resulting in a confusing lookup failure. - constexpr auto _regSize = register_size<_Tp, _Np>; + constexpr auto _regSize = register_size<_Tp, _Np>; static_assert(_Np <= _regSize, "Converting to a smaller register loses data"); - if constexpr (_Np == _regSize) - return v; - else - { - return [=](std::index_sequence<_Idx...>) { + + auto r = [=](std::index_sequence<_Idx...>) { return __builtin_shufflevector(v, v, (_Idx < _Np ? int(_Idx) : -1)...); }(std::make_index_sequence<_regSize>()); - } + + return register_type(r); } /// @brief A named constructor for creating a builtin type from a register value. @@ -470,12 +473,17 @@ struct vec_traits constexpr auto _regSize = register_size<_Tp, _Np>; static_assert(_Np <= _regSize, "Converting from a smaller register creates undefined values"); if constexpr (_Np == _regSize) - return r; + return builtin_type(r); else { - return [=](std::index_sequence<_Idx...>) { - return __builtin_shufflevector(r, r, ((_Idx < _numDataElements) ? int(_Idx) : -1)...); - }(std::make_index_sequence<_numStorageElements>()); + // Convert to a full register of the builtin elements. + using FullRegBuiltin = typename vec::builtin_type; + auto b = FullRegBuiltin(r); + + // Then shuffle down to the correct number of elements. + return builtin_type([=](std::index_sequence<_Idx...>) { + return __builtin_shufflevector(b, b, _Idx...); + }(std::make_index_sequence<_numStorageElements>())); } } diff --git a/include/xvec/detail/mask.hpp b/include/xvec/detail/mask.hpp index 39955d1..3c84813 100644 --- a/include/xvec/detail/mask.hpp +++ b/include/xvec/detail/mask.hpp @@ -91,6 +91,13 @@ class basic_mask constexpr basic_mask& operator=(const basic_mask&) = default; constexpr basic_mask& operator=(basic_mask&&) noexcept = default; + /// Allow a mask to be constructed from the compiler's own representation. The + /// compiler may use multiple or partial target registers in its + /// representation. This constructor allows a mask to be built from the output + /// of an overloaded operator, a builtin, and so on. + /// @ingroup simd_constructor + constexpr basic_mask(const builtin_type& m) noexcept : mask(m) {} + /// @brief Build a mask from a boolean value. /// @param b The boolean value to broadcast to all mask elements. constexpr explicit basic_mask(std::same_as auto b) noexcept : @@ -183,7 +190,7 @@ class basic_mask /// be a type which can be used for a standard constructor (e.g., /// integral-like) so the `from_builtin` ensures that the desired effect is /// achieved directly. - static constexpr basic_mask from_builtin(const auto& m) { basic_mask bm; bm.mask = traits::truncate_to_size(m); return bm; } + static constexpr basic_mask from_builtin(const builtin_type& m) { basic_mask bm; bm.mask = traits::truncate_to_size(m); return bm; } /// Convert the mask into a register which can be passed to an intrinsic. constexpr auto to_register() const noexcept { return traits::to_register(mask); } diff --git a/include/xvec/detail/operators.hpp b/include/xvec/detail/operators.hpp index 570f4c3..e22ea9a 100644 --- a/include/xvec/detail/operators.hpp +++ b/include/xvec/detail/operators.hpp @@ -63,7 +63,7 @@ constexpr auto builtin_operator(generic_tag, _Vp lhs, _Vp rhs, _Op op) { r = op(lhsd, rhsd); #endif - if constexpr (mask_operator<_Op>) return _Vp::mask_type::from_builtin(r); + if constexpr (mask_operator<_Op>) return typename _Vp::mask_type(typename _Vp::mask_type::builtin_type(r)); else return _Vp::from_builtin(r); } diff --git a/include/xvec/detail/target_config.hpp b/include/xvec/detail/target_config.hpp index 4daa0a0..5780b29 100644 --- a/include/xvec/detail/target_config.hpp +++ b/include/xvec/detail/target_config.hpp @@ -8,6 +8,8 @@ #pragma once +#include + #include #include @@ -20,6 +22,10 @@ namespace _XVEC_NAMESPACE::simd /// generic operations. struct generic_tag {}; +/// If a type supplied to get_register_type is invalid, mark that as such with +/// this special type. +struct invalid_register {}; + #if defined(__SSE__) and !defined(_XVEC_FORCE_SCALAR) // These target tags are specific to x86 and provide a hierarchy of @@ -78,6 +84,54 @@ inline constexpr int register_size = (hasAvx && (sizeof(_Tp) * _Np) > 16) ? (32 / sizeof(_Tp)) : 16 / sizeof(_Tp); +/// Generate the appropriate register type for the given vec type. For example, +/// a small vector of ints would be converted to the smallest native register +/// capable of storing all the bits, with padding added to bring it up to the +/// register size. the type of the element is also taken into account (e.g., +/// `double` elements would convert to a `d` suffix like __m256d). +template +consteval auto get_register_type() { + constexpr auto bytes = sizeof(T); + + using _Tp = std::remove_cvref_t()[0])>; + + constexpr bool isFloat = std::same_as; + constexpr bool isDouble = std::same_as; + + if constexpr (bytes > maxBytesInVec) + return invalid_register{}; + else +#if defined(__AVX512FP16__) + if constexpr (std::same_as<_Tp, _Float16>) + { + if constexpr (bytes > 32) return __m512h(); + else if constexpr (bytes > 16) return __m256h(); + else return __m128h(); + } else +#endif +#if defined(__AVX512F__) + if constexpr (bytes > 32) + { + if constexpr (isFloat) return __m512(); + else if constexpr (isDouble) return __m512d(); + else return __m512i(); + } else +#endif +#if defined(__AVX__) + if constexpr (bytes > 16) + { + if constexpr (isFloat) return __m256(); + else if constexpr (isDouble) return __m256d(); + else return __m256i(); + } else +#endif + { + if constexpr (isFloat) return __m128(); + else if constexpr (isDouble) return __m128d(); + else return __m128i(); + } +} + } // namespace detail #else @@ -93,6 +147,15 @@ namespace detail /// target-specific code. inline constexpr int maxBytesInVec = 16; template inline constexpr int register_size = maxBytesInVec / sizeof(_Tp); + +/// In generic targets, there is no way to know what the register type should +/// be, so return an invalid_register type to indicate that the target does not +/// support the requested type. This will cause a compile-time error if the user +/// tries to use a type which is not supported by the target. Targets which +/// support a specific register type should provide a specialization of this +/// function which returns the appropriate register type. +template consteval auto get_register_type() { return invalid_register{}; } + } struct compact_mask_tag : public generic_tag {}; diff --git a/include/xvec/generic/generic_impl.hpp b/include/xvec/generic/generic_impl.hpp index c1465f7..64b7f93 100644 --- a/include/xvec/generic/generic_impl.hpp +++ b/include/xvec/generic/generic_impl.hpp @@ -1153,16 +1153,19 @@ constexpr basic_vec<_Tp, _Abi> mask_to_vec(generic_tag, const basic_mask<_Bytes, static_assert(_Value == 1 || _Value == -1, "Only some values are permitted for mask to vec output"); static_assert(_Bytes == sizeof(_Tp), "Invalid mask or destination - differing sizes"); + using _To = basic_vec<_Tp, _Abi>; + using _B = typename _To::builtin_type; + if constexpr (std::integral<_Tp> && _Value == -1) // The mask is already entirely 0 or 1 bits, so nothing more to do - return basic_vec<_Tp, _Abi>(mask.to_builtin()); + return _To(_B(mask.to_builtin())); else if constexpr (std::integral<_Tp> && _Value == 1) // The mask is entirely 0 or 1 bits so a logical shift will convert the MSB into an LSB. - return basic_vec<_Tp, _Abi>(mask.to_builtin() >> (sizeof(_Tp) * 8 - 1)); + return _To(_B(mask.to_builtin() >> (sizeof(_Tp) * 8 - 1))); else { - constexpr auto _vs = basic_vec<_Tp, _Abi>(_Tp(_Value)); - return select(mask, _vs, basic_vec<_Tp, _Abi>()); + constexpr auto _vs = _To(_Tp(_Value)); + return select(mask, _vs, _To()); } } @@ -1368,7 +1371,7 @@ constexpr auto get_n_bit_mask(compact_mask_tag, const _M&, simd_size_type _n) if constexpr (_M::size <= 64) { const auto clamped = std::min(_M::size(), _n); - return _M::from_builtin(clamped == 64 ? ~0ULL : (1ULL << clamped) - 1); + return _M::from_builtin(typename _M::builtin_type(clamped == 64 ? ~0ULL : (1ULL << clamped) - 1)); } else return _M::from_builtin(_M(true).to_builtin() >> (_M::size - std::min(_n, _M::size()))); diff --git a/include/xvec/simd b/include/xvec/simd index d56a36a..788f12e 100644 --- a/include/xvec/simd +++ b/include/xvec/simd @@ -153,15 +153,16 @@ public: constexpr basic_vec(const builtin_type& v) noexcept : data(v) {} /// Allow a vec to be constructed from a target register, such as the return - /// value of an intrinsic. If the vec object is smaller than the register - /// type then some values may be discarded, so the constructor is explicit to - /// make the user be clear about this. Note that in the special case that the - /// vec is already the exact size of the target register, this constructor - /// will be disabled because the register type will match the compiler's own - /// builtin type. + /// value of an intrinsic. If the vec object is smaller than the register type + /// then some values may be discarded, so the constructor is explicit to make + /// the user be clear about this. Note that there is also a constructor which + /// accepts a builtin type, so in the case where a register is directly + /// convertible to a builtin, this would introduce ambiguity and so is + /// disabled. /// @ingroup simd_constructor explicit constexpr basic_vec(register_type r) noexcept - requires (!std::same_as) : data(traits::from_register(r)) {} + requires (!std::is_convertible_v) + : data(traits::from_register(r)) {} /// @brief Constructs a basic_vec by reading the contents of an object with the same /// number of values (e.g., span, C-array, std::array). Both unmasked and masked variants are provided. @@ -266,9 +267,9 @@ public: /// This will return the smallest target register that can contain all the elements of the vec. explicit constexpr operator typename traits::register_type() const { return traits::to_register(data); } - /// @brief Convert the vec into the nearest register size which is large enough to store all the vec elements. - /// This is more convenient to use than the explicit register_type cast since there is no need to work out the register - /// type. + /// Convert the mask into a register which can be passed to an intrinsic + /// (e.g., on x86 `vec` would become `__m256d`). Vectors which + /// don't fit into a native register must be broken into smaller pieces first. constexpr typename traits::register_type to_register() const { return traits::to_register(data); } /// @brief Return the builtin value. diff --git a/include/xvec/x86/permute.hpp b/include/xvec/x86/permute.hpp index 1519da3..85b01c0 100644 --- a/include/xvec/x86/permute.hpp +++ b/include/xvec/x86/permute.hpp @@ -76,7 +76,7 @@ constexpr auto permute_byte_shuffle (const basic_vec<_Tp, _AbiT>& v, const basic static_assert(basic_vec<_Tp, _AbiT>::size <= 16 && basic_vec<_Ip, _AbiI>::size <= 16, "Data and indexes must fit first lane"); // :TODO: In future both indexes and values could extend beyond lanes through // duplication or multi-lane index merging, which will be faster than using permutex[2]var_epi8. - return vec<_Tp, basic_vec<_Ip, _AbiI>::size>(_mm_shuffle_epi8(v.to_register(), indexes.to_register() & 0xf)); + return vec<_Tp, basic_vec<_Ip, _AbiI>::size>(_mm_shuffle_epi8(v.to_register(), (indexes & uint8_t(0xf)).to_register())); } /// Permute one register of Intel AVX2 values. @@ -129,8 +129,8 @@ _mm512_permutex2var_epi8_emulated(__m512i v_data_0, // for that permutation comes from the upper 5-bits of the incoming index (i.e., discard the LSB). The permute // instruction masks the index bits itself, so we only need to put the upper 5-bits of the index // into the correct location. - __v32hi evenPerm = _mm512_permutex2var_epi16(v_data_0, v_shuf_idxs >> 1, v_data_1); - __v32hi oddPerm = _mm512_permutex2var_epi16(v_data_0, v_shuf_idxs >> 9, v_data_1); + auto evenPerm = _mm512_permutex2var_epi16(v_data_0, v_shuf_idxs >> 1, v_data_1); + auto oddPerm = _mm512_permutex2var_epi16(v_data_0, v_shuf_idxs >> 9, v_data_1); // evenPerm and oddPerm now contain pairs of bytes. We need to select which byte of each pair to use, // and that selection is driven by the LSB of each index (i.e., LSB == 0 means use the lower byte of the @@ -162,7 +162,7 @@ inline target_overloads permutexvar { [] _Vec>(_Vec v, _Vec i) { return _mm256_permutexvar_epi8(i.to_register(), v.to_register()); }, [] _Vec>(_Vec v, _Vec i) { return _mm512_permutexvar_epi8(i.to_register(), v.to_register()); }, #else - [] _Vec>(_Vec v, _Vec i) { return __m256i(_mm512_permutex2var_epi8_emulated(__m512i(v.to_register()), __m512i(i.to_register()), __m512i())); }, + [] _Vec>(_Vec v, _Vec i) { return __m256i(_mm512_permutex2var_epi8_emulated(v.to_register(), i.to_register(), __m512i())); }, [] _Vec>(_Vec v, _Vec i) { return _mm512_permutex2var_epi8_emulated(v.to_register(), i.to_register(), __m512i()); }, #endif @@ -180,7 +180,7 @@ inline target_overloads permutex2var { [] _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm256_permutex2var_epi8(v0.to_register(), i.to_register(), v1.to_register()); }, [] _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi8(v0.to_register(), i.to_register(), v1.to_register()); }, #else - [] _Vec>(_Vec v0, _Vec i, _Vec v1) { return __m256i(_mm512_permutex2var_epi8_emulated(__m512i(v0.to_register()), __m512i(i.to_register()), v1.to_register())); }, + [] _Vec>(_Vec v0, _Vec i, _Vec v1) { return __m256i(_mm512_permutex2var_epi8_emulated(v0.to_register(), i.to_register(), v1.to_register())); }, [] _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi8_emulated(v0.to_register(), i.to_register(), v1.to_register()); }, #endif @@ -394,7 +394,7 @@ inline target_overloads compress_one_register { [] _Vec>(_Vec v, auto m, auto f) { return _Vec(_mm512_mask_compress_epi8(f.to_register(), m.to_register(), v.to_register())); }, - // int 16 + // int 16 [] _Vec>(_Vec v, auto m, auto f) { return _Vec(_mm_mask_compress_epi16(f.to_register(), m.to_register(), v.to_register())); }, [] _Vec>(_Vec v, auto m, auto f) diff --git a/include/xvec/x86/x86_impl.hpp b/include/xvec/x86/x86_impl.hpp index e714360..ed6b7c3 100644 --- a/include/xvec/x86/x86_impl.hpp +++ b/include/xvec/x86/x86_impl.hpp @@ -440,7 +440,7 @@ constexpr void store_masked(x86_tag, const _Vp& value_original, std::span<_Up, _ constexpr auto numSseElements = 16 / sizeof(_Up); auto impl = [=](_Vec x, auto m, auto idx) { // Note that the mask register is zero extended to ensure no extra mask elements appear in it during `to_register'. - _mm_maskmoveu_si128(__m128i(x.to_register()), __m128i(grow(m).to_register()), (char*)(to.data() + idx)); + _mm_maskmoveu_si128(x.to_register(), grow(m).to_register(), (char*)(to.data() + idx)); }; // Use the mask to limit the bounds of the write if requested by the user. @@ -478,9 +478,9 @@ constexpr void store_masked(x86_avx2_tag, const _Vp& value_original, std::span<_ const auto rc = permute(s, perm_uninitResize); if constexpr (sizeof(_Up) == 4) - _mm256_maskstore_epi32(reinterpret_cast(to.data() + idx), rm.to_register(), __m256i(rc.to_register())); + _mm256_maskstore_epi32(reinterpret_cast(to.data() + idx), rm.to_register(), rc.to_register()); else if constexpr (sizeof(_Up) == 8) - _mm256_maskstore_epi64(reinterpret_cast(to.data() + idx), rm.to_register(), __m256i(rc.to_register())); + _mm256_maskstore_epi64(reinterpret_cast(to.data() + idx), rm.to_register(), rc.to_register()); else static_assert(dependent_false<_Vec>, "Unimplemented masked store"); }; @@ -505,24 +505,24 @@ constexpr void store_masked(x86_avx512_tag, const _Vp& value_original, std::span mandates_for_store(value_original, to, flags); target_overloads call_insn { - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi8(p, m, __m128i(v.to_register())); }, - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi16(p, m, __m128i(v.to_register())); }, - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi32(p, m, __m128i(v.to_register())); }, - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, m, __m128i(v.to_register())); }, - - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi8(p, m, __m256i(v.to_register())); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi16(p, m, __m256i(v.to_register())); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi32(p, m, __m256i(v.to_register())); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, m, __m256i(v.to_register())); }, - - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi8(p, m, __m512i(v.to_register())); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi16(p, m, __m512i(v.to_register())); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi32(p, m, __m512i(v.to_register())); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, m, __m512i(v.to_register())); }, - - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, dupBits(m), __m128i(v.to_register())); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, dupBits(m), __m256i(v.to_register())); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, dupBits(m), __m512i(v.to_register())); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi8(p, m, v.to_register()); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi16(p, m, v.to_register()); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi32(p, m, v.to_register()); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, m, v.to_register()); }, + + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi8(p, m, v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi16(p, m, v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi32(p, m, v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, m, v.to_register()); }, + + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi8(p, m, v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi16(p, m, v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi32(p, m, v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, m, v.to_register()); }, + + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, dupBits(m), v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, dupBits(m), v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, dupBits(m), v.to_register()); }, [=](auto unhandled, auto, auto) { static_assert(dependent_false, "Unimplemented partial masked store"); } }; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 584202e..646f3f6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -250,7 +250,6 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") xvec_test_common INTERFACE -Wno-unknown-pragmas - -flax-vector-conversions ) endif()