Skip to content

Rewrote to_register utility #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
danieltowner wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Rewrote to_register utility #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 32 additions & 24 deletions include/xvec/detail/core.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,30 +437,33 @@ struct vec_traits
/// The compiler's representation of the simd::vec type.
using builtin_type __attribute__((__vector_size__(sizeof(element_type) * _numStorageElements))) = element_type;

/// @brief The best register size to use for operations on this builtin type.
using register_type __attribute__((__vector_size__(sizeof(element_type) * register_size<element_type, _numDataElements>))) = element_type;

/// @brief Allow the builtin data to be turned into the closest matching
/// target register type. This will return the smallest target register that
/// can contain all the elements of the builtin type. It is an error to try to
/// store the data in a register which is too small. In that case the user
/// must break the builtin type into pieces which do fit in a register (e.g.,
/// using fit_to_size, or extract). If the builtin type contains less data
/// than the register then the unused register elements will have undefined
/// values.
/// @brief The register to use for intrinsic operations on this vec (e.g., for
/// vec<double, 3> it would be __m256d). Note that unininitialised elements
/// may be inserted to grow the vector to a size which can be represented in a
/// register.
using register_type = decltype(detail::get_register_type<builtin_type>());

/// Convert the vector data into a native register value which can be passed
/// directly into an intrinsic. Firstly the data is expanded to full a
/// complete native register (e.g., 128-bit, 256-bit, etc) using uninitialised
/// bits. Secondly, the register is converted into the most appropriate native
/// register type which can be used directly by an intrinsic. For example,
/// vec<int, 3> would become __m128i, vec<float, 5> would become __m256, and
/// so on. This function must not be used on a vec which is too big to fit a
/// native register (use simd::chunk to break it into register-sized pieces).
/// Note that while many intrinsics accept a register which contains
/// uninitialised data, this data might lead to incorrect operation. The
/// caller is responsible for ensuring that the uninitialised bits are set to
/// a safe value before calling an intrinsic which requires it.
static constexpr register_type to_register(builtin_type v) {
// Use a static assert rather than a requires to allow this error to be
// reported instead of rather than resulting in a confusing lookup failure.
constexpr auto _regSize = register_size<_Tp, _Np>;
constexpr auto _regSize = register_size<_Tp, _Np>;
static_assert(_Np <= _regSize, "Converting to a smaller register loses data");
if constexpr (_Np == _regSize)
return v;
else
{
return [=]<std::size_t... _Idx>(std::index_sequence<_Idx...>) {

auto r = [=]<std::size_t... _Idx>(std::index_sequence<_Idx...>) {
return __builtin_shufflevector(v, v, (_Idx < _Np ? int(_Idx) : -1)...);
}(std::make_index_sequence<_regSize>());
}

return register_type(r);
}

/// @brief A named constructor for creating a builtin type from a register value.
Expand All @@ -470,12 +473,17 @@ struct vec_traits
constexpr auto _regSize = register_size<_Tp, _Np>;
static_assert(_Np <= _regSize, "Converting from a smaller register creates undefined values");
if constexpr (_Np == _regSize)
return r;
return builtin_type(r);
else
{
return [=]<std::size_t... _Idx>(std::index_sequence<_Idx...>) {
return __builtin_shufflevector(r, r, ((_Idx < _numDataElements) ? int(_Idx) : -1)...);
}(std::make_index_sequence<_numStorageElements>());
// Convert to a full register of the builtin elements.
using FullRegBuiltin = typename vec<element_container_type, _regSize>::builtin_type;
auto b = FullRegBuiltin(r);

// Then shuffle down to the correct number of elements.
return builtin_type([=]<std::size_t... _Idx>(std::index_sequence<_Idx...>) {
return __builtin_shufflevector(b, b, _Idx...);
}(std::make_index_sequence<_numStorageElements>()));
}
}

Expand Down
63 changes: 63 additions & 0 deletions include/xvec/detail/target_config.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@

#pragma once

#include <immintrin.h>

#include <xvec/detail/config.hpp>

#include <concepts>
Expand All @@ -20,6 +22,10 @@ namespace _XVEC_NAMESPACE::simd
/// generic operations.
struct generic_tag {};

/// If a type supplied to get_register_type is invalid, mark that as such with
/// this special type.
struct invalid_register {};

#if defined(__SSE__) and !defined(_XVEC_FORCE_SCALAR)

// These target tags are specific to x86 and provide a hierarchy of
Expand Down Expand Up @@ -78,6 +84,54 @@ inline constexpr int register_size =
(hasAvx && (sizeof(_Tp) * _Np) > 16) ? (32 / sizeof(_Tp)) :
16 / sizeof(_Tp);

/// Generate the appropriate register type for the given vec type. For example,
/// a small vector of ints would be converted to the smallest native register
/// capable of storing all the bits, with padding added to bring it up to the
/// register size. the type of the element is also taken into account (e.g.,
/// `double` elements would convert to a `d` suffix like __m256d).
template<typename T>
consteval auto get_register_type() {
constexpr auto bytes = sizeof(T);

using _Tp = std::remove_cvref_t<decltype(std::declval<T&>()[0])>;

constexpr bool isFloat = std::same_as<float, _Tp>;
constexpr bool isDouble = std::same_as<double, _Tp>;

if constexpr (bytes > maxBytesInVec)
return invalid_register{};
else
#if defined(__AVX512FP16__)
if constexpr (std::same_as<_Tp, _Float16>)
{
if constexpr (bytes > 32) return __m512h();
else if constexpr (bytes > 16) return __m256h();
else return __m128h();
} else
#endif
#if defined(__AVX512F__)
if constexpr (bytes > 32)
{
if constexpr (isFloat) return __m512();
else if constexpr (isDouble) return __m512d();
else return __m512i();
} else
#endif
#if defined(__AVX__)
if constexpr (bytes > 16)
{
if constexpr (isFloat) return __m256();
else if constexpr (isDouble) return __m256d();
else return __m256i();
} else
#endif
{
if constexpr (isFloat) return __m128();
else if constexpr (isDouble) return __m128d();
else return __m128i();
}
}

} // namespace detail

#else
Expand All @@ -93,6 +147,15 @@ namespace detail
/// target-specific code.
inline constexpr int maxBytesInVec = 16;
template<typename _Tp, int _Np> inline constexpr int register_size = maxBytesInVec / sizeof(_Tp);

/// In generic targets, there is no way to know what the register type should
/// be, so return an invalid_register type to indicate that the target does not
/// support the requested type. This will cause a compile-time error if the user
/// tries to use a type which is not supported by the target. Targets which
/// support a specific register type should provide a specialization of this
/// function which returns the appropriate register type.
template<typename> consteval auto get_register_type() { return invalid_register{}; }

}

struct compact_mask_tag : public generic_tag {};
Expand Down
21 changes: 11 additions & 10 deletions include/xvec/simd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,15 +153,16 @@ public:
constexpr basic_vec(const builtin_type& v) noexcept : data(v) {}

/// Allow a vec to be constructed from a target register, such as the return
/// value of an intrinsic. If the vec object is smaller than the register
/// type then some values may be discarded, so the constructor is explicit to
/// make the user be clear about this. Note that in the special case that the
/// vec is already the exact size of the target register, this constructor
/// will be disabled because the register type will match the compiler's own
/// builtin type.
/// value of an intrinsic. If the vec object is smaller than the register type
/// then some values may be discarded, so the constructor is explicit to make
/// the user be clear about this. Note that there is also a constructor which
/// accepts a builtin type, so in the case where a register is directly
/// convertible to a builtin, this would introduce ambiguity and so is
/// disabled.
/// @ingroup simd_constructor
explicit constexpr basic_vec(register_type r) noexcept
requires (!std::same_as<register_type, builtin_type>) : data(traits::from_register(r)) {}
requires (!std::is_convertible_v<register_type, const builtin_type&>)
: data(traits::from_register(r)) {}

/// @brief Constructs a basic_vec by reading the contents of an object with the same
/// number of values (e.g., span, C-array, std::array). Both unmasked and masked variants are provided.
Expand Down Expand Up @@ -266,9 +267,9 @@ public:
/// This will return the smallest target register that can contain all the elements of the vec.
explicit constexpr operator typename traits::register_type() const { return traits::to_register(data); }

/// @brief Convert the vec into the nearest register size which is large enough to store all the vec elements.
/// This is more convenient to use than the explicit register_type cast since there is no need to work out the register
/// type.
/// Convert the mask into a register which can be passed to an intrinsic
/// (e.g., on x86 `vec<double, 3>` would become `__m256d`). Vectors which
/// don't fit into a native register must be broken into smaller pieces first.
constexpr typename traits::register_type to_register() const { return traits::to_register(data); }

/// @brief Return the builtin value.
Expand Down
8 changes: 4 additions & 4 deletions include/xvec/x86/permute.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ constexpr auto permute_byte_shuffle (const basic_vec<_Tp, _AbiT>& v, const basic
static_assert(basic_vec<_Tp, _AbiT>::size <= 16 && basic_vec<_Ip, _AbiI>::size <= 16, "Data and indexes must fit first lane");
// :TODO: In future both indexes and values could extend beyond lanes through
// duplication or multi-lane index merging, which will be faster than using permutex[2]var_epi8.
return vec<_Tp, basic_vec<_Ip, _AbiI>::size>(_mm_shuffle_epi8(v.to_register(), indexes.to_register() & 0xf));
return vec<_Tp, basic_vec<_Ip, _AbiI>::size>(_mm_shuffle_epi8(v.to_register(), (indexes & uint8_t(0xf)).to_register()));
}

/// Permute one register of Intel AVX2 values.
Expand Down Expand Up @@ -162,7 +162,7 @@ inline target_overloads permutexvar {
[]<ymm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) { return _mm256_permutexvar_epi8(i.to_register(), v.to_register()); },
[]<zmm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) { return _mm512_permutexvar_epi8(i.to_register(), v.to_register()); },
#else
[]<ymm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) { return __m256i(_mm512_permutex2var_epi8_emulated(__m512i(v.to_register()), __m512i(i.to_register()), __m512i())); },
[]<ymm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) { return __m256i(_mm512_permutex2var_epi8_emulated(v.to_register(), i.to_register(), __m512i())); },
[]<zmm_register<std::uint8_t> _Vec>(_Vec v, _Vec i) { return _mm512_permutex2var_epi8_emulated(v.to_register(), i.to_register(), __m512i()); },
#endif

Expand All @@ -180,7 +180,7 @@ inline target_overloads permutex2var {
[]<ymm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm256_permutex2var_epi8(v0.to_register(), i.to_register(), v1.to_register()); },
[]<zmm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi8(v0.to_register(), i.to_register(), v1.to_register()); },
#else
[]<ymm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return __m256i(_mm512_permutex2var_epi8_emulated(__m512i(v0.to_register()), __m512i(i.to_register()), v1.to_register())); },
[]<ymm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return __m256i(_mm512_permutex2var_epi8_emulated(v0.to_register(), i.to_register(), v1.to_register())); },
[]<zmm_register<std::uint8_t> _Vec>(_Vec v0, _Vec i, _Vec v1) { return _mm512_permutex2var_epi8_emulated(v0.to_register(), i.to_register(), v1.to_register()); },
#endif

Expand Down Expand Up @@ -394,7 +394,7 @@ inline target_overloads compress_one_register {
[]<zmm_register<std::uint8_t> _Vec>(_Vec v, auto m, auto f)
{ return _Vec(_mm512_mask_compress_epi8(f.to_register(), m.to_register(), v.to_register())); },

// int 16
// int 16
[]<xmm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto f)
{ return _Vec(_mm_mask_compress_epi16(f.to_register(), m.to_register(), v.to_register())); },
[]<ymm_register<std::uint16_t> _Vec>(_Vec v, auto m, auto f)
Expand Down
42 changes: 21 additions & 21 deletions include/xvec/x86/x86_impl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,7 @@ constexpr void store_masked(x86_tag, const _Vp& value_original, std::span<_Up, _
constexpr auto numSseElements = 16 / sizeof(_Up);
auto impl = [=]<typename _Vec>(_Vec x, auto m, auto idx) {
// Note that the mask register is zero extended to ensure no extra mask elements appear in it during `to_register'.
_mm_maskmoveu_si128(__m128i(x.to_register()), __m128i(grow<numSseElements>(m).to_register()), (char*)(to.data() + idx));
_mm_maskmoveu_si128(x.to_register(), grow<numSseElements>(m).to_register(), (char*)(to.data() + idx));
};

// Use the mask to limit the bounds of the write if requested by the user.
Expand Down Expand Up @@ -478,9 +478,9 @@ constexpr void store_masked(x86_avx2_tag, const _Vp& value_original, std::span<_
const auto rc = permute<numAvxElements>(s, perm_uninitResize);

if constexpr (sizeof(_Up) == 4)
_mm256_maskstore_epi32(reinterpret_cast<int*>(to.data() + idx), rm.to_register(), __m256i(rc.to_register()));
_mm256_maskstore_epi32(reinterpret_cast<int*>(to.data() + idx), rm.to_register(), rc.to_register());
else if constexpr (sizeof(_Up) == 8)
_mm256_maskstore_epi64(reinterpret_cast<long long*>(to.data() + idx), rm.to_register(), __m256i(rc.to_register()));
_mm256_maskstore_epi64(reinterpret_cast<long long*>(to.data() + idx), rm.to_register(), rc.to_register());
else
static_assert(dependent_false<_Vec>, "Unimplemented masked store");
};
Expand All @@ -505,24 +505,24 @@ constexpr void store_masked(x86_avx512_tag, const _Vp& value_original, std::span
mandates_for_store(value_original, to, flags);

target_overloads call_insn {
[=](xmm_register<uint8_t> auto v, auto m, auto p) { _mm_mask_storeu_epi8(p, m, __m128i(v.to_register())); },
[=](xmm_register<uint16_t> auto v, auto m, auto p) { _mm_mask_storeu_epi16(p, m, __m128i(v.to_register())); },
[=](xmm_register<uint32_t> auto v, auto m, auto p) { _mm_mask_storeu_epi32(p, m, __m128i(v.to_register())); },
[=](xmm_register<uint64_t> auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, m, __m128i(v.to_register())); },

[=](ymm_register<uint8_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi8(p, m, __m256i(v.to_register())); },
[=](ymm_register<uint16_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi16(p, m, __m256i(v.to_register())); },
[=](ymm_register<uint32_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi32(p, m, __m256i(v.to_register())); },
[=](ymm_register<uint64_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, m, __m256i(v.to_register())); },

[=](zmm_register<uint8_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi8(p, m, __m512i(v.to_register())); },
[=](zmm_register<uint16_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi16(p, m, __m512i(v.to_register())); },
[=](zmm_register<uint32_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi32(p, m, __m512i(v.to_register())); },
[=](zmm_register<uint64_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, m, __m512i(v.to_register())); },

[=](xmm_register<unsigned __int128> auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, dupBits(m), __m128i(v.to_register())); },
[=](ymm_register<unsigned __int128> auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, dupBits(m), __m256i(v.to_register())); },
[=](zmm_register<unsigned __int128> auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, dupBits(m), __m512i(v.to_register())); },
[=](xmm_register<uint8_t> auto v, auto m, auto p) { _mm_mask_storeu_epi8(p, m, v.to_register()); },
[=](xmm_register<uint16_t> auto v, auto m, auto p) { _mm_mask_storeu_epi16(p, m, v.to_register()); },
[=](xmm_register<uint32_t> auto v, auto m, auto p) { _mm_mask_storeu_epi32(p, m, v.to_register()); },
[=](xmm_register<uint64_t> auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, m, v.to_register()); },

[=](ymm_register<uint8_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi8(p, m, v.to_register()); },
[=](ymm_register<uint16_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi16(p, m, v.to_register()); },
[=](ymm_register<uint32_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi32(p, m, v.to_register()); },
[=](ymm_register<uint64_t> auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, m, v.to_register()); },

[=](zmm_register<uint8_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi8(p, m, v.to_register()); },
[=](zmm_register<uint16_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi16(p, m, v.to_register()); },
[=](zmm_register<uint32_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi32(p, m, v.to_register()); },
[=](zmm_register<uint64_t> auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, m, v.to_register()); },

[=](xmm_register<unsigned __int128> auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, dupBits(m), v.to_register()); },
[=](ymm_register<unsigned __int128> auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, dupBits(m), v.to_register()); },
[=](zmm_register<unsigned __int128> auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, dupBits(m), v.to_register()); },

[=](auto unhandled, auto, auto) { static_assert(dependent_false<decltype(unhandled)>, "Unimplemented partial masked store"); }
};
Expand Down
Loading