diff --git a/include/xvec/generic/generic_impl.hpp b/include/xvec/generic/generic_impl.hpp index 0299d40..e06bc3c 100644 --- a/include/xvec/generic/generic_impl.hpp +++ b/include/xvec/generic/generic_impl.hpp @@ -686,66 +686,65 @@ compress_mask_by_mask(compact_mask_tag, const _M& v, const _M& selector, bool fi /// Gather the contents of memory from the supplied index positions. // :TODO: :COMPILER: Expose the gather llvm primitive and use it directly? /// @tparam _Range The type of the range -/// @tparam _Idx The index type -/// @tparam _IdxAbi The ABI of the index simd::vec +/// @tparam _Idx The simd index type /// @tparam _Flags The type of the flags /// @param r The range from which to gather /// @param indexes The simd of indexes /// @param flags The flags controlling the gather /// @return The gathered simd value -template -constexpr auto gather_from(generic_tag, const _Range& r, const basic_vec<_Idx, _IdxAbi>& indexes, flags<_Flags...> flags) +template +constexpr auto gather_from(generic_tag, const _Range& r, const _Idx& indexes, flags<_Flags...> flags) { using _Tp = std::ranges::range_value_t<_Range>; - const auto rmax = _Idx(std::min(r.size(), size_t(std::numeric_limits<_Idx>::max()))); + const auto rmax = r.size(); if constexpr (contains_flag(flags)) checkStaticMemoryBounds("gather", indexes, rmax); - constexpr int numIndexes = basic_vec<_Idx, _IdxAbi>::size(); + constexpr int numIndexes = _Idx::size(); // Put the indexes and the result into memory, as this generates better code // than trying to pull out individual elements of a simd register to pass to a // separate load instruction, and to put the results back into a register. - const auto indexPtr = reinterpret_cast(&indexes); + const auto indexPtr = reinterpret_cast(&indexes); _Tp result[numIndexes]; for (int i=0; i(flags) || (idx < rmax) ? r[idx] : _Tp(); + bool validIndex = (idx >= 0 && std::cmp_less(+idx, rmax)); + result[i] = contains_flag(flags) || validIndex ? r[idx] : _Tp(); } - return vec<_Tp, numIndexes>(result); + return rebind_t<_Tp, _Idx>(result); } /// Gather the contents of memory from the supplied index positions, using a mask. /// @tparam _Range The type of the range -/// @tparam _Idx The index type -/// @tparam _IdxAbi The ABI of the index simd::vec +/// @tparam _Idx The simd index type /// @tparam _Flags The type of the flags /// @param r The range from which to gather /// @param mask The mask to use for gathering /// @param indexes The simd of indexes /// @param flags The flags controlling the gather /// @return The gathered simd value -template +template constexpr auto gather_from(generic_tag, const _Range& r, - const typename basic_vec<_Idx, _IdxAbi>::mask_type& mask, - const basic_vec<_Idx, _IdxAbi>& indexes, flags<_Flags...> flags) + const typename _Idx::mask_type& mask, + const _Idx& indexes, flags<_Flags...> flags) { using _Tp = std::ranges::range_value_t<_Range>; - const auto rmax = _Idx(std::min(r.size(), size_t(std::numeric_limits<_Idx>::max()))); + const auto rmax = r.size(); if constexpr (contains_flag(flags)) checkStaticMemoryBounds("gather", indexes, rmax, mask); - constexpr int numIndexes = basic_vec<_Idx, _IdxAbi>::size(); + constexpr int numIndexes = _Idx::size(); // Put the indexes and the result into memory, as this generates better code // than trying to pull out individual elements of a simd register to pass to a // separate load instruction, and to put the results back into a register. - const auto indexPtr = reinterpret_cast(&indexes); + const auto indexPtr = reinterpret_cast(&indexes); _Tp result[numIndexes]; // Need simple mask which can be queried by index. @@ -754,83 +753,79 @@ constexpr auto gather_from(generic_tag, const _Range& r, for (int i=0; i(flags) || (idx < rmax); + bool validIndex = contains_flag(flags) || (idx >= 0 && std::cmp_less(+idx, rmax)); result[i] = validIndex && mb[i] ? r[idx] : _Tp(); } - return vec<_Tp, numIndexes>(result); + return rebind_t<_Tp, _Idx>(result); } /// Scatter the contents of a simd to the supplied index positions. /// @tparam _Range The type of the range -/// @tparam _Tp The element type -/// @tparam _TpAbi The ABI of the vec -/// @tparam _Idx The index type -/// @tparam _IdxAbi The ABI of the index simd::vec +/// @tparam _Vp The values to scatter to the range. +/// @tparam _Idx The indexes type /// @tparam _Flags The type of the flags /// @param values The simd of values to scatter /// @param r The range to which to scatter /// @param indexes The simd of indexes /// @param flags The flags controlling the scatter -template +template constexpr void -scatter_to(generic_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, - const basic_vec<_Idx, _IdxAbi>& indexes, flags<_Flags...> flags) +scatter_to(generic_tag, const _Vp& values, _Range&& r, const _Idx& indexes, flags<_Flags...> flags) { - static_assert(basic_vec<_Tp, _TpAbi>::size() == basic_vec<_Idx, _IdxAbi>::size(), - "Caller should ensure sizes match"); - using _Out = std::ranges::range_value_t<_Range>; - - const auto rmax = _Idx(std::min(r.size(), size_t(std::numeric_limits<_Idx>::max()))); - const auto mb = (indexes < rmax).to_bitset(); + const auto rmax = r.size(); auto data_ptr = std::to_address(r.begin()); if constexpr (contains_flag(flags)) checkStaticMemoryBounds("scatter", indexes, rmax); - [=](std::index_sequence<_Iota...>) { - - if constexpr (detail::contains_flag(flags)) - ((data_ptr[indexes[_Iota]] = values[_Iota]), ...); - else - ((mb[_Iota] ? (data_ptr[indexes[_Iota]] = values[_Iota]) : _Out()), ...); - - }(std::make_index_sequence::size>()); + auto doWrite = [&](auto i) { + const auto idx = indexes[i]; + const bool isValidIndex = idx >= 0 && std::cmp_less(+idx, rmax); + if (contains_flag(flags) || isValidIndex) + data_ptr[idx] = values[i]; + }; + [&](std::index_sequence<_Iota...>) { + (doWrite(_Iota), ...); + }(std::make_index_sequence<_Idx::size>()); } /// Scatter the contents of a simd to the supplied index positions, using a mask. /// @tparam _Range The type of the range -/// @tparam _Tp The element type -/// @tparam _TpAbi The ABI of the vec -/// @tparam _Idx The index type -/// @tparam _IdxAbi The ABI of the index simd::vec +/// @tparam _Vp The values to scatter to the range +/// @tparam _Idx The indexes type /// @tparam _Flags The type of the flags /// @param values The simd of values to scatter /// @param r The range to which to scatter /// @param mask The mask to use for scattering /// @param indexes The simd of indexes /// @param flags The flags controlling the scatter -template +template constexpr void -scatter_to(generic_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, - const typename basic_vec<_Idx, _IdxAbi>::mask_type& mask, - const basic_vec<_Idx, _IdxAbi>& indexes, flags<_Flags...> flags) +scatter_to(generic_tag, const _Vp& values, _Range&& r, + const typename _Idx::mask_type& mask, + const _Idx& indexes, flags<_Flags...> flags) { - using _Out = std::ranges::range_value_t<_Range>; - const auto rmax = _Idx(std::min(r.size(), size_t(std::numeric_limits<_Idx>::max()))); - auto rangeCheckedMask = - contains_flag(flags) ? mask : (indexes < rmax) && mask; + const auto rmax = r.size(); auto data_ptr = std::to_address(r.begin()); if constexpr (contains_flag(flags)) checkStaticMemoryBounds("scatter", indexes, rmax, mask); - const auto mb = rangeCheckedMask.to_bitset(); + auto doWrite = [&](auto i) { + if (!mask[i]) + return; + + const auto idx = indexes[i]; + const bool isValidIndex = idx >= 0 && std::cmp_less(+idx, rmax); + if (contains_flag(flags) || isValidIndex) + data_ptr[idx] = values[i]; + }; [=](std::index_sequence<_Iota...>) { - ((mb[_Iota] ? (data_ptr[indexes[_Iota]] = values[_Iota]) : _Out()), ...); - }(std::make_index_sequence::size>()); + (doWrite(_Iota), ...); + }(std::make_index_sequence<_Idx::size>()); } /// :TODO: FMA - Don't provide generic - should always match a real instruction to get the right precision? diff --git a/include/xvec/x86/x86_impl.hpp b/include/xvec/x86/x86_impl.hpp index c67836f..8082510 100644 --- a/include/xvec/x86/x86_impl.hpp +++ b/include/xvec/x86/x86_impl.hpp @@ -690,35 +690,34 @@ constexpr auto gather_from(x86_avx2_tag, const _Range& r, { using _Tp = std::ranges::range_value_t<_Range>; auto data_ptr = std::to_address(r.begin()); - const auto rmax = _Idx(std::min(r.size(), size_t(std::numeric_limits<_Idx>::max()))); target_overloads impl { #if defined(__AVX512F__) // 32-bit Intel AVX-512 - [=](xmm_register auto i, auto m) { return _mm_mmask_i32gather_epi32(__m128i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, - [=](ymm_register auto i, auto m) { return _mm256_mmask_i32gather_epi32(__m256i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](xmm_register auto i, auto m) { return _mm_mmask_i32gather_epi32(__m128i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](ymm_register auto i, auto m) { return _mm256_mmask_i32gather_epi32(__m256i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, - [=](xmm_register auto i, auto m) { return _mm_mmask_i64gather_epi64(__m128i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, - [=](ymm_register auto i, auto m) { return _mm256_mmask_i64gather_epi64(__m256i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](xmm_register auto i, auto m) { return _mm_mmask_i64gather_epi64(__m128i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](ymm_register auto i, auto m) { return _mm256_mmask_i64gather_epi64(__m256i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, - [=](zmm_register auto i, auto m) { return _mm512_mask_i32gather_epi32(__m512i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, - [=](zmm_register auto i, auto m) { return _mm512_mask_i64gather_epi64(__m512(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](zmm_register auto i, auto m) { return _mm512_mask_i32gather_epi32(__m512i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](zmm_register auto i, auto m) { return _mm512_mask_i64gather_epi64(__m512(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, #elif defined(__AVX2__) // Note the explicit growth of the mask to the correct register size to // avoid inadvertently leaving uninitialised elements active. // 32-bit Intel AVX2 - [=](xmm_register auto i, auto m) + [=](xmm_register auto i, auto m) { return _mm_mask_i32gather_epi32(__m128i(), data_ptr, i.to_register(), grow<4>(m).to_register(), sizeof(_Tp)); }, - [=](ymm_register auto i, auto m) + [=](ymm_register auto i, auto m) { return _mm256_mask_i32gather_epi32(__m256i(), data_ptr, i.to_register(), grow<8>(m).to_register(), sizeof(_Tp)); }, // 64-bit Intel AVX2 - [=](xmm_register auto i, auto m) + [=](xmm_register auto i, auto m) { return _mm_mask_i64gather_epi64(__m128i(), data_ptr, i.to_register(), grow<2>(m).to_register(), sizeof(_Tp)); }, - [=](ymm_register auto i, auto m) + [=](ymm_register auto i, auto m) { return _mm256_mask_i64gather_epi64(__m256i(), data_ptr, i.to_register(), grow<4>(m).to_register(), sizeof(_Tp)); }, #endif @@ -726,14 +725,14 @@ constexpr auto gather_from(x86_avx2_tag, const _Range& r, [=](auto unhandled, auto) { static_assert(dependent_false); } }; - // Convert the indexes to at least 32-bit, since no instruction exists - // operating at a smaller granularity. - constexpr auto idxNumBytes = std::max({sizeof(uint32_t), sizeof(_Tp), sizeof(_Idx)}); - using _IW = container_for_num_bytes; + // The indexes and the values to gether must be the same size, and at least + // 32-bit signed to match the hardware expectation. + constexpr auto idxNumBytes = std::max({sizeof(int32_t), sizeof(_Tp), sizeof(_Idx)}); + using _IW = std::make_signed_t>; + const auto rmax = std::min<_IW>(r.size(), std::numeric_limits<_Idx>::max()); auto wrapper = [=](_VecIdx idx, auto m) { - // Bump the indexes and mask up to the preferred size if necessary. - using RetypedIndex = vec<_IW, _VecIdx::size>; + using RetypedIndex = rebind_t<_IW, _VecIdx>; auto ri = RetypedIndex(idx); // Note that the mask is actively resized to zero out unwanted positions. @@ -744,7 +743,7 @@ constexpr auto gather_from(x86_avx2_tag, const _Range& r, // Add a bounds check if necessary. if constexpr (!contains_flag(flags)) - rm = rm && (ri < rmax); + rm = rm && (ri >= _IW(0) && ri < rmax); // Do the gather, reading back elements which are at least as big as the required type, and then downcasting to the types size if necessary. auto t = RetypedIndex(impl(ri, rm)); @@ -774,7 +773,7 @@ constexpr auto gather_from(x86_avx2_tag, const _Range& r, inline auto scatterImpl = [](basic_vec<_Tp, _VT> v, - basic_vec idx, auto ptr, uint64_t m) + basic_vec idx, auto ptr, uint64_t m) { constexpr uint64_t justMaskBits = (1ULL << basic_vec<_Tp, _VT>::size()) - 1; const uint64_t nm = justMaskBits & m; @@ -807,7 +806,7 @@ scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, const basic_vec<_Idx, _IdxAbi>& indexes, flags<_Flags...> flags) { auto ptr = std::to_address(r.begin()); - const auto rmax = _Idx(std::min(r.size(), size_t(std::numeric_limits<_Idx>::max()))); + const auto rmax = int32_t(std::min(r.size(), size_t(std::numeric_limits<_Idx>::max()))); // Bind extra pointer and default mask parameters to the scatter lambda. Note // that a range check is added if necessary too. @@ -815,7 +814,7 @@ scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, if constexpr (contains_flag(flags)) scatterImpl(v, i, ptr, ~0ULL); else - scatterImpl(v, i, ptr, (i < rmax).to_register()); + scatterImpl(v, i, ptr, (i >= 0 && i < rmax).to_register()); }; if constexpr (contains_flag(flags)) @@ -823,7 +822,7 @@ scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, // All scatter instructions can be made to use 32-bit indexes. // :TODO: A special case might be 64-bit values using 64-bit indexes - auto indexAs32 = vec::size>(indexes); + auto indexAs32 = vec::size>(indexes); chunked_invoke<(int)basic_vec<_Tp>::size>(impl, vec_as_container(values), indexAs32); } @@ -840,7 +839,7 @@ scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, // Range check if required. const auto rmax = _Idx(std::min(r.size(), size_t(std::numeric_limits<_Idx>::max()))); auto rangeCheckedMask = - contains_flag(flags) ? mask : (indexes < rmax) && mask; + contains_flag(flags) ? mask : (indexes >= _Idx(0) && indexes < rmax) && mask; if constexpr (contains_flag(flags)) checkStaticMemoryBounds("scatter", indexes, rmax, mask); @@ -850,7 +849,7 @@ scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, // All scatter instructions can be made to use 32-bit indexes. // :TODO: A special case might be 64-bit values using 64-bit indexes - auto indexAs32 = vec::size>(indexes); + auto indexAs32 = vec::size>(indexes); chunked_invoke<(int)basic_vec<_Tp>::size()>(impl, vec_as_container(values), indexAs32, rangeCheckedMask); } #endif diff --git a/test/TestSimdGatherScatter.cpp b/test/TestSimdGatherScatter.cpp index 298b4a1..fbe9f79 100644 --- a/test/TestSimdGatherScatter.cpp +++ b/test/TestSimdGatherScatter.cpp @@ -29,30 +29,40 @@ constexpr std::array<_Tp, _MemSize> build_memory() { BOOST_AUTO_TEST_CASE_TEMPLATE(GatherFromContiguousRange, TypeParam, AllSimdTypes) { using xvec::rebind_cast; - - // Create a contiguous lookup table, and a range into that memory which is - // half the size. The gather should only permit values in the range to be - // read, not those in the whole table. - constexpr int memorySize = 1024; - const auto memTable = build_memory(); - constexpr int rangeSize = 200; - const auto table = std::ranges::subrange(memTable.begin(), memTable.begin() + rangeSize); - - // Create a set of random indexes into the entire memory table, not just the smaller range. - const auto indexes = GetRandomVector>(memorySize - 1); - - const auto checkedExpected = applyUnary(indexes, [=](auto idx) { return idx < rangeSize ? memTable[idx] : typename TypeParam::value_type(); }); - const auto uncheckedExpected = applyUnary(indexes, [=](auto idx) { return memTable[idx]; }); - const auto expectedByteIndexed = applyUnary(indexes, [=](auto idx) { return (idx & 0xff) < rangeSize ? memTable[idx & 0xff] : typename TypeParam::value_type(); }); - - // Gather using 4 different index sizes. - BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes)), expectedByteIndexed); - BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes)), checkedExpected); - BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes)), checkedExpected); - BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes)), checkedExpected); - - // One unchecked variant - it uses the same pathway through the functions, just with a different check flag. - BOOST_SIMD_EQUAL(unchecked_gather_from(table, rebind_cast(indexes)), uncheckedExpected); + + using _Tp = typename TypeParam::value_type; + + // Create a backing allocation containing a smaller readable range. + constexpr int memorySize = 256; + const auto memTable = build_memory<_Tp, memorySize>(); + + constexpr int tableOffset = 44; // Allow space before the range for negative indexes. + constexpr int rangeSize = 100; + constexpr int maximumIndex = 127; + const auto table = std::ranges::subrange(memTable.begin() + tableOffset, memTable.begin() + tableOffset + rangeSize); + + // Most random indexes are valid, while values from 100 through 127 exercise + // the upper bound. Every value remains representable by a signed byte. + auto indexes = rebind_cast(GetRandomVector>(maximumIndex)); + indexes = set_element(indexes, 0, -2); + + // Generate the expected values for every index width. + const auto checkedExpected = applyUnary(indexes, [=](auto idx) { + return idx >= 0 && idx < rangeSize ? table[idx] : _Tp{}; + }); + const auto uncheckedExpected = applyUnary(indexes, [=](auto idx) { + return memTable[tableOffset + idx]; + }); + + // Gather using four different signed index sizes. + BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes)), checkedExpected); + BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes)), checkedExpected); + BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes)), checkedExpected); + BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes)), checkedExpected); + + // The unchecked variant may read outside the logical range, but every index + // still addresses valid storage in the backing allocation. + BOOST_SIMD_EQUAL(unchecked_gather_from(table, rebind_cast(indexes)), uncheckedExpected); } BOOST_AUTO_TEST_CASE_TEMPLATE(GatherFromContiguousRangeWithConversion, TypeParam, LoadableSimdTypes) @@ -61,68 +71,81 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(GatherFromContiguousRangeWithConversion, TypeParam using _Tp = typename TypeParam::value_type; - // Create a contiguous lookup table of bytes (since they can be converted into - // any other loadable type), and a range into that memory which is half the - // size. The gather should only permit values in the range to be read, not - // those in the whole table. - constexpr int memorySize = 1024; + // Create a backing allocation of bytes containing a smaller readable range. + constexpr int memorySize = 256; const auto memTable = build_memory(); - constexpr int rangeSize = 200; - const auto table = std::ranges::subrange(memTable.begin(), memTable.begin() + rangeSize); - // Create a set of random indexes into the entire memory table, not just the smaller range. - const auto indexes = GetRandomVector>(memorySize - 1); + constexpr int tableOffset = 44; // Allow space before the range for negative indexes. + constexpr int rangeSize = 100; + constexpr int maximumIndex = 127; + const auto table = std::ranges::subrange(memTable.begin() + tableOffset, memTable.begin() + tableOffset + rangeSize); + + // Most random indexes are valid, while values from 100 through 127 exercise + // the upper bound. Every value remains representable by a signed byte. + auto indexes = rebind_cast(GetRandomVector>(maximumIndex)); + indexes = set_element(indexes, 0, -2); - const auto checkedExpected = applyUnary(indexes, [=](auto idx) -> _Tp { return idx < rangeSize ? memTable[idx] : _Tp(); }); - const auto uncheckedExpected = applyUnary(indexes, [=](auto idx) -> _Tp { return memTable[idx]; }); - const auto expectedByteIndexed = applyUnary(indexes, [=](auto idx) -> _Tp { return (idx & 0xff) < rangeSize ? memTable[idx & 0xff] : _Tp(); }); + // Generate the expected converted values for every index width. + const auto checkedExpected = applyUnary(indexes, [=](auto idx) -> _Tp { + return idx >= 0 && idx < rangeSize ? table[idx] : _Tp{}; + }); + const auto uncheckedExpected = applyUnary(indexes, [=](auto idx) -> _Tp { + return memTable[tableOffset + idx]; + }); - // Gather using 4 different index sizes. - BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes), xvec::simd::flag_convert), expectedByteIndexed); - BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes), xvec::simd::flag_convert), checkedExpected); - BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes), xvec::simd::flag_convert), checkedExpected); - BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes), xvec::simd::flag_convert), checkedExpected); + // Gather using four different signed index sizes. + BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes), xvec::simd::flag_convert), checkedExpected); + BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes), xvec::simd::flag_convert), checkedExpected); + BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes), xvec::simd::flag_convert), checkedExpected); + BOOST_SIMD_EQUAL(partial_gather_from(table, rebind_cast(indexes), xvec::simd::flag_convert), checkedExpected); - // One unchecked variant - it uses the same pathway through the functions, just with a different check flag. - BOOST_SIMD_EQUAL(unchecked_gather_from(table, rebind_cast(indexes), xvec::simd::flag_convert), uncheckedExpected); + // The unchecked variant may read outside the logical range, but every index + // still addresses valid storage in the backing allocation. + BOOST_SIMD_EQUAL(unchecked_gather_from(table, rebind_cast(indexes), xvec::simd::flag_convert), uncheckedExpected); } BOOST_AUTO_TEST_CASE_TEMPLATE(MaskedGatherFromContiguousRange, TypeParam, AllSimdTypes) { using xvec::rebind_cast; - // Create a contiguous lookup table. - constexpr int memorySize = 1024; - const auto memTable = build_memory(); - constexpr int rangeSize = 200; - const auto table = std::ranges::subrange(memTable.begin(), memTable.begin() + rangeSize); + using _Tp = typename TypeParam::value_type; + + // Create a backing allocation containing a smaller readable range. + constexpr int memorySize = 256; + const auto memTable = build_memory<_Tp, memorySize>(); - // Create a set of random indexes into the memory from which to read. - const auto indexes = GetRandomVector>(memorySize - 1); + constexpr int tableOffset = 44; // Allow space before the range for negative indexes. + constexpr int rangeSize = 100; + constexpr int maximumIndex = 127; + const auto table = std::ranges::subrange(memTable.begin() + tableOffset, memTable.begin() + tableOffset + rangeSize); - // Create a random mask. + // Most random indexes are valid, while values from 100 through 127 exercise + // the upper bound. Every value remains representable by a signed byte. + auto indexes = rebind_cast(GetRandomVector>(maximumIndex)); + indexes = set_element(indexes, 0, -2); + + // Create a random mask and ensure the negative index is enabled. auto maskBits = getRandomBitset(); - auto mask = xvec::simd::mask(maskBits); + maskBits[0] = true; + const auto mask = xvec::simd::mask(maskBits); - auto expected = TypeParam([=](auto i) { - return (maskBits[i] && indexes[i] < rangeSize) ? table[indexes[i]] : typename TypeParam::value_type{}; - }); - auto expectedByteIndexed = TypeParam([=](auto i) { - return maskBits[i] && ((indexes[i] & 0xFF) < rangeSize) ? table[indexes[i] & 0xFF] : typename TypeParam::value_type{}; + // Generate the expected values for every index width. + const auto expected = TypeParam([=](auto i) { + return maskBits[i] && indexes[i] >= 0 && indexes[i] < rangeSize ? table[indexes[i]] : _Tp{}; }); - auto indexesAs8 = rebind_cast(indexes); - BOOST_SIMD_EQUAL(partial_gather_from(table, typename decltype(indexesAs8)::mask_type(mask), indexesAs8), expectedByteIndexed); + // Masked gather using four different signed index sizes. + const auto indexesAs8 = rebind_cast(indexes); + BOOST_SIMD_EQUAL(partial_gather_from(table, typename decltype(indexesAs8)::mask_type(mask), indexesAs8), expected); - auto indexesAs16 = rebind_cast(indexes); + const auto indexesAs16 = rebind_cast(indexes); BOOST_SIMD_EQUAL(partial_gather_from(table, typename decltype(indexesAs16)::mask_type(mask), indexesAs16), expected); - auto indexesAs32 = rebind_cast(indexes); + const auto indexesAs32 = rebind_cast(indexes); BOOST_SIMD_EQUAL(partial_gather_from(table, typename decltype(indexesAs32)::mask_type(mask), indexesAs32), expected); - auto indexesAs64 = rebind_cast(indexes); + const auto indexesAs64 = rebind_cast(indexes); BOOST_SIMD_EQUAL(partial_gather_from(table, typename decltype(indexesAs64)::mask_type(mask), indexesAs64), expected); - } BOOST_AUTO_TEST_CASE_TEMPLATE(MaskedGatherFromContiguousRangeWithConversion, TypeParam, LoadableSimdTypes) @@ -131,44 +154,49 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(MaskedGatherFromContiguousRangeWithConversion, Typ using _Tp = typename TypeParam::value_type; - // Create a contiguous lookup table from bytes, which can be converted to all the other types. - constexpr int memorySize = 1024; + // Create a backing allocation of bytes containing a smaller readable range. + constexpr int memorySize = 256; const auto memTable = build_memory(); - constexpr int rangeSize = 200; - const auto table = std::ranges::subrange(memTable.begin(), memTable.begin() + rangeSize); - // Create a set of random indexes into the memory from which to read. - const auto indexes = GetRandomVector>(memorySize - 1); + constexpr int tableOffset = 44; // Allow space before the range for negative indexes. + constexpr int rangeSize = 100; + constexpr int maximumIndex = 127; + const auto table = std::ranges::subrange(memTable.begin() + tableOffset, memTable.begin() + tableOffset + rangeSize); - // Create a random mask. + // Most random indexes are valid, while values from 100 through 127 exercise + // the upper bound. Every value remains representable by a signed byte. + auto indexes = rebind_cast(GetRandomVector>(maximumIndex)); + indexes = set_element(indexes, 0, -2); + + // Create a random mask and ensure the negative index is enabled. auto maskBits = getRandomBitset(); - auto mask = xvec::simd::mask(maskBits); + maskBits[0] = true; + const auto mask = xvec::simd::mask(maskBits); - auto checkedExpected = TypeParam([=](auto i) -> _Tp { - return (maskBits[i] && indexes[i] < rangeSize) ? table[indexes[i]] : _Tp{}; - }); - auto uncheckedExpected = TypeParam([=](auto i) -> _Tp { - return (maskBits[i]) ? table[indexes[i]] : _Tp{}; + // Generate the expected converted values for every index width. + const auto checkedExpected = TypeParam([=](auto i) -> _Tp { + return maskBits[i] && indexes[i] >= 0 && indexes[i] < rangeSize ? table[indexes[i]] : _Tp{}; }); - auto expectedByteIndexed = TypeParam([=](auto i) -> _Tp{ - return maskBits[i] && ((indexes[i] & 0xFF) < rangeSize) ? table[indexes[i] & 0xFF] : _Tp{}; + const auto uncheckedExpected = TypeParam([=](auto i) -> _Tp { + return maskBits[i] ? memTable[tableOffset + indexes[i]] : _Tp{}; }); - auto indexesAs8 = rebind_cast(indexes); - BOOST_SIMD_EQUAL(partial_gather_from(table, typename decltype(indexesAs8)::mask_type(mask), indexesAs8, xvec::simd::flag_convert), expectedByteIndexed); + // Masked gather using four different signed index sizes. + const auto indexesAs8 = rebind_cast(indexes); + BOOST_SIMD_EQUAL(partial_gather_from(table, typename decltype(indexesAs8)::mask_type(mask), indexesAs8, xvec::simd::flag_convert), checkedExpected); - auto indexesAs16 = rebind_cast(indexes); + const auto indexesAs16 = rebind_cast(indexes); BOOST_SIMD_EQUAL(partial_gather_from(table, typename decltype(indexesAs16)::mask_type(mask), indexesAs16, xvec::simd::flag_convert), checkedExpected); - auto indexesAs32 = rebind_cast(indexes); + const auto indexesAs32 = rebind_cast(indexes); BOOST_SIMD_EQUAL(partial_gather_from(table, typename decltype(indexesAs32)::mask_type(mask), indexesAs32, xvec::simd::flag_convert), checkedExpected); - auto indexesAs64 = rebind_cast(indexes); + const auto indexesAs64 = rebind_cast(indexes); BOOST_SIMD_EQUAL(partial_gather_from(table, typename decltype(indexesAs64)::mask_type(mask), indexesAs64, xvec::simd::flag_convert), checkedExpected); - - // One unchecked variant - it uses the same pathway through the functions, just with a different check flag. - BOOST_SIMD_EQUAL(unchecked_gather_from(table, typename decltype(indexesAs64)::mask_type(mask), indexesAs64, xvec::simd::flag_convert), uncheckedExpected); + // The unchecked variant may read outside the logical range, but every index + // still addresses valid storage in the backing allocation. + BOOST_SIMD_EQUAL(unchecked_gather_from(table, typename decltype(indexesAs64)::mask_type(mask), indexesAs64, xvec::simd::flag_convert), uncheckedExpected); } #if defined(_XVEC_HAS_CONSTEXPR) @@ -199,57 +227,113 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(ConstexprGatherFromContiguousRange, TypeParam, Loa BOOST_AUTO_TEST_CASE_TEMPLATE(ScatterToRangeWithSameType, TypeParam, AllSimdTypes) { + using xvec::rebind_cast; + SimdTestFixture f; - // Create a contiguous table to write into. Only part of that table will be - // written, to check that the bounds checks work. - constexpr int memorySize = 1024; + // Create a backing allocation containing a smaller writable range. Comparing + // the entire allocation verifies that out-of-range writes are rejected. + constexpr int memorySize = 256; const auto originalMemory = build_memory(); - constexpr int rangeSize = 200; - // Create a set of random indexes into the memory to which to write. - const auto indexes = GetRandomVector>(memorySize - 1); + constexpr int tableOffset = 44; // Allow space before the range for negative indexes. + constexpr int rangeSize = 100; + constexpr int maximumIndex = 127; + + // Most random indexes are valid, while values from 100 through 127 exercise + // the upper bound. Every value remains representable by a signed byte. + auto indexes = rebind_cast(GetRandomVector>(maximumIndex)); + indexes = set_element(indexes, 0, -2); // Compute the expected memory after scattering into it. auto expectedMemory = originalMemory; - for (int i=0; i= 0 && indexes[i] < rangeSize) + expectedMemory[tableOffset + indexes[i]] = f.v0[i]; + + // Scatter using 8-bit signed indexes. + auto memory8 = originalMemory; + auto table8 = std::ranges::subrange(memory8.begin() + tableOffset, memory8.begin() + tableOffset + rangeSize); + partial_scatter_to(f.v0, table8, rebind_cast(indexes)); + BOOST_TEST(memory8 == expectedMemory, boost::test_tools::per_element()); + + // Scatter using 16-bit signed indexes. + auto memory16 = originalMemory; + auto table16 = std::ranges::subrange(memory16.begin() + tableOffset, memory16.begin() + tableOffset + rangeSize); + partial_scatter_to(f.v0, table16, rebind_cast(indexes)); + BOOST_TEST(memory16 == expectedMemory, boost::test_tools::per_element()); + + // Scatter using 32-bit signed indexes. + auto memory32 = originalMemory; + auto table32 = std::ranges::subrange(memory32.begin() + tableOffset, memory32.begin() + tableOffset + rangeSize); + partial_scatter_to(f.v0, table32, rebind_cast(indexes)); + BOOST_TEST(memory32 == expectedMemory, boost::test_tools::per_element()); + + // Scatter using 64-bit signed indexes. + auto memory64 = originalMemory; + auto table64 = std::ranges::subrange(memory64.begin() + tableOffset, memory64.begin() + tableOffset + rangeSize); + partial_scatter_to(f.v0, table64, rebind_cast(indexes)); + BOOST_TEST(memory64 == expectedMemory, boost::test_tools::per_element()); } BOOST_AUTO_TEST_CASE_TEMPLATE(MaskedScatterToContiguousRange, TypeParam, AllSimdTypes) { + using xvec::rebind_cast; + SimdTestFixture f; - // Create a contiguous table to write into. Only part of that table will be - // written, to check that the bounds checks work. - constexpr int memorySize = 1024; + // Create a backing allocation containing a smaller writable range. Comparing + // the entire allocation verifies that out-of-range writes are rejected. + constexpr int memorySize = 256; const auto originalMemory = build_memory(); - constexpr int rangeSize = 200; - // Create a set of random indexes into the memory to which to write. - const auto indexes = GetRandomVector>(memorySize - 1); + constexpr int tableOffset = 44; // Allow space before the range for negative indexes. + constexpr int rangeSize = 100; + constexpr int maximumIndex = 127; + + // Most random indexes are valid, while values from 100 through 127 exercise + // the upper bound. Every value remains representable by a signed byte. + auto indexes = rebind_cast(GetRandomVector>(maximumIndex)); + indexes = set_element(indexes, 0, -2); - // Create a random mask. + // Create a random mask and ensure the negative index is enabled. auto maskBits = getRandomBitset(); - auto mask = xvec::simd::mask(maskBits); + maskBits[0] = true; + const auto mask = xvec::simd::mask(maskBits); - // Compute the expected memory after scattering into unmasked locations. + // Compute the expected memory for every index width. auto expectedMemory = originalMemory; - for (int i=0; i= 0 && indexes[i] < rangeSize) + expectedMemory[tableOffset + indexes[i]] = f.v0[i]; + + // Scatter using 8-bit signed indexes. + const auto indexesAs8 = rebind_cast(indexes); + auto memory8 = originalMemory; + auto table8 = std::ranges::subrange(memory8.begin() + tableOffset, memory8.begin() + tableOffset + rangeSize); + partial_scatter_to(f.v0, table8, typename decltype(indexesAs8)::mask_type(mask), indexesAs8); + BOOST_TEST(memory8 == expectedMemory, boost::test_tools::per_element()); + + // Scatter using 16-bit signed indexes. + const auto indexesAs16 = rebind_cast(indexes); + auto memory16 = originalMemory; + auto table16 = std::ranges::subrange(memory16.begin() + tableOffset, memory16.begin() + tableOffset + rangeSize); + partial_scatter_to(f.v0, table16, typename decltype(indexesAs16)::mask_type(mask), indexesAs16); + BOOST_TEST(memory16 == expectedMemory, boost::test_tools::per_element()); + + // Scatter using 32-bit signed indexes. + const auto indexesAs32 = rebind_cast(indexes); + auto memory32 = originalMemory; + auto table32 = std::ranges::subrange(memory32.begin() + tableOffset, memory32.begin() + tableOffset + rangeSize); + partial_scatter_to(f.v0, table32, typename decltype(indexesAs32)::mask_type(mask), indexesAs32); + BOOST_TEST(memory32 == expectedMemory, boost::test_tools::per_element()); + + // Scatter using 64-bit signed indexes. + const auto indexesAs64 = rebind_cast(indexes); + auto memory64 = originalMemory; + auto table64 = std::ranges::subrange(memory64.begin() + tableOffset, memory64.begin() + tableOffset + rangeSize); + partial_scatter_to(f.v0, table64, typename decltype(indexesAs64)::mask_type(mask), indexesAs64); + BOOST_TEST(memory64 == expectedMemory, boost::test_tools::per_element()); } #if defined(_XVEC_HAS_CONSTEXPR) diff --git a/test/TestSimdIterator.cpp b/test/TestSimdIterator.cpp index e894772..ff6063c 100644 --- a/test/TestSimdIterator.cpp +++ b/test/TestSimdIterator.cpp @@ -10,11 +10,13 @@ #include // make iterators printable for the test results. +namespace xvec::simd { template std::ostream& operator<<(std::ostream& stream, const xvec::simd::simd_iterator<_Simd>& iter) { stream << "simd_iterator<" << (void*)iter.data << ", " << iter.index << ">"; return stream; } +} // Namespace xvec::simd #include "SimdTestUtilities.hpp" #include "TypesToTest.hpp"