diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index eba91cc493502a..e772a4c4e538a3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -11,6 +11,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; namespace System { @@ -25,6 +26,30 @@ internal static partial class SpanHelpers // .ByteMemOps #endif private const nuint ZeroMemoryNativeThreshold = 1024; +#if (TARGET_AMD64 || TARGET_ARM64) && !MONO + // Blocks are copied with a constant-length Memmove, which the JIT expands into "load the whole + // block into registers, then store it" - correct for any overlap. 64 bytes fits its unrolling + // budget on every target here, and bounds re-entry: a block-sized call skips the loops below and + // is finished by the tail, which never calls Memmove back. + private const nuint OverlappedBlockSize = 64; + + // Aligning the destination only pays for the extra leading block on larger copies. + private const nuint OverlappedAlignThreshold = 2048; + + // Past this the platform memmove's backward loop beats a managed descending one. + private const nuint OverlappedBackwardThreshold = 256; + +#if TARGET_ARM64 + // What makes the platform memmove worth replacing for long forward copies is 'rep movsb', which + // is x86-only: arm64 uses a plain vector loop that has no such cliff and is well tuned, so past + // this size it wins. Below it the P/Invoke is still the dominant cost and we come out ahead. + private const nuint OverlappedForwardThreshold = 1024; +#else + // ulong rather than nuint because nuint.MaxValue is not a compile-time constant, same as + // MemmoveNativeThreshold above. The comparison widens to ulong and folds away. + private const ulong OverlappedForwardThreshold = ulong.MaxValue; +#endif +#endif #if HAS_CUSTOM_BLOCKS [StructLayout(LayoutKind.Sequential, Size = 16)] @@ -35,6 +60,7 @@ private struct Block64 {} #endif // HAS_CUSTOM_BLOCKS [Intrinsic] // Unrolled for small constant lengths + [MethodImpl(MethodImplOptions.NoInlining)] // keeping the call is what lets the JIT unroll it internal static void Memmove(ref byte dest, ref byte src, nuint len) { // P/Invoke into the native version when the buffers are overlapping. @@ -235,6 +261,37 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) return; } +#if (TARGET_AMD64 || TARGET_ARM64) && !MONO + // We're better off here than calling the platform memmove: for short copies the P/Invoke alone + // costs more than the copy, and its forward loop is tuned for disjoint buffers - 'rep movsb' + // collapses when the buffers are less than a cache line apart, and the non-temporal stores it + // uses for large copies evict the lines the rest of the copy reads back. Its backward loop has + // neither problem, so long right shifts are left to it. + // + // Gated on SIMD being available, which is what the blocks below are expanded into. It folds + // away under the JIT, R2R and AOT; it is only false in the interpreter-only mode, where the + // blocks would each cost a call and the platform memmove is the better deal. + if (Vector128.IsHardwareAccelerated) + { + if ((nuint)Unsafe.ByteOffset(ref dest, ref src) < len) + { + // dest is below src, so copying upwards never overwrites a byte we still have to read. + // Note the 'else': once we know the buffers overlap this way, a length over the + // threshold has to fall through to the platform memmove, not to the backward copy. + if (len <= OverlappedForwardThreshold) + { + CopyOverlappedForward(ref dest, ref src, len); + return; + } + } + else if (len <= OverlappedBackwardThreshold) + { + CopyOverlappedBackward(ref dest, ref src, len); + return; + } + } +#endif + PInvoke: // Implicit nullchecks Debug.Assert(len > 0); @@ -243,6 +300,154 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) MemmoveNative(ref dest, ref src, len); } +#if (TARGET_AMD64 || TARGET_ARM64) && !MONO + // Every step below either is a constant-length Memmove, which the JIT expands so that the whole + // block is loaded before any of it is stored, or a single load feeding a single store, where the + // data dependency does the same for one register's worth. Steps then run in strictly ascending, + // non-overlapping order, so a store can never reach a source byte a later step has to read. That + // also rules out the trailing "block anchored at the end of the buffer" shortcut the + // non-overlapping paths use - those bytes may already have been rewritten. + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyOverlappedForward(ref byte dest, ref byte src, nuint len) + { + Debug.Assert(len > 0); + + // Only one of the two can be aligned, and an unaligned store costs more than an unaligned load. + if (len >= OverlappedAlignThreshold) + { + nuint head = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); + if (head != 64) + { + CopyOverlappedForwardTail(ref dest, ref src, head); + dest = ref Unsafe.Add(ref dest, head); + src = ref Unsafe.Add(ref src, head); + len -= head; + } + } + + // Take the widest block the JIT will still expand: its budget is four vector registers, so + // 256 bytes under AVX512 and 128 under AVX2. Wider blocks mean fewer loop iterations, and + // fewer boundaries where one block's store and the next block's load share a cache line, + // which is what a tight overlap is sensitive to. arm64 tops out at the 64-byte loop below. + if (Vector512.IsHardwareAccelerated) + { + while (len > 256) + { + Memmove(ref dest, ref src, 256); + dest = ref Unsafe.Add(ref dest, 256); + src = ref Unsafe.Add(ref src, 256); + len -= 256; + } + } + else if (Vector256.IsHardwareAccelerated) + { + while (len > 128) + { + Memmove(ref dest, ref src, 128); + dest = ref Unsafe.Add(ref dest, 128); + src = ref Unsafe.Add(ref src, 128); + len -= 128; + } + } + + while (len > OverlappedBlockSize) + { + Memmove(ref dest, ref src, OverlappedBlockSize); + dest = ref Unsafe.Add(ref dest, OverlappedBlockSize); + src = ref Unsafe.Add(ref src, OverlappedBlockSize); + len -= OverlappedBlockSize; + } + + CopyOverlappedForwardTail(ref dest, ref src, len); + } + + // Mirror image: steps run in strictly descending order, walking down from the end. + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyOverlappedBackward(ref byte dest, ref byte src, nuint len) + { + Debug.Assert(len > 0); + + while (len > OverlappedBlockSize) + { + len -= OverlappedBlockSize; + Memmove(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len), OverlappedBlockSize); + } + + CopyOverlappedBackwardTail(ref dest, ref src, len); + } + + // Finishes at most OverlappedBlockSize bytes, ascending. Deliberately doesn't call Memmove - this + // is where a block the JIT didn't expand comes back around, so calling it again is what recursion + // would look like. One step per set bit of len, so the steps tile the range without overlapping. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CopyOverlappedForwardTail(ref byte dest, ref byte src, nuint len) + { + Debug.Assert(len <= OverlappedBlockSize); + + while (len >= 16) + { + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 16); + src = ref Unsafe.Add(ref src, 16); + len -= 16; + } + if ((len & 8) != 0) + { + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 8); + src = ref Unsafe.Add(ref src, 8); + } + if ((len & 4) != 0) + { + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 4); + src = ref Unsafe.Add(ref src, 4); + } + if ((len & 2) != 0) + { + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 2); + src = ref Unsafe.Add(ref src, 2); + } + if ((len & 1) != 0) + { + dest = src; + } + } + + // Mirror image: the steps tile the range from the end downwards. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CopyOverlappedBackwardTail(ref byte dest, ref byte src, nuint len) + { + Debug.Assert(len <= OverlappedBlockSize); + + while (len >= 16) + { + len -= 16; + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); + } + if ((len & 8) != 0) + { + len -= 8; + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); + } + if ((len & 4) != 0) + { + len -= 4; + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); + } + if ((len & 2) != 0) + { + len -= 2; + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); + } + if ((len & 1) != 0) + { + dest = src; + } + } +#endif + // Non-inlinable wrapper around the QCall that avoids polluting the fast path // with P/Invoke prolog/epilog. [MethodImpl(MethodImplOptions.NoInlining)]