Memmove: handle overlapping forward copies in managed code

src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs

@@ -11,6 +11,7 @@
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics;
 
 namespace System
 {
@@ -25,6 +26,18 @@ internal static partial class SpanHelpers // .ByteMemOps
 #endif
         private const nuint ZeroMemoryNativeThreshold = 1024;
 
+#if (TARGET_AMD64 || TARGET_ARM64) && !MONO
+        // Blocks are copied with a constant-length Memmove, which the JIT unrolls into "load the whole block
+        // into registers, then store it" - correct for any overlap. 64 bytes fits its unrolling budget on
+        // every target here, and bounds re-entry: a shorter length is finished without calling Memmove.
+        private const nuint MemmoveOverlappedBlock = 64;
+
+        // Aligning the destination only pays for the extra leading block on larger copies.
+        private const nuint MemmoveOverlappedAlignThreshold = 2048;
+
+        // Past this the platform memmove's backward loop is about twice as fast as a managed descending one.
+        private const nuint MemmoveOverlappedBackwardThreshold = 256;
+#endif
 
 #if HAS_CUSTOM_BLOCKS
         [StructLayout(LayoutKind.Sequential, Size = 16)]
@@ -34,7 +47,10 @@ private struct Block16 {}
         private struct Block64 {}
 #endif // HAS_CUSTOM_BLOCKS
 
+        // Too big to be worth inlining: it burns a lot of the caller's budget, and keeping the call is also
+        // what lets the JIT unroll it when the length is constant.
         [Intrinsic] // Unrolled for small constant lengths
+        [MethodImpl(MethodImplOptions.NoInlining)]
         internal static void Memmove(ref byte dest, ref byte src, nuint len)
         {
             // P/Invoke into the native version when the buffers are overlapping.
@@ -235,6 +251,23 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len)
                 return;
             }
 
+#if (TARGET_AMD64 || TARGET_ARM64) && !MONO
+            // memmove's forward loop is tuned for disjoint buffers: 'rep movsb' collapses when they are less
+            // than a cache line apart, and the non-temporal stores it uses for large copies evict the lines
+            // the rest of the copy reads back. Its backward loop has neither problem and beats us above 256.
+            if ((nuint)Unsafe.ByteOffset(ref dest, ref src) < len)
+            {
+                CopyOverlappedForward(ref dest, ref src, len);
+                return;
+            }
+
+            if (len <= MemmoveOverlappedBackwardThreshold)
+            {
+                CopyOverlappedBackward(ref dest, ref src, len);
+                return;
+            }
+#endif
+
         PInvoke:
             // Implicit nullchecks
             Debug.Assert(len > 0);
@@ -243,6 +276,113 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len)
             MemmoveNative(ref dest, ref src, len);
         }
 
+#if (TARGET_AMD64 || TARGET_ARM64) && !MONO
+        // Blocks run in strictly ascending order, so a store can never reach a source byte a later block
+        // still has to read. That also rules out the trailing "block anchored at the end of the buffer"
+        // shortcut the non-overlapping paths use - those bytes would already have been rewritten.
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static void CopyOverlappedForward(ref byte dest, ref byte src, nuint len)
+        {
+            Debug.Assert(len > 0);
+
+            // Only one of the two can be aligned, and an unaligned store costs more than an unaligned load.
+            if (len >= MemmoveOverlappedAlignThreshold)
+            {
+                nuint head = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64);
+                if (head != 64)
+                {
+                    CopyOverlappedForwardTail(ref dest, ref src, head);
+                    dest = ref Unsafe.Add(ref dest, head);
+                    src = ref Unsafe.Add(ref src, head);
+                    len -= head;
+                }
+            }
+
+            // The JIT unrolls a constant-length Memmove up to four vector registers wide, so take the widest
+            // block it will still expand. Bigger blocks also mean fewer boundaries where a store and the
+            // next block's load land in the same cache line, which is what a tight overlap is sensitive to.
+            if (Vector512.IsHardwareAccelerated)
+            {
+                while (len > 256)
+                {
+                    Memmove(ref dest, ref src, 256);
+                    dest = ref Unsafe.Add(ref dest, 256);
+                    src = ref Unsafe.Add(ref src, 256);
+                    len -= 256;
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated)
+            {
+                while (len > 128)
+                {
+                    Memmove(ref dest, ref src, 128);
+                    dest = ref Unsafe.Add(ref dest, 128);
+                    src = ref Unsafe.Add(ref src, 128);
+                    len -= 128;
+                }
+            }
+
+            while (len > MemmoveOverlappedBlock)
+            {
+                Memmove(ref dest, ref src, MemmoveOverlappedBlock);
+                dest = ref Unsafe.Add(ref dest, MemmoveOverlappedBlock);
+                src = ref Unsafe.Add(ref src, MemmoveOverlappedBlock);
+                len -= MemmoveOverlappedBlock;
+            }
+
+            CopyOverlappedForwardTail(ref dest, ref src, len);
+        }
+
+        // Mirror image: blocks run in strictly descending order, walking down from the end.
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static void CopyOverlappedBackward(ref byte dest, ref byte src, nuint len)
+        {
+            Debug.Assert(len > 0);
+
+            while (len > MemmoveOverlappedBlock)
+            {
+                len -= MemmoveOverlappedBlock;
+                Memmove(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len), MemmoveOverlappedBlock);
+            }
+
+            while (len >= sizeof(ulong))
+            {
+                len -= sizeof(ulong);
+                Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned<ulong>(ref Unsafe.Add(ref src, len)));
+            }
+
+            while (len != 0)
+            {
+                len--;
+                Unsafe.Add(ref dest, len) = Unsafe.Add(ref src, len);
+            }
+        }
+
+        // Finishes at most MemmoveOverlappedBlock bytes. Deliberately doesn't call Memmove - this is where a
+        // call the JIT didn't unroll lands, so calling it again is what recursion would look like. Every step
+        // is one load feeding one store, so the ordering comes from the data dependency.
+        private static void CopyOverlappedForwardTail(ref byte dest, ref byte src, nuint len)
+        {
+            Debug.Assert(len <= MemmoveOverlappedBlock);
+
+            while (len >= sizeof(ulong))
+            {
+                Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned<ulong>(ref src));
+                dest = ref Unsafe.Add(ref dest, sizeof(ulong));
+                src = ref Unsafe.Add(ref src, sizeof(ulong));
+                len -= sizeof(ulong);
+            }
+
+            while (len != 0)
+            {
+                dest = src;
+                dest = ref Unsafe.Add(ref dest, 1);
+                src = ref Unsafe.Add(ref src, 1);
+                len--;
+            }
+        }
+#endif
+
         // Non-inlinable wrapper around the QCall that avoids polluting the fast path
         // with P/Invoke prolog/epilog.
         [MethodImpl(MethodImplOptions.NoInlining)]