From 14baba1de3e81f88a219f55f5729769ee421a2fb Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Mon, 24 Aug 2026 14:49:50 +0200 Subject: [PATCH 1/8] Memmove: handle overlapping forward copies in managed code Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 20467548-69a0-453c-af07-8259bf6415fb --- .../src/System/SpanHelpers.ByteMemOps.cs | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index eba91cc493502a..db5e989963eab3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -11,6 +11,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; namespace System { @@ -25,6 +26,14 @@ internal static partial class SpanHelpers // .ByteMemOps #endif private const nuint ZeroMemoryNativeThreshold = 1024; + // The platform's forward memmove ('rep movsb' on x86) loses most of its throughput when the source + // and the destination are less than a cache line apart, which is exactly what a shift by a single + // array element looks like. Copy those overlapping buffers ourselves instead. + private const nuint MemmoveOverlappedNativeMinDistance = 64; + + // Largest block handed to the platform's memmove when the buffers overlap: big enough for it to + // amortize its own set-up cost, small enough to keep it away from non-temporal stores. + private const nuint MemmoveOverlappedNativeChunk = 32 * 1024; #if HAS_CUSTOM_BLOCKS [StructLayout(LayoutKind.Sequential, Size = 16)] @@ -235,6 +244,17 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) return; } + // 'dest' below 'src' means the data is shifted towards the start of the buffer, which is by + // far the most common overlapping shape (List.RemoveAt/RemoveRange, Queue, overlapping + // Span.CopyTo, ...). Such a copy can run in strictly ascending order, which lets us pick a + // better strategy than blindly handing it to the platform's memmove. Shifts towards the end of + // the buffer keep using memmove, whose backward copy loop doesn't have the problems below. + if ((nuint)Unsafe.ByteOffset(ref dest, ref src) < len) + { + MemmoveOverlappedForward(ref dest, ref src, len); + return; + } + PInvoke: // Implicit nullchecks Debug.Assert(len > 0); @@ -243,6 +263,129 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) MemmoveNative(ref dest, ref src, len); } + // Copies overlapping buffers where 'dest' is at a lower address than 'src', i.e. the data is + // shifted towards the start of the buffer. Both of the platform memmove's problems with this + // shape come from it being tuned for disjoint buffers, so we route around them here. + [MethodImpl(MethodImplOptions.NoInlining)] + private static void MemmoveOverlappedForward(ref byte dest, ref byte src, nuint len) + { + Debug.Assert(len > 0); + + nuint distance = (nuint)Unsafe.ByteOffset(ref dest, ref src); + + // A forward 'rep movsb' loses most of its throughput when it has to feed itself, i.e. when the + // two buffers are less than a cache line apart - which is precisely a shift by one element. And + // below the cut-off the non-overlapping paths use, the QCall costs more than the copy itself. + // Targets that never call into the platform's memmove (MemmoveNativeThreshold is unbounded + // there) consequently always take this path, just like their non-overlapping copies do. + if (Vector128.IsHardwareAccelerated && + (len <= MemmoveNativeThreshold || distance < MemmoveOverlappedNativeMinDistance)) + { + CopyForwardVectorized(ref dest, ref src, len); + return; + } + + // Implicit nullchecks + _ = Unsafe.ReadUnaligned(ref dest); + _ = Unsafe.ReadUnaligned(ref src); + + // Large copies make memmove switch to non-temporal stores, which is exactly wrong here: the + // lines it pushes out of the cache are the ones the rest of the copy is about to read back. + // Feeding it one chunk at a time keeps it on its cached - and far faster - copy loop. Walking + // the chunks from the start is safe because 'dest' trails 'src'. + while (len > MemmoveOverlappedNativeChunk) + { + MemmoveNative(ref dest, ref src, MemmoveOverlappedNativeChunk); + dest = ref Unsafe.Add(ref dest, MemmoveOverlappedNativeChunk); + src = ref Unsafe.Add(ref src, MemmoveOverlappedNativeChunk); + len -= MemmoveOverlappedNativeChunk; + } + + MemmoveNative(ref dest, ref src, len); + } + + // Copies 'src' to 'dest' in strictly ascending order, so a byte is always read before the copy can + // overwrite it. That also rules out the "copy a final block anchored at the end of the buffer" + // trick the non-overlapping paths use - that block may already have been rewritten by then. + private static void CopyForwardVectorized(ref byte dest, ref byte src, nuint len) + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + // The blocks are addressed off 'dest'/'src' with constant offsets rather than off a running + // index so that targets with load/store-pair instructions can fold them (arm64 'ldp'/'stp'). + if (Vector256.IsHardwareAccelerated) + { + while (len >= 128) + { + // All of the blocks are loaded before any of them is stored, so a store can never + // clobber source bytes that this iteration still has to read. + Vector256 block0 = Vector256.LoadUnsafe(ref src); + Vector256 block1 = Vector256.LoadUnsafe(ref src, 32); + Vector256 block2 = Vector256.LoadUnsafe(ref src, 64); + Vector256 block3 = Vector256.LoadUnsafe(ref src, 96); + Vector256.StoreUnsafe(block0, ref dest); + Vector256.StoreUnsafe(block1, ref dest, 32); + Vector256.StoreUnsafe(block2, ref dest, 64); + Vector256.StoreUnsafe(block3, ref dest, 96); + dest = ref Unsafe.Add(ref dest, 128); + src = ref Unsafe.Add(ref src, 128); + len -= 128; + } + + while (len >= 32) + { + Vector256.StoreUnsafe(Vector256.LoadUnsafe(ref src), ref dest); + dest = ref Unsafe.Add(ref dest, 32); + src = ref Unsafe.Add(ref src, 32); + len -= 32; + } + } + else + { + while (len >= 64) + { + Vector128 block0 = Vector128.LoadUnsafe(ref src); + Vector128 block1 = Vector128.LoadUnsafe(ref src, 16); + Vector128 block2 = Vector128.LoadUnsafe(ref src, 32); + Vector128 block3 = Vector128.LoadUnsafe(ref src, 48); + Vector128.StoreUnsafe(block0, ref dest); + Vector128.StoreUnsafe(block1, ref dest, 16); + Vector128.StoreUnsafe(block2, ref dest, 32); + Vector128.StoreUnsafe(block3, ref dest, 48); + dest = ref Unsafe.Add(ref dest, 64); + src = ref Unsafe.Add(ref src, 64); + len -= 64; + } + } + + // Drain the remainder with progressively smaller blocks, each one picking up exactly where the + // previous one ended. Every block is a single load followed by a single store, so no partial + // block can be written before the bytes it overlaps have been read. + while (len >= 16) + { + Vector128.StoreUnsafe(Vector128.LoadUnsafe(ref src), ref dest); + dest = ref Unsafe.Add(ref dest, 16); + src = ref Unsafe.Add(ref src, 16); + len -= 16; + } + + while (len >= 4) + { + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 4); + src = ref Unsafe.Add(ref src, 4); + len -= 4; + } + + while (len != 0) + { + dest = src; + dest = ref Unsafe.Add(ref dest, 1); + src = ref Unsafe.Add(ref src, 1); + len--; + } + } + // Non-inlinable wrapper around the QCall that avoids polluting the fast path // with P/Invoke prolog/epilog. [MethodImpl(MethodImplOptions.NoInlining)] From 5d58eecbcf061bc42c98a445462a4ff01f756a35 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Mon, 24 Aug 2026 17:06:38 +0200 Subject: [PATCH 2/8] Align the destination and drop the native fallback Aligning the destination closes the gap with rep movsb at every size, so the distance heuristic and the chunked memmove are no longer needed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 20467548-69a0-453c-af07-8259bf6415fb --- .../src/System/SpanHelpers.ByteMemOps.cs | 100 +++++++++--------- 1 file changed, 48 insertions(+), 52 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index db5e989963eab3..895a0fe83d49e4 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -26,14 +26,9 @@ internal static partial class SpanHelpers // .ByteMemOps #endif private const nuint ZeroMemoryNativeThreshold = 1024; - // The platform's forward memmove ('rep movsb' on x86) loses most of its throughput when the source - // and the destination are less than a cache line apart, which is exactly what a shift by a single - // array element looks like. Copy those overlapping buffers ourselves instead. - private const nuint MemmoveOverlappedNativeMinDistance = 64; - - // Largest block handed to the platform's memmove when the buffers overlap: big enough for it to - // amortize its own set-up cost, small enough to keep it away from non-temporal stores. - private const nuint MemmoveOverlappedNativeChunk = 32 * 1024; + // Copy size at which aligning the destination of an overlapping copy starts to pay for the extra + // leading block. Below it the alignment prologue costs more than the misaligned stores it avoids. + private const nuint MemmoveOverlappedAlignThreshold = 2048; #if HAS_CUSTOM_BLOCKS [StructLayout(LayoutKind.Sequential, Size = 16)] @@ -246,12 +241,15 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) // 'dest' below 'src' means the data is shifted towards the start of the buffer, which is by // far the most common overlapping shape (List.RemoveAt/RemoveRange, Queue, overlapping - // Span.CopyTo, ...). Such a copy can run in strictly ascending order, which lets us pick a - // better strategy than blindly handing it to the platform's memmove. Shifts towards the end of - // the buffer keep using memmove, whose backward copy loop doesn't have the problems below. - if ((nuint)Unsafe.ByteOffset(ref dest, ref src) < len) + // Span.CopyTo, ...). Such a copy can run in strictly ascending order, so we do it ourselves + // rather than handing it to the platform's memmove, whose forward copy loop is tuned for + // disjoint buffers: 'rep movsb' collapses when the two buffers are less than a cache line + // apart, and the non-temporal stores it switches to for large copies push out exactly the + // lines the rest of the copy is about to read back. Shifts towards the end of the buffer keep + // using memmove, whose backward copy loop doesn't have either problem. + if (Vector128.IsHardwareAccelerated && (nuint)Unsafe.ByteOffset(ref dest, ref src) < len) { - MemmoveOverlappedForward(ref dest, ref src, len); + CopyForwardVectorized(ref dest, ref src, len); return; } @@ -263,53 +261,51 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) MemmoveNative(ref dest, ref src, len); } - // Copies overlapping buffers where 'dest' is at a lower address than 'src', i.e. the data is - // shifted towards the start of the buffer. Both of the platform memmove's problems with this - // shape come from it being tuned for disjoint buffers, so we route around them here. + // Copies 'src' to 'dest' in strictly ascending order, so a byte is always read before the copy can + // overwrite it. That also rules out the "copy a final block anchored at the end of the buffer" + // trick the non-overlapping paths use - that block may already have been rewritten by then. [MethodImpl(MethodImplOptions.NoInlining)] - private static void MemmoveOverlappedForward(ref byte dest, ref byte src, nuint len) + private static void CopyForwardVectorized(ref byte dest, ref byte src, nuint len) { Debug.Assert(len > 0); + Debug.Assert(Vector128.IsHardwareAccelerated); - nuint distance = (nuint)Unsafe.ByteOffset(ref dest, ref src); - - // A forward 'rep movsb' loses most of its throughput when it has to feed itself, i.e. when the - // two buffers are less than a cache line apart - which is precisely a shift by one element. And - // below the cut-off the non-overlapping paths use, the QCall costs more than the copy itself. - // Targets that never call into the platform's memmove (MemmoveNativeThreshold is unbounded - // there) consequently always take this path, just like their non-overlapping copies do. - if (Vector128.IsHardwareAccelerated && - (len <= MemmoveNativeThreshold || distance < MemmoveOverlappedNativeMinDistance)) + // Align the destination. An unaligned store costs more than an unaligned load, and an + // overlapping copy can only ever have one of the two aligned. The leading bytes have to be + // copied ascending as well, so unlike the non-overlapping paths this can't be done with a + // single oversized leading block - that block would read source bytes it just overwrote. + if (len >= MemmoveOverlappedAlignThreshold) { - CopyForwardVectorized(ref dest, ref src, len); - return; - } - - // Implicit nullchecks - _ = Unsafe.ReadUnaligned(ref dest); - _ = Unsafe.ReadUnaligned(ref src); + nuint head = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); + if (head != 64) + { + len -= head; - // Large copies make memmove switch to non-temporal stores, which is exactly wrong here: the - // lines it pushes out of the cache are the ones the rest of the copy is about to read back. - // Feeding it one chunk at a time keeps it on its cached - and far faster - copy loop. Walking - // the chunks from the start is safe because 'dest' trails 'src'. - while (len > MemmoveOverlappedNativeChunk) - { - MemmoveNative(ref dest, ref src, MemmoveOverlappedNativeChunk); - dest = ref Unsafe.Add(ref dest, MemmoveOverlappedNativeChunk); - src = ref Unsafe.Add(ref src, MemmoveOverlappedNativeChunk); - len -= MemmoveOverlappedNativeChunk; - } + while (head >= 16) + { + Vector128.StoreUnsafe(Vector128.LoadUnsafe(ref src), ref dest); + dest = ref Unsafe.Add(ref dest, 16); + src = ref Unsafe.Add(ref src, 16); + head -= 16; + } - MemmoveNative(ref dest, ref src, len); - } + while (head >= 4) + { + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 4); + src = ref Unsafe.Add(ref src, 4); + head -= 4; + } - // Copies 'src' to 'dest' in strictly ascending order, so a byte is always read before the copy can - // overwrite it. That also rules out the "copy a final block anchored at the end of the buffer" - // trick the non-overlapping paths use - that block may already have been rewritten by then. - private static void CopyForwardVectorized(ref byte dest, ref byte src, nuint len) - { - Debug.Assert(Vector128.IsHardwareAccelerated); + while (head != 0) + { + dest = src; + dest = ref Unsafe.Add(ref dest, 1); + src = ref Unsafe.Add(ref src, 1); + head--; + } + } + } // The blocks are addressed off 'dest'/'src' with constant offsets rather than off a running // index so that targets with load/store-pair instructions can fold them (arm64 'ldp'/'stp'). From 7a510dfcc31f84ee17860c5e8a09310583618dd1 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Mon, 24 Aug 2026 18:02:19 +0200 Subject: [PATCH 3/8] Handle both overlap directions in managed code below a size threshold Factors the copy into shared block primitives so the forward and backward routines are mirror images of each other. Backward overlapping copies up to 256 bytes now skip the QCall as well; above that the platform memmove's backward loop is still about twice as fast as a managed descending one. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 20467548-69a0-453c-af07-8259bf6415fb --- .../src/System/SpanHelpers.ByteMemOps.cs | 306 +++++++++++++----- 1 file changed, 227 insertions(+), 79 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index 895a0fe83d49e4..fb08cd53f70683 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -26,10 +26,17 @@ internal static partial class SpanHelpers // .ByteMemOps #endif private const nuint ZeroMemoryNativeThreshold = 1024; - // Copy size at which aligning the destination of an overlapping copy starts to pay for the extra - // leading block. Below it the alignment prologue costs more than the misaligned stores it avoids. + // Copy size at which aligning the destination of an overlapping forward copy starts to pay for the + // extra leading block. Below it the alignment prologue costs more than the misaligned stores it + // avoids. private const nuint MemmoveOverlappedAlignThreshold = 2048; + // The platform memmove's backward copy loop stays about twice as fast as a managed descending one: + // a descending stream defeats the hardware prefetcher, and unlike the forward direction, aligning + // the destination doesn't recover it. So a copy towards the end of the buffer is only done here + // while it is short enough for the QCall itself to dominate. + private const nuint MemmoveOverlappedBackwardThreshold = 256; + #if HAS_CUSTOM_BLOCKS [StructLayout(LayoutKind.Sequential, Size = 16)] private struct Block16 {} @@ -239,18 +246,28 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) return; } - // 'dest' below 'src' means the data is shifted towards the start of the buffer, which is by - // far the most common overlapping shape (List.RemoveAt/RemoveRange, Queue, overlapping - // Span.CopyTo, ...). Such a copy can run in strictly ascending order, so we do it ourselves - // rather than handing it to the platform's memmove, whose forward copy loop is tuned for - // disjoint buffers: 'rep movsb' collapses when the two buffers are less than a cache line - // apart, and the non-temporal stores it switches to for large copies push out exactly the - // lines the rest of the copy is about to read back. Shifts towards the end of the buffer keep - // using memmove, whose backward copy loop doesn't have either problem. - if (Vector128.IsHardwareAccelerated && (nuint)Unsafe.ByteOffset(ref dest, ref src) < len) + if (Vector128.IsHardwareAccelerated) { - CopyForwardVectorized(ref dest, ref src, len); - return; + // 'dest' below 'src' means the data is shifted towards the start of the buffer, which is by + // far the most common overlapping shape (List.RemoveAt/RemoveRange, Queue, overlapping + // Span.CopyTo, ...). Those are always copied here: the platform memmove's forward loop is + // tuned for disjoint buffers and both of the tricks it uses backfire on an in-place shift. + // 'rep movsb' collapses when the two buffers are less than a cache line apart, and the + // non-temporal stores it switches to for large copies evict the very lines the rest of the + // copy is about to read back. + if ((nuint)Unsafe.ByteOffset(ref dest, ref src) < len) + { + CopyForwardVectorized(ref dest, ref src, len); + return; + } + + // 'dest' above 'src'. The platform memmove's backward loop has neither problem and outruns a + // managed descending loop, so this is only worth doing while the QCall dominates the copy. + if (len <= MemmoveOverlappedBackwardThreshold) + { + CopyBackwardVectorized(ref dest, ref src, len); + return; + } } PInvoke: @@ -261,9 +278,10 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) MemmoveNative(ref dest, ref src, len); } - // Copies 'src' to 'dest' in strictly ascending order, so a byte is always read before the copy can - // overwrite it. That also rules out the "copy a final block anchored at the end of the buffer" - // trick the non-overlapping paths use - that block may already have been rewritten by then. + // Copies overlapping buffers where 'dest' is at a lower address than 'src'. The blocks run in + // strictly ascending order, so a byte is always read before the copy can overwrite it. That also + // rules out the "copy a final block anchored at the end of the buffer" trick the non-overlapping + // paths use - by the time that block ran, the bytes it reads would already have been rewritten. [MethodImpl(MethodImplOptions.NoInlining)] private static void CopyForwardVectorized(ref byte dest, ref byte src, nuint len) { @@ -271,39 +289,16 @@ private static void CopyForwardVectorized(ref byte dest, ref byte src, nuint len Debug.Assert(Vector128.IsHardwareAccelerated); // Align the destination. An unaligned store costs more than an unaligned load, and an - // overlapping copy can only ever have one of the two aligned. The leading bytes have to be - // copied ascending as well, so unlike the non-overlapping paths this can't be done with a - // single oversized leading block - that block would read source bytes it just overwrote. + // overlapping copy can only ever have one of the two aligned. if (len >= MemmoveOverlappedAlignThreshold) { nuint head = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); if (head != 64) { + CopyBlocksForward(ref dest, ref src, head); + dest = ref Unsafe.Add(ref dest, head); + src = ref Unsafe.Add(ref src, head); len -= head; - - while (head >= 16) - { - Vector128.StoreUnsafe(Vector128.LoadUnsafe(ref src), ref dest); - dest = ref Unsafe.Add(ref dest, 16); - src = ref Unsafe.Add(ref src, 16); - head -= 16; - } - - while (head >= 4) - { - Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); - dest = ref Unsafe.Add(ref dest, 4); - src = ref Unsafe.Add(ref src, 4); - head -= 4; - } - - while (head != 0) - { - dest = src; - dest = ref Unsafe.Add(ref dest, 1); - src = ref Unsafe.Add(ref src, 1); - head--; - } } } @@ -313,75 +308,228 @@ private static void CopyForwardVectorized(ref byte dest, ref byte src, nuint len { while (len >= 128) { - // All of the blocks are loaded before any of them is stored, so a store can never - // clobber source bytes that this iteration still has to read. - Vector256 block0 = Vector256.LoadUnsafe(ref src); - Vector256 block1 = Vector256.LoadUnsafe(ref src, 32); - Vector256 block2 = Vector256.LoadUnsafe(ref src, 64); - Vector256 block3 = Vector256.LoadUnsafe(ref src, 96); - Vector256.StoreUnsafe(block0, ref dest); - Vector256.StoreUnsafe(block1, ref dest, 32); - Vector256.StoreUnsafe(block2, ref dest, 64); - Vector256.StoreUnsafe(block3, ref dest, 96); + CopyBlock128(ref dest, ref src); dest = ref Unsafe.Add(ref dest, 128); src = ref Unsafe.Add(ref src, 128); len -= 128; } + } + else + { + while (len >= 64) + { + CopyBlock64(ref dest, ref src); + dest = ref Unsafe.Add(ref dest, 64); + src = ref Unsafe.Add(ref src, 64); + len -= 64; + } + } + + CopyBlocksForward(ref dest, ref src, len); + } - while (len >= 32) + // Copies overlapping buffers where 'dest' is at a higher address than 'src'. Mirror image of the + // above: the blocks run in strictly descending order, walking down from the end of the buffer. + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CopyBackwardVectorized(ref byte dest, ref byte src, nuint len) + { + Debug.Assert(len > 0); + Debug.Assert(Vector128.IsHardwareAccelerated); + + if (Vector256.IsHardwareAccelerated) + { + while (len >= 128) { - Vector256.StoreUnsafe(Vector256.LoadUnsafe(ref src), ref dest); - dest = ref Unsafe.Add(ref dest, 32); - src = ref Unsafe.Add(ref src, 32); - len -= 32; + len -= 128; + CopyBlock128(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); } } else { while (len >= 64) { - Vector128 block0 = Vector128.LoadUnsafe(ref src); - Vector128 block1 = Vector128.LoadUnsafe(ref src, 16); - Vector128 block2 = Vector128.LoadUnsafe(ref src, 32); - Vector128 block3 = Vector128.LoadUnsafe(ref src, 48); - Vector128.StoreUnsafe(block0, ref dest); - Vector128.StoreUnsafe(block1, ref dest, 16); - Vector128.StoreUnsafe(block2, ref dest, 32); - Vector128.StoreUnsafe(block3, ref dest, 48); - dest = ref Unsafe.Add(ref dest, 64); - src = ref Unsafe.Add(ref src, 64); len -= 64; + CopyBlock64(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); } } - // Drain the remainder with progressively smaller blocks, each one picking up exactly where the - // previous one ended. Every block is a single load followed by a single store, so no partial - // block can be written before the bytes it overlaps have been read. - while (len >= 16) + CopyBlocksBackward(ref dest, ref src, len); + } + + // Copies fewer than 128 bytes, largest block first, so that the blocks run in ascending order. + private static void CopyBlocksForward(ref byte dest, ref byte src, nuint len) + { + Debug.Assert(len < 128); + + if ((len & 64) != 0) + { + CopyBlock64(ref dest, ref src); + dest = ref Unsafe.Add(ref dest, 64); + src = ref Unsafe.Add(ref src, 64); + } + + if ((len & 32) != 0) { - Vector128.StoreUnsafe(Vector128.LoadUnsafe(ref src), ref dest); + CopyBlock32(ref dest, ref src); + dest = ref Unsafe.Add(ref dest, 32); + src = ref Unsafe.Add(ref src, 32); + } + + if ((len & 16) != 0) + { + CopyBlock16(ref dest, ref src); dest = ref Unsafe.Add(ref dest, 16); src = ref Unsafe.Add(ref src, 16); - len -= 16; } - while (len >= 4) + if ((len & 8) != 0) + { + CopyBlock8(ref dest, ref src); + dest = ref Unsafe.Add(ref dest, 8); + src = ref Unsafe.Add(ref src, 8); + } + + if ((len & 4) != 0) { Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); dest = ref Unsafe.Add(ref dest, 4); src = ref Unsafe.Add(ref src, 4); + } + + if ((len & 2) != 0) + { + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 2); + src = ref Unsafe.Add(ref src, 2); + } + + if ((len & 1) != 0) + { + dest = src; + } + } + + // Copies fewer than 128 bytes, largest block first, so that the blocks run in descending order. + private static void CopyBlocksBackward(ref byte dest, ref byte src, nuint len) + { + Debug.Assert(len < 128); + + if ((len & 64) != 0) + { + len -= 64; + CopyBlock64(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); + } + + if ((len & 32) != 0) + { + len -= 32; + CopyBlock32(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); + } + + if ((len & 16) != 0) + { + len -= 16; + CopyBlock16(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); + } + + if ((len & 8) != 0) + { + len -= 8; + CopyBlock8(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); + } + + if ((len & 4) != 0) + { len -= 4; + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); } - while (len != 0) + if ((len & 2) != 0) + { + len -= 2; + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); + } + + if ((len & 1) != 0) { + Debug.Assert(len == 1); dest = src; - dest = ref Unsafe.Add(ref dest, 1); - src = ref Unsafe.Add(ref src, 1); - len--; } } + // Every block below is fully loaded before any of it is stored, so it can be copied in either + // direction without one of its own stores clobbering source bytes it still has to read. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CopyBlock128(ref byte dest, ref byte src) + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 block0 = Vector256.LoadUnsafe(ref src); + Vector256 block1 = Vector256.LoadUnsafe(ref src, 32); + Vector256 block2 = Vector256.LoadUnsafe(ref src, 64); + Vector256 block3 = Vector256.LoadUnsafe(ref src, 96); + Vector256.StoreUnsafe(block0, ref dest); + Vector256.StoreUnsafe(block1, ref dest, 32); + Vector256.StoreUnsafe(block2, ref dest, 64); + Vector256.StoreUnsafe(block3, ref dest, 96); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CopyBlock64(ref byte dest, ref byte src) + { + if (Vector256.IsHardwareAccelerated) + { + Vector256 block0 = Vector256.LoadUnsafe(ref src); + Vector256 block1 = Vector256.LoadUnsafe(ref src, 32); + Vector256.StoreUnsafe(block0, ref dest); + Vector256.StoreUnsafe(block1, ref dest, 32); + } + else + { + Vector128 block0 = Vector128.LoadUnsafe(ref src); + Vector128 block1 = Vector128.LoadUnsafe(ref src, 16); + Vector128 block2 = Vector128.LoadUnsafe(ref src, 32); + Vector128 block3 = Vector128.LoadUnsafe(ref src, 48); + Vector128.StoreUnsafe(block0, ref dest); + Vector128.StoreUnsafe(block1, ref dest, 16); + Vector128.StoreUnsafe(block2, ref dest, 32); + Vector128.StoreUnsafe(block3, ref dest, 48); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CopyBlock32(ref byte dest, ref byte src) + { + if (Vector256.IsHardwareAccelerated) + { + Vector256.StoreUnsafe(Vector256.LoadUnsafe(ref src), ref dest); + } + else + { + Vector128 block0 = Vector128.LoadUnsafe(ref src); + Vector128 block1 = Vector128.LoadUnsafe(ref src, 16); + Vector128.StoreUnsafe(block0, ref dest); + Vector128.StoreUnsafe(block1, ref dest, 16); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CopyBlock16(ref byte dest, ref byte src) => + Vector128.StoreUnsafe(Vector128.LoadUnsafe(ref src), ref dest); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CopyBlock8(ref byte dest, ref byte src) + { +#if TARGET_64BIT + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); +#else + uint block0 = Unsafe.ReadUnaligned(ref src); + uint block1 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 4)); + Unsafe.WriteUnaligned(ref dest, block0); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), block1); +#endif + } + // Non-inlinable wrapper around the QCall that avoids polluting the fast path // with P/Invoke prolog/epilog. [MethodImpl(MethodImplOptions.NoInlining)] From 3d7581844d34cd2cb111a9febdf2435ba76ce3dc Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Mon, 24 Aug 2026 18:31:54 +0200 Subject: [PATCH 4/8] Copy overlapping blocks with the JIT's unrolled Memmove Replaces the hand-written vector blocks, which assumed the JIT would keep all of a block's loads ahead of its stores, with constant-length Memmove calls. The JIT unrolls those into exactly that shape by design, and picks the widest vector the target has. Blocks are capped at 64 bytes so a call the JIT did not unroll re-enters with a length the tails handle without calling Memmove again, which keeps these routines from recursing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 20467548-69a0-453c-af07-8259bf6415fb --- .../src/System/SpanHelpers.ByteMemOps.cs | 226 +++++------------- 1 file changed, 66 insertions(+), 160 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index fb08cd53f70683..7b535a93fcde81 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -26,6 +26,15 @@ internal static partial class SpanHelpers // .ByteMemOps #endif private const nuint ZeroMemoryNativeThreshold = 1024; + // Block size used by the overlapping copies below. Each block is copied with a constant-length + // Memmove, which the JIT unrolls into "load the whole block into registers, then store it" - the + // one shape that stays correct however the two buffers overlap. That unrolling has a target + // dependent budget of four vector registers, and 64 bytes is within it even where those are only + // 16 bytes wide. Keeping the block this small also bounds re-entry: should the JIT not unroll the + // call after all, it arrives back here with a length that is handled without another Memmove, so + // these routines cannot recurse. + private const nuint MemmoveOverlappedBlock = 64; + // Copy size at which aligning the destination of an overlapping forward copy starts to pay for the // extra leading block. Below it the alignment prologue costs more than the misaligned stores it // avoids. @@ -257,7 +266,7 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) // copy is about to read back. if ((nuint)Unsafe.ByteOffset(ref dest, ref src) < len) { - CopyForwardVectorized(ref dest, ref src, len); + CopyOverlappedForward(ref dest, ref src, len); return; } @@ -265,7 +274,7 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) // managed descending loop, so this is only worth doing while the QCall dominates the copy. if (len <= MemmoveOverlappedBackwardThreshold) { - CopyBackwardVectorized(ref dest, ref src, len); + CopyOverlappedBackward(ref dest, ref src, len); return; } } @@ -278,113 +287,81 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) MemmoveNative(ref dest, ref src, len); } - // Copies overlapping buffers where 'dest' is at a lower address than 'src'. The blocks run in - // strictly ascending order, so a byte is always read before the copy can overwrite it. That also - // rules out the "copy a final block anchored at the end of the buffer" trick the non-overlapping - // paths use - by the time that block ran, the bytes it reads would already have been rewritten. + // Copies overlapping buffers where 'dest' is at a lower address than 'src', i.e. the data is + // shifted towards the start of the buffer. Blocks run in strictly ascending order, so a store can + // never reach a source byte a later block still has to read. That also rules out the "copy a final + // block anchored at the end of the buffer" trick the non-overlapping paths use - by the time such a + // block ran, the bytes it reads would already have been rewritten. [MethodImpl(MethodImplOptions.NoInlining)] - private static void CopyForwardVectorized(ref byte dest, ref byte src, nuint len) + private static void CopyOverlappedForward(ref byte dest, ref byte src, nuint len) { Debug.Assert(len > 0); - Debug.Assert(Vector128.IsHardwareAccelerated); - // Align the destination. An unaligned store costs more than an unaligned load, and an - // overlapping copy can only ever have one of the two aligned. - if (len >= MemmoveOverlappedAlignThreshold) + if (len > MemmoveOverlappedBlock) { - nuint head = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); - if (head != 64) + // Align the destination. An unaligned store costs more than an unaligned load, and an + // overlapping copy can only ever have one of the two aligned. + if (len >= MemmoveOverlappedAlignThreshold) { - CopyBlocksForward(ref dest, ref src, head); - dest = ref Unsafe.Add(ref dest, head); - src = ref Unsafe.Add(ref src, head); - len -= head; + nuint head = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); + if (head != 64) + { + CopyOverlappedForwardTail(ref dest, ref src, head); + dest = ref Unsafe.Add(ref dest, head); + src = ref Unsafe.Add(ref src, head); + len -= head; + } } - } - // The blocks are addressed off 'dest'/'src' with constant offsets rather than off a running - // index so that targets with load/store-pair instructions can fold them (arm64 'ldp'/'stp'). - if (Vector256.IsHardwareAccelerated) - { - while (len >= 128) - { - CopyBlock128(ref dest, ref src); - dest = ref Unsafe.Add(ref dest, 128); - src = ref Unsafe.Add(ref src, 128); - len -= 128; - } - } - else - { - while (len >= 64) + do { - CopyBlock64(ref dest, ref src); - dest = ref Unsafe.Add(ref dest, 64); - src = ref Unsafe.Add(ref src, 64); - len -= 64; + Memmove(ref dest, ref src, MemmoveOverlappedBlock); + dest = ref Unsafe.Add(ref dest, MemmoveOverlappedBlock); + src = ref Unsafe.Add(ref src, MemmoveOverlappedBlock); + len -= MemmoveOverlappedBlock; } + while (len > MemmoveOverlappedBlock); } - CopyBlocksForward(ref dest, ref src, len); + CopyOverlappedForwardTail(ref dest, ref src, len); } - // Copies overlapping buffers where 'dest' is at a higher address than 'src'. Mirror image of the - // above: the blocks run in strictly descending order, walking down from the end of the buffer. + // Copies overlapping buffers where 'dest' is at a higher address than 'src', i.e. the data is + // shifted towards the end of the buffer. Mirror image of the above, walking down from the end. [MethodImpl(MethodImplOptions.NoInlining)] - private static void CopyBackwardVectorized(ref byte dest, ref byte src, nuint len) + private static void CopyOverlappedBackward(ref byte dest, ref byte src, nuint len) { Debug.Assert(len > 0); - Debug.Assert(Vector128.IsHardwareAccelerated); - if (Vector256.IsHardwareAccelerated) + while (len > MemmoveOverlappedBlock) { - while (len >= 128) - { - len -= 128; - CopyBlock128(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); - } - } - else - { - while (len >= 64) - { - len -= 64; - CopyBlock64(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); - } + len -= MemmoveOverlappedBlock; + Memmove(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len), MemmoveOverlappedBlock); } - CopyBlocksBackward(ref dest, ref src, len); + CopyOverlappedBackwardTail(ref dest, ref src, len); } - // Copies fewer than 128 bytes, largest block first, so that the blocks run in ascending order. - private static void CopyBlocksForward(ref byte dest, ref byte src, nuint len) + // The two routines below finish a copy of at most MemmoveOverlappedBlock bytes. They deliberately + // don't call Memmove: they are where an un-unrolled Memmove from the loops above lands, so calling + // it again is what recursion would look like. Every step is a single load followed by a single + // store of the same width, so the ordering the copy depends on comes from the data dependency + // rather than from how the register allocator happened to schedule a wider block. + private static void CopyOverlappedForwardTail(ref byte dest, ref byte src, nuint len) { - Debug.Assert(len < 128); - - if ((len & 64) != 0) - { - CopyBlock64(ref dest, ref src); - dest = ref Unsafe.Add(ref dest, 64); - src = ref Unsafe.Add(ref src, 64); - } - - if ((len & 32) != 0) - { - CopyBlock32(ref dest, ref src); - dest = ref Unsafe.Add(ref dest, 32); - src = ref Unsafe.Add(ref src, 32); - } + Debug.Assert(len <= MemmoveOverlappedBlock); - if ((len & 16) != 0) + while (len >= 16) { - CopyBlock16(ref dest, ref src); + Vector128.StoreUnsafe(Vector128.LoadUnsafe(ref src), ref dest); dest = ref Unsafe.Add(ref dest, 16); src = ref Unsafe.Add(ref src, 16); + len -= 16; } if ((len & 8) != 0) { - CopyBlock8(ref dest, ref src); + CopyStep8(ref dest, ref src); dest = ref Unsafe.Add(ref dest, 8); src = ref Unsafe.Add(ref src, 8); } @@ -409,33 +386,20 @@ private static void CopyBlocksForward(ref byte dest, ref byte src, nuint len) } } - // Copies fewer than 128 bytes, largest block first, so that the blocks run in descending order. - private static void CopyBlocksBackward(ref byte dest, ref byte src, nuint len) + private static void CopyOverlappedBackwardTail(ref byte dest, ref byte src, nuint len) { - Debug.Assert(len < 128); - - if ((len & 64) != 0) - { - len -= 64; - CopyBlock64(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); - } + Debug.Assert(len <= MemmoveOverlappedBlock); - if ((len & 32) != 0) - { - len -= 32; - CopyBlock32(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); - } - - if ((len & 16) != 0) + while (len >= 16) { len -= 16; - CopyBlock16(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); + Vector128.StoreUnsafe(Vector128.LoadUnsafe(ref Unsafe.Add(ref src, len)), ref Unsafe.Add(ref dest, len)); } if ((len & 8) != 0) { len -= 8; - CopyBlock8(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); + CopyStep8(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); } if ((len & 4) != 0) @@ -457,76 +421,18 @@ private static void CopyBlocksBackward(ref byte dest, ref byte src, nuint len) } } - // Every block below is fully loaded before any of it is stored, so it can be copied in either - // direction without one of its own stores clobbering source bytes it still has to read. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void CopyBlock128(ref byte dest, ref byte src) - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 block0 = Vector256.LoadUnsafe(ref src); - Vector256 block1 = Vector256.LoadUnsafe(ref src, 32); - Vector256 block2 = Vector256.LoadUnsafe(ref src, 64); - Vector256 block3 = Vector256.LoadUnsafe(ref src, 96); - Vector256.StoreUnsafe(block0, ref dest); - Vector256.StoreUnsafe(block1, ref dest, 32); - Vector256.StoreUnsafe(block2, ref dest, 64); - Vector256.StoreUnsafe(block3, ref dest, 96); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void CopyBlock64(ref byte dest, ref byte src) - { - if (Vector256.IsHardwareAccelerated) - { - Vector256 block0 = Vector256.LoadUnsafe(ref src); - Vector256 block1 = Vector256.LoadUnsafe(ref src, 32); - Vector256.StoreUnsafe(block0, ref dest); - Vector256.StoreUnsafe(block1, ref dest, 32); - } - else - { - Vector128 block0 = Vector128.LoadUnsafe(ref src); - Vector128 block1 = Vector128.LoadUnsafe(ref src, 16); - Vector128 block2 = Vector128.LoadUnsafe(ref src, 32); - Vector128 block3 = Vector128.LoadUnsafe(ref src, 48); - Vector128.StoreUnsafe(block0, ref dest); - Vector128.StoreUnsafe(block1, ref dest, 16); - Vector128.StoreUnsafe(block2, ref dest, 32); - Vector128.StoreUnsafe(block3, ref dest, 48); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void CopyBlock32(ref byte dest, ref byte src) - { - if (Vector256.IsHardwareAccelerated) - { - Vector256.StoreUnsafe(Vector256.LoadUnsafe(ref src), ref dest); - } - else - { - Vector128 block0 = Vector128.LoadUnsafe(ref src); - Vector128 block1 = Vector128.LoadUnsafe(ref src, 16); - Vector128.StoreUnsafe(block0, ref dest); - Vector128.StoreUnsafe(block1, ref dest, 16); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void CopyBlock16(ref byte dest, ref byte src) => - Vector128.StoreUnsafe(Vector128.LoadUnsafe(ref src), ref dest); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void CopyBlock8(ref byte dest, ref byte src) + private static void CopyStep8(ref byte dest, ref byte src) { #if TARGET_64BIT Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); #else - uint block0 = Unsafe.ReadUnaligned(ref src); - uint block1 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 4)); - Unsafe.WriteUnaligned(ref dest, block0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), block1); + // Two independent halves: both are read before either is written, so this stays correct for a + // backward copy whose buffers are less than 8 bytes apart. + uint lower = Unsafe.ReadUnaligned(ref src); + uint upper = Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 4)); + Unsafe.WriteUnaligned(ref dest, lower); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), upper); #endif } From 10094b75d857d01e0ca4d90e3d04121f710fdf9d Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Mon, 24 Aug 2026 18:57:51 +0200 Subject: [PATCH 5/8] Simplify: restrict to 64-bit x64/arm64 and trim the helpers Other targets keep using the CRT memmove exactly as before, which removes the 32-bit split in the tail, the Vector128 gate and the Intrinsics using. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 20467548-69a0-453c-af07-8259bf6415fb --- .../src/System/SpanHelpers.ByteMemOps.cs | 203 ++++++------------ 1 file changed, 63 insertions(+), 140 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index 7b535a93fcde81..30d5673acffca4 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -11,7 +11,6 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; namespace System { @@ -26,25 +25,19 @@ internal static partial class SpanHelpers // .ByteMemOps #endif private const nuint ZeroMemoryNativeThreshold = 1024; - // Block size used by the overlapping copies below. Each block is copied with a constant-length - // Memmove, which the JIT unrolls into "load the whole block into registers, then store it" - the - // one shape that stays correct however the two buffers overlap. That unrolling has a target - // dependent budget of four vector registers, and 64 bytes is within it even where those are only - // 16 bytes wide. Keeping the block this small also bounds re-entry: should the JIT not unroll the - // call after all, it arrives back here with a length that is handled without another Memmove, so - // these routines cannot recurse. +#if TARGET_AMD64 || TARGET_ARM64 + // Overlapping copies are done in blocks of this size with a constant-length Memmove, which the JIT + // unrolls into "load the whole block into registers, then store it" - correct for any overlap. 64 + // bytes stays within its unrolling budget everywhere, and bounds re-entry: a call it didn't unroll + // comes back with a length the tail handles without calling Memmove again, so these can't recurse. private const nuint MemmoveOverlappedBlock = 64; - // Copy size at which aligning the destination of an overlapping forward copy starts to pay for the - // extra leading block. Below it the alignment prologue costs more than the misaligned stores it - // avoids. + // Aligning the destination only pays for the extra leading block on larger copies. private const nuint MemmoveOverlappedAlignThreshold = 2048; - // The platform memmove's backward copy loop stays about twice as fast as a managed descending one: - // a descending stream defeats the hardware prefetcher, and unlike the forward direction, aligning - // the destination doesn't recover it. So a copy towards the end of the buffer is only done here - // while it is short enough for the QCall itself to dominate. + // Past this the platform memmove's backward loop is about twice as fast as a managed descending one. private const nuint MemmoveOverlappedBackwardThreshold = 256; +#endif #if HAS_CUSTOM_BLOCKS [StructLayout(LayoutKind.Sequential, Size = 16)] @@ -255,29 +248,24 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) return; } - if (Vector128.IsHardwareAccelerated) +#if TARGET_AMD64 || TARGET_ARM64 + // The platform memmove's forward loop is tuned for disjoint buffers: 'rep movsb' collapses when + // they are less than a cache line apart - a shift by one element - and the non-temporal stores it + // uses for large copies evict the lines the rest of the copy reads back. Its backward loop has + // neither problem and outruns a managed descending one, so that direction is only handled here + // while the QCall dominates the copy. + if ((nuint)Unsafe.ByteOffset(ref dest, ref src) < len) { - // 'dest' below 'src' means the data is shifted towards the start of the buffer, which is by - // far the most common overlapping shape (List.RemoveAt/RemoveRange, Queue, overlapping - // Span.CopyTo, ...). Those are always copied here: the platform memmove's forward loop is - // tuned for disjoint buffers and both of the tricks it uses backfire on an in-place shift. - // 'rep movsb' collapses when the two buffers are less than a cache line apart, and the - // non-temporal stores it switches to for large copies evict the very lines the rest of the - // copy is about to read back. - if ((nuint)Unsafe.ByteOffset(ref dest, ref src) < len) - { - CopyOverlappedForward(ref dest, ref src, len); - return; - } + CopyOverlappedForward(ref dest, ref src, len); + return; + } - // 'dest' above 'src'. The platform memmove's backward loop has neither problem and outruns a - // managed descending loop, so this is only worth doing while the QCall dominates the copy. - if (len <= MemmoveOverlappedBackwardThreshold) - { - CopyOverlappedBackward(ref dest, ref src, len); - return; - } + if (len <= MemmoveOverlappedBackwardThreshold) + { + CopyOverlappedBackward(ref dest, ref src, len); + return; } +#endif PInvoke: // Implicit nullchecks @@ -287,47 +275,41 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) MemmoveNative(ref dest, ref src, len); } - // Copies overlapping buffers where 'dest' is at a lower address than 'src', i.e. the data is - // shifted towards the start of the buffer. Blocks run in strictly ascending order, so a store can - // never reach a source byte a later block still has to read. That also rules out the "copy a final - // block anchored at the end of the buffer" trick the non-overlapping paths use - by the time such a - // block ran, the bytes it reads would already have been rewritten. +#if TARGET_AMD64 || TARGET_ARM64 + // Blocks run in strictly ascending order, so a store can never reach a source byte a later block + // still has to read. That also rules out the "copy a final block anchored at the end of the buffer" + // trick the non-overlapping paths use - those bytes would already have been rewritten. [MethodImpl(MethodImplOptions.NoInlining)] private static void CopyOverlappedForward(ref byte dest, ref byte src, nuint len) { Debug.Assert(len > 0); - if (len > MemmoveOverlappedBlock) + // An unaligned store costs more than an unaligned load, and an overlapping copy can only ever + // have one of the two aligned. + if (len >= MemmoveOverlappedAlignThreshold) { - // Align the destination. An unaligned store costs more than an unaligned load, and an - // overlapping copy can only ever have one of the two aligned. - if (len >= MemmoveOverlappedAlignThreshold) + nuint head = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); + if (head != 64) { - nuint head = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); - if (head != 64) - { - CopyOverlappedForwardTail(ref dest, ref src, head); - dest = ref Unsafe.Add(ref dest, head); - src = ref Unsafe.Add(ref src, head); - len -= head; - } + CopyOverlappedForwardTail(ref dest, ref src, head); + dest = ref Unsafe.Add(ref dest, head); + src = ref Unsafe.Add(ref src, head); + len -= head; } + } - do - { - Memmove(ref dest, ref src, MemmoveOverlappedBlock); - dest = ref Unsafe.Add(ref dest, MemmoveOverlappedBlock); - src = ref Unsafe.Add(ref src, MemmoveOverlappedBlock); - len -= MemmoveOverlappedBlock; - } - while (len > MemmoveOverlappedBlock); + while (len > MemmoveOverlappedBlock) + { + Memmove(ref dest, ref src, MemmoveOverlappedBlock); + dest = ref Unsafe.Add(ref dest, MemmoveOverlappedBlock); + src = ref Unsafe.Add(ref src, MemmoveOverlappedBlock); + len -= MemmoveOverlappedBlock; } CopyOverlappedForwardTail(ref dest, ref src, len); } - // Copies overlapping buffers where 'dest' is at a higher address than 'src', i.e. the data is - // shifted towards the end of the buffer. Mirror image of the above, walking down from the end. + // Mirror image: blocks run in strictly descending order, walking down from the end. [MethodImpl(MethodImplOptions.NoInlining)] private static void CopyOverlappedBackward(ref byte dest, ref byte src, nuint len) { @@ -339,102 +321,43 @@ private static void CopyOverlappedBackward(ref byte dest, ref byte src, nuint le Memmove(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len), MemmoveOverlappedBlock); } - CopyOverlappedBackwardTail(ref dest, ref src, len); - } - - // The two routines below finish a copy of at most MemmoveOverlappedBlock bytes. They deliberately - // don't call Memmove: they are where an un-unrolled Memmove from the loops above lands, so calling - // it again is what recursion would look like. Every step is a single load followed by a single - // store of the same width, so the ordering the copy depends on comes from the data dependency - // rather than from how the register allocator happened to schedule a wider block. - private static void CopyOverlappedForwardTail(ref byte dest, ref byte src, nuint len) - { - Debug.Assert(len <= MemmoveOverlappedBlock); - - while (len >= 16) - { - Vector128.StoreUnsafe(Vector128.LoadUnsafe(ref src), ref dest); - dest = ref Unsafe.Add(ref dest, 16); - src = ref Unsafe.Add(ref src, 16); - len -= 16; - } - - if ((len & 8) != 0) + while (len >= sizeof(ulong)) { - CopyStep8(ref dest, ref src); - dest = ref Unsafe.Add(ref dest, 8); - src = ref Unsafe.Add(ref src, 8); + len -= sizeof(ulong); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); } - if ((len & 4) != 0) + while (len != 0) { - Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); - dest = ref Unsafe.Add(ref dest, 4); - src = ref Unsafe.Add(ref src, 4); - } - - if ((len & 2) != 0) - { - Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); - dest = ref Unsafe.Add(ref dest, 2); - src = ref Unsafe.Add(ref src, 2); - } - - if ((len & 1) != 0) - { - dest = src; + len--; + Unsafe.Add(ref dest, len) = Unsafe.Add(ref src, len); } } - private static void CopyOverlappedBackwardTail(ref byte dest, ref byte src, nuint len) + // Finishes at most MemmoveOverlappedBlock bytes. Deliberately doesn't call Memmove - this is where a + // call the JIT didn't unroll lands, so calling it again is what recursion would look like. Every step + // is one load feeding one store, so the ordering comes from the data dependency. + private static void CopyOverlappedForwardTail(ref byte dest, ref byte src, nuint len) { Debug.Assert(len <= MemmoveOverlappedBlock); - while (len >= 16) + while (len >= sizeof(ulong)) { - len -= 16; - Vector128.StoreUnsafe(Vector128.LoadUnsafe(ref Unsafe.Add(ref src, len)), ref Unsafe.Add(ref dest, len)); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, sizeof(ulong)); + src = ref Unsafe.Add(ref src, sizeof(ulong)); + len -= sizeof(ulong); } - if ((len & 8) != 0) - { - len -= 8; - CopyStep8(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len)); - } - - if ((len & 4) != 0) + while (len != 0) { - len -= 4; - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); - } - - if ((len & 2) != 0) - { - len -= 2; - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); - } - - if ((len & 1) != 0) - { - Debug.Assert(len == 1); dest = src; + dest = ref Unsafe.Add(ref dest, 1); + src = ref Unsafe.Add(ref src, 1); + len--; } } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void CopyStep8(ref byte dest, ref byte src) - { -#if TARGET_64BIT - Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); -#else - // Two independent halves: both are read before either is written, so this stays correct for a - // backward copy whose buffers are less than 8 bytes apart. - uint lower = Unsafe.ReadUnaligned(ref src); - uint upper = Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, 4)); - Unsafe.WriteUnaligned(ref dest, lower); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), upper); #endif - } // Non-inlinable wrapper around the QCall that avoids polluting the fast path // with P/Invoke prolog/epilog. From 9d21a81978852026b1510a0f707d0bb375f47785 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Mon, 24 Aug 2026 19:18:22 +0200 Subject: [PATCH 6/8] Exclude Mono and tighten comments Mono turns every SpanHelpers.Memmove call into OP_MEMMOVE rather than an unrolled block, so the block loops would degrade into one native call per 64 bytes there. Leave Mono on the CRT memmove. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 20467548-69a0-453c-af07-8259bf6415fb --- .../src/System/SpanHelpers.ByteMemOps.cs | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index 30d5673acffca4..9af2ce56987dbe 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -25,11 +25,10 @@ internal static partial class SpanHelpers // .ByteMemOps #endif private const nuint ZeroMemoryNativeThreshold = 1024; -#if TARGET_AMD64 || TARGET_ARM64 - // Overlapping copies are done in blocks of this size with a constant-length Memmove, which the JIT - // unrolls into "load the whole block into registers, then store it" - correct for any overlap. 64 - // bytes stays within its unrolling budget everywhere, and bounds re-entry: a call it didn't unroll - // comes back with a length the tail handles without calling Memmove again, so these can't recurse. +#if (TARGET_AMD64 || TARGET_ARM64) && !MONO + // Blocks are copied with a constant-length Memmove, which the JIT unrolls into "load the whole block + // into registers, then store it" - correct for any overlap. 64 bytes fits its unrolling budget on + // every target here, and bounds re-entry: a shorter length is finished without calling Memmove. private const nuint MemmoveOverlappedBlock = 64; // Aligning the destination only pays for the extra leading block on larger copies. @@ -248,12 +247,10 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) return; } -#if TARGET_AMD64 || TARGET_ARM64 - // The platform memmove's forward loop is tuned for disjoint buffers: 'rep movsb' collapses when - // they are less than a cache line apart - a shift by one element - and the non-temporal stores it - // uses for large copies evict the lines the rest of the copy reads back. Its backward loop has - // neither problem and outruns a managed descending one, so that direction is only handled here - // while the QCall dominates the copy. +#if (TARGET_AMD64 || TARGET_ARM64) && !MONO + // memmove's forward loop is tuned for disjoint buffers: 'rep movsb' collapses when they are less + // than a cache line apart, and the non-temporal stores it uses for large copies evict the lines + // the rest of the copy reads back. Its backward loop has neither problem and beats us above 256. if ((nuint)Unsafe.ByteOffset(ref dest, ref src) < len) { CopyOverlappedForward(ref dest, ref src, len); @@ -275,17 +272,16 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) MemmoveNative(ref dest, ref src, len); } -#if TARGET_AMD64 || TARGET_ARM64 +#if (TARGET_AMD64 || TARGET_ARM64) && !MONO // Blocks run in strictly ascending order, so a store can never reach a source byte a later block - // still has to read. That also rules out the "copy a final block anchored at the end of the buffer" - // trick the non-overlapping paths use - those bytes would already have been rewritten. + // still has to read. That also rules out the trailing "block anchored at the end of the buffer" + // shortcut the non-overlapping paths use - those bytes would already have been rewritten. [MethodImpl(MethodImplOptions.NoInlining)] private static void CopyOverlappedForward(ref byte dest, ref byte src, nuint len) { Debug.Assert(len > 0); - // An unaligned store costs more than an unaligned load, and an overlapping copy can only ever - // have one of the two aligned. + // Only one of the two can be aligned, and an unaligned store costs more than an unaligned load. if (len >= MemmoveOverlappedAlignThreshold) { nuint head = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); From 0d16eb636c9227da04c46ed53d36511b7b0dbbba Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Mon, 24 Aug 2026 21:13:49 +0200 Subject: [PATCH 7/8] Never inline Memmove, and widen the overlapped block Memmove is big enough that inlining it burns a lot of the caller's budget, and when it was inlined into the overlapped copies the JIT no longer had a call to unroll - the inlined body re-dispatched on overlap and called straight back in, which cost ~8x at tier1. Keeping the call also makes the unroll reliable, so the block can be the widest the JIT still expands: 256 bytes with Vector512. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 20467548-69a0-453c-af07-8259bf6415fb --- .../src/System/SpanHelpers.ByteMemOps.cs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index 9af2ce56987dbe..ccd3ad5227bdbb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -11,6 +11,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; namespace System { @@ -46,7 +47,10 @@ private struct Block16 {} private struct Block64 {} #endif // HAS_CUSTOM_BLOCKS + // Too big to be worth inlining: it burns a lot of the caller's budget, and keeping the call is also + // what lets the JIT unroll it when the length is constant. [Intrinsic] // Unrolled for small constant lengths + [MethodImpl(MethodImplOptions.NoInlining)] internal static void Memmove(ref byte dest, ref byte src, nuint len) { // P/Invoke into the native version when the buffers are overlapping. @@ -294,6 +298,30 @@ private static void CopyOverlappedForward(ref byte dest, ref byte src, nuint len } } + // The JIT unrolls a constant-length Memmove up to four vector registers wide, so take the widest + // block it will still expand. Bigger blocks also mean fewer boundaries where a store and the + // next block's load land in the same cache line, which is what a tight overlap is sensitive to. + if (Vector512.IsHardwareAccelerated) + { + while (len > 256) + { + Memmove(ref dest, ref src, 256); + dest = ref Unsafe.Add(ref dest, 256); + src = ref Unsafe.Add(ref src, 256); + len -= 256; + } + } + else if (Vector256.IsHardwareAccelerated) + { + while (len > 128) + { + Memmove(ref dest, ref src, 128); + dest = ref Unsafe.Add(ref dest, 128); + src = ref Unsafe.Add(ref src, 128); + len -= 128; + } + } + while (len > MemmoveOverlappedBlock) { Memmove(ref dest, ref src, MemmoveOverlappedBlock); From 9b7bae4d4bbad51da2ee4eac611fecf6192758e5 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Mon, 24 Aug 2026 22:58:06 +0200 Subject: [PATCH 8/8] Make the overlapped copy sound, and speed up the small cases The block loops keep using a constant-length Memmove, which the JIT expands into "load the whole block into registers, then store it" - the only thing that gives that ordering. The tails no longer assume it: C# doesn't order a separate load and store, so the previous head/tail pair, which relied on both halves being loaded before either was stored, wasn't guaranteed. Each tail step is now a single load feeding a single store, and steps run in strictly ascending (or descending) non-overlapping order, so the ordering comes from the data dependency instead. That also splits the tail per direction - an ascending tail is wrong for a right shift - and drops the block-size fast path, so a re-entrant 64-byte Memmove still skips the loop and lands in a tail that never calls Memmove back. Without that bound it recurses until the stack overflows in tier0, where the constant length isn't expanded. The tail moves 16 bytes at a time instead of 8, which is where the short copies gain. osx-arm64, vs main, int[] shifts (ratio, lower is better): | bytes | ShiftLeft | ShiftRight | |--------|-----------|------------| | 16 | 0.46 | 0.44 | | 40 | 0.51 | 0.44 | | 64 | 0.63 | 0.51 | | 400+ | parity | parity | Blocks stay at 64 bytes on every target: that is what arm64's unrolling budget allows, and it is what bounds the re-entry above. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/System/SpanHelpers.ByteMemOps.cs | 165 ++++++++++-------- 1 file changed, 91 insertions(+), 74 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs index ccd3ad5227bdbb..1e5c81bbc27938 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs @@ -11,7 +11,6 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; namespace System { @@ -27,16 +26,17 @@ internal static partial class SpanHelpers // .ByteMemOps private const nuint ZeroMemoryNativeThreshold = 1024; #if (TARGET_AMD64 || TARGET_ARM64) && !MONO - // Blocks are copied with a constant-length Memmove, which the JIT unrolls into "load the whole block - // into registers, then store it" - correct for any overlap. 64 bytes fits its unrolling budget on - // every target here, and bounds re-entry: a shorter length is finished without calling Memmove. - private const nuint MemmoveOverlappedBlock = 64; + // Blocks are copied with a constant-length Memmove, which the JIT expands into "load the whole + // block into registers, then store it" - correct for any overlap. 64 bytes fits its unrolling + // budget on every target here, and bounds re-entry: a block-sized call skips the loop below and + // is finished by the tail, which never calls Memmove back. + private const nuint OverlappedBlockSize = 64; // Aligning the destination only pays for the extra leading block on larger copies. - private const nuint MemmoveOverlappedAlignThreshold = 2048; + private const nuint OverlappedAlignThreshold = 2048; - // Past this the platform memmove's backward loop is about twice as fast as a managed descending one. - private const nuint MemmoveOverlappedBackwardThreshold = 256; + // Past this the platform memmove's backward loop beats a managed descending one. + private const nuint OverlappedBackwardThreshold = 256; #endif #if HAS_CUSTOM_BLOCKS @@ -47,10 +47,8 @@ private struct Block16 {} private struct Block64 {} #endif // HAS_CUSTOM_BLOCKS - // Too big to be worth inlining: it burns a lot of the caller's budget, and keeping the call is also - // what lets the JIT unroll it when the length is constant. [Intrinsic] // Unrolled for small constant lengths - [MethodImpl(MethodImplOptions.NoInlining)] + [MethodImpl(MethodImplOptions.NoInlining)] // keeping the call is what lets the JIT unroll it internal static void Memmove(ref byte dest, ref byte src, nuint len) { // P/Invoke into the native version when the buffers are overlapping. @@ -252,16 +250,19 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) } #if (TARGET_AMD64 || TARGET_ARM64) && !MONO - // memmove's forward loop is tuned for disjoint buffers: 'rep movsb' collapses when they are less - // than a cache line apart, and the non-temporal stores it uses for large copies evict the lines - // the rest of the copy reads back. Its backward loop has neither problem and beats us above 256. + // We're better off here than calling the platform memmove: for short copies the P/Invoke alone + // costs more than the copy, and its forward loop is tuned for disjoint buffers - 'rep movsb' + // collapses when the buffers are less than a cache line apart, and the non-temporal stores it + // uses for large copies evict the lines the rest of the copy reads back. Its backward loop has + // neither problem, so long right shifts are left to it. if ((nuint)Unsafe.ByteOffset(ref dest, ref src) < len) { + // dest is below src, so copying upwards never overwrites a byte we still have to read. CopyOverlappedForward(ref dest, ref src, len); return; } - if (len <= MemmoveOverlappedBackwardThreshold) + if (len <= OverlappedBackwardThreshold) { CopyOverlappedBackward(ref dest, ref src, len); return; @@ -277,16 +278,19 @@ internal static void Memmove(ref byte dest, ref byte src, nuint len) } #if (TARGET_AMD64 || TARGET_ARM64) && !MONO - // Blocks run in strictly ascending order, so a store can never reach a source byte a later block - // still has to read. That also rules out the trailing "block anchored at the end of the buffer" - // shortcut the non-overlapping paths use - those bytes would already have been rewritten. + // Every step below either is a constant-length Memmove, which the JIT expands so that the whole + // block is loaded before any of it is stored, or a single load feeding a single store, where the + // data dependency does the same for one register's worth. Steps then run in strictly ascending, + // non-overlapping order, so a store can never reach a source byte a later step has to read. That + // also rules out the trailing "block anchored at the end of the buffer" shortcut the + // non-overlapping paths use - those bytes may already have been rewritten. [MethodImpl(MethodImplOptions.NoInlining)] private static void CopyOverlappedForward(ref byte dest, ref byte src, nuint len) { Debug.Assert(len > 0); // Only one of the two can be aligned, and an unaligned store costs more than an unaligned load. - if (len >= MemmoveOverlappedAlignThreshold) + if (len >= OverlappedAlignThreshold) { nuint head = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); if (head != 64) @@ -298,87 +302,100 @@ private static void CopyOverlappedForward(ref byte dest, ref byte src, nuint len } } - // The JIT unrolls a constant-length Memmove up to four vector registers wide, so take the widest - // block it will still expand. Bigger blocks also mean fewer boundaries where a store and the - // next block's load land in the same cache line, which is what a tight overlap is sensitive to. - if (Vector512.IsHardwareAccelerated) + while (len > OverlappedBlockSize) { - while (len > 256) - { - Memmove(ref dest, ref src, 256); - dest = ref Unsafe.Add(ref dest, 256); - src = ref Unsafe.Add(ref src, 256); - len -= 256; - } - } - else if (Vector256.IsHardwareAccelerated) - { - while (len > 128) - { - Memmove(ref dest, ref src, 128); - dest = ref Unsafe.Add(ref dest, 128); - src = ref Unsafe.Add(ref src, 128); - len -= 128; - } - } - - while (len > MemmoveOverlappedBlock) - { - Memmove(ref dest, ref src, MemmoveOverlappedBlock); - dest = ref Unsafe.Add(ref dest, MemmoveOverlappedBlock); - src = ref Unsafe.Add(ref src, MemmoveOverlappedBlock); - len -= MemmoveOverlappedBlock; + Memmove(ref dest, ref src, OverlappedBlockSize); + dest = ref Unsafe.Add(ref dest, OverlappedBlockSize); + src = ref Unsafe.Add(ref src, OverlappedBlockSize); + len -= OverlappedBlockSize; } CopyOverlappedForwardTail(ref dest, ref src, len); } - // Mirror image: blocks run in strictly descending order, walking down from the end. + // Mirror image: steps run in strictly descending order, walking down from the end. [MethodImpl(MethodImplOptions.NoInlining)] private static void CopyOverlappedBackward(ref byte dest, ref byte src, nuint len) { Debug.Assert(len > 0); - while (len > MemmoveOverlappedBlock) + while (len > OverlappedBlockSize) { - len -= MemmoveOverlappedBlock; - Memmove(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len), MemmoveOverlappedBlock); + len -= OverlappedBlockSize; + Memmove(ref Unsafe.Add(ref dest, len), ref Unsafe.Add(ref src, len), OverlappedBlockSize); } - while (len >= sizeof(ulong)) + CopyOverlappedBackwardTail(ref dest, ref src, len); + } + + // Finishes at most OverlappedBlockSize bytes, ascending. Deliberately doesn't call Memmove - this + // is where a block the JIT didn't expand comes back around, so calling it again is what recursion + // would look like. One step per set bit of len, so the steps tile the range without overlapping. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CopyOverlappedForwardTail(ref byte dest, ref byte src, nuint len) + { + Debug.Assert(len <= OverlappedBlockSize); + + while (len >= 16) { - len -= sizeof(ulong); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 16); + src = ref Unsafe.Add(ref src, 16); + len -= 16; } - - while (len != 0) + if ((len & 8) != 0) + { + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 8); + src = ref Unsafe.Add(ref src, 8); + } + if ((len & 4) != 0) { - len--; - Unsafe.Add(ref dest, len) = Unsafe.Add(ref src, len); + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 4); + src = ref Unsafe.Add(ref src, 4); + } + if ((len & 2) != 0) + { + Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); + dest = ref Unsafe.Add(ref dest, 2); + src = ref Unsafe.Add(ref src, 2); + } + if ((len & 1) != 0) + { + dest = src; } } - // Finishes at most MemmoveOverlappedBlock bytes. Deliberately doesn't call Memmove - this is where a - // call the JIT didn't unroll lands, so calling it again is what recursion would look like. Every step - // is one load feeding one store, so the ordering comes from the data dependency. - private static void CopyOverlappedForwardTail(ref byte dest, ref byte src, nuint len) + // Mirror image: the steps tile the range from the end downwards. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CopyOverlappedBackwardTail(ref byte dest, ref byte src, nuint len) { - Debug.Assert(len <= MemmoveOverlappedBlock); + Debug.Assert(len <= OverlappedBlockSize); - while (len >= sizeof(ulong)) + while (len >= 16) { - Unsafe.WriteUnaligned(ref dest, Unsafe.ReadUnaligned(ref src)); - dest = ref Unsafe.Add(ref dest, sizeof(ulong)); - src = ref Unsafe.Add(ref src, sizeof(ulong)); - len -= sizeof(ulong); + len -= 16; + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); } - - while (len != 0) + if ((len & 8) != 0) + { + len -= 8; + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); + } + if ((len & 4) != 0) + { + len -= 4; + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); + } + if ((len & 2) != 0) + { + len -= 2; + Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, len), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, len))); + } + if ((len & 1) != 0) { dest = src; - dest = ref Unsafe.Add(ref dest, 1); - src = ref Unsafe.Add(ref src, 1); - len--; } } #endif