Prims, sculpts, GLTF primitives and avatar meshes are ordered for the vertex cache

indra/llappearance/llpolymesh.cpp

@@ -39,6 +39,7 @@
 #include "llvolume.h"
 #include "llendianswizzle.h"
 
+#include <meshoptimizer.h>
 
 #define HEADER_ASCII "Linden Mesh 1.0"
 #define HEADER_BINARY "Linden Binary Mesh 1.0"
@@ -559,6 +560,22 @@ bool LLPolyMeshSharedData::loadMesh( const std::string& fileName )
                         numTris++;
                 }
 
+                // The triangles in an order a vertex cache reuses, once for
+                // the file and every avatar that shares it. Morphs and skin
+                // weights address vertices, which keep their order, and an
+                // LOD's faces are its own.
+                const U32 vertex_count = (U32)mNumVertices;
+                bool in_range = numFaces > 0;
+                for (i = 0; i < (U32)numFaces * 3 && in_range; i++)
+                {
+                        in_range = (U32)mFaces[i / 3][i % 3] < vertex_count;
+                }
+                if (in_range)
+                {
+                        unsigned int* indices = reinterpret_cast<unsigned int*>(mFaces);
+                        meshopt_optimizeVertexCache(indices, indices, (size_t)numFaces * 3, vertex_count);
+                }
+
                 LL_DEBUGS() << "verts: " << numVertices
                          << ", faces: "   << numFaces
                          << ", tris: "    << numTris

indra/llcommon/alsimd.h

@@ -1405,6 +1405,70 @@ AL_SIMD_INLINE u16x8 add_u16(u16x8 a, u16x8 b)
 #endif
 }
 
+////////////////////////////////////
+// Four signed 32-bit lanes
+////////////////////////////////////
+
+AL_SIMD_INLINE i32x4 loadu_i32(const int32_t* p)
+{
+#if AL_SIMD_X86
+    return _mm_loadu_si128(reinterpret_cast<const __m128i*>(p));
+#else
+    return vld1q_s32(p);
+#endif
+}
+
+AL_SIMD_INLINE void storeu_i32(int32_t* p, i32x4 v)
+{
+#if AL_SIMD_X86
+    _mm_storeu_si128(reinterpret_cast<__m128i*>(p), v);
+#else
+    vst1q_s32(p, v);
+#endif
+}
+
+AL_SIMD_INLINE i32x4 set1_i32(int32_t x)
+{
+#if AL_SIMD_X86
+    return _mm_set1_epi32(x);
+#else
+    return vdupq_n_s32(x);
+#endif
+}
+
+AL_SIMD_INLINE i32x4 and_i32(i32x4 a, i32x4 b)
+{
+#if AL_SIMD_X86
+    return _mm_and_si128(a, b);
+#else
+    return vandq_s32(a, b);
+#endif
+}
+
+// All ones in each lane where a and b hold the same bits.
+AL_SIMD_INLINE mask4 cmpeq_i32(i32x4 a, i32x4 b)
+{
+#if AL_SIMD_X86
+    return _mm_castsi128_ps(_mm_cmpeq_epi32(a, b));
+#else
+    return vceqq_s32(a, b);
+#endif
+}
+
+// Every lane of a, b, c and d clamped to 0..255, as sixteen bytes in that
+// order: lane 0 of the result holds a's four, its lane 0 the low byte,
+// lane 1 b's, lane 2 c's and lane 3 d's.
+AL_SIMD_INLINE i32x4 pack_u8(i32x4 a, i32x4 b, i32x4 c, i32x4 d)
+{
+#if AL_SIMD_X86
+    return _mm_packus_epi16(_mm_packs_epi32(a, b), _mm_packs_epi32(c, d));
+#else
+    const uint8x8_t ab = vqmovun_s16(vcombine_s16(vqmovn_s32(a), vqmovn_s32(b)));
+    const uint8x8_t cd = vqmovun_s16(vcombine_s16(vqmovn_s32(c), vqmovn_s32(d)));
+    return vreinterpretq_s32_u8(vcombine_u8(ab, cd));
+#endif
+}
+
 ////////////////////////////////////
 // Memory hints
 ////////////////////////////////////

indra/llcommon/tests/alsimd_test.cpp

@@ -122,10 +122,13 @@ namespace
     }
 #endif
 
-    // Four int32 lanes, built from their bits.
+    // Four int32 lanes, loaded as integers: built through floats, a lane of
+    // the sign bit alone is -0, whose sign /fp:fast need not keep, and
+    // MSVC's arm64 build makes it +0.
     i32x4 ints(S32 x, S32 y, S32 z, S32 w)
     {
-        return as_i32(set(std::bit_cast<F32>(x), std::bit_cast<F32>(y), std::bit_cast<F32>(z), std::bit_cast<F32>(w)));
+        const int32_t values[4] = { x, y, z, w };
+        return loadu_i32(values);
     }
 
     void require_lanes(const std::string& what, f32x4 got, F32 x, F32 y, F32 z, F32 w)
@@ -562,6 +565,47 @@ namespace
         CHECK((AL_ISA_LEVEL <= 2 || (AL_ISA_LEVEL == 3 && AL_SIMD_AVX2) || (AL_ISA_LEVEL == 4 && AL_SIMD_AVX512)));
     }
 
+    // Four 32-bit lanes: unaligned loads and stores move bits, the
+    // comparison is of whole lanes, and the pack clamps each lane to a byte
+    // in argument order.
+    TEST_CASE_METHOD(alsimd_data, "alsimd: integer lanes load and compare whole lanes and pack to clamped bytes", "[alsimd]")
+    {
+        const int32_t src[6] = {7, -1, 0x7fffffff, (int32_t)0x80000000, 42, 99};
+        const Lanes loaded = lanes_of(loadu_i32(src + 1));
+        CHECK(int_lane(loaded, 0) == -1);
+        CHECK(int_lane(loaded, 1) == 0x7fffffff);
+        CHECK(int_lane(loaded, 2) == (int32_t)0x80000000);
+        CHECK(int_lane(loaded, 3) == 42);
+
+        int32_t dst[6] = {1, 1, 1, 1, 1, 1};
+        storeu_i32(dst + 1, ints(5, -6, 7, -8));
+        CHECK(dst[0] == 1);
+        CHECK(dst[1] == 5);
+        CHECK(dst[2] == -6);
+        CHECK(dst[3] == 7);
+        CHECK(dst[4] == -8);
+        CHECK(dst[5] == 1);
+
+        const Lanes ones = lanes_of(set1_i32(-3));
+        CHECK(int_lane(ones, 0) == -3);
+        CHECK(int_lane(ones, 3) == -3);
+        const Lanes anded = lanes_of(and_i32(ints(0x0ff0, -1, 0x00ff00ff, 6), ints(0x00ff, 0x1234, -1, 3)));
+        CHECK(int_lane(anded, 0) == 0x00f0);
+        CHECK(int_lane(anded, 1) == 0x1234);
+        CHECK(int_lane(anded, 2) == 0x00ff00ff);
+        CHECK(int_lane(anded, 3) == 2);
+
+        // a lane that differs in one bit, or only in the sign bit, is unequal
+        CHECK(bits(cmpeq_i32(ints(1, 2, 0, (int32_t)0x80000000), ints(1, 3, 0, 0))) == 0x5u);
+        CHECK(bits(cmpeq_i32(ints(-1, -1, -1, -1), ints(-1, -1, -1, -1))) == 0xFu);
+
+        const Lanes packed = lanes_of(pack_u8(ints(0, 1, 254, 255), ints(-1, 256, 0x7fffffff, (int32_t)0x80000000), ints(10, 20, 30, 40), ints(128, 0, 0, 0)));
+        CHECK((U32)int_lane(packed, 0) == 0xFFFE0100u);
+        CHECK((U32)int_lane(packed, 1) == 0x00FFFF00u);
+        CHECK((U32)int_lane(packed, 2) == 0x281E140Au);
+        CHECK((U32)int_lane(packed, 3) == 0x00000080u);
+    }
+
     // The aligned copy moves every byte and no other, at every size the
     // three loops can be entered with, both ends at every sixteen-byte
     // offset within a line, and at the sizes the slabs come in.

indra/llmath/alsimdkernels.cpp

@@ -133,4 +133,14 @@ bool normalize3_or_mark(LLVector4a* v, size_t n)
     return kernels::normalize3_or_mark_impl<ALSimdLanes>(v, n);
 }
 
+void unit_cube_codes(const LLVector4a* src, size_t n, U32* dst)
+{
+    kernels::unit_cube_codes_impl<ALSimdWide>(src, n, dst);
+}
+
+size_t grid_coincident_triangles(const U32* codes, size_t columns, size_t rows, U64* upper, U64* lower, size_t words)
+{
+    return kernels::grid_coincident_triangles_impl<ALSimdWide>(codes, columns, rows, upper, lower, words);
+}
+
 } // namespace alsimd

indra/llmath/alsimdkernels.h

@@ -71,6 +71,20 @@ void transform_texcoords(const F32* src, F32* dst, size_t n,
 // be aligned.
 void offset_indices_u16(const U16* src, U16* dst, size_t n, U16 offset);
 
+// Each point's x, y and z from [-0.5, 0.5] to the nearest of the 256
+// steps across it, packed x | y << 8 | z << 16, w left out; a coordinate
+// outside the cube clamps to its end. Points that are the same get one
+// code, and a sculpt's points, eight bits an axis, get a code each.
+void unit_cube_codes(const LLVector4a* src, size_t n, U32* dst);
+
+// The triangles of a grid of columns by rows vertices, vertex
+// s + columns * t, that have two corners of one code. The quad at column s
+// and row t is bottom left, top right, top left, flagged in bit s % 64 of
+// upper[t * words + s / 64], and bottom left, bottom right, top right, in
+// lower; words is at least (columns + 62) / 64. codes is read up to eight
+// past its columns * rows. Returns how many rows of quads have a flag.
+size_t grid_coincident_triangles(const U32* codes, size_t columns, size_t rows, U64* upper, U64* lower, size_t words);
+
 // Three little-endian 16-bit values per vertex, six bytes apart, to
 // {x, y, z, 0} * scale + bias; the fourth lane of scale is ignored, so w
 // comes out as bias's.

indra/llmath/alsimdkernels.inl

@@ -540,6 +540,118 @@ void offset_indices_u16_impl(const U16* src, U16* dst, size_t n, U16 offset)
     }
 }
 
+////////////////////////////////////
+// Coincident corners
+////////////////////////////////////
+
+// Four points a step, eight under AVX2, the rest one at a time.
+template <class W>
+void unit_cube_codes_impl(const LLVector4a* src, size_t n, U32* dst)
+{
+    size_t i = 0;
+#if AL_SIMD_AVX2
+    if constexpr (W::lanes >= 2)
+    {
+        const __m256 half = _mm256_set1_ps(0.5f);
+        const __m256 steps = _mm256_set1_ps(255.f);
+        const __m256i xyz = _mm256_set1_epi32(0x00FFFFFF);
+        // The packs keep each 128-bit half apart: the codes come out as
+        // points 0, 2, 4, 6, 1, 3, 5, 7.
+        const __m256i order = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
+        const auto code = [&](const F32* p) { return _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_add_ps(_mm256_loadu_ps(p), half), steps)); };
+        for (; i + 8 <= n; i += 8)
+        {
+            const F32* p = src[i].getF32ptr();
+            const __m256i packed = _mm256_packus_epi16(_mm256_packs_epi32(code(p), code(p + 8)), _mm256_packs_epi32(code(p + 16), code(p + 24)));
+            _mm256_storeu_si256(reinterpret_cast<__m256i*>(dst + i), _mm256_and_si256(_mm256_permutevar8x32_epi32(packed, order), xyz));
+        }
+    }
+#endif
+    const f32x4 half = alsimd::set1(0.5f);
+    const f32x4 steps = alsimd::set1(255.f);
+    const i32x4 xyz = alsimd::set1_i32(0x00FFFFFF);
+    const auto code = [&](size_t k) { return alsimd::cvt_round(alsimd::mul(alsimd::add(alsimd::load(src[k].getF32ptr()), half), steps)); };
+    for (; i + 4 <= n; i += 4)
+    {
+        alsimd::storeu_i32(reinterpret_cast<int32_t*>(dst + i), alsimd::and_i32(alsimd::pack_u8(code(i), code(i + 1), code(i + 2), code(i + 3)), xyz));
+    }
+    for (; i < n; ++i)
+    {
+        int32_t lanes[4];
+        const i32x4 c = code(i);
+        alsimd::storeu_i32(lanes, alsimd::and_i32(alsimd::pack_u8(c, c, c, c), xyz));
+        dst[i] = static_cast<U32>(lanes[0]);
+    }
+}
+
+// A row of quads at a time, four quads a step, eight under AVX2. A step
+// reads its bottom left, bottom right, top left and top right corners'
+// codes four or eight at once, which runs past the row's last quad: the
+// flags past it are cleared.
+template <class W>
+size_t grid_coincident_triangles_impl(const U32* codes, size_t columns, size_t rows, U64* upper, U64* lower, size_t words)
+{
+    if (columns < 2 || rows < 2)
+    {
+        return 0;
+    }
+    const size_t quads = columns - 1;
+    const size_t last_word = (quads - 1) / 64;
+    const U64 last_bits = (quads % 64) ? (U64(1) << (quads % 64)) - 1 : ~U64(0);
+    size_t flagged = 0;
+    for (size_t t = 0; t + 1 < rows; ++t)
+    {
+        const int32_t* bottom = reinterpret_cast<const int32_t*>(codes + columns * t);
+        const int32_t* top = bottom + columns;
+        U64* up = upper + words * t;
+        U64* lo = lower + words * t;
+        for (size_t w = 0; w < words; ++w)
+        {
+            up[w] = 0;
+            lo[w] = 0;
+        }
+        size_t s = 0;
+#if AL_SIMD_AVX2
+        if constexpr (W::lanes >= 2)
+        {
+            for (; s < quads; s += 8)
+            {
+                const __m256i bl = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(bottom + s));
+                const __m256i br = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(bottom + s + 1));
+                const __m256i tl = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(top + s));
+                const __m256i tr = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(top + s + 1));
+                const __m256i diagonal = _mm256_cmpeq_epi32(bl, tr);
+                const __m256i u = _mm256_or_si256(_mm256_or_si256(diagonal, _mm256_cmpeq_epi32(tl, tr)), _mm256_cmpeq_epi32(bl, tl));
+                const __m256i l = _mm256_or_si256(_mm256_or_si256(diagonal, _mm256_cmpeq_epi32(bl, br)), _mm256_cmpeq_epi32(br, tr));
+                up[s / 64] |= U64(_mm256_movemask_ps(_mm256_castsi256_ps(u))) << (s % 64);
+                lo[s / 64] |= U64(_mm256_movemask_ps(_mm256_castsi256_ps(l))) << (s % 64);
+            }
+        }
+#endif
+        for (; s < quads; s += 4)
+        {
+            const i32x4 bl = alsimd::loadu_i32(bottom + s);
+            const i32x4 br = alsimd::loadu_i32(bottom + s + 1);
+            const i32x4 tl = alsimd::loadu_i32(top + s);
+            const i32x4 tr = alsimd::loadu_i32(top + s + 1);
+            const mask4 diagonal = alsimd::cmpeq_i32(bl, tr);
+            const mask4 u = alsimd::or_(alsimd::or_(diagonal, alsimd::cmpeq_i32(tl, tr)), alsimd::cmpeq_i32(bl, tl));
+            const mask4 l = alsimd::or_(alsimd::or_(diagonal, alsimd::cmpeq_i32(bl, br)), alsimd::cmpeq_i32(br, tr));
+            up[s / 64] |= U64(alsimd::bits(u)) << (s % 64);
+            lo[s / 64] |= U64(alsimd::bits(l)) << (s % 64);
+        }
+        up[last_word] &= last_bits;
+        lo[last_word] &= last_bits;
+        U64 any = 0;
+        for (size_t w = 0; w <= last_word; ++w)
+        {
+            any |= up[w] | lo[w];
+        }
+        flagged += any != 0;
+    }
+    return flagged;
+}
+
 ////////////////////////////////////
 // Dequantization
 ////////////////////////////////////