diff --git a/indra/llappearance/llpolymesh.cpp b/indra/llappearance/llpolymesh.cpp index 1b5261e7e3..0fe7648e7b 100644 --- a/indra/llappearance/llpolymesh.cpp +++ b/indra/llappearance/llpolymesh.cpp @@ -39,6 +39,7 @@ #include "llvolume.h" #include "llendianswizzle.h" +#include #define HEADER_ASCII "Linden Mesh 1.0" #define HEADER_BINARY "Linden Binary Mesh 1.0" @@ -559,6 +560,22 @@ bool LLPolyMeshSharedData::loadMesh( const std::string& fileName ) numTris++; } + // The triangles in an order a vertex cache reuses, once for + // the file and every avatar that shares it. Morphs and skin + // weights address vertices, which keep their order, and an + // LOD's faces are its own. + const U32 vertex_count = (U32)mNumVertices; + bool in_range = numFaces > 0; + for (i = 0; i < (U32)numFaces * 3 && in_range; i++) + { + in_range = (U32)mFaces[i / 3][i % 3] < vertex_count; + } + if (in_range) + { + unsigned int* indices = reinterpret_cast(mFaces); + meshopt_optimizeVertexCache(indices, indices, (size_t)numFaces * 3, vertex_count); + } + LL_DEBUGS() << "verts: " << numVertices << ", faces: " << numFaces << ", tris: " << numTris diff --git a/indra/llcommon/alsimd.h b/indra/llcommon/alsimd.h index 1f08c3d9da..23c150e268 100644 --- a/indra/llcommon/alsimd.h +++ b/indra/llcommon/alsimd.h @@ -1405,6 +1405,70 @@ AL_SIMD_INLINE u16x8 add_u16(u16x8 a, u16x8 b) #endif } +//////////////////////////////////// +// Four signed 32-bit lanes +//////////////////////////////////// + +AL_SIMD_INLINE i32x4 loadu_i32(const int32_t* p) +{ +#if AL_SIMD_X86 + return _mm_loadu_si128(reinterpret_cast(p)); +#else + return vld1q_s32(p); +#endif +} + +AL_SIMD_INLINE void storeu_i32(int32_t* p, i32x4 v) +{ +#if AL_SIMD_X86 + _mm_storeu_si128(reinterpret_cast<__m128i*>(p), v); +#else + vst1q_s32(p, v); +#endif +} + +AL_SIMD_INLINE i32x4 set1_i32(int32_t x) +{ +#if AL_SIMD_X86 + return _mm_set1_epi32(x); +#else + return vdupq_n_s32(x); +#endif +} + +AL_SIMD_INLINE i32x4 and_i32(i32x4 a, i32x4 b) +{ +#if AL_SIMD_X86 + return _mm_and_si128(a, b); +#else + return vandq_s32(a, b); +#endif +} + +// All ones in each lane where a and b hold the same bits. +AL_SIMD_INLINE mask4 cmpeq_i32(i32x4 a, i32x4 b) +{ +#if AL_SIMD_X86 + return _mm_castsi128_ps(_mm_cmpeq_epi32(a, b)); +#else + return vceqq_s32(a, b); +#endif +} + +// Every lane of a, b, c and d clamped to 0..255, as sixteen bytes in that +// order: lane 0 of the result holds a's four, its lane 0 the low byte, +// lane 1 b's, lane 2 c's and lane 3 d's. +AL_SIMD_INLINE i32x4 pack_u8(i32x4 a, i32x4 b, i32x4 c, i32x4 d) +{ +#if AL_SIMD_X86 + return _mm_packus_epi16(_mm_packs_epi32(a, b), _mm_packs_epi32(c, d)); +#else + const uint8x8_t ab = vqmovun_s16(vcombine_s16(vqmovn_s32(a), vqmovn_s32(b))); + const uint8x8_t cd = vqmovun_s16(vcombine_s16(vqmovn_s32(c), vqmovn_s32(d))); + return vreinterpretq_s32_u8(vcombine_u8(ab, cd)); +#endif +} + //////////////////////////////////// // Memory hints //////////////////////////////////// diff --git a/indra/llcommon/tests/alsimd_test.cpp b/indra/llcommon/tests/alsimd_test.cpp index d722e83246..552775fbd4 100644 --- a/indra/llcommon/tests/alsimd_test.cpp +++ b/indra/llcommon/tests/alsimd_test.cpp @@ -122,10 +122,13 @@ namespace } #endif - // Four int32 lanes, built from their bits. + // Four int32 lanes, loaded as integers: built through floats, a lane of + // the sign bit alone is -0, whose sign /fp:fast need not keep, and + // MSVC's arm64 build makes it +0. i32x4 ints(S32 x, S32 y, S32 z, S32 w) { - return as_i32(set(std::bit_cast(x), std::bit_cast(y), std::bit_cast(z), std::bit_cast(w))); + const int32_t values[4] = { x, y, z, w }; + return loadu_i32(values); } void require_lanes(const std::string& what, f32x4 got, F32 x, F32 y, F32 z, F32 w) @@ -562,6 +565,47 @@ namespace CHECK((AL_ISA_LEVEL <= 2 || (AL_ISA_LEVEL == 3 && AL_SIMD_AVX2) || (AL_ISA_LEVEL == 4 && AL_SIMD_AVX512))); } + // Four 32-bit lanes: unaligned loads and stores move bits, the + // comparison is of whole lanes, and the pack clamps each lane to a byte + // in argument order. + TEST_CASE_METHOD(alsimd_data, "alsimd: integer lanes load and compare whole lanes and pack to clamped bytes", "[alsimd]") + { + const int32_t src[6] = {7, -1, 0x7fffffff, (int32_t)0x80000000, 42, 99}; + const Lanes loaded = lanes_of(loadu_i32(src + 1)); + CHECK(int_lane(loaded, 0) == -1); + CHECK(int_lane(loaded, 1) == 0x7fffffff); + CHECK(int_lane(loaded, 2) == (int32_t)0x80000000); + CHECK(int_lane(loaded, 3) == 42); + + int32_t dst[6] = {1, 1, 1, 1, 1, 1}; + storeu_i32(dst + 1, ints(5, -6, 7, -8)); + CHECK(dst[0] == 1); + CHECK(dst[1] == 5); + CHECK(dst[2] == -6); + CHECK(dst[3] == 7); + CHECK(dst[4] == -8); + CHECK(dst[5] == 1); + + const Lanes ones = lanes_of(set1_i32(-3)); + CHECK(int_lane(ones, 0) == -3); + CHECK(int_lane(ones, 3) == -3); + const Lanes anded = lanes_of(and_i32(ints(0x0ff0, -1, 0x00ff00ff, 6), ints(0x00ff, 0x1234, -1, 3))); + CHECK(int_lane(anded, 0) == 0x00f0); + CHECK(int_lane(anded, 1) == 0x1234); + CHECK(int_lane(anded, 2) == 0x00ff00ff); + CHECK(int_lane(anded, 3) == 2); + + // a lane that differs in one bit, or only in the sign bit, is unequal + CHECK(bits(cmpeq_i32(ints(1, 2, 0, (int32_t)0x80000000), ints(1, 3, 0, 0))) == 0x5u); + CHECK(bits(cmpeq_i32(ints(-1, -1, -1, -1), ints(-1, -1, -1, -1))) == 0xFu); + + const Lanes packed = lanes_of(pack_u8(ints(0, 1, 254, 255), ints(-1, 256, 0x7fffffff, (int32_t)0x80000000), ints(10, 20, 30, 40), ints(128, 0, 0, 0))); + CHECK((U32)int_lane(packed, 0) == 0xFFFE0100u); + CHECK((U32)int_lane(packed, 1) == 0x00FFFF00u); + CHECK((U32)int_lane(packed, 2) == 0x281E140Au); + CHECK((U32)int_lane(packed, 3) == 0x00000080u); + } + // The aligned copy moves every byte and no other, at every size the // three loops can be entered with, both ends at every sixteen-byte // offset within a line, and at the sizes the slabs come in. diff --git a/indra/llmath/alsimdkernels.cpp b/indra/llmath/alsimdkernels.cpp index 3656f81d69..a7826f7b5f 100644 --- a/indra/llmath/alsimdkernels.cpp +++ b/indra/llmath/alsimdkernels.cpp @@ -133,4 +133,14 @@ bool normalize3_or_mark(LLVector4a* v, size_t n) return kernels::normalize3_or_mark_impl(v, n); } +void unit_cube_codes(const LLVector4a* src, size_t n, U32* dst) +{ + kernels::unit_cube_codes_impl(src, n, dst); +} + +size_t grid_coincident_triangles(const U32* codes, size_t columns, size_t rows, U64* upper, U64* lower, size_t words) +{ + return kernels::grid_coincident_triangles_impl(codes, columns, rows, upper, lower, words); +} + } // namespace alsimd diff --git a/indra/llmath/alsimdkernels.h b/indra/llmath/alsimdkernels.h index 79ddfd4ddf..3373121331 100644 --- a/indra/llmath/alsimdkernels.h +++ b/indra/llmath/alsimdkernels.h @@ -71,6 +71,20 @@ void transform_texcoords(const F32* src, F32* dst, size_t n, // be aligned. void offset_indices_u16(const U16* src, U16* dst, size_t n, U16 offset); +// Each point's x, y and z from [-0.5, 0.5] to the nearest of the 256 +// steps across it, packed x | y << 8 | z << 16, w left out; a coordinate +// outside the cube clamps to its end. Points that are the same get one +// code, and a sculpt's points, eight bits an axis, get a code each. +void unit_cube_codes(const LLVector4a* src, size_t n, U32* dst); + +// The triangles of a grid of columns by rows vertices, vertex +// s + columns * t, that have two corners of one code. The quad at column s +// and row t is bottom left, top right, top left, flagged in bit s % 64 of +// upper[t * words + s / 64], and bottom left, bottom right, top right, in +// lower; words is at least (columns + 62) / 64. codes is read up to eight +// past its columns * rows. Returns how many rows of quads have a flag. +size_t grid_coincident_triangles(const U32* codes, size_t columns, size_t rows, U64* upper, U64* lower, size_t words); + // Three little-endian 16-bit values per vertex, six bytes apart, to // {x, y, z, 0} * scale + bias; the fourth lane of scale is ignored, so w // comes out as bias's. diff --git a/indra/llmath/alsimdkernels.inl b/indra/llmath/alsimdkernels.inl index 88d012212e..01ed18641f 100644 --- a/indra/llmath/alsimdkernels.inl +++ b/indra/llmath/alsimdkernels.inl @@ -540,6 +540,118 @@ void offset_indices_u16_impl(const U16* src, U16* dst, size_t n, U16 offset) } } +//////////////////////////////////// +// Coincident corners +//////////////////////////////////// + +// Four points a step, eight under AVX2, the rest one at a time. +template +void unit_cube_codes_impl(const LLVector4a* src, size_t n, U32* dst) +{ + size_t i = 0; +#if AL_SIMD_AVX2 + if constexpr (W::lanes >= 2) + { + const __m256 half = _mm256_set1_ps(0.5f); + const __m256 steps = _mm256_set1_ps(255.f); + const __m256i xyz = _mm256_set1_epi32(0x00FFFFFF); + // The packs keep each 128-bit half apart: the codes come out as + // points 0, 2, 4, 6, 1, 3, 5, 7. + const __m256i order = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); + const auto code = [&](const F32* p) { return _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_add_ps(_mm256_loadu_ps(p), half), steps)); }; + for (; i + 8 <= n; i += 8) + { + const F32* p = src[i].getF32ptr(); + const __m256i packed = _mm256_packus_epi16(_mm256_packs_epi32(code(p), code(p + 8)), _mm256_packs_epi32(code(p + 16), code(p + 24))); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(dst + i), _mm256_and_si256(_mm256_permutevar8x32_epi32(packed, order), xyz)); + } + } +#endif + const f32x4 half = alsimd::set1(0.5f); + const f32x4 steps = alsimd::set1(255.f); + const i32x4 xyz = alsimd::set1_i32(0x00FFFFFF); + const auto code = [&](size_t k) { return alsimd::cvt_round(alsimd::mul(alsimd::add(alsimd::load(src[k].getF32ptr()), half), steps)); }; + for (; i + 4 <= n; i += 4) + { + alsimd::storeu_i32(reinterpret_cast(dst + i), alsimd::and_i32(alsimd::pack_u8(code(i), code(i + 1), code(i + 2), code(i + 3)), xyz)); + } + for (; i < n; ++i) + { + int32_t lanes[4]; + const i32x4 c = code(i); + alsimd::storeu_i32(lanes, alsimd::and_i32(alsimd::pack_u8(c, c, c, c), xyz)); + dst[i] = static_cast(lanes[0]); + } +} + +// A row of quads at a time, four quads a step, eight under AVX2. A step +// reads its bottom left, bottom right, top left and top right corners' +// codes four or eight at once, which runs past the row's last quad: the +// flags past it are cleared. +template +size_t grid_coincident_triangles_impl(const U32* codes, size_t columns, size_t rows, U64* upper, U64* lower, size_t words) +{ + if (columns < 2 || rows < 2) + { + return 0; + } + const size_t quads = columns - 1; + const size_t last_word = (quads - 1) / 64; + const U64 last_bits = (quads % 64) ? (U64(1) << (quads % 64)) - 1 : ~U64(0); + size_t flagged = 0; + for (size_t t = 0; t + 1 < rows; ++t) + { + const int32_t* bottom = reinterpret_cast(codes + columns * t); + const int32_t* top = bottom + columns; + U64* up = upper + words * t; + U64* lo = lower + words * t; + for (size_t w = 0; w < words; ++w) + { + up[w] = 0; + lo[w] = 0; + } + size_t s = 0; +#if AL_SIMD_AVX2 + if constexpr (W::lanes >= 2) + { + for (; s < quads; s += 8) + { + const __m256i bl = _mm256_loadu_si256(reinterpret_cast(bottom + s)); + const __m256i br = _mm256_loadu_si256(reinterpret_cast(bottom + s + 1)); + const __m256i tl = _mm256_loadu_si256(reinterpret_cast(top + s)); + const __m256i tr = _mm256_loadu_si256(reinterpret_cast(top + s + 1)); + const __m256i diagonal = _mm256_cmpeq_epi32(bl, tr); + const __m256i u = _mm256_or_si256(_mm256_or_si256(diagonal, _mm256_cmpeq_epi32(tl, tr)), _mm256_cmpeq_epi32(bl, tl)); + const __m256i l = _mm256_or_si256(_mm256_or_si256(diagonal, _mm256_cmpeq_epi32(bl, br)), _mm256_cmpeq_epi32(br, tr)); + up[s / 64] |= U64(_mm256_movemask_ps(_mm256_castsi256_ps(u))) << (s % 64); + lo[s / 64] |= U64(_mm256_movemask_ps(_mm256_castsi256_ps(l))) << (s % 64); + } + } +#endif + for (; s < quads; s += 4) + { + const i32x4 bl = alsimd::loadu_i32(bottom + s); + const i32x4 br = alsimd::loadu_i32(bottom + s + 1); + const i32x4 tl = alsimd::loadu_i32(top + s); + const i32x4 tr = alsimd::loadu_i32(top + s + 1); + const mask4 diagonal = alsimd::cmpeq_i32(bl, tr); + const mask4 u = alsimd::or_(alsimd::or_(diagonal, alsimd::cmpeq_i32(tl, tr)), alsimd::cmpeq_i32(bl, tl)); + const mask4 l = alsimd::or_(alsimd::or_(diagonal, alsimd::cmpeq_i32(bl, br)), alsimd::cmpeq_i32(br, tr)); + up[s / 64] |= U64(alsimd::bits(u)) << (s % 64); + lo[s / 64] |= U64(alsimd::bits(l)) << (s % 64); + } + up[last_word] &= last_bits; + lo[last_word] &= last_bits; + U64 any = 0; + for (size_t w = 0; w <= last_word; ++w) + { + any |= up[w] | lo[w]; + } + flagged += any != 0; + } + return flagged; +} + //////////////////////////////////// // Dequantization //////////////////////////////////// diff --git a/indra/llmath/llvolume.cpp b/indra/llmath/llvolume.cpp index 5b91624281..93e94ac27f 100644 --- a/indra/llmath/llvolume.cpp +++ b/indra/llmath/llvolume.cpp @@ -58,6 +58,8 @@ #include +#include + constexpr F32 MIN_CUT_DELTA = 0.02f; constexpr F32 HOLLOW_MIN = 0.f; @@ -6077,60 +6079,409 @@ namespace return collapse; } - // A side's triangles, from the quads of its grid, vertex s + num_s * t, - // whose left columns are listed, or from every quad: bottom left, top - // right, top left and bottom left, bottom right, top right, but for - // those whose corners a collapsed row or column puts in one place. - // Returns the end of what it wrote. - template - U16* grid_indices(U16* out, S32 num_s, S32 num_t, const S32* lefts, S32 quads, const Collapse& collapse) + // A side's quads go in an order a 16-entry vertex cache reuses. A grid + // of up to COLUMN_ROWS rows goes a column at a time, every row of a + // column before the next: a column's right edge is still cached when the + // next column takes it as its left. A taller grid of up to LONE_BAND_QUADS + // quads across goes row by row; a wider one in bands of at most + // BAND_QUADS quads across, as near equal as the quads divide, each band + // row by row, every other band from the last row back to the first: a + // band's row reuses the row before it, and a band starts where the band + // before it ended. That is on a first or last row, where a collapse or a + // pinch leaves triangles out and loads the row's vertices in another + // order; two rows of a band take 14 entries, so the band's next row still + // finds them. The order depends on the grid's layout alone, never on + // positions, so the indices a full build writes hold for the partial + // builds after it. + constexpr S32 COLUMN_ROWS = 7; + constexpr S32 LONE_BAND_QUADS = 7; + constexpr S32 BAND_QUADS = 6; + + // What a side keeps of its quads' triangles: all of them; all but those + // a collapsed row or column leaves out, a collapsed column being the + // first quad's left or the last quad's right; or all but those flagged + // as having two corners in one place. whole(t) is whether every + // triangle of row t is kept. WHOLE_ROWS is whether rows are whole often + // enough to write a band's row at once: a sphere's collapsed poles + // leave a triangle out of every row. + struct KeepAll + { + static constexpr bool ALL = true; + static constexpr bool WHOLE_ROWS = true; + + bool upper(S32, S32) const { return true; } + bool lower(S32, S32) const { return true; } + bool whole(S32) const { return true; } + }; + + struct Kept { - const auto upper = [&](S32 s, S32 t) + static constexpr bool ALL = false; + static constexpr bool WHOLE_ROWS = false; + + bool first_row; + bool last_row; + bool first_column; + bool last_column; + S32 last_quad_column; + S32 last_quad_row; + + bool upper(S32 s, S32 t) const { return !((last_row && t == last_quad_row) || (first_column && s == 0)); } + bool lower(S32 s, S32 t) const { return !((first_row && t == 0) || (last_column && s == last_quad_column)); } + bool whole(S32 t) const { return !first_column && !last_column && !(first_row && t == 0) && !(last_row && t == last_quad_row); } + }; + + struct Unpinched + { + static constexpr bool ALL = false; + static constexpr bool WHOLE_ROWS = true; + + // Bit s % 64 of word t * words + s / 64 for the quad at column s + // and row t. + const U64* upper_pinched; + const U64* lower_pinched; + S32 words; + + bool upper(S32 s, S32 t) const { return !((upper_pinched[t * words + s / 64] >> (s % 64)) & 1); } + bool lower(S32 s, S32 t) const { return !((lower_pinched[t * words + s / 64] >> (s % 64)) & 1); } + bool whole(S32 t) const { - *out++ = (U16)(s + num_s*t); //bottom left - *out++ = (U16)(s+1 + num_s*(t+1)); //top right - *out++ = (U16)(s + num_s*(t+1)); //top left - }; - const auto lower = [&](S32 s, S32 t) + U64 any = 0; + for (S32 w = 0; w < words; ++w) + { + any |= upper_pinched[t * words + w] | lower_pinched[t * words + w]; + } + return any == 0; + } + }; + + // The quad at column s and row t of a grid of vertices s + num_s * t, as + // its two triangles, bottom left, top right, top left and bottom left, + // bottom right, top right, but for those keep leaves out. Returns the end + // of what it wrote. + template + LL_FORCE_INLINE U16* quad_indices(U16* out, S32 s, S32 t, S32 num_s, const KEEP& keep) + { + const S32 bottom = s + num_s * t; + const S32 top = bottom + num_s; + if (KEEP::ALL || keep.upper(s, t)) { - *out++ = (U16)(s + num_s*t); //bottom left - *out++ = (U16)(s+1 + num_s*t); //bottom right - *out++ = (U16)(s+1 + num_s*(t+1)); //top right - }; - // The last quad's column. - const S32 last = num_s - 2; - for (S32 t = 0; t < num_t - 1; t++) + out[0] = (U16)bottom; //bottom left + out[1] = (U16)(top + 1); //top right + out[2] = (U16)top; //top left + out += 3; + } + if (KEEP::ALL || keep.lower(s, t)) + { + out[0] = (U16)bottom; //bottom left + out[1] = (U16)(bottom + 1); //bottom right + out[2] = (U16)(top + 1); //top right + out += 3; + } + return out; + } + + // Whole quads written together, a band's row or a column: each quad's + // indices, the quad step vertices on from the one before, as offsets + // from the first quad's bottom left vertex, padded to whole vectors. + struct QuadUnit + { + static constexpr S32 VECTORS = 6; + static constexpr S32 MAX_QUADS = VECTORS * 8 / 6; + + alignas(16) U16 offsets[VECTORS * 8]; + S32 quads; + S32 length; + S32 vectors; + + // quad_count is from 1 to MAX_QUADS. + QuadUnit(S32 quad_count, S32 step, S32 num_s) + : quads(quad_count), length(quad_count * 6), vectors((quad_count * 6 + 7) / 8) { - const bool keep_upper = !COLLAPSED || !(collapse.last_row && t + 1 == num_t - 1); - const bool keep_lower = !COLLAPSED || !(collapse.first_row && t == 0); - if (!COLLAPSED || (keep_upper && keep_lower && !collapse.first_column && !collapse.last_column)) + for (S32 q = 0; q < quads; ++q) { - for (S32 q = 0; q < quads; ++q) + const S32 bottom = q * step; + const S32 top = bottom + num_s; + U16* quad = offsets + 6 * q; + quad[0] = (U16)bottom; //bottom left + quad[1] = (U16)(top + 1); //top right + quad[2] = (U16)top; //top left + quad[3] = (U16)bottom; //bottom left + quad[4] = (U16)(bottom + 1); //bottom right + quad[5] = (U16)(top + 1); //top right + } + std::fill(offsets + length, offsets + vectors * 8, (U16)0); + } + + }; + + // A unit while it is written, its offsets held as values: a store of an + // index could otherwise be one to the offsets, and each would be loaded + // again after it. + struct UnitWriter + { + alsimd::u16x8 v0, v1, v2, v3, v4, v5; + const U16* offsets; + S32 length; + S32 vectors; + + explicit UnitWriter(const QuadUnit& unit) + : v0(alsimd::loadu_u16(unit.offsets)), v1(alsimd::loadu_u16(unit.offsets + 8)), v2(alsimd::loadu_u16(unit.offsets + 16)), + v3(alsimd::loadu_u16(unit.offsets + 24)), v4(alsimd::loadu_u16(unit.offsets + 32)), v5(alsimd::loadu_u16(unit.offsets + 40)), + offsets(unit.offsets), length(unit.length), vectors(unit.vectors) + { + } + + // The unit whose first quad's bottom left vertex is base, at out. + // Its whole vectors run past its length into what is written after + // it, and are written one index at a time where they would run past + // limit. Returns the end of the unit. + LL_FORCE_INLINE U16* write(U16* out, S32 base, const U16* limit) const + { + if (out + vectors * 8 > limit) + { + for (S32 k = 0; k < length; ++k) { - const S32 s = LISTED ? lefts[q] : q; - upper(s, t); - lower(s, t); + out[k] = (U16)(offsets[k] + base); } - continue; + return out + length; } - // A collapsed column is the first quad's left or the last - // quad's right. - for (S32 q = 0; q < quads; ++q) + const alsimd::u16x8 b = alsimd::set1_u16((U16)base); + switch (vectors) + { + case 6: alsimd::storeu_u16(out + 40, alsimd::add_u16(v5, b)); [[fallthrough]]; + case 5: alsimd::storeu_u16(out + 32, alsimd::add_u16(v4, b)); [[fallthrough]]; + case 4: alsimd::storeu_u16(out + 24, alsimd::add_u16(v3, b)); [[fallthrough]]; + case 3: alsimd::storeu_u16(out + 16, alsimd::add_u16(v2, b)); [[fallthrough]]; + case 2: alsimd::storeu_u16(out + 8, alsimd::add_u16(v1, b)); [[fallthrough]]; + default: alsimd::storeu_u16(out, alsimd::add_u16(v0, b)); + } + return out + length; + } + }; + static_assert(LONE_BAND_QUADS <= QuadUnit::MAX_QUADS && BAND_QUADS <= QuadUnit::MAX_QUADS && COLUMN_ROWS - 1 <= QuadUnit::MAX_QUADS); + + // A unit pays for its offsets once it has this many quads and is written + // this many times. + constexpr S32 UNIT_QUADS = 4; + constexpr S32 UNIT_USES = 3; + + // The index order is called, not inlined, so that createSide's own loops + // keep one layout however the order is written. +#if LL_WINDOWS +#define VOLUME_NOINLINE __declspec(noinline) +#else +#define VOLUME_NOINLINE __attribute__((noinline)) +#endif + + // count columns of rows quads, the listed ones or the first count, a + // unit each. Nothing is written past limit. Returns the end of what it + // wrote. + template + VOLUME_NOINLINE U16* unit_columns(U16* out, const U16* limit, const S32* lefts, S32 count, S32 num_s, S32 rows) + { + const QuadUnit column(rows, num_s, num_s); + const UnitWriter writer(column); + for (S32 q = 0; q < count; ++q) + { + out = writer.write(out, LISTED ? lefts[q] : q, limit); + } + return out; + } + + // The quads from grid column begin on in bands, the first wide of them + // a quad wider than narrow, of rows quads up: each band row by row, + // every other band down, a unit a band's row where keep keeps the whole + // row and a quad at a time where it does not. Nothing is written past + // limit. Returns the end of what it wrote. + template + VOLUME_NOINLINE U16* unit_bands(U16* out, const U16* limit, S32 begin, S32 bands, S32 narrow, S32 wide, S32 num_s, S32 rows, const KEEP& keep) + { + const QuadUnit narrow_row(narrow, 1, num_s); + const QuadUnit wide_row(wide > 0 ? narrow + 1 : narrow, 1, num_s); + S32 first = begin; + for (S32 band = 0; band < bands; ++band) + { + const S32 width = band < wide ? narrow + 1 : narrow; + const UnitWriter writer(band < wide ? wide_row : narrow_row); + const auto band_row = [&](U16* at, S32 t) { - const S32 s = LISTED ? lefts[q] : q; - if (keep_upper && !(s == 0 && collapse.first_column)) + if (KEEP::ALL || keep.whole(t)) { - upper(s, t); + return writer.write(at, first + num_s * t, limit); } - if (keep_lower && !(s == last && collapse.last_column)) + for (S32 s = first; s < first + width; ++s) { - lower(s, t); + at = quad_indices(at, s, t, num_s, keep); + } + return at; + }; + if (band & 1) + { + for (S32 t = rows - 1; t >= 0; --t) + { + out = band_row(out, t); } } + else + { + for (S32 t = 0; t < rows; ++t) + { + out = band_row(out, t); + } + } + first += width; } return out; } + // A grid of up to COLUMN_ROWS rows, a column of quads at a time: the + // count columns listed, or the first count. Nothing is written past + // limit. Returns the end of what it wrote. + template + U16* grid_columns(U16* out, const U16* limit, const S32* lefts, S32 count, S32 num_s, S32 rows, const KEEP& keep) + { + if (KEEP::ALL && rows >= UNIT_QUADS && count >= UNIT_USES) + { + return unit_columns(out, limit, lefts, count, num_s, rows); + } + if (rows == 1) + { + for (S32 q = 0; q < count; ++q) + { + out = quad_indices(out, LISTED ? lefts[q] : q, 0, num_s, keep); + } + return out; + } + for (S32 q = 0; q < count; ++q) + { + const S32 s = LISTED ? lefts[q] : q; + for (S32 t = 0; t < rows; ++t) + { + out = quad_indices(out, s, t, num_s, keep); + } + } + return out; + } + + // The count quads from grid column begin on, each sharing its left edge + // with the one before it, in bands of rows quads up: each band row by + // row, every other band down. Nothing is written past limit. Returns the + // end of what it wrote. + template + U16* grid_bands(U16* out, const U16* limit, S32 begin, S32 count, S32 num_s, S32 rows, const KEEP& keep) + { + const S32 bands = count <= LONE_BAND_QUADS ? 1 : (count + BAND_QUADS - 1) / BAND_QUADS; + const S32 narrow = bands == 1 ? count : count / bands; + const S32 wide = bands == 1 ? 0 : count % bands; + if constexpr (KEEP::WHOLE_ROWS) + { + if (narrow >= UNIT_QUADS) + { + return unit_bands(out, limit, begin, bands, narrow, wide, num_s, rows, keep); + } + } + if (bands == 1) + { + for (S32 t = 0; t < rows; ++t) + { + for (S32 s = begin; s < begin + count; ++s) + { + out = quad_indices(out, s, t, num_s, keep); + } + } + return out; + } + S32 first = begin; + for (S32 band = 0; band < bands; ++band) + { + const S32 end = first + narrow + (band < wide ? 1 : 0); + const bool down = band & 1; + for (S32 row = 0; row < rows; ++row) + { + const S32 t = down ? rows - 1 - row : row; + for (S32 s = first; s < end; ++s) + { + out = quad_indices(out, s, t, num_s, keep); + } + } + first = end; + } + return out; + } + + // A side's triangles, from the quads of its grid whose left columns are + // listed, or from every quad, but for those keep leaves out, in the order + // above. Writes nothing past limit. Returns the end of what it wrote. + template + VOLUME_NOINLINE U16* grid_indices(U16* out, const U16* limit, S32 num_s, S32 num_t, const S32* lefts, S32 quads, const KEEP& keep) + { + const S32 rows = num_t - 1; + if (num_t <= COLUMN_ROWS) + { + return grid_columns(out, limit, lefts, quads, num_s, rows, keep); + } + if (!LISTED) + { + return grid_bands(out, limit, 0, quads, num_s, rows, keep); + } + // A flat inner side's quads run between its corners, and two runs + // share no vertex, so each is a grid of its own. + S32 begin = 0; + for (S32 q = 1; q <= quads; ++q) + { + if (q == quads || lefts[q] != lefts[q - 1] + 1) + { + out = grid_bands(out, limit, lefts[begin], q - begin, num_s, rows, keep); + begin = q; + } + } + return out; + } + + bool same_point(const LLVector4a& a, const LLVector4a& b) + { + return a.equal(b).areAllSet(LLVector4Logical::MASK_XYZ); + } + + // Leaves set only the flags of triangles two of whose corners are the + // same point, of those grid_coincident_triangles found two corners of + // one code in, and returns how many are left set. + S32 confirm_pinches(const LLVector4a* pos, S32 num_s, S32 num_t, U64* upper, U64* lower, S32 words) + { + S32 pinched = 0; + // other is the corner off the triangle's diagonal, as a step from + // its bottom left one. + const auto confirm = [&](U64& flags, S32 first, S32 other) + { + U64 left = flags; + while (left) + { + const S32 bit = std::countr_zero(left); + left &= left - 1; + const S32 bl = first + bit; + const S32 tr = bl + num_s + 1; + if (same_point(pos[bl], pos[tr]) || same_point(pos[bl], pos[bl + other]) || same_point(pos[bl + other], pos[tr])) + { + ++pinched; + } + else + { + flags &= ~(U64(1) << bit); + } + } + }; + for (S32 t = 0; t < num_t - 1; ++t) + { + for (S32 w = 0; w < words; ++w) + { + confirm(upper[t * words + w], num_s * t + w * 64, num_s); + confirm(lower[t * words + w], num_s * t + w * 64, 1); + } + } + return pinched; + } + // The path station a cap closes: the last for a top cap, the first for a // bottom one. S32 cap_station(const LLVolume* volume, bool top) @@ -7345,20 +7696,52 @@ bool LLVolumeFace::createSide(LLVolume* volume, bool partial_build) { LL_PROFILE_ZONE_NAMED_CATEGORY_VOLUME("llvfcs - generate indices"); - // Now we generate the indices. + // Now we generate the indices, into resizeIndices' whole 16-byte + // vectors. + const U16* limit = mIndices + ((mNumIndices + 7) & ~7); U16* end_index = mIndices; if (collapsed) { - end_index = doubled ? grid_indices(mIndices, mNumS, mNumT, lefts, quads, collapse) - : grid_indices(mIndices, mNumS, mNumT, lefts, quads, collapse); + const Kept kept = { collapse.first_row, collapse.last_row, collapse.first_column, collapse.last_column, mNumS - 2, mNumT - 2 }; + end_index = doubled ? grid_indices(mIndices, limit, mNumS, mNumT, lefts, quads, kept) + : grid_indices(mIndices, limit, mNumS, mNumT, lefts, quads, kept); + } + else if (sculpt_stitching != LL_SCULPT_TYPE_NONE && !volume->isUnique() && quads > 0 && rows > 0) + { + // A sculpt's triangles two of whose corners are one point have + // no area, and are left out: the points' codes find them and the + // points confirm them. A flexible sculpt's points move after its + // full build, and it keeps them. A face keeps at least one + // triangle. + ALScratchArena::Scope scratch; + U32* codes = scratch.allocate(num_vertices + 8); + alsimd::unit_cube_codes(pos, num_vertices, codes); + std::fill(codes + num_vertices, codes + num_vertices + 8, 0u); + const S32 words = (quads + 63) / 64; + U64* upper = scratch.allocate((size_t)words * rows); + U64* lower = scratch.allocate((size_t)words * rows); + S32 pinched = 0; + if (alsimd::grid_coincident_triangles(codes, mNumS, mNumT, upper, lower, words) > 0) + { + pinched = confirm_pinches(pos, mNumS, mNumT, upper, lower, words); + } + if (pinched > 0 && pinched < triangles) + { + end_index = grid_indices(mIndices, limit, mNumS, mNumT, lefts, quads, Unpinched{ upper, lower, words }); + } + else + { + end_index = grid_indices(mIndices, limit, mNumS, mNumT, lefts, quads, KeepAll()); + } } else { - end_index = doubled ? grid_indices(mIndices, mNumS, mNumT, lefts, quads, collapse) - : grid_indices(mIndices, mNumS, mNumT, lefts, quads, collapse); + end_index = doubled ? grid_indices(mIndices, limit, mNumS, mNumT, lefts, quads, KeepAll()) + : grid_indices(mIndices, limit, mNumS, mNumT, lefts, quads, KeepAll()); } cur_index = (S32)(end_index - mIndices); - llassert(cur_index == mNumIndices); + llassert(cur_index <= mNumIndices); + mNumIndices = cur_index; } // The normals come from the grid's quads, not from the triangles the diff --git a/indra/llmath/tests/alsimdkernels_test.cpp b/indra/llmath/tests/alsimdkernels_test.cpp index 60a71af603..3faabbb062 100644 --- a/indra/llmath/tests/alsimdkernels_test.cpp +++ b/indra/llmath/tests/alsimdkernels_test.cpp @@ -387,6 +387,129 @@ namespace } } + // Codes of points in the unit cube: a sculpt's points, eight bits an + // axis and mirrored or not, come out as their bytes; others round to + // the nearest step and clamp at the cube's ends; w never counts; every + // width bit for bit, unaligned, at every count, writing nothing past it. + TEST_CASE("alsimdkernels: unit_cube_codes gives a sculpt's points their bytes and rounds and clamps the rest", "[alsimdkernels]") + { + std::mt19937 rng(14); + std::uniform_int_distribution byte(0, 255); + std::uniform_real_distribution near_cube(-0.7f, 0.7f); + const U32 UNWRITTEN = 0xDEADBEEFu; + for (size_t n : COUNTS) + { + std::vector points(n + 1); + std::vector expected(n + 2, UNWRITTEN); + for (size_t i = 0; i < n; ++i) + { + const F32 w = near_cube(rng) * 1000.f; + if (i % 3 == 2) + { + LLVector4a p(near_cube(rng), near_cube(rng), near_cube(rng), w); + points[i] = p; + U32 code = 0; + for (int axis = 0; axis < 3; ++axis) + { + const F32 scaled = (p[axis] + 0.5f) * 255.f; + code |= (U32)llclamp((S32)std::nearbyint(scaled), 0, 255) << (8 * axis); + } + expected[1 + i] = code; + } + else + { + // as LLVolume::sculpt makes them, mirrored in x on every + // other point + const int r = byte(rng), g = byte(rng), b = byte(rng); + LLVector4a p((F32)r, (F32)g, (F32)b); + p.mul(1.f / 255.f); + p.sub(LLVector4a(0.5f, 0.5f, 0.5f)); + const bool mirrored = i % 3 == 1; + if (mirrored) + { + p.mul(LLVector4a(-1.f, 1.f, 1.f, 1.f)); + } + p.getF32ptr()[3] = w; + points[i] = p; + expected[1 + i] = (U32)(mirrored ? 255 - r : r) | ((U32)g << 8) | ((U32)b << 16); + } + } + for_each_width([&](const char* width) + { + std::vector codes(n + 2, UNWRITTEN); + unit_cube_codes_impl(points.data(), n, codes.data() + 1); + for (size_t i = 0; i < n + 2; ++i) + { + CAPTURE(width, n, i); + REQUIRE(codes[i] == expected[i]); + } + }); + } + } + + // A grid's triangles with two corners of one code, against every + // pair of every quad's corners: codes from a few values so that many + // corners match, rows of quads across one, two and three words, the + // flags past the row's last quad and in the words past it clear, and + // the rows counted. The codes past the grid are what the kernel may + // read and must not flag from. + TEST_CASE("alsimdkernels: grid_coincident_triangles flags every triangle with two corners of one code and no other", "[alsimdkernels]") + { + std::mt19937 rng(15); + const size_t columns_tried[] = {2, 3, 4, 5, 8, 9, 16, 17, 33, 64, 65, 66, 70, 129, 130}; + const size_t rows_tried[] = {2, 3, 8}; + for (size_t columns : columns_tried) + { + for (size_t rows : rows_tried) + { + for (int few : {3, 50}) + { + std::uniform_int_distribution value(0, few); + std::vector codes(columns * rows + 8); + for (U32& c : codes) + { + c = (U32)value(rng); + } + const size_t quads = columns - 1; + const size_t words = (columns + 62) / 64 + 1; + std::vector want_upper(words * rows, 0), want_lower(words * rows, 0); + size_t want_rows = 0; + for (size_t t = 0; t + 1 < rows; ++t) + { + bool any = false; + for (size_t s = 0; s < quads; ++s) + { + const U32 bl = codes[s + columns * t], br = codes[s + 1 + columns * t]; + const U32 tl = codes[s + columns * (t + 1)], tr = codes[s + 1 + columns * (t + 1)]; + const bool upper = bl == tr || tl == tr || bl == tl; + const bool lower = bl == br || br == tr || bl == tr; + want_upper[words * t + s / 64] |= U64(upper) << (s % 64); + want_lower[words * t + s / 64] |= U64(lower) << (s % 64); + any = any || upper || lower; + } + want_rows += any ? 1 : 0; + } + for_each_width([&](const char* width) + { + std::vector upper(words * rows, ~U64(0)), lower(words * rows, ~U64(0)); + const size_t got_rows = grid_coincident_triangles_impl(codes.data(), columns, rows, upper.data(), lower.data(), words); + CAPTURE(width, columns, rows, few); + CHECK(got_rows == want_rows); + for (size_t t = 0; t + 1 < rows; ++t) + { + for (size_t w = 0; w < words; ++w) + { + CAPTURE(t, w); + REQUIRE(upper[words * t + w] == want_upper[words * t + w]); + REQUIRE(lower[words * t + w] == want_lower[words * t + w]); + } + } + }); + } + } + } + } + // Three 16-bit values per vertex: every value and count, the double // reference within the two roundings a tier without FMA takes, w from // bias, the last vertex read without its neighbour, every width bit diff --git a/indra/llmath/tests/llvolume_bench.cpp b/indra/llmath/tests/llvolume_bench.cpp index 16fb13d50d..cd732147ec 100644 --- a/indra/llmath/tests/llvolume_bench.cpp +++ b/indra/llmath/tests/llvolume_bench.cpp @@ -425,7 +425,7 @@ namespace out.push_back(s + 1 + columns * (t + 1)); } - // createSide's order: every quad of a row, row after row. + // The grid's own order: every quad of a row, row after row. void rowMajor(std::vector& out, U32 columns, U32 rows) { out.clear(); @@ -438,24 +438,60 @@ namespace } } - // Bands of BAND quads across, each band row by row before the next: a - // band's row reuses the BAND + 1 vertices the row before it loaded, - // and two rows of a band fit a 16-entry cache. - constexpr U32 BAND = 7; - - void banded(std::vector& out, U32 columns, U32 rows) + // Every quad of a column, column after column: a column's right edge + // is still cached when the next column takes it, while a column of + // vertices fits the cache. + void byColumn(std::vector& out, U32 columns, U32 rows) { out.clear(); - for (U32 first = 0; first + 1 < columns; first += BAND) + for (U32 s = 0; s + 1 < columns; ++s) { - const U32 last = llmin(first + BAND, columns - 1); for (U32 t = 0; t + 1 < rows; ++t) { - for (U32 s = first; s < last; ++s) + emitQuad(out, columns, s, t); + } + } + } + + // One band up to LONE_BAND quads across, else bands of at most BAND + // quads across, as near equal as the quads divide, each band row by row + // and every other band back up: a band's row reuses the vertices the row + // before it loaded, two rows of a band fit a 16-entry cache with two to + // spare, and a band starts where the last one ended. + constexpr U32 LONE_BAND = 7; + constexpr U32 BAND = 6; + + void inBands(std::vector& out, U32 columns, U32 rows) + { + out.clear(); + const U32 quads = columns - 1; + const U32 bands = quads <= LONE_BAND ? 1 : (quads + BAND - 1) / BAND; + U32 first = 0; + for (U32 band = 0; band < bands; ++band) + { + const U32 end = first + quads / bands + (band < quads % bands ? 1 : 0); + for (U32 row = 0; row + 1 < rows; ++row) + { + const U32 t = (band & 1) ? rows - 2 - row : row; + for (U32 s = first; s < end; ++s) { emitQuad(out, columns, s, t); } } + first = end; + } + } + + // createSide's order: by column up to seven rows, in bands above. + void sideOrder(std::vector& out, U32 columns, U32 rows) + { + if (rows <= 7) + { + byColumn(out, columns, rows); + } + else + { + inBands(out, columns, rows); } } @@ -471,8 +507,22 @@ namespace U32 rows; }; - // The sweep's grids of more than two rows, from the prims and sculpts - // that have them. + // The first side of more than two rows, if the volume has one. + const LLVolumeFace* tallSide(const LLVolume& volume) + { + for (S32 i = 0; i < volume.getNumVolumeFaces(); ++i) + { + const LLVolumeFace& face = volume.getVolumeFace(i); + if (!(face.mTypeMask & LLVolumeFace::CAP_MASK) && face.mNumT > 2) + { + return &face; + } + } + return nullptr; + } + + // The sweep's grids of more than two rows, from the prims, flexible + // prims and sculpts that have them. std::vector sweepGrids() { std::vector out; @@ -481,18 +531,24 @@ namespace for (S32 lod = 2; lod < LOD_COUNT; ++lod) { LLPointer volume = new LLVolume(prim.params, LOD_DETAIL[lod]); - for (S32 i = 0; i < volume->getNumVolumeFaces(); ++i) + if (const LLVolumeFace* face = tallSide(*volume)) { - const LLVolumeFace& face = volume->getVolumeFace(i); - if (!(face.mTypeMask & LLVolumeFace::CAP_MASK) && face.mNumT > 2) - { - out.push_back({ fmt::format("{} side at LOD {}, {} by {}", prim.name, lod, face.mNumS, face.mNumT), (U32)face.mNumS, - (U32)face.mNumT }); - break; - } + out.push_back({ fmt::format("{} side at LOD {}, {} by {}", prim.name, lod, face->mNumS, face->mNumT), (U32)face->mNumS, + (U32)face->mNumT }); } } } + for (S32 sections : { 4, 16 }) + { + LLPointer volume = flexiVolume(LL_PCODE_PROFILE_CIRCLE, 3); + bendPath(*volume, sections, 0.4f); + volume->regen(); + if (const LLVolumeFace* face = tallSide(*volume)) + { + out.push_back({ fmt::format("flexible cylinder side, {} sections, LOD 3, {} by {}", sections, face->mNumS, face->mNumT), (U32)face->mNumS, + (U32)face->mNumT }); + } + } LLPointer sculpted = sculptVolume(LL_SCULPT_TYPE_SPHERE, 3); const SculptMap map = sculptMap(LL_SCULPT_TYPE_SPHERE); sculpted->sculpt(map.width, map.height, 3, map.rgb.data(), 0, false); @@ -770,24 +826,29 @@ namespace std::fflush(stdout); } - // Each grid's index order three ways: createSide's rows, bands of seven - // columns made the same way, and meshoptimizer's cache order over the + // Each grid's index order: row by row, by column, in bands, createSide's + // choice of the last two, and meshoptimizer's cache order over the // rows. The vertices each transforms a triangle in a 16-entry cache are // printed first. TEST_CASE("llvolume_bench: a grid's index order", "[llvolume_bench][!benchmark]") { ll_test::skipUnlessOptimised(); const std::vector grids = sweepGrids(); - std::printf("vertices transformed a triangle, 16-entry cache\n %-40s %10s %10s %13s\n", "grid", "row-major", "banded", "meshoptimizer"); + std::printf("vertices transformed a triangle, 16-entry cache\n %-56s %8s %8s %8s %10s %13s\n", "grid", "rows", "columns", "bands", + "createSide", "meshoptimizer"); for (const Grid& grid : grids) { - std::vector rows, bands, optimized; + const U32 vertices = grid.columns * grid.rows; + std::vector rows, columns, bands, side, optimized; rowMajor(rows, grid.columns, grid.rows); - banded(bands, grid.columns, grid.rows); + byColumn(columns, grid.columns, grid.rows); + inBands(bands, grid.columns, grid.rows); + sideOrder(side, grid.columns, grid.rows); optimized.resize(rows.size()); - meshopt_optimizeVertexCache(optimized.data(), rows.data(), rows.size(), grid.columns * grid.rows); - std::printf(" %-40s %10.3f %10.3f %13.3f\n", grid.name.c_str(), transformedPerTriangle(rows, grid.columns * grid.rows), - transformedPerTriangle(bands, grid.columns * grid.rows), transformedPerTriangle(optimized, grid.columns * grid.rows)); + meshopt_optimizeVertexCache(optimized.data(), rows.data(), rows.size(), vertices); + std::printf(" %-56s %8.3f %8.3f %8.3f %10.3f %13.3f\n", grid.name.c_str(), transformedPerTriangle(rows, vertices), + transformedPerTriangle(columns, vertices), transformedPerTriangle(bands, vertices), transformedPerTriangle(side, vertices), + transformedPerTriangle(optimized, vertices)); } std::fflush(stdout); @@ -795,20 +856,20 @@ namespace { std::vector out; out.reserve((grid.columns - 1) * (grid.rows - 1) * 6); - BENCHMARK(fmt::format("row-major, {}", grid.name)) + BENCHMARK(fmt::format("row by row, {}", grid.name)) { rowMajor(out, grid.columns, grid.rows); return out.back(); }; - BENCHMARK(fmt::format("banded, {}", grid.name)) + BENCHMARK(fmt::format("createSide's order, {}", grid.name)) { - banded(out, grid.columns, grid.rows); + sideOrder(out, grid.columns, grid.rows); return out.back(); }; std::vector rows; rowMajor(rows, grid.columns, grid.rows); std::vector optimized(rows.size()); - BENCHMARK(fmt::format("meshoptimizer over row-major, {}", grid.name)) + BENCHMARK(fmt::format("meshoptimizer over row by row, {}", grid.name)) { meshopt_optimizeVertexCache(optimized.data(), rows.data(), rows.size(), grid.columns * grid.rows); return optimized.back(); diff --git a/indra/llmath/tests/llvolume_test.cpp b/indra/llmath/tests/llvolume_test.cpp index 9e6e27f831..ef1632b56d 100644 --- a/indra/llmath/tests/llvolume_test.cpp +++ b/indra/llmath/tests/llvolume_test.cpp @@ -2845,6 +2845,111 @@ namespace CHECK(report.failures == 0); } + // Each triangle of a face, turned to start at its least index with its + // winding kept, sorted. + std::vector> sortedTriangles(const std::vector>& in) + { + std::vector> out; + for (std::array t : in) + { + while (t[0] > t[1] || t[0] > t[2]) + { + t = { t[1], t[2], t[0] }; + } + out.push_back(t); + } + std::sort(out.begin(), out.end()); + return out; + } + + bool samePoint(const LLVector4a& a, const LLVector4a& b) + { + return a[0] == b[0] && a[1] == b[1] && a[2] == b[2]; + } + + // A sculpt on a shared volume keeps each of its grid's triangles but + // those two of whose corners are one point, once each, whatever its + // stitching, map and flags. A unique volume, as a flexible sculpt's is, + // keeps every one. Normals come from the grid's quads either way. + TEST_CASE("llvolume: a sculpt leaves out the triangles two of whose corners are one point", "[llvolume]") + { + Report report; + S32 left_out = 0; + const auto check = [&](U8 type, const SculptMap& map, S32 lod, const std::string& name) + { + LLPointer shared = buildSculpt(type, map, lod); + LLVolumeParams p = shared->getParams(); + LLPointer unique = new LLVolume(p, LOD_DETAIL[lod], false, true); + unique->sculpt(map.width, map.height, 3, map.rgb.data(), 0, true); + REQUIRE(shared->getNumVolumeFaces() == 1); + REQUIRE(unique->getNumVolumeFaces() == 1); + const LLVolumeFace& face = shared->getVolumeFace(0); + const LLVolumeFace& whole = unique->getVolumeFace(0); + REQUIRE(isGrid(face)); + std::vector> grid, kept; + for (S32 t = 0; t + 1 < face.mNumT; ++t) + { + for (S32 s = 0; s + 1 < face.mNumS; ++s) + { + const U32 bl = gridIndex(face, s, t), br = bl + 1, tl = bl + face.mNumS, tr = tl + 1; + for (const std::array& tri : { std::array{ bl, tr, tl }, std::array{ bl, br, tr } }) + { + grid.push_back(tri); + const LLVector4a* pos = face.mPositions; + if (!samePoint(pos[tri[0]], pos[tri[1]]) && !samePoint(pos[tri[1]], pos[tri[2]]) && !samePoint(pos[tri[0]], pos[tri[2]])) + { + kept.push_back(tri); + } + } + } + } + if (kept.empty()) + { + kept = grid; + } + std::vector> got, all; + for (S32 i = 0; i + 2 < face.mNumIndices; i += 3) + { + got.push_back({ face.mIndices[i], face.mIndices[i + 1], face.mIndices[i + 2] }); + } + for (S32 i = 0; i + 2 < whole.mNumIndices; i += 3) + { + all.push_back({ whole.mIndices[i], whole.mIndices[i + 1], whole.mIndices[i + 2] }); + } + if (sortedTriangles(got) != sortedTriangles(kept)) + { + report.fail(fmt::format("{}: {} triangles where {} of the grid's {} have three corners apart", name, got.size(), kept.size(), grid.size())); + } + if (sortedTriangles(all) != sortedTriangles(grid)) + { + report.fail(fmt::format("{}: the unique volume has {} triangles of the grid's {}", name, all.size(), grid.size())); + } + if (whole.mNumVertices != face.mNumVertices || farthest(whole.mPositions, face.mPositions, face.mNumVertices) != 0.f || + farthest(whole.mNormals, face.mNormals, face.mNumVertices) != 0.f) + { + report.fail(fmt::format("{}: the unique volume's points or normals differ", name)); + } + left_out += (S32)(grid.size() - got.size()); + }; + const U8 flags[] = { 0, LL_SCULPT_FLAG_MIRROR, LL_SCULPT_FLAG_INVERT }; + for (S32 lod = 0; lod < LOD_COUNT; ++lod) + { + for (U8 flag : flags) + { + check((U8)(LL_SCULPT_TYPE_PLANE | flag), pinchedMap(), lod, fmt::format("pinched plane, flags {}, LOD {}", flag, lod)); + check((U8)(LL_SCULPT_TYPE_PLANE | flag), foldedMap(), lod, fmt::format("folded plane, flags {}, LOD {}", flag, lod)); + for (U8 stitching : STITCHINGS) + { + check((U8)(stitching | flag), mapFor(stitching, 64, 64), lod, fmt::format("{}, flags {}, LOD {}", stitchingName(stitching), flag, lod)); + } + } + } + INFO(fmt::format("{} triangles left out", left_out)); + INFO(report.text()); + CHECK(left_out > 0); + CHECK(report.failures == 0); + } + TEST_CASE("llvolume: a sculpt's copied row is a hard edge", "[llvolume]") { LLPointer volume = buildSculpt(LL_SCULPT_TYPE_PLANE, foldedMap(), 3); @@ -3131,18 +3236,21 @@ namespace struct Bound { const char* name; - F64 acmr[LOD_COUNT]; + F64 transformed[LOD_COUNT]; F64 overfetch[LOD_COUNT]; }; - // Vertices transformed a triangle with a 16-entry FIFO cache, and - // bytes fetched over the buffer's, per LOD. + // Vertices transformed with a 16-entry FIFO cache over all the + // class's faces, and bytes fetched over the buffer's, per LOD. The + // vertices, not the vertices a triangle: a triangle left out for + // having no area takes no vertex, and must not count against the + // order of the rest. const Bound bounds[] = { - { "box", { 2.0000, 2.0000, 2.0000, 2.0000 }, { 1.2000, 1.2000, 1.2000, 1.2000 } }, - { "cylinder", { 1.2500, 1.1667, 1.1000, 1.0833 }, { 1.0133, 1.0286, 1.0182, 1.0353 } }, - { "sphere", { 1.0833, 0.8889, 0.7000, 1.0909 }, { 1.0286, 1.0240, 1.0000, 1.0043 } }, - { "torus", { 0.6806, 1.1111, 1.0667, 1.0417 }, { 1.0122, 1.0080, 1.0000, 1.0010 } }, - { "sculpt sphere", { 0.6806, 1.1250, 1.0625, 1.0312 }, { 1.0122, 1.0074, 1.0021, 1.0006 } }, - { "sculpt plane", { 0.6806, 1.1250, 1.0625, 1.0312 }, { 1.0122, 1.0074, 1.0021, 1.0006 } }, + { "box", { 24, 24, 24, 24 }, { 1.2000, 1.2000, 1.2000, 1.2000 } }, + { "cylinder", { 30, 42, 66, 104 }, { 1.0133, 1.0286, 1.0182, 1.0353 } }, + { "sphere", { 26, 48, 126, 347 }, { 1.0286, 1.0240, 1.0000, 1.0043 } }, + { "torus", { 49, 108, 286, 697 }, { 1.0122, 1.0080, 1.0000, 1.0010 } }, + { "sculpt sphere", { 47, 85, 317, 1242 }, { 1.0122, 1.0074, 1.0021, 1.0006 } }, + { "sculpt plane", { 49, 88, 321, 1249 }, { 1.0122, 1.0074, 1.0021, 1.0006 } }, }; const Shape shapes[] = { box(), cylinder(), sphere(), torus() }; for (const Bound& bound : bounds) @@ -3164,8 +3272,9 @@ namespace const Shape* s = std::find_if(std::begin(shapes), std::end(shapes), [&](const Shape& x) { return x.name == name; }); figures.add(*build(*s, lod)); } - INFO(fmt::format("{} at LOD {}: {:.4f} transformed a triangle, overfetch {:.4f}", name, lod, figures.acmr(), figures.overfetch())); - CHECK(figures.acmr() <= bound.acmr[lod] + 1e-4); + INFO(fmt::format("{} at LOD {}: {:.0f} transformed for {:.0f} triangles, overfetch {:.4f}", name, lod, figures.transformed, + figures.triangles, figures.overfetch())); + CHECK(figures.transformed <= bound.transformed[lod]); CHECK(figures.overfetch() <= bound.overfetch[lod] + 1e-4); } } @@ -3205,6 +3314,85 @@ namespace } } + // createSide builds its index order where meshoptimizer would compute + // one: every side of the sweep, of the flexible prims and of the + // sculpts transforms at most five percent more vertices a triangle than + // meshoptimizer's cache order of the same triangles. + TEST_CASE("llvolume: every side's index order is within five percent of meshoptimizer's", "[llvolume]") + { + Report report; + S32 sides = 0; + F64 worst = 0; + std::string worst_at; + const auto check = [&](const LLVolume& volume, const std::string& name) + { + for (S32 i = 0; i < volume.getNumVolumeFaces(); ++i) + { + const LLVolumeFace& face = volume.getVolumeFace(i); + if (isCap(face) || face.mNumIndices == 0) + { + continue; + } + const std::vector own(face.mIndices, face.mIndices + face.mNumIndices); + std::vector optimized(own.size()); + meshopt_optimizeVertexCache(optimized.data(), own.data(), own.size(), face.mNumVertices); + const F64 ours = meshopt_analyzeVertexCache(own.data(), own.size(), face.mNumVertices, 16, 0, 0).acmr; + const F64 theirs = meshopt_analyzeVertexCache(optimized.data(), optimized.size(), face.mNumVertices, 16, 0, 0).acmr; + ++sides; + if (ours / theirs > worst) + { + worst = ours / theirs; + worst_at = fmt::format("{}: {}, {} by {}", name, faceName(face), face.mNumS, face.mNumT); + } + if (ours > theirs * 1.05 + 1e-6) + { + report.fail(fmt::format("{}: {}, {} by {}: {:.3f} vertices a triangle against meshoptimizer's {:.3f}", name, faceName(face), face.mNumS, + face.mNumT, ours, theirs)); + } + } + }; + for (const Shape& s : sweepShapes()) + { + for (S32 lod = 0; lod < LOD_COUNT; ++lod) + { + check(*build(s, lod), lodName(s, lod)); + } + } + for (const Shape& base : sweepShapes()) + { + if (base.path != LL_PCODE_PATH_LINE || (base.name.find(", plain") == std::string::npos && base.name.find(", profile cut") == std::string::npos)) + { + continue; + } + for (S32 sections : { 4, 16 }) + { + for (S32 lod = 0; lod < LOD_COUNT; ++lod) + { + LLPointer volume = flexiVolume(base, lod); + bendPath(*volume, sections, 1.1f); + volume->regen(); + check(*volume, fmt::format("flexible {}, {} sections at LOD {}", base.name, sections, lod)); + } + } + } + const std::array sizes[] = { { 64, 64 }, { 128, 32 }, { 32, 128 }, { 64, 16 } }; + for (U8 stitching : STITCHINGS) + { + for (const auto& size : sizes) + { + for (S32 lod = 0; lod < LOD_COUNT; ++lod) + { + check(*buildSculpt(stitching, mapFor(stitching, size[0], size[1]), lod), + fmt::format("{} sculpt from a {} by {} map at LOD {}", stitchingName(stitching), size[0], size[1], lod)); + } + } + } + INFO(fmt::format("{} sides; the most against meshoptimizer {:.3f}, {}", sides, worst, worst_at)); + INFO(report.text()); + CHECK(sides > 0); + CHECK(report.failures == 0); + } + TEST_CASE("llvolume: resizePath builds the flexible volume again in full", "[llvolume]") { for (const auto& [profile, hole, hollow, name] : { std::tuple{ LL_PCODE_PROFILE_SQUARE, LL_PCODE_HOLE_SAME, 0.f, "box" }, diff --git a/indra/newview/gltf/primitive.cpp b/indra/newview/gltf/primitive.cpp index d0e8c8810f..a30942efed 100644 --- a/indra/newview/gltf/primitive.cpp +++ b/indra/newview/gltf/primitive.cpp @@ -29,6 +29,7 @@ #include "asset.h" #include "buffer_util.h" #include "../llviewershadermgr.h" +#include "alscratcharena.h" #include @@ -283,6 +284,59 @@ struct MikktMesh }; +// An attribute's values in the order remap gives them, and only those it +// keeps. +template +static void remap_attribute(std::vector& attribute, const U32* remap, size_t kept) +{ + if (!attribute.empty()) + { + meshopt_remapVertexBuffer(attribute.data(), attribute.data(), attribute.size(), sizeof(T), remap); + attribute.resize(kept); + } +} + +// A triangle list's triangles in an order a vertex cache reuses, then its +// vertices in the order those triangles first take them, every attribute +// moved alike; a vertex no triangle takes is dropped. The remap and +// meshoptimizer's scratch come from the thread's arena. A primitive whose +// attributes are not one value a vertex, or whose indices are not whole +// triangles, is left as it is. +static void order_for_vertex_cache(Primitive& prim) +{ + const size_t index_count = prim.mIndexArray.size(); + const size_t vertex_count = prim.mPositions.size(); + if (prim.mGLMode != LLRender::TRIANGLES || index_count < 3 || index_count % 3 != 0 || vertex_count == 0) + { + return; + } + for (size_t size : { prim.mNormals.size(), prim.mTangents.size(), prim.mTexCoords0.size(), prim.mTexCoords1.size(), prim.mColors.size(), + prim.mWeights.size(), prim.mJoints.size() }) + { + if (size != 0 && size != vertex_count) + { + return; + } + } + + U32* indices = prim.mIndexArray.data(); + meshopt_optimizeVertexCache(indices, indices, index_count, vertex_count); + + ALScratchArena::Scope scratch; + U32* remap = scratch.allocate(vertex_count); + const size_t kept = meshopt_optimizeVertexFetchRemap(remap, indices, index_count, vertex_count); + meshopt_remapIndexBuffer(indices, indices, index_count, remap); + + remap_attribute(prim.mPositions, remap, kept); + remap_attribute(prim.mNormals, remap, kept); + remap_attribute(prim.mTangents, remap, kept); + remap_attribute(prim.mTexCoords0, remap, kept); + remap_attribute(prim.mTexCoords1, remap, kept); + remap_attribute(prim.mColors, remap, kept); + remap_attribute(prim.mWeights, remap, kept); + remap_attribute(prim.mJoints, remap, kept); +} + static void vertical_flip(std::vector& texcoords) { for (auto& tc : texcoords) @@ -468,6 +522,8 @@ bool Primitive::prep(Asset& asset) } } + order_for_vertex_cache(*this); + if (!mNormals.empty()) { mask |= LLVertexBuffer::MAP_NORMAL;