diff --git a/Source/UnitTest/test_simd.cpp b/Source/UnitTest/test_simd.cpp index 105e9760..9431e779 100644 --- a/Source/UnitTest/test_simd.cpp +++ b/Source/UnitTest/test_simd.cpp @@ -195,8 +195,8 @@ TEST(SuiteVint, match8) /** @brief Pack four-lane float groups in order at every VLA width. */ TEST(SuiteVfloat, pack_quads) { - vfloat4 groups[ASTCENC_SIMD_WIDTH / 4]; - for (unsigned int c = 0; c < ASTCENC_SIMD_WIDTH / 4; c++) + vfloat4 groups[ASTCENC_SIMD_BATCH_WIDTH]; + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { float base = static_cast(c * 4); groups[c] = vfloat4(base, base + 1, base + 2, base + 3); @@ -213,7 +213,7 @@ TEST(SuiteVfloat, pack_quads) /** @brief Load unaligned consecutive floats and replicate each into a four-lane group. */ TEST(SuiteVfloat, load1_quads) { - constexpr unsigned int count = ASTCENC_SIMD_WIDTH / 4; + constexpr unsigned int count = ASTCENC_SIMD_BATCH_WIDTH; ASTCENC_ALIGNAS float values[count + 1]; for (unsigned int i = 0; i < count + 1; i++) { @@ -239,7 +239,7 @@ TEST(SuiteVfloat, store_lane0_quads) values[i] = static_cast(i) + 0.5f; } - constexpr unsigned int count = ASTCENC_SIMD_WIDTH / 4; + constexpr unsigned int count = ASTCENC_SIMD_BATCH_WIDTH; constexpr float sentinel = -1.0f; ASTCENC_ALIGNAS float result[count + 2]; for (float& value : result) @@ -261,7 +261,7 @@ TEST(SuiteVfloat, store_lane0_quads) /** @brief Load unaligned consecutive integers and replicate each into a four-lane group. */ TEST(SuiteVint, load1_quads) { - constexpr unsigned int count = ASTCENC_SIMD_WIDTH / 4; + constexpr unsigned int count = ASTCENC_SIMD_BATCH_WIDTH; ASTCENC_ALIGNAS int values[count + 1]; for (unsigned int i = 0; i < count + 1; i++) { @@ -295,8 +295,8 @@ TEST(SuiteVint, broadcast_quad) /** @brief Pack four-lane integer groups in order at every VLA width. */ TEST(SuiteVint, pack_quads) { - vint4 groups[ASTCENC_SIMD_WIDTH / 4]; - for (unsigned int c = 0; c < ASTCENC_SIMD_WIDTH / 4; c++) + vint4 groups[ASTCENC_SIMD_BATCH_WIDTH]; + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { int base = static_cast(c * 4) - 8; groups[c] = vint4(base, base + 1, base + 2, base + 3); @@ -355,7 +355,7 @@ TEST(SuiteVfloat, group_reductions) { ASTCENC_ALIGNAS float result[ASTCENC_SIMD_WIDTH]; storea(reduced[test], result); - for (unsigned int c = 0; c < ASTCENC_SIMD_WIDTH / 4; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { vfloat4 group = vfloat4::loada(values + c * 4); float expected[] { hadd_s(group), hadd_rgb_s(group), hmin_s(group), hmax_s(group) }; diff --git a/Source/astcenc_averages_and_directions.cpp b/Source/astcenc_averages_and_directions.cpp index 6cccab52..23eaec2e 100644 --- a/Source/astcenc_averages_and_directions.cpp +++ b/Source/astcenc_averages_and_directions.cpp @@ -1299,8 +1299,254 @@ void compute_error_squared_rgb( samec_error = hadd_s(samec_errorsumv); } +/* See header for documentation. */ +void compute_partition_errors_luminance_batch( + const partition_info* const* candidates, + const image_block& blk, + float weight_imprecision, + float* uncor_errors, + float* samec_errors +) { + // Luminance texels lie on the line (L, L, L) through the origin. Both endpoint models + // fit this line exactly, so only the range-dependent weight quantization error remains. + // Compute it directly from luminance ranges, avoiding averages, directions, and projections + + assert(blk.grayscale); + assert(blk.is_constant_channel(3)); + + unsigned int partition_count = candidates[0]->partition_count; + vfloat error_sum = vfloat::zero(); + vfloat ew_r(blk.channel_weight.lane<0>()); + vfloat ew_g(blk.channel_weight.lane<1>()); + vfloat ew_b(blk.channel_weight.lane<2>()); + + vgatherf_table tr = vgatherf_load(blk.data_r, blk.texel_count); + + // Process partition p from every candidate together, preserving each candidate's + // partition order + for (unsigned int p = 0; p < partition_count; p++) + { + vfloat4 counts[ASTCENC_SIMD_BATCH_WIDTH]; + unsigned int max_count = 0; + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) + { + unsigned int count = candidates[c]->partition_texel_count[p]; + max_count = astc::max(max_count, count); + counts[c] = vfloat4(static_cast(count)); + } + + vfloat low(1e10f); + vfloat high(-1e10f); + + for (unsigned int i = 0; i < max_count; i += 4) + { + vint4 indices[ASTCENC_SIMD_BATCH_WIDTH]; + + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) + { + // Repeat the final padded quad for exhausted candidates to preserve min/max + unsigned int last = (candidates[c]->partition_texel_count[p] - 1u) & ~3u; + indices[c] = vint4(candidates[c]->texels_of_partition[p] + astc::min(i, last)); + } + + vfloat r = gatherf(tr, pack_quads(indices)); + low = min(r, low); + high = max(r, high); + } + + // Keep the channel weights separate and retain the RGB addition order + vfloat length = max_quads(high) - min_quads(low); + vfloat length_squared = length * length; + vfloat error_weight = pack_quads(counts) * vfloat(weight_imprecision); + vfloat error = (length_squared * (ew_r * error_weight)) + + (length_squared * (ew_g * error_weight)) + + (length_squared * (ew_b * error_weight)); + error_sum += error; + } + + store_lane0_quads(error_sum, uncor_errors); + store_lane0_quads(error_sum, samec_errors); +} + #if defined(ASTCENC_HAS_NATIVE_QUAD_OPS) +/* See header for documentation. */ +void compute_partition_errors_luminance_alpha_batch( + const partition_info* const* candidates, + const image_block& blk, + float weight_imprecision, + float* uncor_errors, + float* samec_errors +) { + // Apply the RGB batch strategy to luminance with alpha. Direction finding uses + // RRRA lanes per candidate, while error scoring uses four texels per candidate quad. + // + // Identical RGB components share direction and distance calculations, but retain separate + // channel weights. Keep the original addition order, four running error sums, and partition + // order for quantization errors to preserve floating-point invariance + + assert(blk.grayscale); + + unsigned int partition_count = candidates[0]->partition_count; + + // Reuse the original per-channel texel accumulation and reduction for partition averages + vfloat4 averages[ASTCENC_SIMD_BATCH_WIDTH][BLOCK_MAX_PARTITIONS]; + + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) + { + compute_partition_averages_rgba_grayscale(*candidates[c], blk, averages[c]); + } + + vfloat safe = broadcast_quad(unit4()); + vfloat channel_weight = broadcast_quad(blk.channel_weight); + vfloat zero = vfloat::zero(); + + // Preserve four running texel-error sums per candidate across partition boundaries + vfloat uncor_sum = zero; + vfloat samec_sum = zero; + + vfloat uncor_quant[BLOCK_MAX_PARTITIONS]; + vfloat samec_quant[BLOCK_MAX_PARTITIONS]; + + vfloat ew_r(blk.channel_weight.lane<0>()); + vfloat ew_g(blk.channel_weight.lane<1>()); + vfloat ew_b(blk.channel_weight.lane<2>()); + vfloat ew_a(blk.channel_weight.lane<3>()); + + vgatherf_table tr = vgatherf_load(blk.data_r, blk.texel_count); + vgatherf_table ta = vgatherf_load(blk.data_a, blk.texel_count); + + // Process partition p from every candidate together, preserving each candidate's + // partition order + for (unsigned int p = 0; p < partition_count; p++) + { + vfloat4 colors[ASTCENC_SIMD_BATCH_WIDTH]; + vfloat4 counts[ASTCENC_SIMD_BATCH_WIDTH]; + unsigned int max_count = 0; + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) + { + unsigned int count = candidates[c]->partition_texel_count[p]; + max_count = astc::max(max_count, count); + colors[c] = averages[c][p]; + counts[c] = vfloat4(static_cast(count)); + } + + // Direction estimation uses RRRA lanes within each candidate quad + vfloat mean = pack_quads(colors); + vfloat countv = pack_quads(counts); + vfloat sum_xp = zero; + vfloat sum_wp = zero; + vfloat index = zero; + + for (unsigned int i = 0; i < max_count; i++) + { + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) + { + unsigned int last = candidates[c]->partition_texel_count[p] - 1u; + unsigned int texel = candidates[c]->texels_of_partition[p][astc::min(i, last)]; + float r = blk.data_r[texel]; + colors[c] = vfloat4(r, r, r, blk.data_a[texel]); + } + + // Preserve texel order and mask exhausted candidates without rebuilding RGB vectors + vfloat datum = select(zero, pack_quads(colors) - mean, index < countv); + index += vfloat(1.0f); + + sum_xp += select(zero, datum, swizzle_quads<0, 0, 0, 0>(datum) > zero); + sum_wp += select(zero, datum, swizzle_quads<3, 3, 3, 3>(datum) > zero); + } + + // Select the dominant direction independently for each candidate + vfloat prod_xp = dot_quads(sum_xp, sum_xp); + vfloat best = sum_xp; + + vfloat prod_wp = dot_quads(sum_wp, sum_wp); + best = select(best, sum_wp, prod_wp > prod_xp); + + // Keep direction selection, normalization, and line setup in the same candidate groups + vfloat bs = normalize_safe_quads(best, safe); + vfloat samec_bs = normalize_safe_quads(mean, safe); + vfloat amod = mean - bs * dot_quads(mean, bs); + + // Scoring uses four texels per candidate quad, with separate channel vectors + vfloat bs0 = swizzle_quads<0, 0, 0, 0>(bs); + vfloat bs3 = swizzle_quads<3, 3, 3, 3>(bs); + + vfloat amod0 = swizzle_quads<0, 0, 0, 0>(amod); + vfloat amod3 = swizzle_quads<3, 3, 3, 3>(amod); + + vfloat samec0 = swizzle_quads<0, 0, 0, 0>(samec_bs); + vfloat samec3 = swizzle_quads<3, 3, 3, 3>(samec_bs); + vfloat lane_ids = int_to_float(vint::lane_id() & vint(3)); + + vfloat low(1e10f); + vfloat high(-1e10f); + + for (unsigned int i = 0; i < max_count; i += 4) + { + vint4 indices[ASTCENC_SIMD_BATCH_WIDTH]; + + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) + { + // Repeat the final padded quad for exhausted candidates to preserve min/max + unsigned int last = (candidates[c]->partition_texel_count[p] - 1u) & ~3u; + indices[c] = vint4(candidates[c]->texels_of_partition[p] + astc::min(i, last)); + } + + vint texels = pack_quads(indices); + vmask active = lane_ids < countv; + lane_ids += vfloat(4.0f); + + vfloat r = gatherf(tr, texels); + vfloat a = gatherf(ta, texels); + + // Retain the three RGB additions and separately weighted channel errors + vfloat rgb_param = r * bs0; + vfloat param = rgb_param + rgb_param + rgb_param + (a * bs3); + + low = min(param, low); + high = max(param, high); + + vfloat dr = (amod0 - r) + (param * bs0); + vfloat da = (amod3 - a) + (param * bs3); + vfloat error = (ew_r * dr * dr) + (ew_g * dr * dr) + + (ew_b * dr * dr) + (ew_a * da * da); + uncor_sum += select(zero, error, active); + + rgb_param = r * samec0; + param = rgb_param + rgb_param + rgb_param + (a * samec3); + + dr = param * samec0 - r; + da = param * samec3 - a; + error = (ew_r * dr * dr) + (ew_g * dr * dr) + + (ew_b * dr * dr) + (ew_a * da * da); + samec_sum += select(zero, error, active); + } + + // Defer adding quantization errors until after the texel sums are reduced + vfloat length = max(max_quads(high) - min_quads(low), 1e-7f); + vfloat error_weights = channel_weight * (countv * vfloat(weight_imprecision)); + + vfloat uncor_vector = bs * length; + vfloat samec_vector = samec_bs * length; + uncor_quant[p] = sum_quads((uncor_vector * uncor_vector) * error_weights); + samec_quant[p] = sum_quads((samec_vector * samec_vector) * error_weights); + } + + // Reduce each candidate's texel sums as (s0 + s2) + (s1 + s3), matching hadd_s(vfloat4) + // Then add quantization errors in the original partition order + uncor_sum = sum_quads(uncor_sum); + samec_sum = sum_quads(samec_sum); + for (unsigned int p = 0; p < partition_count; p++) + { + uncor_sum += uncor_quant[p]; + samec_sum += samec_quant[p]; + } + + store_lane0_quads(uncor_sum, uncor_errors); + store_lane0_quads(samec_sum, samec_errors); +} + /* See header for documentation. */ void compute_partition_errors_rgb_batch( const partition_info* const* candidates, @@ -1339,23 +1585,18 @@ void compute_partition_errors_rgb_batch( // partitions, using (s0 + s2) + (s1 + s3), matching the original hadd_s() reduction. Weight // quantization errors are then added in the original partition order. - constexpr unsigned int batch_size = ASTCENC_SIMD_WIDTH / 4; unsigned int partition_count = candidates[0]->partition_count; // Reuse the original per-channel texel accumulation and reduction for partition averages - vfloat4 averages[batch_size][BLOCK_MAX_PARTITIONS]; - vfloat4 units[batch_size]; - vfloat4 channel_weights[batch_size]; + vfloat4 averages[ASTCENC_SIMD_BATCH_WIDTH][BLOCK_MAX_PARTITIONS]; - for (unsigned int c = 0; c < batch_size; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { compute_partition_averages_rgb(*candidates[c], blk, averages[c]); - units[c] = unit3(); - channel_weights[c] = blk.channel_weight; } - vfloat safe = pack_quads(units); - vfloat channel_weight = pack_quads(channel_weights); + vfloat safe = broadcast_quad(unit3()); + vfloat channel_weight = broadcast_quad(blk.channel_weight); vfloat zero = vfloat::zero(); // Preserve four running texel-error sums per candidate across partition boundaries @@ -1377,10 +1618,10 @@ void compute_partition_errors_rgb_batch( // partition order for (unsigned int p = 0; p < partition_count; p++) { - vfloat4 colors[batch_size]; - vfloat4 counts[batch_size]; + vfloat4 colors[ASTCENC_SIMD_BATCH_WIDTH]; + vfloat4 counts[ASTCENC_SIMD_BATCH_WIDTH]; unsigned int max_count = 0; - for (unsigned int c = 0; c < batch_size; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { unsigned int count = candidates[c]->partition_texel_count[p]; max_count = astc::max(max_count, count); @@ -1399,7 +1640,7 @@ void compute_partition_errors_rgb_batch( for (unsigned int i = 0; i < max_count; i++) { - for (unsigned int c = 0; c < batch_size; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { unsigned int last = candidates[c]->partition_texel_count[p] - 1u; unsigned int texel = candidates[c]->texels_of_partition[p][astc::min(i, last)]; @@ -1455,9 +1696,9 @@ void compute_partition_errors_rgb_batch( for (unsigned int i = 0; i < max_count; i += 4) { - vint4 indices[batch_size]; + vint4 indices[ASTCENC_SIMD_BATCH_WIDTH]; - for (unsigned int c = 0; c < batch_size; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { // Repeat the final padded quad for exhausted candidates to preserve min/max unsigned int last = (candidates[c]->partition_texel_count[p] - 1u) & ~3u; @@ -1523,39 +1764,31 @@ void compute_partition_errors_rgb_batch( } /* See header for documentation. */ -void compute_partition_errors_rgba_grayscale_batch( +void compute_partition_errors_rgba_batch( const partition_info* const* candidates, const image_block& blk, float weight_imprecision, float* uncor_errors, float* samec_errors ) { - // Apply the RGB batch strategy above to grayscale RGB with alpha. Direction finding uses - // RRRA lanes per candidate, while error scoring uses four texels per candidate quad. + // Apply the RGB batch strategy above to general RGBA data. Direction finding uses + // RGBA lanes per candidate, while error scoring uses four texels per candidate quad. // - // Identical RGB components share direction and distance calculations, but retain separate - // channel weights. Keep the original addition order, four running error sums, and partition - // order for quantization errors to preserve floating-point invariance + // Preserve the original partition averages, texel order for direction sums, four running + // error sums, and partition order for quantization errors to preserve floating-point invariance - assert(blk.grayscale); - - constexpr unsigned int batch_size = ASTCENC_SIMD_WIDTH / 4; unsigned int partition_count = candidates[0]->partition_count; // Reuse the original per-channel texel accumulation and reduction for partition averages - vfloat4 averages[batch_size][BLOCK_MAX_PARTITIONS]; - vfloat4 units[batch_size]; - vfloat4 channel_weights[batch_size]; + vfloat4 averages[ASTCENC_SIMD_BATCH_WIDTH][BLOCK_MAX_PARTITIONS]; - for (unsigned int c = 0; c < batch_size; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { - compute_partition_averages_rgba_grayscale(*candidates[c], blk, averages[c]); - units[c] = unit4(); - channel_weights[c] = blk.channel_weight; + compute_partition_averages_rgba(*candidates[c], blk, averages[c]); } - vfloat safe = pack_quads(units); - vfloat channel_weight = pack_quads(channel_weights); + vfloat safe = broadcast_quad(unit4()); + vfloat channel_weight = broadcast_quad(blk.channel_weight); vfloat zero = vfloat::zero(); // Preserve four running texel-error sums per candidate across partition boundaries @@ -1571,16 +1804,18 @@ void compute_partition_errors_rgba_grayscale_batch( vfloat ew_a(blk.channel_weight.lane<3>()); vgatherf_table tr = vgatherf_load(blk.data_r, blk.texel_count); + vgatherf_table tg = vgatherf_load(blk.data_g, blk.texel_count); + vgatherf_table tb = vgatherf_load(blk.data_b, blk.texel_count); vgatherf_table ta = vgatherf_load(blk.data_a, blk.texel_count); // Process partition p from every candidate together, preserving each candidate's // partition order for (unsigned int p = 0; p < partition_count; p++) { - vfloat4 colors[batch_size]; - vfloat4 counts[batch_size]; + vfloat4 colors[ASTCENC_SIMD_BATCH_WIDTH]; + vfloat4 counts[ASTCENC_SIMD_BATCH_WIDTH]; unsigned int max_count = 0; - for (unsigned int c = 0; c < batch_size; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { unsigned int count = candidates[c]->partition_texel_count[p]; max_count = astc::max(max_count, count); @@ -1588,51 +1823,75 @@ void compute_partition_errors_rgba_grayscale_batch( counts[c] = vfloat4(static_cast(count)); } - // Direction estimation uses RRRA lanes within each candidate quad + // Direction estimation uses RGBA lanes per candidate, e.g. candidates 0 and 1 on AVX2: + // [c0.R, c0.G, c0.B, c0.A | c1.R, c1.G, c1.B, c1.A] vfloat mean = pack_quads(colors); vfloat countv = pack_quads(counts); vfloat sum_xp = zero; + vfloat sum_yp = zero; + vfloat sum_zp = zero; vfloat sum_wp = zero; vfloat index = zero; for (unsigned int i = 0; i < max_count; i++) { - for (unsigned int c = 0; c < batch_size; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { unsigned int last = candidates[c]->partition_texel_count[p] - 1u; unsigned int texel = candidates[c]->texels_of_partition[p][astc::min(i, last)]; - float r = blk.data_r[texel]; - colors[c] = vfloat4(r, r, r, blk.data_a[texel]); + colors[c] = blk.texel(texel); } - // Preserve texel order and mask exhausted candidates without rebuilding RGB vectors + // Preserve texel order and mask exhausted candidates without rebuilding RGBA vectors vfloat datum = select(zero, pack_quads(colors) - mean, index < countv); index += vfloat(1.0f); sum_xp += select(zero, datum, swizzle_quads<0, 0, 0, 0>(datum) > zero); + sum_yp += select(zero, datum, swizzle_quads<1, 1, 1, 1>(datum) > zero); + sum_zp += select(zero, datum, swizzle_quads<2, 2, 2, 2>(datum) > zero); sum_wp += select(zero, datum, swizzle_quads<3, 3, 3, 3>(datum) > zero); } - // Select the dominant direction independently for each candidate + // Reduce squared RGBA components within each candidate vfloat prod_xp = dot_quads(sum_xp, sum_xp); + vfloat prod_yp = dot_quads(sum_yp, sum_yp); + vfloat prod_zp = dot_quads(sum_zp, sum_zp); + vfloat prod_wp = dot_quads(sum_wp, sum_wp); + vfloat best = sum_xp; + vfloat best_sum = prod_xp; - vfloat prod_wp = dot_quads(sum_wp, sum_wp); - best = select(best, sum_wp, prod_wp > prod_xp); + vmask mask = prod_yp > best_sum; + best = select(best, sum_yp, mask); + best_sum = select(best_sum, prod_yp, mask); + + mask = prod_zp > best_sum; + best = select(best, sum_zp, mask); + best_sum = select(best_sum, prod_zp, mask); + + mask = prod_wp > best_sum; + best = select(best, sum_wp, mask); // Keep direction selection, normalization, and line setup in the same candidate groups vfloat bs = normalize_safe_quads(best, safe); vfloat samec_bs = normalize_safe_quads(mean, safe); vfloat amod = mean - bs * dot_quads(mean, bs); - // Scoring uses four texels per candidate quad, with separate channel vectors + // Scoring uses four texels per candidate quad, with separate R, G, B, and A vectors + // Broadcast each line component to all four texels of its candidate vfloat bs0 = swizzle_quads<0, 0, 0, 0>(bs); + vfloat bs1 = swizzle_quads<1, 1, 1, 1>(bs); + vfloat bs2 = swizzle_quads<2, 2, 2, 2>(bs); vfloat bs3 = swizzle_quads<3, 3, 3, 3>(bs); vfloat amod0 = swizzle_quads<0, 0, 0, 0>(amod); + vfloat amod1 = swizzle_quads<1, 1, 1, 1>(amod); + vfloat amod2 = swizzle_quads<2, 2, 2, 2>(amod); vfloat amod3 = swizzle_quads<3, 3, 3, 3>(amod); vfloat samec0 = swizzle_quads<0, 0, 0, 0>(samec_bs); + vfloat samec1 = swizzle_quads<1, 1, 1, 1>(samec_bs); + vfloat samec2 = swizzle_quads<2, 2, 2, 2>(samec_bs); vfloat samec3 = swizzle_quads<3, 3, 3, 3>(samec_bs); vfloat lane_ids = int_to_float(vint::lane_id() & vint(3)); @@ -1641,9 +1900,9 @@ void compute_partition_errors_rgba_grayscale_batch( for (unsigned int i = 0; i < max_count; i += 4) { - vint4 indices[batch_size]; + vint4 indices[ASTCENC_SIMD_BATCH_WIDTH]; - for (unsigned int c = 0; c < batch_size; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { // Repeat the final padded quad for exhausted candidates to preserve min/max unsigned int last = (candidates[c]->partition_texel_count[p] - 1u) & ~3u; @@ -1655,28 +1914,36 @@ void compute_partition_errors_rgba_grayscale_batch( lane_ids += vfloat(4.0f); vfloat r = gatherf(tr, texels); + vfloat g = gatherf(tg, texels); + vfloat b = gatherf(tb, texels); vfloat a = gatherf(ta, texels); - // Retain the three RGB additions and separately weighted channel errors - vfloat rgb_param = r * bs0; - vfloat param = rgb_param + rgb_param + rgb_param + (a * bs3); + vfloat param = (r * bs0) + (g * bs1) + (b * bs2) + (a * bs3); low = min(param, low); high = max(param, high); - vfloat dr = (amod0 - r) + (param * bs0); - vfloat da = (amod3 - a) + (param * bs3); - vfloat error = (ew_r * dr * dr) + (ew_g * dr * dr) - + (ew_b * dr * dr) + (ew_a * da * da); + vfloat d0 = (amod0 - r) + (param * bs0); + vfloat d1 = (amod1 - g) + (param * bs1); + vfloat d2 = (amod2 - b) + (param * bs2); + vfloat d3 = (amod3 - a) + (param * bs3); + + vfloat error = (ew_r * d0 * d0) + (ew_g * d1 * d1) + + (ew_b * d2 * d2) + (ew_a * d3 * d3); + + // Match haccumulate() by adding successive texel quads lane-wise, without reduction uncor_sum += select(zero, error, active); - rgb_param = r * samec0; - param = rgb_param + rgb_param + rgb_param + (a * samec3); + param = (r * samec0) + (g * samec1) + (b * samec2) + (a * samec3); + + d0 = param * samec0 - r; + d1 = param * samec1 - g; + d2 = param * samec2 - b; + d3 = param * samec3 - a; + + error = (ew_r * d0 * d0) + (ew_g * d1 * d1) + + (ew_b * d2 * d2) + (ew_a * d3 * d3); - dr = param * samec0 - r; - da = param * samec3 - a; - error = (ew_r * dr * dr) + (ew_g * dr * dr) - + (ew_b * dr * dr) + (ew_a * da * da); samec_sum += select(zero, error, active); } @@ -1685,8 +1952,9 @@ void compute_partition_errors_rgba_grayscale_batch( vfloat error_weights = channel_weight * (countv * vfloat(weight_imprecision)); vfloat uncor_vector = bs * length; - vfloat samec_vector = samec_bs * length; uncor_quant[p] = sum_quads((uncor_vector * uncor_vector) * error_weights); + + vfloat samec_vector = samec_bs * length; samec_quant[p] = sum_quads((samec_vector * samec_vector) * error_weights); } @@ -1703,6 +1971,7 @@ void compute_partition_errors_rgba_grayscale_batch( store_lane0_quads(uncor_sum, uncor_errors); store_lane0_quads(samec_sum, samec_errors); } + #endif #endif diff --git a/Source/astcenc_find_best_partitioning.cpp b/Source/astcenc_find_best_partitioning.cpp index 8b5851b4..28291eee 100644 --- a/Source/astcenc_find_best_partitioning.cpp +++ b/Source/astcenc_find_best_partitioning.cpp @@ -619,31 +619,39 @@ unsigned int find_best_partition_candidates( #if defined(ASTCENC_HAS_NATIVE_QUAD_OPS) - if (blk.grayscale && uses_alpha) + if (uses_alpha) { - constexpr unsigned int batch_size = ASTCENC_SIMD_WIDTH / 4; - - unsigned int padded_limit = (partition_search_limit + batch_size - 1) & ~(batch_size - 1); + unsigned int padded_limit = partition_search_limit + ASTCENC_SIMD_BATCH_WIDTH - 1; + padded_limit &= ~(ASTCENC_SIMD_BATCH_WIDTH - 1); for (unsigned int i = partition_search_limit; i < padded_limit; i++) { partition_sequence[i] = partition_sequence[partition_search_limit - 1]; } - for (unsigned int i = 0; i < partition_search_limit; i += batch_size) + for (unsigned int i = 0; i < partition_search_limit; i += ASTCENC_SIMD_BATCH_WIDTH) { - const partition_info* candidates[batch_size]; - for (unsigned int c = 0; c < batch_size; c++) + const partition_info* candidates[ASTCENC_SIMD_BATCH_WIDTH]; + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { candidates[c] = &bsd.get_raw_partition_info(partition_count, partition_sequence[i + c]); } - float uncor_errors[batch_size]; - float samec_errors[batch_size]; - compute_partition_errors_rgba_grayscale_batch(candidates, blk, weight_imprecision_estim, - uncor_errors, samec_errors); + float uncor_errors[ASTCENC_SIMD_BATCH_WIDTH]; + float samec_errors[ASTCENC_SIMD_BATCH_WIDTH]; + if (blk.grayscale) + { + compute_partition_errors_luminance_alpha_batch(candidates, blk, weight_imprecision_estim, + uncor_errors, samec_errors); + } + else + { + compute_partition_errors_rgba_batch(candidates, blk, weight_imprecision_estim, + uncor_errors, samec_errors); + } // Keep candidate insertion order and exclude padded results from the rankings - unsigned int active_count = astc::min(batch_size, partition_search_limit - i); + unsigned int active_count = astc::min(ASTCENC_SIMD_BATCH_WIDTH, + partition_search_limit - i); for (unsigned int c = 0; c < active_count; c++) { unsigned int partition = partition_sequence[i + c]; @@ -748,100 +756,119 @@ unsigned int find_best_partition_candidates( } else { -#if defined(ASTCENC_HAS_NATIVE_QUAD_OPS) - constexpr unsigned int batch_size = ASTCENC_SIMD_WIDTH / 4; - - unsigned int padded_limit = (partition_search_limit + batch_size - 1) & ~(batch_size - 1); - for (unsigned int i = partition_search_limit; i < padded_limit; i++) - { - partition_sequence[i] = partition_sequence[partition_search_limit - 1]; - } - - for (unsigned int i = 0; i < partition_search_limit; i += batch_size) +#if !defined(ASTCENC_HAS_NATIVE_QUAD_OPS) + if (blk.grayscale) +#endif { - const partition_info* candidates[batch_size]; - for (unsigned int c = 0; c < batch_size; c++) + unsigned int padded_limit = partition_search_limit + ASTCENC_SIMD_BATCH_WIDTH - 1; + padded_limit &= ~(ASTCENC_SIMD_BATCH_WIDTH - 1); + for (unsigned int i = partition_search_limit; i < padded_limit; i++) { - candidates[c] = &bsd.get_raw_partition_info(partition_count, partition_sequence[i + c]); + partition_sequence[i] = partition_sequence[partition_search_limit - 1]; } - float uncor_errors[batch_size]; - float samec_errors[batch_size]; - compute_partition_errors_rgb_batch(candidates, blk, weight_imprecision_estim, - uncor_errors, samec_errors); - - // Ignore padded candidates so they cannot displace real results or change tie ordering - unsigned int active_count = astc::min(batch_size, partition_search_limit - i); - for (unsigned int c = 0; c < active_count; c++) + for (unsigned int i = 0; i < partition_search_limit; i += ASTCENC_SIMD_BATCH_WIDTH) { - unsigned int partition = partition_sequence[i + c]; - insert_result(requested_candidates, uncor_errors[c], partition, - uncor_best_errors, uncor_best_partitions); - insert_result(requested_candidates, samec_errors[c], partition, - samec_best_errors, samec_best_partitions); + const partition_info* candidates[ASTCENC_SIMD_BATCH_WIDTH]; + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) + { + candidates[c] = &bsd.get_raw_partition_info(partition_count, partition_sequence[i + c]); + } + + float uncor_errors[ASTCENC_SIMD_BATCH_WIDTH]; + float samec_errors[ASTCENC_SIMD_BATCH_WIDTH]; +#if defined(ASTCENC_HAS_NATIVE_QUAD_OPS) + if (blk.grayscale) +#endif + { + compute_partition_errors_luminance_batch(candidates, blk, weight_imprecision_estim, + uncor_errors, samec_errors); + } +#if defined(ASTCENC_HAS_NATIVE_QUAD_OPS) + else + { + compute_partition_errors_rgb_batch(candidates, blk, weight_imprecision_estim, + uncor_errors, samec_errors); + } +#endif + + // Ignore padded candidates so they cannot displace real results or change tie ordering + unsigned int active_count = astc::min(ASTCENC_SIMD_BATCH_WIDTH, + partition_search_limit - i); + for (unsigned int c = 0; c < active_count; c++) + { + unsigned int partition = partition_sequence[i + c]; + insert_result(requested_candidates, uncor_errors[c], partition, + uncor_best_errors, uncor_best_partitions); + insert_result(requested_candidates, samec_errors[c], partition, + samec_best_errors, samec_best_partitions); + } } } -#else - for (unsigned int i = 0; i < partition_search_limit; i++) +#if !defined(ASTCENC_HAS_NATIVE_QUAD_OPS) + else { - unsigned int partition = partition_sequence[i]; - const auto& pi = bsd.get_raw_partition_info(partition_count, partition); + for (unsigned int i = 0; i < partition_search_limit; i++) + { + unsigned int partition = partition_sequence[i]; + const auto& pi = bsd.get_raw_partition_info(partition_count, partition); - // Compute weighting to give to each component in each partition - partition_metrics pms[BLOCK_MAX_PARTITIONS]; - compute_avgs_and_dirs_3_comp_rgb(pi, blk, pms); + // Compute weighting to give to each component in each partition + partition_metrics pms[BLOCK_MAX_PARTITIONS]; + compute_avgs_and_dirs_3_comp_rgb(pi, blk, pms); - partition_lines3 plines[BLOCK_MAX_PARTITIONS]; + partition_lines3 plines[BLOCK_MAX_PARTITIONS]; - for (unsigned int j = 0; j < partition_count; j++) - { - partition_metrics& pm = pms[j]; - partition_lines3& pl = plines[j]; + for (unsigned int j = 0; j < partition_count; j++) + { + partition_metrics& pm = pms[j]; + partition_lines3& pl = plines[j]; - pl.uncor_line.a = pm.avg; - pl.uncor_line.b = normalize_safe(pm.dir, unit3()); + pl.uncor_line.a = pm.avg; + pl.uncor_line.b = normalize_safe(pm.dir, unit3()); - pl.samec_line.a = vfloat4::zero(); - pl.samec_line.b = normalize_safe(pm.avg, unit3()); + pl.samec_line.a = vfloat4::zero(); + pl.samec_line.b = normalize_safe(pm.avg, unit3()); - pl.uncor_pline.amod = pl.uncor_line.a - pl.uncor_line.b * dot3(pl.uncor_line.a, pl.uncor_line.b); - pl.uncor_pline.bs = pl.uncor_line.b; + pl.uncor_pline.amod = pl.uncor_line.a - pl.uncor_line.b * dot3(pl.uncor_line.a, pl.uncor_line.b); + pl.uncor_pline.bs = pl.uncor_line.b; - pl.samec_pline.amod = vfloat4::zero(); - pl.samec_pline.bs = pl.samec_line.b; - } + pl.samec_pline.amod = vfloat4::zero(); + pl.samec_pline.bs = pl.samec_line.b; + } - float uncor_error = 0.0f; - float samec_error = 0.0f; + float uncor_error = 0.0f; + float samec_error = 0.0f; - compute_error_squared_rgb(pi,blk, plines, uncor_error, samec_error); + compute_error_squared_rgb(pi,blk, plines, uncor_error, samec_error); - // Compute an estimate of error introduced by weight quantization imprecision. This - // error is computed as follows, for each partition: - // 1: compute the principal-axis vector (full length) in error-space - // 2: convert the principal-axis vector to regular RGB-space - // 3: scale the vector by a constant that estimates average quantization error - // 4: for each texel, square the vector, then do a dot-product with the texel's - // error weight; sum up the results across all texels. - // 4(optimized): square the vector once, then do a dot-product with the average - // texel error, then multiply by the number of texels. + // Compute an estimate of error introduced by weight quantization imprecision. This + // error is computed as follows, for each partition: + // 1: compute the principal-axis vector (full length) in error-space + // 2: convert the principal-axis vector to regular RGB-space + // 3: scale the vector by a constant that estimates average quantization error + // 4: for each texel, square the vector, then do a dot-product with the texel's + // error weight; sum up the results across all texels. + // 4(optimized): square the vector once, then do a dot-product with the average + // texel error, then multiply by the number of texels. - for (unsigned int j = 0; j < partition_count; j++) - { - partition_lines3& pl = plines[j]; + for (unsigned int j = 0; j < partition_count; j++) + { + partition_lines3& pl = plines[j]; - float tpp = static_cast(pi.partition_texel_count[j]); - vfloat4 error_weights = blk.channel_weight * (tpp * weight_imprecision_estim); + float tpp = static_cast(pi.partition_texel_count[j]); + vfloat4 error_weights = blk.channel_weight * (tpp * weight_imprecision_estim); - vfloat4 uncor_vector = pl.uncor_line.b * pl.line_length; - vfloat4 samec_vector = pl.samec_line.b * pl.line_length; + vfloat4 uncor_vector = pl.uncor_line.b * pl.line_length; + vfloat4 samec_vector = pl.samec_line.b * pl.line_length; - uncor_error += dot3_s(uncor_vector * uncor_vector, error_weights); - samec_error += dot3_s(samec_vector * samec_vector, error_weights); - } + uncor_error += dot3_s(uncor_vector * uncor_vector, error_weights); + samec_error += dot3_s(samec_vector * samec_vector, error_weights); + } - insert_result(requested_candidates, uncor_error, partition, uncor_best_errors, uncor_best_partitions); - insert_result(requested_candidates, samec_error, partition, samec_best_errors, samec_best_partitions); + insert_result(requested_candidates, uncor_error, partition, uncor_best_errors, uncor_best_partitions); + insert_result(requested_candidates, samec_error, partition, samec_best_errors, samec_best_partitions); + } } #endif } diff --git a/Source/astcenc_internal.h b/Source/astcenc_internal.h index e11027a1..b3ebe1d7 100644 --- a/Source/astcenc_internal.h +++ b/Source/astcenc_internal.h @@ -1580,14 +1580,52 @@ void compute_error_squared_rgb( float& uncor_error, float& samec_error); +/** + * @brief Estimate partition errors for luminance data in candidate quads. + * + * All candidates have the same partition count. RGB components must be identical + * and alpha must be constant; channel weights may differ. + * + * @param candidates The ASTCENC_SIMD_BATCH_WIDTH partition candidates. + * @param blk The source luminance image block. + * @param weight_imprecision The squared weight quantization error estimate. + * @param[out] uncor_errors The uncorrelated-line errors for each candidate. + * @param[out] samec_errors The same-chroma-line errors for each candidate. + */ +void compute_partition_errors_luminance_batch( + const partition_info* const* candidates, + const image_block& blk, + float weight_imprecision, + float* uncor_errors, + float* samec_errors); + #if defined(ASTCENC_HAS_NATIVE_QUAD_OPS) +/** + * @brief Estimate partition errors for luminance with alpha in candidate quads. + * + * All candidates have the same partition count. RGB components must be identical; + * channel weights may differ. + * + * @param candidates The ASTCENC_SIMD_BATCH_WIDTH partition candidates. + * @param blk The source luminance image block with alpha. + * @param weight_imprecision The squared weight quantization error estimate. + * @param[out] uncor_errors The uncorrelated-line errors for each candidate. + * @param[out] samec_errors The same-chroma-line errors for each candidate. + */ +void compute_partition_errors_luminance_alpha_batch( + const partition_info* const* candidates, + const image_block& blk, + float weight_imprecision, + float* uncor_errors, + float* samec_errors); + /** * @brief Estimate partition errors while retaining candidate data in wide vectors. * * All candidates have the same partition count. * - * @param candidates The ASTCENC_SIMD_WIDTH / 4 partition candidates. + * @param candidates The ASTCENC_SIMD_BATCH_WIDTH partition candidates. * @param blk The source image block. * @param weight_imprecision The squared weight quantization error estimate. * @param[out] uncor_errors The uncorrelated-line errors for each candidate. @@ -1601,18 +1639,17 @@ void compute_partition_errors_rgb_batch( float* samec_errors); /** - * @brief Estimate partition errors for grayscale RGB with alpha in candidate quads. + * @brief Estimate partition errors for general RGBA data in candidate quads. * - * All candidates have the same partition count. RGB components must be identical; - * channel weights may differ. + * All candidates have the same partition count. * - * @param candidates The ASTCENC_SIMD_WIDTH / 4 partition candidates. - * @param blk The source grayscale RGB image block with alpha. + * @param candidates The ASTCENC_SIMD_BATCH_WIDTH partition candidates. + * @param blk The source image block. * @param weight_imprecision The squared weight quantization error estimate. * @param[out] uncor_errors The uncorrelated-line errors for each candidate. * @param[out] samec_errors The same-chroma-line errors for each candidate. */ -void compute_partition_errors_rgba_grayscale_batch( +void compute_partition_errors_rgba_batch( const partition_info* const* candidates, const image_block& blk, float weight_imprecision, diff --git a/Source/astcenc_lz_optimize.cpp b/Source/astcenc_lz_optimize.cpp index 840d3235..ff5d3aed 100644 --- a/Source/astcenc_lz_optimize.cpp +++ b/Source/astcenc_lz_optimize.cpp @@ -824,7 +824,7 @@ static void decompress_symbolic_block_raw( vint lane_id = vint::lane_id(); vint float_count(bsd.texel_count * 4); - for (unsigned int j = 0; j < bsd.texel_count; j += ASTCENC_SIMD_WIDTH / 4) + for (unsigned int j = 0; j < bsd.texel_count; j += ASTCENC_SIMD_BATCH_WIDTH) { vint weight = load1_quads(plane1_weights + j); vint colori = base + asr<6>(delta * weight + vint(32)); @@ -1017,7 +1017,7 @@ struct block_error_calculator store(select(vfloat::zero(), pixels_rgba, mask), original_rgba + i); store(select(vfloat::zero(), weights_rgba, mask), rgba_weights + i); - texel_lanes += vint(ASTCENC_SIMD_WIDTH / 4); + texel_lanes += vint(ASTCENC_SIMD_BATCH_WIDTH); } channel_weights = blk.channel_weight; diff --git a/Source/astcenc_vecmathlib.h b/Source/astcenc_vecmathlib.h index d410199c..abc72244 100644 --- a/Source/astcenc_vecmathlib.h +++ b/Source/astcenc_vecmathlib.h @@ -96,7 +96,7 @@ template T gatherf_byte_inds(const float* base, const uint8_t* indic #include "astcenc_vecmathlib_common_4.h" #include "astcenc_vecmathlib_avx512_16.h" - #define ASTCENC_SIMD_WIDTH 16 + #define ASTCENC_SIMD_WIDTH 16u using vfloat = vfloat16; @@ -124,7 +124,7 @@ template T gatherf_byte_inds(const float* base, const uint8_t* indic #include "astcenc_vecmathlib_avx2_8.h" #include "astcenc_vecmathlib_common_8.h" - #define ASTCENC_SIMD_WIDTH 8 + #define ASTCENC_SIMD_WIDTH 8u using vfloat = vfloat8; @@ -150,7 +150,7 @@ template T gatherf_byte_inds(const float* base, const uint8_t* indic #include "astcenc_vecmathlib_sse_4.h" #include "astcenc_vecmathlib_common_4.h" - #define ASTCENC_SIMD_WIDTH 4 + #define ASTCENC_SIMD_WIDTH 4u using vfloat = vfloat4; using vfloatacc = vfloat4; @@ -177,7 +177,7 @@ template T gatherf_byte_inds(const float* base, const uint8_t* indic #include "astcenc_vecmathlib_sve_8.h" #include "astcenc_vecmathlib_common_8.h" - #define ASTCENC_SIMD_WIDTH 8 + #define ASTCENC_SIMD_WIDTH 8u using vfloat = vfloat8; @@ -203,7 +203,7 @@ template T gatherf_byte_inds(const float* base, const uint8_t* indic #include "astcenc_vecmathlib_neon_4.h" #include "astcenc_vecmathlib_common_4.h" - #define ASTCENC_SIMD_WIDTH 4 + #define ASTCENC_SIMD_WIDTH 4u using vfloat = vfloat4; using vfloatacc = vfloat4; @@ -258,7 +258,7 @@ template T gatherf_byte_inds(const float* base, const uint8_t* indic #include "astcenc_vecmathlib_none_4.h" #include "astcenc_vecmathlib_common_4.h" - #define ASTCENC_SIMD_WIDTH 4 + #define ASTCENC_SIMD_WIDTH 4u using vfloat = vfloat4; using vfloatacc = vfloat4; @@ -274,6 +274,9 @@ template T gatherf_byte_inds(const float* base, const uint8_t* indic constexpr auto vint_from_size = vint4_from_size; #endif +/** @brief The number of four-lane quads in a SIMD vector. */ +#define ASTCENC_SIMD_BATCH_WIDTH (ASTCENC_SIMD_WIDTH / 4u) + #if !defined(ASTCENC_HAS_NATIVE_MATCH8) /** @@ -298,7 +301,7 @@ ASTCENC_SIMD_INLINE unsigned int match8(const uint8_t* p, uint8_t value) * @brief Store lane zero from each quad as consecutive floats. * * @param a The input vector. - * @param[out] p The unaligned output array of ASTCENC_SIMD_WIDTH / 4 floats. + * @param[out] p The unaligned output array of ASTCENC_SIMD_BATCH_WIDTH floats. */ ASTCENC_SIMD_INLINE void store_lane0_quads(vfloat a, float* p) { @@ -307,7 +310,7 @@ ASTCENC_SIMD_INLINE void store_lane0_quads(vfloat a, float* p) #else ASTCENC_ALIGNAS float values[ASTCENC_SIMD_WIDTH]; storea(a, values); - for (unsigned int c = 0; c < ASTCENC_SIMD_WIDTH / 4; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { p[c] = values[c * 4]; } @@ -320,7 +323,7 @@ ASTCENC_SIMD_INLINE void store_lane0_quads(vfloat a, float* p) * A quad is four consecutive lanes. Quad c occupies lanes 4*c through 4*c+3. * Four-wide backends return quad zero. * - * @param groups The ASTCENC_SIMD_WIDTH / 4 source groups. + * @param groups The ASTCENC_SIMD_BATCH_WIDTH source groups. * * @return The packed vector with source groups in consecutive quads. */ @@ -330,7 +333,7 @@ ASTCENC_SIMD_INLINE vfloat pack_quads(const vfloat4* groups) return groups[0]; #else ASTCENC_ALIGNAS float values[ASTCENC_SIMD_WIDTH]; - for (unsigned int c = 0; c < ASTCENC_SIMD_WIDTH / 4; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { storea(groups[c], values + c * 4); } @@ -344,7 +347,7 @@ ASTCENC_SIMD_INLINE vfloat pack_quads(const vfloat4* groups) * A quad is four consecutive lanes. Quad c occupies lanes 4*c through 4*c+3. * Four-wide backends return quad zero. * - * @param groups The ASTCENC_SIMD_WIDTH / 4 source groups. + * @param groups The ASTCENC_SIMD_BATCH_WIDTH source groups. * * @return The packed vector with source groups in consecutive quads. */ @@ -354,7 +357,7 @@ ASTCENC_SIMD_INLINE vint pack_quads(const vint4* groups) return groups[0]; #else ASTCENC_ALIGNAS int values[ASTCENC_SIMD_WIDTH]; - for (unsigned int c = 0; c < ASTCENC_SIMD_WIDTH / 4; c++) + for (unsigned int c = 0; c < ASTCENC_SIMD_BATCH_WIDTH; c++) { storea(groups[c], values + c * 4); } @@ -443,6 +446,18 @@ ASTCENC_SIMD_INLINE vint broadcast_quad(vint4 a) #endif +/** + * @brief Replicate a four-lane float vector into every quad. + * + * @param a The source quad. + * + * @return The vector with the source quad repeated. + */ +ASTCENC_SIMD_INLINE vfloat broadcast_quad(vfloat4 a) +{ + return int_as_float(broadcast_quad(float_as_int(a))); +} + /** * @brief Sum each quad independently, replicating the result within its four lanes. * diff --git a/Source/astcenc_vecmathlib_rvv_n.h b/Source/astcenc_vecmathlib_rvv_n.h index 8baae371..dc86c44e 100644 --- a/Source/astcenc_vecmathlib_rvv_n.h +++ b/Source/astcenc_vecmathlib_rvv_n.h @@ -351,7 +351,7 @@ ASTCENC_SIMD_INLINE vint broadcast_quad(vint4 a) /** * @brief Load consecutive integers, replicating each into a four-lane group. * - * @param p The unaligned source array of ASTCENC_SIMD_WIDTH / 4 integers. + * @param p The unaligned source array of ASTCENC_SIMD_BATCH_WIDTH integers. * * @return The vector containing p[0] in lanes 0-3, p[1] in lanes 4-7, and so on. */ @@ -366,7 +366,7 @@ ASTCENC_SIMD_INLINE vint load1_quads(const int* p) /** * @brief Load consecutive floats, replicating each into a four-lane group. * - * @param p The unaligned source array of ASTCENC_SIMD_WIDTH / 4 floats. + * @param p The unaligned source array of ASTCENC_SIMD_BATCH_WIDTH floats. * * @return The vector containing p[0] in lanes 0-3, p[1] in lanes 4-7, and so on. */