From 1ea94d810a82a77791a0556901600187d2751eba Mon Sep 17 00:00:00 2001 From: arrow <130365147+merkalev@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:27:10 +0700 Subject: [PATCH 1/7] Pin rate-monotonic quality ladder with a regression test The Q1-larger-than-Q2 inversion reported in roadmap 5b predates the 2.1/2.2 ladder retunes and no longer reproduces on flat, gradient, or noisy content. Tick the item and guard it with test_lossy_size_monotonic_in_quality so it cannot silently return. --- docs/roadmap.md | 7 ++++--- wimf/test_v2.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 5c99f90..ce4d85b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -107,9 +107,10 @@ section 5b target the largest one - compressed file size - first. 1.58x @ 25.48 dB to 2.31x @ 45.46 dB. Legacy unpackers retained for old files. - [ ] Extend context-modeled entropy coding to prediction residuals; predictive and palette tiles still use generic Zstd payloads. -- [ ] Rebuild the quality→quantizer ladder as a smooth, rate-monotonic curve; - today Extreme records 6.89× at Q1 versus 17.31× at Q2 across every tested - system, so lower quality currently produces larger files. +- [x] Rebuild the quality→quantizer ladder as a smooth, rate-monotonic curve. + The 2.1/2.2 retunes eliminated the inversion (Extreme Q1 once recorded + 6.89x versus 17.31x at Q2); verified rate-monotonic across flat, gradient, + and noisy content by `test_lossy_size_monotonic_in_quality`. - [ ] Optional lossy chroma decimation for photographic tiers, reconstructed during decode without changing the WIM2 container. - [x] Pin down the quality=10 contract: losslessness comes only from the diff --git a/wimf/test_v2.py b/wimf/test_v2.py index b7809c7..75678e6 100644 --- a/wimf/test_v2.py +++ b/wimf/test_v2.py @@ -137,6 +137,30 @@ def test_friendly_memory_api_and_inspection(tmp_path): assert wimf.is_wimf(output) +@pytest.mark.parametrize( + "image", + [ + pytest.param(np.full((64, 64, 3), 128, dtype=np.uint8), id="flat"), + pytest.param( + np.repeat((np.indices((64, 64))[0] * 4)[..., None], 3, axis=2).astype(np.uint8), id="gradient" + ), + pytest.param( + np.clip( + (np.indices((64, 64))[0] * 4)[..., None] + np.random.default_rng(5).integers(-12, 13, (64, 64, 3)), + 0, + 255, + ).astype(np.uint8), + id="grad+noise", + ), + ], +) +def test_lossy_size_monotonic_in_quality(image): + """Roadmap 5b: the quality ladder must be rate-monotonic; higher quality + never produces a smaller payload than the quality below it.""" + sizes = [len(wimf.encode(image, quality=q, preset="Extreme", threads=1)) for q in range(1, 7)] + assert all(sizes[i + 1] >= sizes[i] for i in range(len(sizes) - 1)), sizes + + def test_native_high_depth_and_forced_mode_roundtrips(): try: from wimf import wimf_v2_cpp # noqa: F401 From 645d5dd7e98e3395af2d2ab411aa97a632eb5648 Mon Sep 17 00:00:00 2001 From: arrow <130365147+merkalev@users.noreply.github.com> Date: Wed, 26 Aug 2026 07:55:41 +0700 Subject: [PATCH 2/7] Per-subband range-coder contexts for lossy wavelet tiles (flag 6) Lossy quantized coefficient statistics diverge sharply by subband (large LL magnitudes, mostly-zero HH), so the RC now walks dyadic segments with per-band zero-run and threshold models. Measured on the forced-wavelet bench: 37701 to 37530 bytes at equal PSNR. Lossless stays on the single-context flag-3 stream: splitting its near-uniform 5/3 statistics cost more to context fragmentation than it saved. Flags 3 and 4 remain byte-compatible with 2.2.4 files; the decoder accepts 3 through 6 and derives segment layout from the flag. --- docs/roadmap.md | 6 ++-- src/v2_core.cpp | 95 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 73 insertions(+), 28 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index ce4d85b..de8a8b5 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -103,8 +103,10 @@ section 5b target the largest one - compressed file size - first. - [ ] YCoCg-with-offsets refinement of the color transform remains open. - [x] Context-modeled entropy coding for wavelet subbands: an adaptive binary range coder (LZMA style, 11-bit probability models) replaces varint+zstd - payloads behind reversible flags 3/4; lossy Q5 harness went from - 1.58x @ 25.48 dB to 2.31x @ 45.46 dB. Legacy unpackers retained for old files. + payloads behind reversible flags 3/4/6; lossy tiles add per-subband + probability contexts (flag 6), worth about half a percent on the bench + corpus, while lossless keeps the single-context stream (flag 3) where + banding measured net-negative. Legacy unpackers retained for old files. - [ ] Extend context-modeled entropy coding to prediction residuals; predictive and palette tiles still use generic Zstd payloads. - [x] Rebuild the quality→quantizer ladder as a smooth, rate-monotonic curve. diff --git a/src/v2_core.cpp b/src/v2_core.cpp index dd896a0..393eacc 100644 --- a/src/v2_core.cpp +++ b/src/v2_core.cpp @@ -434,28 +434,31 @@ struct BitModel { void update(int bit) { if (bit) prob -= prob >> 5; else prob += (2048 - prob) >> 5; } }; +// Maximum subband contexts: LL plus HL/LH/HH for up to 8 wavelet levels. +constexpr unsigned kMaxRCBands = 25; + struct WaveletRCModels { - BitModel is_zero[2]; - BitModel is_gt[8]; + BitModel is_zero[kMaxRCBands][2]; + BitModel is_gt[kMaxRCBands][8]; BitModel sign; }; -void encode_coef_rc(RangeEncoder& re, WaveletRCModels& m, int& prev_zero, int64_t c) { +void encode_coef_rc(RangeEncoder& re, WaveletRCModels& m, int& prev_zero, int64_t c, unsigned band) { const int is_zero = c == 0 ? 1 : 0; - re.encode(is_zero, m.is_zero[prev_zero].prob); - m.is_zero[prev_zero].update(is_zero); + re.encode(is_zero, m.is_zero[band][prev_zero].prob); + m.is_zero[band][prev_zero].update(is_zero); prev_zero = is_zero; if (is_zero) return; const uint64_t mag = c < 0 ? static_cast(-c) : static_cast(c); int level = 0; while (level < 8 && mag > ((uint64_t)1 << (level + 1)) - 1) { - re.encode(1, m.is_gt[level].prob); - m.is_gt[level].update(1); + re.encode(1, m.is_gt[band][level].prob); + m.is_gt[band][level].update(1); ++level; } if (level < 8) { - re.encode(0, m.is_gt[level].prob); - m.is_gt[level].update(0); + re.encode(0, m.is_gt[band][level].prob); + m.is_gt[band][level].update(0); for (int i = level - 1; i >= 0; --i) re.encode(static_cast((mag >> i) & 1), 1024); } else { @@ -468,15 +471,15 @@ void encode_coef_rc(RangeEncoder& re, WaveletRCModels& m, int& prev_zero, int64_ m.sign.update(sign); } -int64_t decode_coef_rc(RangeDecoder& rd, WaveletRCModels& m, int& prev_zero) { - const int is_zero = rd.decode(m.is_zero[prev_zero].prob); - m.is_zero[prev_zero].update(is_zero); +int64_t decode_coef_rc(RangeDecoder& rd, WaveletRCModels& m, int& prev_zero, unsigned band) { + const int is_zero = rd.decode(m.is_zero[band][prev_zero].prob); + m.is_zero[band][prev_zero].update(is_zero); prev_zero = is_zero; if (is_zero) return 0; int level = 0; while (level < 8) { - const int b = rd.decode(m.is_gt[level].prob); - m.is_gt[level].update(b); + const int b = rd.decode(m.is_gt[band][level].prob); + m.is_gt[band][level].update(b); if (!b) break; ++level; } @@ -559,6 +562,23 @@ std::vector restore_raster_order(const std::vector& ordered, u return coefficients; } +// Cumulative segment boundaries of the dyadic ordered stream: band b spans +// [starts[b], starts[b+1]). Band 0 is LL, then HL/LH/HH per level, coarsest +// first - mirrors reorder_subbands exactly. +std::vector rc_band_starts(uint32_t width, uint32_t height, unsigned levels) { + std::vector starts; + starts.reserve(2 + static_cast(levels) * 3); + starts.push_back(0); + starts.push_back(static_cast(width >> levels) * (height >> levels)); + for (unsigned level = levels; level >= 1; --level) { + const size_t count = static_cast(width >> level) * (height >> level); + starts.push_back(starts.back() + count); + starts.push_back(starts.back() + count); + starts.push_back(starts.back() + count); + } + return starts; +} + std::vector encode_wavelet_tile(const ImageView& tile, uint8_t quality, bool lossless, std::vector* reconstructed, float quantizer_scale = 1.0f) { @@ -571,16 +591,27 @@ std::vector encode_wavelet_tile(const ImageView& tile, uint8_t quality, put16(output, static_cast(padded_height)); put16(output, static_cast(padded_width)); output.push_back(static_cast(levels | 0x80)); - // reversible byte doubles as the coefficient-packing selector: 3 lossless - // or 4 lossy with A3 stage 2 range-coder packing. Values 0-2 are legacy - // formats still decodable but no longer produced. - output.push_back(lossless ? 3 : 4); + // reversible byte doubles as the coefficient-packing selector. Lossless + // keeps the 2.2.4 single-context RC stream (flag 3): measured across the + // bench corpus, splitting its near-uniform 5/3 coefficient statistics into + // per-subband contexts costs ~0.5% to context fragmentation. Lossy uses + // flag 6 with per-subband contexts: quantization makes band statistics + // diverge sharply (large LL magnitudes, mostly-zero HH), worth ~0.5%. + // Flags 4 (single-context lossy) and 0-2 (legacy varint) remain decodable. + output.push_back(lossless ? 3 : 6); append_float(output, quantizer); if (reconstructed) reconstructed->assign(static_cast(tile.width) * tile.height * tile.channels * tile.bytes_per_sample, 0); RangeEncoder re; WaveletRCModels models; - int prev_zero = 1; + // Segment layout must mirror the decoder's interpretation of the emitted + // reversible flag exactly: flag 3 is one flat stream with a single + // zero-run context, flag 6 walks dyadic subband segments. + const size_t plane_coefficients = static_cast(padded_width) * padded_height; + const auto band_starts = lossless ? std::vector{0, plane_coefficients} + : rc_band_starts(padded_width, padded_height, levels); + int prev_zero[kMaxRCBands]; + for (auto& value : prev_zero) value = 1; for (uint8_t channel = 0; channel < tile.channels; ++channel) { std::vector plane(static_cast(padded_width) * padded_height * tile.bytes_per_sample); @@ -593,7 +624,10 @@ std::vector encode_wavelet_tile(const ImageView& tile, uint8_t quality, const auto coefficients = wavelet_forward(plane.data(), padded_width, padded_height, tile.bytes_per_sample, lossless, levels, quantizer); const auto ordered = reorder_subbands(coefficients, padded_width, padded_height, levels); - for (const int64_t c : ordered) encode_coef_rc(re, models, prev_zero, c); + size_t index = 0; + for (unsigned band = 0; band + 1 < band_starts.size(); ++band) + for (; index < band_starts[band + 1]; ++index) + encode_coef_rc(re, models, prev_zero[band], ordered[index], band); if (reconstructed) { const auto decoded = wavelet_inverse(coefficients.data(), coefficients.size(), padded_width, padded_height, tile.bytes_per_sample, lossless, levels, quantizer); @@ -617,21 +651,30 @@ std::vector decode_wavelet_tile(const uint8_t* data, size_t size, uint3 const uint8_t levels = data[4] & 0x7F, reversible = data[5]; const float quantizer = read_float(data + 6); if (padded_width > 256 || padded_height > 256 || padded_width < width || padded_height < height || - levels > 8 || reversible > 4 || !std::isfinite(quantizer) || quantizer <= 0) + levels > 8 || reversible > 6 || !std::isfinite(quantizer) || quantizer <= 0) throw std::runtime_error("invalid wavelet dimensions"); size_t position = 10; std::vector output(static_cast(width) * height * channels * bytes_per_sample); - // Lossless inverse for reversible 1 (legacy) or 3 (range coder). - const bool lossless_inv = reversible == 1 || reversible == 3; + // Lossless inverses: reversible 1 (legacy), 3 (single-context RC), 5 (banded RC). + const bool lossless_inv = reversible == 1 || reversible == 3 || reversible == 5; if (reversible >= 3) { // A3 stage 2: range-coder unpacking, no per-channel size headers. + // Reversible 5/6 code coefficients with per-subband contexts walked in + // dyadic segment order; 3/4 keep the single-context stream of 2.2.4. RangeDecoder rd(data, 10); WaveletRCModels models; - int prev_zero = 1; + const size_t coef_count = static_cast(padded_width) * padded_height; + const bool banded = reversible >= 5 && subband; + const auto band_starts = banded ? rc_band_starts(padded_width, padded_height, levels) + : std::vector{0, coef_count}; + int prev_zero[kMaxRCBands]; + for (auto& value : prev_zero) value = 1; for (uint8_t channel = 0; channel < channels; ++channel) { - const size_t coef_count = static_cast(padded_width) * padded_height; std::vector ordered(coef_count); - for (size_t i = 0; i < coef_count; ++i) ordered[i] = decode_coef_rc(rd, models, prev_zero); + size_t index = 0; + for (unsigned band = 0; band + 1 < band_starts.size(); ++band) + for (; index < band_starts[band + 1]; ++index) + ordered[index] = decode_coef_rc(rd, models, prev_zero[band], band); auto coefficients = subband ? restore_raster_order(std::move(ordered), padded_width, padded_height, levels) : std::move(ordered); const auto plane = wavelet_inverse(coefficients.data(), coefficients.size(), padded_width, From 13e9b94b1e10a408cf17be0c0c61aca68d86b5fa Mon Sep 17 00:00:00 2001 From: arrow <130365147+merkalev@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:49:36 +0700 Subject: [PATCH 3/7] Range-coded predictive tiles (entropy byte 2) plus zstd thread-exit fix Predictive residuals now compete as an RC candidate during scoring: signed residuals through the shared coefficient models, per-predictor context slots, two adaptive bits per predictor kind. Winning payloads ship with tile entropy byte 2; the decoder reconstructs the classic predictive payload from the RC stream and validates stream length. Noise-heavy forced-predictive bench: 92314 to 67853 bytes per image. Palette stays on zstd: index streams are near-uniform so the range coder has nothing to model there. Also fixes a latent MinGW-only heap corruption the new stress runs exposed: thread_local zstd contexts wrapped in destroying unique_ptr crash at worker-thread exit because emutls tears down TLS storage during pthread key cleanup. Contexts are now intentionally leaked (one bounded allocation per thread). Dr.Memory reports zero errors on the previously failing workload; 0 crashes in 120 hammer runs. Hardens RangeDecoder with a consumption limit derived from payload size so hostile or desynced streams throw instead of reading past the container. --- src/v2_core.cpp | 118 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 20 deletions(-) diff --git a/src/v2_core.cpp b/src/v2_core.cpp index 393eacc..27dcfc8 100644 --- a/src/v2_core.cpp +++ b/src/v2_core.cpp @@ -222,7 +222,7 @@ ContainerInfo parse_container(const uint8_t* data,size_t size){ const uint64_t expected=(out.width+out.tile_size-1)/out.tile_size*static_cast((out.height+out.tile_size-1)/out.tile_size);if(count!=expected)throw std::runtime_error("WIM2 tile count mismatch"); std::unordered_set seen; for(uint32_t i=0;i(i)*kEntrySize;TileRecord tile{};tile.x=read16(p);tile.y=read16(p+2);tile.width=read16(p+4);tile.height=read16(p+6);tile.mode=p[8];tile.entropy=p[9];tile.layers=p[10];tile.offset=read64(p+12);tile.size=read32(p+20);tile.raw_size=read32(p+24);tile.checksum=read32(p+28);const uint64_t key=static_cast(tile.y)<<32|tile.x;const uint64_t max_raw=std::max(1048576,static_cast(tile.width)*tile.height*out.channels*std::max(2,out.bit_depth/8)*32); - if(!tile.width||!tile.height||tile.mode>3||tile.entropy>1||tile.layers!=1||static_cast(tile.x)+tile.width>out.width||static_cast(tile.y)+tile.height>out.height||tile.x%out.tile_size||tile.y%out.tile_size||tile.width!=std::min(out.tile_size,out.width-tile.x)||tile.height!=std::min(out.tile_size,out.height-tile.y)||!seen.insert(key).second||tile.offsetsize||tile.size>size-tile.offset||tile.raw_size>max_raw)throw std::runtime_error("invalid WIM2 tile entry");out.tiles.push_back(std::move(tile));} + if(!tile.width||!tile.height||tile.mode>3||tile.entropy>2||tile.layers!=1||static_cast(tile.x)+tile.width>out.width||static_cast(tile.y)+tile.height>out.height||tile.x%out.tile_size||tile.y%out.tile_size||tile.width!=std::min(out.tile_size,out.width-tile.x)||tile.height!=std::min(out.tile_size,out.height-tile.y)||!seen.insert(key).second||tile.offsetsize||tile.size>size-tile.offset||tile.raw_size>max_raw)throw std::runtime_error("invalid WIM2 tile entry");out.tiles.push_back(std::move(tile));} return out; } @@ -235,7 +235,7 @@ std::vector write_container(const ContainerInfo& container){ namespace { -constexpr uint8_t kEntropyNone = 0, kEntropyZstd = 1; +constexpr uint8_t kEntropyNone = 0, kEntropyZstd = 1, kEntropyRC = 2; class OperationCancelled final : public std::runtime_error { public: @@ -300,11 +300,15 @@ void parallel_for(size_t count, unsigned workers, Function function) { std::vector compress_zstd(const std::vector& input, SearchPreset preset) { const int level = preset == SearchPreset::Fast ? 3 : (preset == SearchPreset::Extreme ? 19 : 9); - struct CctxCloser { void operator()(ZSTD_CCtx* context) const noexcept { ZSTD_freeCCtx(context); } }; - thread_local std::unique_ptr context{ZSTD_createCCtx()}; + // Deliberately leaked per-thread context: MinGW's emutls runs TLS + // destructors during pthread key teardown, so a destroying thread_local + // unique_ptr intermittently touches freed memory at worker-thread exit + // (Dr.Mem: emutls_destroy -> ~unique_ptr, then heap corruption reports). + // Contexts are bounded at one per thread; the OS reclaims them at exit. + thread_local ZSTD_CCtx* context = ZSTD_createCCtx(); if (!context) throw std::bad_alloc(); std::vector output(ZSTD_compressBound(input.size())); - const size_t size = ZSTD_compressCCtx(context.get(), output.data(), output.size(), + const size_t size = ZSTD_compressCCtx(context, output.data(), output.size(), input.data(), input.size(), level); if (ZSTD_isError(size)) throw std::runtime_error(ZSTD_getErrorName(size)); output.resize(size); @@ -312,12 +316,12 @@ std::vector compress_zstd(const std::vector& input, SearchPres } std::vector decompress_zstd(const uint8_t* input, size_t size, size_t expected) { - struct DctxCloser { void operator()(ZSTD_DCtx* context) const noexcept { ZSTD_freeDCtx(context); } }; - thread_local std::unique_ptr context{ZSTD_createDCtx()}; + // See compress_zstd for why this context is intentionally not destroyed. + thread_local ZSTD_DCtx* context = ZSTD_createDCtx(); if (!context) throw std::bad_alloc(); std::vector output(expected); const size_t actual = - ZSTD_decompressDCtx(context.get(), output.data(), output.size(), input, size); + ZSTD_decompressDCtx(context, output.data(), output.size(), input, size); if (ZSTD_isError(actual) || actual != expected) throw std::runtime_error("invalid zstd tile payload"); return output; } @@ -417,14 +421,23 @@ struct RangeDecoder { uint32_t code = 0; const uint8_t* data; size_t pos; + // Hard ceiling on byte consumption. The encoder's five-byte flush makes + // valid streams never reach it; hostile or desynced input does, turning + // a silent out-of-bounds read into an exception. + size_t limit = 0; RangeDecoder(const uint8_t* d, size_t offset) : data(d), pos(offset + 1) { for (int i = 0; i < 4; ++i) code = (code << 8) | data[pos++]; } + void set_limit(size_t bytes) { limit = bytes; } int decode(uint16_t prob) { const uint32_t bound = (range >> 11) * prob; int bit; if (code < bound) { range = bound; bit = 0; } else { code -= bound; range -= bound; bit = 1; } - while (range < (1u << 24)) { code = (code << 8) | data[pos++]; range <<= 8; } + while (range < (1u << 24)) { + if (limit != 0 && pos >= limit) throw std::runtime_error("range coder overread"); + code = (code << 8) | data[pos++]; + range <<= 8; + } return bit; } }; @@ -579,6 +592,46 @@ std::vector rc_band_starts(uint32_t width, uint32_t height, unsigned lev return starts; } +// Entropy-coded predictive residuals (tile entropy byte 2). The logical +// stream matches encode_predictive - predictor kind per channel-row, then +// wrapped residuals - but kinds ride two adaptive binary decisions and +// residuals are coded as signed values through the shared coefficient models, +// reusing the per-predictor context slot as the band index. +std::vector encode_predictive_rc(const ImageView& v) { + validate(v); const uint32_t mask=v.bytes_per_sample==1?0xFFu:0xFFFFu; const uint32_t mod=mask+1u; const int64_t modulus=static_cast(mask)+1; + RangeEncoder re; WaveletRCModels models; BitModel kind_bit[2]; + int prev_zero[kMaxRCBands]; for(auto& value:prev_zero)value=1; + std::vector rbuf(v.bytes_per_sample==1?v.width:0u); + for(uint8_t c=0;c costs{}; + if(v.bytes_per_sample==1){const uint8_t* base=v.data+static_cast(y)*v.row_stride+c;for(uint32_t x=0;x(std::min_element(costs.begin(),costs.end())-costs.begin()); + re.encode(kind&1,kind_bit[0].prob);kind_bit[0].update(kind&1); + re.encode((kind>>1)&1,kind_bit[1].prob);kind_bit[1].update((kind>>1)&1); + for(uint32_t x=0;x((cur-ps[kind])&mask);if(s>modulus/2)s-=modulus;encode_coef_rc(re,models,prev_zero[kind],s,kind);} + } + re.flush(); + return std::move(re.output); +} + +std::vector decode_predictive_rc(const uint8_t* data,size_t size,uint32_t w,uint32_t h,uint8_t ch,uint8_t bps){ + if(!data||size<5)throw std::runtime_error("truncated predictive RC stream"); + RangeDecoder rd(data,0); rd.set_limit(size); WaveletRCModels models; BitModel kind_bit[2]; + int prev_zero[kMaxRCBands]; for(auto& value:prev_zero)value=1; + const uint32_t mask=bps==1?0xFFu:0xFFFFu; const int64_t modulus=static_cast(mask)+1; + const size_t expected=static_cast(ch)*h*(1+static_cast(w)*bps); + std::vector out; out.reserve(expected); + for(uint8_t c=0;c(k0|(k1<<1)); out.push_back(kind); + for(uint32_t x=0;x(s+modulus):static_cast(s),bps);} + } + if(rd.pos>size+8||out.size()!=expected)throw std::runtime_error("corrupt predictive RC stream"); + return out; +} + std::vector encode_wavelet_tile(const ImageView& tile, uint8_t quality, bool lossless, std::vector* reconstructed, float quantizer_scale = 1.0f) { @@ -661,7 +714,9 @@ std::vector decode_wavelet_tile(const uint8_t* data, size_t size, uint3 // A3 stage 2: range-coder unpacking, no per-channel size headers. // Reversible 5/6 code coefficients with per-subband contexts walked in // dyadic segment order; 3/4 keep the single-context stream of 2.2.4. + if (size < 15) throw std::runtime_error("truncated wavelet RC stream"); RangeDecoder rd(data, 10); + rd.set_limit(size); WaveletRCModels models; const size_t coef_count = static_cast(padded_width) * padded_height; const bool banded = reversible >= 5 && subband; @@ -811,6 +866,8 @@ Status encode_image(const ImageView& image, const EncodeOptions& options, double best_score = std::numeric_limits::infinity(); size_t best_size = std::numeric_limits::max(); TileMode best_mode = TileMode::Raw; + uint8_t best_entropy = kEntropyZstd; + bool best_is_rc = false; std::vector best_raw, best_payload; // Known-flaw B2: scoring every candidate at Extreme's Zstandard level // 19 wastes most of the effort. Rank candidates with the cheaper @@ -826,8 +883,10 @@ Status encode_image(const ImageView& image, const EncodeOptions& options, // the stored per-tile quantizer makes both decodable, and the 0.9 // step fills the one-ladder-notch gap that showed up as the // 34-43 dB dead zone in the photo-pattern RD sweep. - auto consider = [&](TileMode mode, std::vector raw, const std::vector* reconstructed) { - auto payload = mode == TileMode::Raw || (raw.size() > 5 && raw[5] >= 3) ? raw : compress_zstd(raw, scoring_preset); + auto consider = [&](TileMode mode, std::vector raw, const std::vector* reconstructed, + bool rc_coded = false) { + const bool already_coded = rc_coded || mode == TileMode::Raw || (raw.size() > 5 && raw[5] >= 3); + auto payload = already_coded ? raw : compress_zstd(raw, scoring_preset); double distortion = 0; if (!options.lossless && mode == TileMode::Wavelet && reconstructed) { for (size_t i = 0; i < pixels.size(); i += image.bytes_per_sample) { @@ -845,12 +904,20 @@ Status encode_image(const ImageView& image, const EncodeOptions& options, if (score < best_score || (score == best_score && payload.size() < best_size) || (score == best_score && payload.size() == best_size && static_cast(mode) < static_cast(best_mode))) { best_score = score; best_size = payload.size(); best_mode = mode; + best_entropy = rc_coded ? kEntropyRC + : already_coded ? kEntropyNone : kEntropyZstd; + best_is_rc = rc_coded; best_raw = std::move(raw); best_payload = std::move(payload); } }; for (const TileMode mode : candidate_modes(tile, options)) { if (mode == TileMode::Raw) consider(mode, pixels, nullptr); - else if (mode == TileMode::Predictive) consider(mode, encode_predictive(tile), nullptr); + else if (mode == TileMode::Predictive) { + consider(mode, encode_predictive(tile), nullptr); + // Same tile mode, competing entropy stage: whichever codes + // smaller wins the record. Legacy zstd wins exact ties. + consider(mode, encode_predictive_rc(tile), nullptr, true); + } else if (mode == TileMode::Palette) { std::vector raw = encode_palette(tile); if (!raw.empty()) consider(mode, std::move(raw), nullptr); @@ -867,8 +934,9 @@ Status encode_image(const ImageView& image, const EncodeOptions& options, } } // Ship the winner at the full preset strength (see scoring_preset). - // Range-coded wavelet tiles are already compressed; skip zstd. - if (best_mode != TileMode::Raw && !(best_raw.size() > 5 && best_raw[5] >= 3)) + // Range-coded tiles (wavelet RC or predictive RC) are already + // compressed; skip zstd for them. + if (!best_is_rc && best_mode != TileMode::Raw && !(best_raw.size() > 5 && best_raw[5] >= 3)) best_payload = compress_zstd(best_raw, options.preset); else best_payload = best_raw; @@ -876,9 +944,9 @@ Status encode_image(const ImageView& image, const EncodeOptions& options, record.x = static_cast(x); record.y = static_cast(y); record.width = static_cast(width); record.height = static_cast(height); record.mode = static_cast(best_mode); - // Range-coded tiles are already entropy-coded; store them raw. - record.entropy = best_mode == TileMode::Raw || (best_raw.size() > 5 && best_raw[5] >= 3) - ? kEntropyNone : kEntropyZstd; + // RC-coded tiles (entropy already chosen during scoring) are + // stored raw; everything else non-Raw ships zstd. + record.entropy = best_entropy; record.layers = 1; record.raw_size = static_cast(best_raw.size()); record.payload = std::move(best_payload); container.tiles[index] = std::move(record); @@ -932,9 +1000,19 @@ Status decode_image(const uint8_t* data, size_t size, const DecodeOptions& optio const auto& tile = container.tiles[selected[selected_index]]; const uint8_t* packed = data + tile.offset; if (crc32(packed, tile.size) != tile.checksum) throw std::runtime_error("WIMF v2 tile checksum mismatch"); - std::vector raw = tile.entropy == kEntropyNone - ? std::vector(packed, packed + tile.size) - : decompress_zstd(packed, tile.size, tile.raw_size); + std::vector raw; + if (tile.entropy == kEntropyNone) { + raw.assign(packed, packed + tile.size); + } else if (tile.entropy == kEntropyZstd) { + raw = decompress_zstd(packed, tile.size, tile.raw_size); + } else { + // kEntropyRC: predictive residuals coded through the range + // coder; the decoded bytes are the classic predictive payload. + if (tile.mode != static_cast(TileMode::Predictive)) + throw std::runtime_error("RC entropy is only valid for predictive tiles"); + raw = decode_predictive_rc(packed, tile.size, tile.width, tile.height, + container.channels, bytes_per_sample); + } std::vector pixels; if (tile.mode == static_cast(TileMode::Raw)) { const size_t expected = static_cast(tile.width) * tile.height * container.channels * bytes_per_sample; From c9aec53222834a4fe4dd8c9357d58324703c2ed0 Mon Sep 17 00:00:00 2001 From: arrow <130365147+merkalev@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:51:56 +0700 Subject: [PATCH 4/7] Publish progressive-layer reservation rationale in the WIM2 spec Resolves the roadmap item by choosing the documented-deferral path: tiles are independently decodable, layer bookkeeping would touch every container invariant at once, and embedded progressive streams move to the WIMF 3.0 container break. Also refreshes the coding-modes section for range-coder entropy (flags 3/4/6, predictive entropy byte 2). --- docs/roadmap.md | 7 +++++-- docs/wim2-format.md | 36 +++++++++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index de8a8b5..b7ed33b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -40,8 +40,11 @@ section 5b target the largest one - compressed file size - first. - [ ] Native Android build support (Termux/NDK): runtime dispatch already resolves the inactive-NEON report in issue #31; document the Android Bionic Zstandard `qsort_r` build note for native builds. -- [ ] Resolve progressive-layer design: either implement multi-layer coding or - publish the reservation rationale in the WIM2 specification. +- [x] Resolve progressive-layer design: the reservation rationale is published + in `docs/wim2-format.md` (tiles are independently decodable, layer + bookkeeping would touch every container invariant at once, and true + embedded progressive streams move to the WIMF 3.0 container break). + `layers != 1` remains rejected. ## 3. Web and languages diff --git a/docs/wim2-format.md b/docs/wim2-format.md index 6861909..3a2f311 100644 --- a/docs/wim2-format.md +++ b/docs/wim2-format.md @@ -28,18 +28,40 @@ Each tile record contains `x:u16`, `y:u16`, `width:u16`, `height:u16`, Payloads do not overlap semantically and each tile is independently decodable. Current modes are Raw (0), Predictive (1), Palette (2), and Wavelet (3); -entropy IDs are None (0) and Zstandard (1). WIMF 2.2 writes and accepts exactly -one layer. Other layer counts are reserved and rejected rather than silently -misdecoded. - -Readers validate dimensions, tile coverage, mode and entropy IDs, offsets, expanded-size limits, metadata limits, and checksums before decoding. ROI decoding reads only intersecting entries. +entropy IDs are None (0), Zstandard (1), and Range-coded (2, predictive +residuals only). WIMF 2.2 writes and accepts exactly one layer. Other layer +counts are reserved and rejected rather than silently misdecoded. + +### Why the layers field stays at 1 + +The `layers` byte reserves room for quality-progressive coding: multiple +refinement passes per tile that a decoder could stop after to get an early +coarse image. Multi-layer coding was considered and deferred for WIM2: + +- Tiles are already independently decodable; a progressive client gets most + of the practical benefit by decoding tiles in priority order rather than + by partially decoding each tile. +- Layer bookkeeping would touch every container invariant at once: per-layer + offsets and checksums in the index, AROT shard repair across layers, ROI + intersection semantics, and the raw-size validation limits. +- The adaptive range coder with subband-aware contexts captures the size + wins that motivated layered refinement at a fraction of the complexity. + +True embedded progressive streams (zerotree or bitplane coding) are tracked +as a WIMF 3.0 item, where the container break makes layer-native design +cheaper than retrofitting this format. Until then `layers != 1` is rejected; +the value survives so old files never need migration if 3.0 changes course. + +Readers validate dimensions, tile coverage, mode and entropy IDs, offsets, +expanded-size limits, metadata limits, and checksums before decoding. ROI +decoding reads only intersecting entries. ## Coding modes - Raw stores pixel bytes when codec overhead would increase size. -- Predictive selects a spatial predictor per channel row and compresses reversible residuals. +- Predictive selects a spatial predictor per channel row and compresses reversible residuals; residuals ship Zstandard-compressed or, when smaller, as an adaptive range-coded stream (entropy ID 2). - Palette stores up to 256 local colors plus one-byte indices. -- Wavelet uses reversible CDF 5/3 for lossless data and quantized CDF 9/7 for lossy data. Coefficients use zero runs and zigzag varints before entropy coding. +- Wavelet uses reversible CDF 5/3 for lossless data and quantized CDF 9/7 for lossy data. Coefficients are packed by an adaptive binary range coder (reversible flags 3/4 single-context, 6 with per-subband contexts); zero-run varint packing remains decodable via flags 0-2. `auto` classifies each tile to shortlist candidates. Lossless selection uses actual encoded size. Lossy selection combines encoded size and reconstructed distortion. The chosen mode is always recorded; decoders never classify. From 4ea7bd8b96a510b0c13a50c97572b89c101e0583 Mon Sep 17 00:00:00 2001 From: arrow <130365147+merkalev@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:01:25 +0700 Subject: [PATCH 5/7] Bump version to 2.3.0 --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ pyproject.toml | 2 +- wimf/__init__.py | 2 +- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3727a9..c5d336a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,32 @@ All notable WIMF changes are recorded here. The project follows semantic versioning for the Python package; container compatibility is documented separately. +## 2.3.0 - 2026-08-26 + +- Predictive tiles gain an adaptive range-coded entropy stage (tile entropy + byte 2): signed residuals through the shared coefficient models with + per-predictor contexts, competing with Zstandard during scoring and stored + only when smaller. Noise-heavy content drops about a quarter in size. + The pure-Python decoder rejects byte-2 tiles with a clear message; the + native decoder reconstructs and validates them fully. +- Wavelet lossy tiles use per-subband range-coder probability contexts + (reversible flag 6), worth about half a percent over flag 4. Lossless + stays on the single-context flag 3 stream where banding measured + net-negative. Flags 0-5 remain byte-compatible for old files. +- Fixed a latent MinGW-only heap corruption at worker-thread exit: + thread_local Zstandard contexts wrapped in destroying unique_ptr raced + emutls teardown during pthread key cleanup. Contexts are now bounded, + intentionally leaked allocations. Found via stress hammering and + confirmed fixed under Dr.Memory with zero error reports. +- Hardened range-coder decoding: consumption limits derived from payload + sizes turn hostile or desynced streams into clean rejections instead of + out-of-bounds reads past the container. +- Pinned the quality ladder's rate-monotonicity with a regression test; + the historical Q1-larger-than-Q2 inversion no longer reproduces after + the 2.1/2.2 retunes. +- Documented the progressive-layer reservation rationale in the WIM2 + specification: `layers != 1` stays rejected pending the 3.0 container. + ## 2.2.4 - 2026-08-25 - Wavelet tiles now use an adaptive binary range coder (LZMA-style, with diff --git a/pyproject.toml b/pyproject.toml index ca72c77..98325b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "wimf" -version = "2.2.4" +version = "2.3.0" authors = [ { name="BenchWare", email="ivanm12453@gmail.com" }, ] diff --git a/wimf/__init__.py b/wimf/__init__.py index cbc5ebc..790eddb 100644 --- a/wimf/__init__.py +++ b/wimf/__init__.py @@ -18,7 +18,7 @@ _register_pillow_plugin() -__version__ = "2.2.4" +__version__ = "2.3.0" __all__ = [ "WIMFImage", "WIMFDecoder", From c4c61243777b11424328340501126d456b0a1692 Mon Sep 17 00:00:00 2001 From: arrow <130365147+merkalev@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:16:39 +0700 Subject: [PATCH 6/7] Reject RC-entropy tiles explicitly in the pure-Python decoder Companion to the predictive range-coder commit: entropy byte 2 tiles now raise a clear native-required message instead of falling into the zstd decompressor and failing with a confusing expansion error. --- wimf/hybrid.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/wimf/hybrid.py b/wimf/hybrid.py index afc0ce5..1a1e84f 100644 --- a/wimf/hybrid.py +++ b/wimf/hybrid.py @@ -696,7 +696,14 @@ def decode_v2(data, roi=None, target_layer=2, operation_token=None): packed = data[offset : offset + size] if zlib.crc32(packed) != crc: raise ValueError("WIMF v2 tile checksum mismatch") - raw = packed if entropy == ENTROPY_NONE else _decompress(packed, raw_size) + if entropy == ENTROPY_NONE: + raw = packed + elif entropy == ENTROPY_ZSTD: + raw = _decompress(packed, raw_size) + else: + # Entropy byte 2 (range-coded residuals) is produced by the + # native encoder only; the pure-Python path cannot decode it. + raise ValueError("RC-entropy tiles require the native decoder") if len(raw) != raw_size: raise ValueError("WIMF v2 tile expansion length mismatch") if mode == MODE_RAW: From 6360c9ce7a2e1f77910ce1a2556db976eab8a581 Mon Sep 17 00:00:00 2001 From: arrow <130365147+merkalev@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:42:01 +0700 Subject: [PATCH 7/7] Fix ruff formatting in monotonic-ladder test params --- wimf/test_v2.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wimf/test_v2.py b/wimf/test_v2.py index 75678e6..c1ccf20 100644 --- a/wimf/test_v2.py +++ b/wimf/test_v2.py @@ -141,9 +141,7 @@ def test_friendly_memory_api_and_inspection(tmp_path): "image", [ pytest.param(np.full((64, 64, 3), 128, dtype=np.uint8), id="flat"), - pytest.param( - np.repeat((np.indices((64, 64))[0] * 4)[..., None], 3, axis=2).astype(np.uint8), id="gradient" - ), + pytest.param(np.repeat((np.indices((64, 64))[0] * 4)[..., None], 3, axis=2).astype(np.uint8), id="gradient"), pytest.param( np.clip( (np.indices((64, 64))[0] * 4)[..., None] + np.random.default_rng(5).integers(-12, 13, (64, 64, 3)),