From 29b99f1792def605a58c9cbe51ca593825d2aa52 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 12 Nov 2026 21:12:11 +0100 Subject: [PATCH 1/3] A JPEG or an icon cut short loses the share of its picture asked for truncate took the whole picture of a JPEG, and of a large icon, whatever share was asked for, because their padding stood in front of the picture. A file a damage will cut now has part of its padding behind the picture as well, so the cut falls in the picture at the share asked for, as png does. - jpg puts the rest of its padding behind the picture as fill bytes before the end of image marker, which ITU T.81 B.1.1.2 allows in front of any marker. A comment behind the last scan is outside the syntax. - ico puts the rest of its gap behind the picture, where no directory entry points, and its entry's offset follows the part in front. - The share in 128 bits and the writer that holds back an end marker move from png to internal/format, which every format that splits shares. png writes the same bytes. - layout.py reads a run of fill bytes and bytes behind an icon's picture as padding. - Guards: jpg and ico leave the ratchet of formats not cut at the share. The map guard cuts every format that splits after its first byte and before its last, and asserts where the padding stands. A small tar.gz is the witness of a cut in the padding before the content. Files nothing cuts are unchanged. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 +-- internal/format/cut.go | 53 ++++++++++++ internal/format/ico/ico.go | 65 +++++++++++---- internal/format/jpg/cut.go | 129 +++++++++++++++++++++++++++++ internal/format/jpg/jpg.go | 15 +++- internal/format/png/cut.go | 18 ++-- internal/format/png/png.go | 40 ++------- internal/guard/cutmanifest_test.go | 21 ++--- internal/guard/cutshare_test.go | 7 +- internal/guard/filemap_test.go | 59 ++++++++++++- internal/guard/mutation/entries.py | 2 +- internal/oracle/layout.py | 24 +++++- 12 files changed, 357 insertions(+), 89 deletions(-) create mode 100644 internal/format/cut.go create mode 100644 internal/format/jpg/cut.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3244b77b..688fde30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,12 +80,13 @@ because it turns other people's test suites red. the length of the content is known. A cut in the padding also gets the note `cut_in_padding`. In a compressed tar.gz a cut in the padding is reported as a cut in the content. -- **`truncate` cuts a PNG's picture by the percentage you set.** - `--damage truncate:missing=90` on a 1 MB PNG used to cut only padding and - leave the whole picture. Now it removes 90% of the picture's content as - well, so a viewer that shows what is left shows only the top of the - picture, the way an interrupted upload leaves it. Files nothing cuts are - unchanged. +- **`truncate` cuts the picture of a PNG, a JPEG or an icon by the + percentage you set.** `--damage truncate:missing=90` on a 1 MB PNG used to + cut only padding and leave the whole picture, and on a JPEG or a large + icon it removed the whole picture whatever percentage you set. Now it + removes 90% of the picture's content, so a viewer that shows what is left + shows the part of the picture that arrived, the way an interrupted upload + leaves it. Files nothing cuts are unchanged. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/format/cut.go b/internal/format/cut.go new file mode 100644 index 00000000..4dbe456b --- /dev/null +++ b/internal/format/cut.go @@ -0,0 +1,53 @@ +package format + +import ( + "io" + "math/bits" +) + +// PaddingInFront is how much padding goes in front of a file's content so that +// a cut after its first cutAt bytes keeps the same share of the content as of +// the whole file (Request.CutAt): cutAt, less the before bytes that stand in +// front of that padding, less content*cutAt/whole rounded down. +// +// It is the answer before a format rounds it to what it can write, and zero or +// less means that even with no padding in front the cut keeps no more than the +// share. Each format that splits its padding around its content for a cut asks +// this, so the share is one rule (docs/TRUNCATE-2026-11-11.md sections 7.4 and +// 8.5). +// +// content*cutAt is worked out in 128 bits, because a large picture times a cut +// near the top of what a file can be passes an int64. The quotient fits as +// long as 0 < cutAt < whole and content <= whole, which the caller sees to. +func PaddingInFront(before, content, cutAt, whole int64) int64 { + hi, lo := bits.Mul64(uint64(content), uint64(cutAt)) + kept, _ := bits.Div64(hi, lo, uint64(whole)) + return cutAt - before - int64(kept) +} + +// TailHolder passes a stream on while keeping its last Keep bytes back, so a +// format can write something in front of them once the stream ends - padding +// in front of the end marker an encoder writes last. +type TailHolder struct { + W io.Writer + Keep int + // Tail is the bytes held back, Keep of them once that many have come. + Tail []byte + // Written is how many bytes went on to W. + Written int64 +} + +func (t *TailHolder) Write(p []byte) (int, error) { + n := len(p) + buf := append(t.Tail, p...) + if len(buf) > t.Keep { + emit := buf[:len(buf)-t.Keep] + if _, err := t.W.Write(emit); err != nil { + return 0, err + } + t.Written += int64(len(emit)) + buf = buf[len(buf)-t.Keep:] + } + t.Tail = append([]byte(nil), buf...) + return n, nil +} diff --git a/internal/format/ico/ico.go b/internal/format/ico/ico.go index 65f66054..3c5c6241 100644 --- a/internal/format/ico/ico.go +++ b/internal/format/ico/ico.go @@ -3,7 +3,9 @@ // An ICO is a directory of images rather than an image, so it carries an // explicit offset saying where each one starts. That offset is the padding // channel: anything between the directory and the picture is space the file -// describes. +// describes. A file a damage will cut has part of that space behind the +// picture instead, where no entry points either, so the cut takes the share +// of the picture asked for (frontGap). // // The picture inside can be a device independent bitmap or a whole PNG, and // both are in use. This generator writes either, because which one a system @@ -105,8 +107,12 @@ type memo struct { // image is the picture as it will sit inside the file. Small by // construction, because an icon side is at most 256 pixels. image []byte - // gap is how many bytes stand between the directory and that picture. + // gap is how many bytes of padding the file carries: all of them between + // the directory and the picture, unless a damage will cut the file. gap int64 + // cutAt is where a damage will cut the file, zero when nothing does + // (format.Request.CutAt). + cutAt int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -137,7 +143,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { m := memo{ width: w, height: h, seed: r.Seed, label: label, embed: embed, - image: blob, gap: r.Bytes - bare, + image: blob, gap: r.Bytes - bare, cutAt: r.CutAt, } p := format.Plan{ @@ -250,25 +256,54 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } - // The map, all of it known from the plan: the directory, the gap that - // makes up the size, and the picture. The gap stands BEFORE the picture, - // so a cut takes the whole picture whatever share is asked for - // (docs/TRUNCATE-2026-11-11.md section 2). + // The map, all of it known from the plan: the directory, the gap in front + // of the picture, the picture, and the rest of the gap behind it when a + // damage will cut the file. + front := frontGap(m, p.Bytes) + picture := int64(len(m.image)) format.Mark(w, format.RoleStructure, 0, preamble) - format.Mark(w, format.RolePadding, preamble, m.gap) - format.Mark(w, format.RoleContent, preamble+m.gap, int64(len(m.image))) + format.Mark(w, format.RolePadding, preamble, front) + format.Mark(w, format.RoleContent, preamble+front, picture) + if back := m.gap - front; back > 0 { + format.Mark(w, format.RolePadding, preamble+front+picture, back) + } - if err := writeDirectory(w, m); err != nil { + if err := writeDirectory(w, m, front); err != nil { return err } - if err := writeGap(ctx, w, m.seed, m.gap); err != nil { + if err := writeGap(ctx, w, m.seed, front); err != nil { return err } - _, err := w.Write(m.image) - return err + if _, err := w.Write(m.image); err != nil { + return err + } + return writeGap(ctx, w, m.seed, m.gap-front) +} + +// frontGap is how much of the gap stands between the directory and the +// picture: all of it, unless a damage will cut the file +// (format.Request.CutAt). Then as much as puts the cut where it keeps the +// share of the picture it keeps of the whole file, and the rest goes behind +// the picture. +// +// With all of the gap in front, a large icon cut at any share lost the whole +// picture. Bytes behind the picture are space no entry points at, as the gap +// is, and Pillow, FFmpeg, ImageMagick, the Windows Imaging Component and +// GDI+ return the same pixels either way (docs/TRUNCATE-2026-11-11.md +// section 8.3). The gap has no header, so the cut lands on the byte - except +// with too small a gap, where none of it goes in front and the cut takes more +// of the picture, the "at least" missing promises. It never needs more than +// the whole gap in front (section 8.5). +func frontGap(m memo, whole int64) int64 { + if m.cutAt <= 0 || m.cutAt >= whole { + return m.gap + } + return max(0, format.PaddingInFront(preamble, int64(len(m.image)), m.cutAt, whole)) } -func writeDirectory(w io.Writer, m memo) error { +// writeDirectory writes the directory of one entry, whose picture starts +// front bytes after it. +func writeDirectory(w io.Writer, m memo, front int64) error { var head [preamble]byte // Two reserved bytes stay zero, then the type: 1 means icon. binary.LittleEndian.PutUint16(head[2:4], 1) @@ -281,7 +316,7 @@ func writeDirectory(w io.Writer, m memo) error { binary.LittleEndian.PutUint16(e[4:6], 1) binary.LittleEndian.PutUint16(e[6:8], 32) binary.LittleEndian.PutUint32(e[8:12], uint32(len(m.image))) - binary.LittleEndian.PutUint32(e[12:16], uint32(int64(preamble)+m.gap)) + binary.LittleEndian.PutUint32(e[12:16], uint32(int64(preamble)+front)) _, err := w.Write(head[:]) return err diff --git a/internal/format/jpg/cut.go b/internal/format/jpg/cut.go new file mode 100644 index 00000000..cafc1933 --- /dev/null +++ b/internal/format/jpg/cut.go @@ -0,0 +1,129 @@ +package jpg + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// writeForACut writes a file a damage will cut after its first m.cutAt bytes, +// with the padding split around the picture so that the cut keeps the share of +// the picture it keeps of the whole file (format.Request.CutAt). +// +// With all of the padding in front of the picture, as an uncut file has it, a +// cut of any share took the whole picture. Behind the picture JPEG has one +// place for bytes that mean nothing: fill bytes, X'FF', which ITU T.81 B.1.1.2 +// lets stand in front of any marker - here the end of image marker. A comment +// behind the last scan is outside the syntax (B.2.1), and bytes after the end +// marker are outside the picture. Pillow, FFmpeg, ImageMagick, Go's own +// decoder, the Windows Imaging Component and GDI+ return the same pixels +// either way (docs/TRUNCATE-2026-11-11.md section 8.3). +// +// The fill is in no file anyone is handed: the cut falls in the picture, and +// the generator stops there. Only a whole file written for a guard has it. +func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { + pad := whole - m.body + front := frontPadding(pad, m.body-soiSize-eoiSize, m.cutAt, whole) + markAroundThePicture(w, m.body, front, pad-front) + + if _, err := w.Write([]byte{0xFF, 0xD8}); err != nil { + return err + } + if front > 0 { + segments, payload := segmentsFor(front) + if err := writeComments(ctx, w, m.seed, segments, payload); err != nil { + return err + } + } + + // The encoder's own start marker is dropped, as in Write, and its end + // marker held back, so the fill goes in front of it. + holder := &format.TailHolder{W: w, Keep: eoiSize} + skip := &headSkipper{w: holder, keep: soiSize} + if err := encode(skip, m); err != nil { + return err + } + if err := skip.check(); err != nil { + return err + } + if skip.written+soiSize != m.body { + return core.Defect(fmt.Errorf("jpg: the picture encoded to %d B where planning said %d B", skip.written+soiSize, m.body)) + } + if !bytes.Equal(holder.Tail, []byte{0xFF, 0xD9}) { + return core.Defect(fmt.Errorf("jpg: the encoded stream ends with % x, not the end of image marker", holder.Tail)) + } + if err := writeFill(ctx, w, pad-front); err != nil { + return err + } + _, err := w.Write(holder.Tail) + return err +} + +// frontPadding is how much of the padding, comment headers included, goes in +// front of the picture, so that a cut after the first cutAt bytes of a whole +// file keeps content*cutAt/whole bytes of the picture's content. +// +// A comment costs four bytes before any padding, so the front is nothing or at +// least that. Where it cannot land exactly it grows, which moves the picture +// later and cuts at most three bytes more of it - the "at least" missing +// promises. The fill behind the picture has no header, so any length is +// possible there. The one way the cut can keep more is padding too small for +// the front it should have: all of it goes in front, and the cut keeps at most +// two bytes more than the share, because the end marker counts in the file and +// not in the content (docs/TRUNCATE-2026-11-11.md section 8.5). The manifest's +// cut reports what was kept either way. +func frontPadding(pad, content, cutAt, whole int64) int64 { + front := format.PaddingInFront(soiSize, content, cutAt, whole) + switch { + case front <= 0: + return 0 + case front < comOverhead: + front = comOverhead + } + return min(front, pad) +} + +// markAroundThePicture marks the map of a file whose padding is split around +// its picture: the start marker, the comments in front, the picture, the fill +// behind and the end marker. A side with no padding has no region. +func markAroundThePicture(w io.Writer, body, front, back int64) { + format.Mark(w, format.RoleStructure, 0, soiSize) + at := int64(soiSize) + if front > 0 { + format.Mark(w, format.RolePadding, at, front) + at += front + } + format.Mark(w, format.RoleContent, at, body-soiSize-eoiSize) + at += body - soiSize - eoiSize + if back > 0 { + format.Mark(w, format.RolePadding, at, back) + at += back + } + format.Mark(w, format.RoleEnd, at, eoiSize) +} + +// writeFill writes n fill bytes, X'FF', in pieces of padChunkSize so that +// cancellation is noticed. +func writeFill(ctx context.Context, w io.Writer, n int64) error { + if n <= 0 { + return nil + } + buf := bytes.Repeat([]byte{0xFF}, int(min(n, padChunkSize))) + for remaining := n; remaining > 0; { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + k := min(remaining, int64(len(buf))) + if _, err := w.Write(buf[:k]); err != nil { + return err + } + remaining -= k + } + return nil +} diff --git a/internal/format/jpg/jpg.go b/internal/format/jpg/jpg.go index fda4cd08..37884806 100644 --- a/internal/format/jpg/jpg.go +++ b/internal/format/jpg/jpg.go @@ -9,7 +9,9 @@ // // Like PNG, the encoded size does not follow from the content, so the exact // size comes from padding. Unlike PNG, the padding goes at the FRONT and may -// need several segments, because one comment carries 65533 bytes at most. +// need several segments, because one comment carries 65533 bytes at most. A +// file a damage will cut has part of it behind the picture as well, so the cut +// takes the share of the picture asked for (cut.go). package jpg import ( @@ -104,6 +106,9 @@ func init() { // // Tk 9.0.4 is not on that list of readers and that is measured // too: it has no JPEG decoder at all. + // + // A file a damage will cut also has fill bytes in front of the + // end of image marker, measured against seven readers (cut.go). Name: "comment segment after the start of image marker", Where: format.PlacementStart, // No limit of its own. One segment carries 65533 B and this @@ -151,6 +156,10 @@ type memo struct { // this generator always writes at least one, which is what removes the // unreachable band every other format has above its floor. segments int + // cutAt is where a damage will cut the file, zero when nothing does + // (format.Request.CutAt). writeForACut splits the padding around the + // picture for it. + cutAt int64 } // checkJointLimits asks the registry's own declaration about a pair of @@ -206,6 +215,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { if err != nil { return format.Plan{}, err } + m.cutAt = r.CutAt w, h := m.width, m.height bare := m.body @@ -287,6 +297,9 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return ctx.Err() default: } + if m.cutAt > 0 && m.cutAt < p.Bytes { + return writeForACut(ctx, w, m, p.Bytes) + } // The map, all of it known from the plan: the start marker, the comments // that make up the size, the picture - its tables, frame and scan as one - diff --git a/internal/format/png/cut.go b/internal/format/png/cut.go index 28b9bc33..367559a3 100644 --- a/internal/format/png/cut.go +++ b/internal/format/png/cut.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "math/bits" "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" @@ -45,14 +44,14 @@ func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { // Both padding chunks draw from the file's seed. Nobody reads padding, and // the one behind the picture lies past any cut. split := &afterTheHead{ctx: ctx, w: w, seed: m.seed, head: headSize, front: front} - holder := &tailHolder{w: split, keep: iendSize} + holder := &format.TailHolder{W: split, Keep: iendSize} if err := encode(holder, m); err != nil { return err } - if holder.written != m.body { - return core.Defect(fmt.Errorf("png: the picture encoded to %d B where planning said %d B", holder.written, m.body)) + if holder.Written != m.body { + return core.Defect(fmt.Errorf("png: the picture encoded to %d B where planning said %d B", holder.Written, m.body)) } - if string(holder.tail[4:8]) != "IEND" { + if string(holder.Tail[4:8]) != "IEND" { return core.Defect(fmt.Errorf("png: the encoded stream does not end with IEND")) } if back := pad - front; back > 0 { @@ -60,7 +59,7 @@ func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { return err } } - _, err := w.Write(holder.tail) + _, err := w.Write(holder.Tail) return err } @@ -118,12 +117,7 @@ func frontPadding(pad, content, cutAt, whole int64) int64 { if pad == 0 || cutAt <= 0 || cutAt >= whole { return 0 } - // content*cutAt in 128 bits, because a file near the top of what one chunk - // carries times a cut near it passes an int64. The quotient fits, because - // cutAt is less than whole. - hi, lo := bits.Mul64(uint64(content), uint64(cutAt)) - kept, _ := bits.Div64(hi, lo, uint64(whole)) - front := cutAt - headSize - int64(kept) + front := format.PaddingInFront(headSize, content, cutAt, whole) switch { case front <= 0: return 0 diff --git a/internal/format/png/png.go b/internal/format/png/png.go index 935270ea..a2c070b0 100644 --- a/internal/format/png/png.go +++ b/internal/format/png/png.go @@ -300,7 +300,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // The closing chunk is always the final twelve bytes, so hold those back // and let everything before them stream straight out. Nothing is buffered // beyond that, which matters because a picture can be very large. - holder := &tailHolder{w: w, keep: iendSize} + holder := &format.TailHolder{W: w, Keep: iendSize} if err := encode(holder, m); err != nil { return err } @@ -310,7 +310,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // largest rung, so this is where the exact size arrives. The padding is // whatever is left, which is the same arithmetic planning would have // done with the same number. - m.body = holder.written + m.body = holder.Written m.padData = p.Bytes - m.body - iendSize - chunkOverhead if m.padData < 0 || m.padData > maxChunkData { // Unreachable unless ladderCeiling is wrong, and then it is better @@ -320,10 +320,10 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { m.body, m.padData, p.Bytes)) } markAfterThePicture(w, m, false) - } else if holder.written != m.body { - return core.Defect(fmt.Errorf("png: the picture encoded to %d B where planning said %d B", holder.written, m.body)) + } else if holder.Written != m.body { + return core.Defect(fmt.Errorf("png: the picture encoded to %d B where planning said %d B", holder.Written, m.body)) } - if string(holder.tail[4:8]) != "IEND" { + if string(holder.Tail[4:8]) != "IEND" { return core.Defect(fmt.Errorf("png: the encoded stream does not end with IEND")) } @@ -333,7 +333,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { } } - _, err := w.Write(holder.tail) + _, err := w.Write(holder.Tail) return err } @@ -501,11 +501,11 @@ func encode(w io.Writer, m memo) error { // couple of milliseconds. Nothing is held in memory - the bytes are counted // and dropped. func encodedBodySize(m memo) (int64, error) { - holder := &tailHolder{w: io.Discard, keep: iendSize} + holder := &format.TailHolder{W: io.Discard, Keep: iendSize} if err := encode(holder, m); err != nil { return 0, err } - return holder.written, nil + return holder.Written, nil } // minimumBytes is the smallest file this generator can produce: a one pixel @@ -581,27 +581,3 @@ func writePaddingChunk(ctx context.Context, w io.Writer, kind string, seed uint6 _, err := w.Write(tail[:]) return err } - -// tailHolder passes bytes through while keeping the last keep bytes back, so -// the caller can insert something in front of them. -type tailHolder struct { - w io.Writer - keep int - tail []byte - written int64 -} - -func (t *tailHolder) Write(p []byte) (int, error) { - n := len(p) - buf := append(t.tail, p...) - if len(buf) > t.keep { - emit := buf[:len(buf)-t.keep] - if _, err := t.w.Write(emit); err != nil { - return 0, err - } - t.written += int64(len(emit)) - buf = buf[len(buf)-t.keep:] - } - t.tail = append([]byte(nil), buf...) - return n, nil -} diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index 6095025e..aa911441 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -34,12 +34,12 @@ type cutCase struct { // A file cut short says in its manifest where the cut fell and how much of // the content is left, and gets a note when the cut missed the content. // -// The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2 and 7). A -// GIF of 1 MiB is nearly all padding after its picture, so 90% missing still -// leaves the whole picture. A PNG splits its padding around the picture for a -// cut, so the cut takes the share asked for. An icon keeps its padding BEFORE -// its picture, so a cut there takes all of it. A text is its words, so the cut -// falls in them. Two cuts in a row end the file where the second one does, and +// The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2, 7 and 8). +// A GIF of 1 MiB is nearly all padding after its picture, so 90% missing +// still leaves the whole picture. A PNG splits its padding around the picture +// for a cut, so the cut takes the share asked for. A small tar.gz keeps its +// padding BEFORE its archive, so a cut there takes all of it. A text is its +// words, so the cut falls in them. Two cuts in a row end the file where the second one does, and // the cut is said of that one. A damage that cuts nothing has no cut to report. // // Canary: the check below must refuse a cut that names the wrong role, and @@ -48,9 +48,10 @@ func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { cases := []cutCase{ {name: "behind", format: "gif", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, {name: "picture", format: "png", size: 1 << 20, missing: "90", in: "content", kept: "share"}, - // At half: the picture of 256x256 takes about 260 KB at the end of the - // file, and a tenth of a megabyte cut away falls inside it. - {name: "icon", format: "ico", size: 1 << 20, missing: "50", in: "padding", kept: "none", note: "before the content"}, + // 4 KiB on disk is 40 KiB whole, and up to 64 KiB the padding is the + // comment in the gzip header, in front of the archive. An icon or a JPEG + // was this case until it split its padding for the cut. + {name: "front", format: "targz", size: 4 << 10, missing: "90", in: "padding", kept: "none", note: "before the content"}, {name: "text", format: "txt", size: 1000, missing: "50", in: "content", kept: "some"}, {name: "twice", format: "txt", size: 1000, missing: "50", in: "content", kept: "some", cuts: 2}, } @@ -176,7 +177,7 @@ func TestTheRecordOfACutRunIsNeverLongerThanItsForecast(t *testing.T) { } opt := engine.Options{OutDir: t.TempDir(), Seed: 7, Command: "test"} planned, err := engine.Plan([]engine.Target{ - cut("picture", "png", 1<<20, "90"), cut("icon", "ico", 1<<20, "50"), cut("text", "txt", 1000, "50"), + cut("picture", "png", 1<<20, "90"), cut("behind", "gif", 1<<20, "90"), cut("text", "txt", 1000, "50"), }, opt) if err != nil { t.Fatalf("planning: %v", err) diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 81d56409..9f259c1a 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -43,8 +43,6 @@ var notYetCutAtTheShare = map[string]notYet{ "xlsx": {paddingBehind, 64 << 10, 50}, "pptx": {paddingBehind, 64 << 10, 50}, "targz": {paddingBehind, 64 << 10, 50}, - "jpg": {paddingInFront, 64 << 10, 50}, - "ico": {paddingInFront, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the // picture is more than a small share, so a cut of 1% keeps all of it. "webp": {smallBehind, 4 << 10, 1}, @@ -52,9 +50,8 @@ var notYetCutAtTheShare = map[string]notYet{ } const ( - paddingBehind = "its padding stands only behind its content, so a large file cut at any share keeps all of it" - paddingInFront = "its padding stands only in front of its content, so a large file cut at any share keeps none of it" - smallBehind = "in a small file the padding or directory behind its content is more than a small share" + paddingBehind = "its padding stands only behind its content, so a large file cut at any share keeps all of it" + smallBehind = "in a small file the padding or directory behind its content is more than a small share" ) // shareSlack is how far, in points of a percent, the share of the content a diff --git a/internal/guard/filemap_test.go b/internal/guard/filemap_test.go index 2975f9b3..bc5ca398 100644 --- a/internal/guard/filemap_test.go +++ b/internal/guard/filemap_test.go @@ -58,11 +58,19 @@ type mappedFile struct { // content from start to end, which a text is and a picture is not, so a new // binary format that forgets its marks is red here. // +// A format that splits its padding for a cut is also cut at the two ends: +// after the first byte, where the content cannot keep its share with any +// padding in front, and before the last byte, where it keeps it only with all +// of the padding in front. Those are the branches that round the front to +// nothing and to everything (docs/TRUNCATE-2026-11-11.md section 8.9), and +// each has to give the layout it promises as well as read back. +// // Canary: the first map with two regions, with the boundary between them moved // by a byte, must not be taken for the map read back. func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { dir := t.TempDir() var files []mappedFile + ends := 0 for _, f := range format.All() { // A format all of whose settings were skipped would leave the sweep // without it and the guard green, so each one has to reach a file. @@ -90,7 +98,13 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { for _, from := range cuts { cut := healthySizeFrom(t, f, setting, max(smallest, from)) name := fmt.Sprintf("%s-%d-%d-cut", f.ID, n, cut) - files = append(files, writeMapped(t, dir, f, name, setting, cut, cut/2)) + half := writeMapped(t, dir, f, name, setting, cut, cut/2) + files = append(files, half) + if n == 0 && from == cutMapSize && paddingOnBothSides(half.marked) { + at := writeTheEnds(t, dir, f, name, setting, cut) + files = append(files, at...) + ends += len(at) + } } } if len(files) == before { @@ -158,7 +172,36 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { if split == 0 { t.Errorf("no file of %d had padding on both sides of its content, so a layout for a cut was never read back", len(files)) } - t.Logf("%d files, %d with padding, %d of them on both sides of the content", len(files), padded, split) + if ends == 0 { + t.Errorf("no format split its padding at %d B, so the ends of the split were never cut", cutMapSize) + } + // Canary of the sides: a map with padding behind its content alone. + if front, back := paddingSides([][3]any{{string(format.RoleStructure), int64(0), int64(2)}, + {string(format.RoleContent), int64(2), int64(5)}, {string(format.RolePadding), int64(7), int64(3)}}); front || !back { + t.Errorf("canary: padding behind the content alone read as front %v, back %v", front, back) + } + t.Logf("%d files, %d with padding, %d of them on both sides of the content, %d cut at the ends of a split", len(files), padded, split, ends) +} + +// writeTheEnds writes a file of f that splits its padding for a cut, cut +// after its first byte and before its last, and reports each that does not +// put all of its padding behind the content and in front of it. +func writeTheEnds(t *testing.T, dir string, f format.Descriptor, name string, setting map[string]string, size int64) []mappedFile { + t.Helper() + var out []mappedFile + for _, end := range []struct { + name string + at int64 + front, back bool + }{{"first", 1, false, true}, {"last", size - 1, true, false}} { + m := writeMapped(t, dir, f, name+"-"+end.name, setting, size, end.at) + if front, back := paddingSides(m.marked); front != end.front || back != end.back { + t.Errorf("%s: cut after %d B of %d B it marked %s - padding in front %v and behind %v, where %v and %v were due", + m.name, end.at, size, mapNames(m.marked), front, back, end.front, end.back) + } + out = append(out, m) + } + return out } // writeMapped writes one healthy file of f through a map recorder, laid out @@ -270,7 +313,17 @@ func mapNames(m [][3]any) string { // paddingOnBothSides says whether a map has padding before its content and // after it, the layout a format writes for a cut (format.Request.CutAt). func paddingOnBothSides(m [][3]any) bool { + front, back := paddingSides(m) + return front && back +} + +// paddingSides says whether a map has padding in front of its content, and +// whether it has padding behind it. +func paddingSides(m [][3]any) (front, back bool) { at := slices.IndexFunc(m, func(r [3]any) bool { return r[0] == string(format.RoleContent) }) + if at < 0 { + return false, false + } isPadding := func(r [3]any) bool { return r[0] == string(format.RolePadding) } - return at > 0 && slices.ContainsFunc(m[:at], isPadding) && slices.ContainsFunc(m[at+1:], isPadding) + return slices.ContainsFunc(m[:at], isPadding), slices.ContainsFunc(m[at+1:], isPadding) } diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index c6bbf000..52c29cf8 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -465,7 +465,7 @@ ( "an icon points at its picture without counting the padding in front of it", "internal/format/ico/ico.go", - 'binary.LittleEndian.PutUint32(e[12:16], uint32(int64(preamble)+m.gap))', + 'binary.LittleEndian.PutUint32(e[12:16], uint32(int64(preamble)+front))', 'binary.LittleEndian.PutUint32(e[12:16], uint32(preamble))', "TestEveryFormatSurvivesItsReferenceTool", ), diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index 9924c060..f8a7090d 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -428,13 +428,25 @@ def layout_gif(data): def layout_jpg(data): """The start marker, the comment segments, then every segment and the scan - up to the end marker as content, and the end marker.""" + up to the end marker as content, fill bytes in front of a marker as + padding, and the end marker. + + Fill bytes are X'FF' bytes, any number of which may stand in front of any + marker (ITU T.81 B.1.1.2): every FF of a run but the last, which starts the + marker itself.""" if data[:2] != b"\xff\xd8": raise NoMap("no start of image marker") out, pos = [[STRUCTURE, 0, 2]], 2 while pos < len(data): if data[pos] != 0xFF: raise NoMap(f"no marker at {pos}") + run = pos + while run + 1 < len(data) and data[run + 1] == 0xFF: + run += 1 + if run > pos: + out.append([PADDING, pos, run - pos]) + pos = run + continue kind = data[pos + 1] if kind == 0xD9: out.append([END, pos, 2]) @@ -454,14 +466,18 @@ def layout_jpg(data): def layout_ico(data): """The directory, then whatever lies before the offset its one entry gives, - then the picture it counts.""" + then the picture it counts, then whatever lies behind it - space no entry + points at, as the space in front is.""" if data[:4] != b"\x00\x00\x01\x00" or struct.unpack(" len(data): raise NoMap(f"the picture runs from {at} for {size} B and the file is {len(data)} B") - return [[STRUCTURE, 0, head], [PADDING, head, at - head], [CONTENT, at, size]] + out = [[STRUCTURE, 0, head], [PADDING, head, at - head], [CONTENT, at, size]] + if at + size < len(data): + out.append([PADDING, at + size, len(data) - at - size]) + return out def tiff_values(data, kind, count, value_at): From 5c4c79ec3158d7515a7763b005c961591aad0a7c Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 12 Nov 2026 21:31:23 +0100 Subject: [PATCH 2/3] gif holds back its trailer with the shared writer The writer that keeps the last bytes of a stream back moved from png to internal/format, and gif had a copy of it. gif now uses the shared one and writes the same bytes. Co-Authored-By: Claude Opus 5.5 --- internal/format/gif/gif.go | 40 ++++++++------------------------------ 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/internal/format/gif/gif.go b/internal/format/gif/gif.go index fe992a2f..aa21b3f6 100644 --- a/internal/format/gif/gif.go +++ b/internal/format/gif/gif.go @@ -337,7 +337,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // The trailer is always the final byte, so hold it back and let the rest // stream out. Nothing else is buffered. - holder := &tailHolder{w: w, keep: trailerSize} + holder := &format.TailHolder{W: w, Keep: trailerSize} if err := encode(holder, m); err != nil { return err } @@ -345,17 +345,17 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // Planning skipped the encoding, so this is where the exact size // arrives and the padding gets settled - the same arithmetic planning // would have done with the same number. - m.body = holder.written + m.body = holder.Written if err := settlePadding(&m, p.Bytes, m.body+trailerSize); err != nil { // Unreachable unless ladderCeiling is wrong, and then saying so // beats writing a file of the wrong length. return core.Defect(fmt.Errorf("gif: %w - ladderCeiling is wrong", err)) } markAfterThePicture(w, m, p.Bytes, false) - } else if holder.written != m.body { - return core.Defect(fmt.Errorf("gif: the picture encoded to %d B where planning said %d B", holder.written, m.body)) + } else if holder.Written != m.body { + return core.Defect(fmt.Errorf("gif: the picture encoded to %d B where planning said %d B", holder.Written, m.body)) } - if holder.tail[0] != 0x3B { + if holder.Tail[0] != 0x3B { return core.Defect(fmt.Errorf("gif: the encoded stream does not end with the trailer")) } @@ -365,7 +365,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { } } - _, err := w.Write(holder.tail) + _, err := w.Write(holder.Tail) return err } @@ -512,11 +512,11 @@ func chooseSize(r format.Request, label string) (memo, error) { } func encodedBodySize(m memo) (int64, error) { - holder := &tailHolder{w: io.Discard, keep: trailerSize} + holder := &format.TailHolder{W: io.Discard, Keep: trailerSize} if err := encode(holder, m); err != nil { return 0, err } - return holder.written, nil + return holder.Written, nil } // minimumBytes is the smallest file this generator can produce: a one pixel @@ -533,27 +533,3 @@ func minimumBytes() int64 { } return body + trailerSize } - -// tailHolder passes bytes through while keeping the last keep bytes back, so -// the caller can insert something in front of them. -type tailHolder struct { - w io.Writer - keep int - tail []byte - written int64 -} - -func (t *tailHolder) Write(p []byte) (int, error) { - n := len(p) - buf := append(t.tail, p...) - if len(buf) > t.keep { - emit := buf[:len(buf)-t.keep] - if _, err := t.w.Write(emit); err != nil { - return 0, err - } - t.written += int64(len(emit)) - buf = buf[len(buf)-t.keep:] - } - t.tail = append([]byte(nil), buf...) - return n, nil -} From b8b2614d976d2f431bd9b99cf50834d8f173d5cd Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 12 Nov 2026 21:31:24 +0100 Subject: [PATCH 3/3] A format added from now on cuts at the share from its first commit The ratchet of formats whose cut does not take the share of their content yet may name only the thirteen formats listed when the rule was set. A new format is held to every size and share by the guard tools/doctor.py runs for it, and stays red until it splits its padding for the cut. Co-Authored-By: Claude Opus 5.5 --- internal/guard/cutshare_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 9f259c1a..c385bbe3 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -49,6 +49,28 @@ var notYetCutAtTheShare = map[string]notYet{ "tiff": {smallBehind, 4 << 10, 1}, } +// listedBeforeTheRule is every format notYetCutAtTheShare may name: the ones +// that missed the share on 2026-11-12, when the owner decided that every format +// cuts at the share before the next release and that a format added later does +// from its first commit (docs/TRUNCATE-2026-11-11.md section 3a). It never +// grows, so a new format cannot wait on the ratchet - tools/doctor.py runs this +// guard for it, and it stays red until the format splits its padding for +// format.Request.CutAt. +var listedBeforeTheRule = []string{"gif", "avif", "jxl", "mp4", "webm", "pdf", "zip", "docx", "xlsx", "pptx", "targz", "webp", "tiff"} + +// listedAfterTheRule is a problem for every format notYetAt names that +// listedBeforeTheRule does not. +func listedAfterTheRule(notYetAt map[string]notYet) []string { + var out []string + for id := range notYetAt { + if !slices.Contains(listedBeforeTheRule, id) { + out = append(out, id+": listed in notYetCutAtTheShare after 2026-11-12 - a format added since cuts at the share from its first commit, so split its padding for format.Request.CutAt instead (docs/TRUNCATE-2026-11-11.md section 3a)") + } + } + slices.Sort(out) + return out +} + const ( paddingBehind = "its padding stands only behind its content, so a large file cut at any share keeps all of it" smallBehind = "in a small file the padding or directory behind its content is more than a small share" @@ -123,6 +145,12 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { for _, problem := range shareVerdicts(problems, notYetCutAtTheShare) { t.Error(problem) } + for _, problem := range listedAfterTheRule(notYetCutAtTheShare) { + t.Error(problem) + } + if len(listedAfterTheRule(map[string]notYet{"newformat": {paddingBehind, 64 << 10, 50}})) != 1 { + t.Errorf("canary: a format listed that was not listed on 2026-11-12 passed") + } // Canaries. whole := int64(1000)