diff --git a/CHANGELOG.md b/CHANGELOG.md index 3244b77b..688fde30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,12 +80,13 @@ because it turns other people's test suites red. the length of the content is known. A cut in the padding also gets the note `cut_in_padding`. In a compressed tar.gz a cut in the padding is reported as a cut in the content. -- **`truncate` cuts a PNG's picture by the percentage you set.** - `--damage truncate:missing=90` on a 1 MB PNG used to cut only padding and - leave the whole picture. Now it removes 90% of the picture's content as - well, so a viewer that shows what is left shows only the top of the - picture, the way an interrupted upload leaves it. Files nothing cuts are - unchanged. +- **`truncate` cuts the picture of a PNG, a JPEG or an icon by the + percentage you set.** `--damage truncate:missing=90` on a 1 MB PNG used to + cut only padding and leave the whole picture, and on a JPEG or a large + icon it removed the whole picture whatever percentage you set. Now it + removes 90% of the picture's content, so a viewer that shows what is left + shows the part of the picture that arrived, the way an interrupted upload + leaves it. Files nothing cuts are unchanged. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/format/cut.go b/internal/format/cut.go new file mode 100644 index 00000000..4dbe456b --- /dev/null +++ b/internal/format/cut.go @@ -0,0 +1,53 @@ +package format + +import ( + "io" + "math/bits" +) + +// PaddingInFront is how much padding goes in front of a file's content so that +// a cut after its first cutAt bytes keeps the same share of the content as of +// the whole file (Request.CutAt): cutAt, less the before bytes that stand in +// front of that padding, less content*cutAt/whole rounded down. +// +// It is the answer before a format rounds it to what it can write, and zero or +// less means that even with no padding in front the cut keeps no more than the +// share. Each format that splits its padding around its content for a cut asks +// this, so the share is one rule (docs/TRUNCATE-2026-11-11.md sections 7.4 and +// 8.5). +// +// content*cutAt is worked out in 128 bits, because a large picture times a cut +// near the top of what a file can be passes an int64. The quotient fits as +// long as 0 < cutAt < whole and content <= whole, which the caller sees to. +func PaddingInFront(before, content, cutAt, whole int64) int64 { + hi, lo := bits.Mul64(uint64(content), uint64(cutAt)) + kept, _ := bits.Div64(hi, lo, uint64(whole)) + return cutAt - before - int64(kept) +} + +// TailHolder passes a stream on while keeping its last Keep bytes back, so a +// format can write something in front of them once the stream ends - padding +// in front of the end marker an encoder writes last. +type TailHolder struct { + W io.Writer + Keep int + // Tail is the bytes held back, Keep of them once that many have come. + Tail []byte + // Written is how many bytes went on to W. + Written int64 +} + +func (t *TailHolder) Write(p []byte) (int, error) { + n := len(p) + buf := append(t.Tail, p...) + if len(buf) > t.Keep { + emit := buf[:len(buf)-t.Keep] + if _, err := t.W.Write(emit); err != nil { + return 0, err + } + t.Written += int64(len(emit)) + buf = buf[len(buf)-t.Keep:] + } + t.Tail = append([]byte(nil), buf...) + return n, nil +} diff --git a/internal/format/gif/gif.go b/internal/format/gif/gif.go index fe992a2f..aa21b3f6 100644 --- a/internal/format/gif/gif.go +++ b/internal/format/gif/gif.go @@ -337,7 +337,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // The trailer is always the final byte, so hold it back and let the rest // stream out. Nothing else is buffered. - holder := &tailHolder{w: w, keep: trailerSize} + holder := &format.TailHolder{W: w, Keep: trailerSize} if err := encode(holder, m); err != nil { return err } @@ -345,17 +345,17 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // Planning skipped the encoding, so this is where the exact size // arrives and the padding gets settled - the same arithmetic planning // would have done with the same number. - m.body = holder.written + m.body = holder.Written if err := settlePadding(&m, p.Bytes, m.body+trailerSize); err != nil { // Unreachable unless ladderCeiling is wrong, and then saying so // beats writing a file of the wrong length. return core.Defect(fmt.Errorf("gif: %w - ladderCeiling is wrong", err)) } markAfterThePicture(w, m, p.Bytes, false) - } else if holder.written != m.body { - return core.Defect(fmt.Errorf("gif: the picture encoded to %d B where planning said %d B", holder.written, m.body)) + } else if holder.Written != m.body { + return core.Defect(fmt.Errorf("gif: the picture encoded to %d B where planning said %d B", holder.Written, m.body)) } - if holder.tail[0] != 0x3B { + if holder.Tail[0] != 0x3B { return core.Defect(fmt.Errorf("gif: the encoded stream does not end with the trailer")) } @@ -365,7 +365,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { } } - _, err := w.Write(holder.tail) + _, err := w.Write(holder.Tail) return err } @@ -512,11 +512,11 @@ func chooseSize(r format.Request, label string) (memo, error) { } func encodedBodySize(m memo) (int64, error) { - holder := &tailHolder{w: io.Discard, keep: trailerSize} + holder := &format.TailHolder{W: io.Discard, Keep: trailerSize} if err := encode(holder, m); err != nil { return 0, err } - return holder.written, nil + return holder.Written, nil } // minimumBytes is the smallest file this generator can produce: a one pixel @@ -533,27 +533,3 @@ func minimumBytes() int64 { } return body + trailerSize } - -// tailHolder passes bytes through while keeping the last keep bytes back, so -// the caller can insert something in front of them. -type tailHolder struct { - w io.Writer - keep int - tail []byte - written int64 -} - -func (t *tailHolder) Write(p []byte) (int, error) { - n := len(p) - buf := append(t.tail, p...) - if len(buf) > t.keep { - emit := buf[:len(buf)-t.keep] - if _, err := t.w.Write(emit); err != nil { - return 0, err - } - t.written += int64(len(emit)) - buf = buf[len(buf)-t.keep:] - } - t.tail = append([]byte(nil), buf...) - return n, nil -} diff --git a/internal/format/ico/ico.go b/internal/format/ico/ico.go index 65f66054..3c5c6241 100644 --- a/internal/format/ico/ico.go +++ b/internal/format/ico/ico.go @@ -3,7 +3,9 @@ // An ICO is a directory of images rather than an image, so it carries an // explicit offset saying where each one starts. That offset is the padding // channel: anything between the directory and the picture is space the file -// describes. +// describes. A file a damage will cut has part of that space behind the +// picture instead, where no entry points either, so the cut takes the share +// of the picture asked for (frontGap). // // The picture inside can be a device independent bitmap or a whole PNG, and // both are in use. This generator writes either, because which one a system @@ -105,8 +107,12 @@ type memo struct { // image is the picture as it will sit inside the file. Small by // construction, because an icon side is at most 256 pixels. image []byte - // gap is how many bytes stand between the directory and that picture. + // gap is how many bytes of padding the file carries: all of them between + // the directory and the picture, unless a damage will cut the file. gap int64 + // cutAt is where a damage will cut the file, zero when nothing does + // (format.Request.CutAt). + cutAt int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -137,7 +143,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { m := memo{ width: w, height: h, seed: r.Seed, label: label, embed: embed, - image: blob, gap: r.Bytes - bare, + image: blob, gap: r.Bytes - bare, cutAt: r.CutAt, } p := format.Plan{ @@ -250,25 +256,54 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } - // The map, all of it known from the plan: the directory, the gap that - // makes up the size, and the picture. The gap stands BEFORE the picture, - // so a cut takes the whole picture whatever share is asked for - // (docs/TRUNCATE-2026-11-11.md section 2). + // The map, all of it known from the plan: the directory, the gap in front + // of the picture, the picture, and the rest of the gap behind it when a + // damage will cut the file. + front := frontGap(m, p.Bytes) + picture := int64(len(m.image)) format.Mark(w, format.RoleStructure, 0, preamble) - format.Mark(w, format.RolePadding, preamble, m.gap) - format.Mark(w, format.RoleContent, preamble+m.gap, int64(len(m.image))) + format.Mark(w, format.RolePadding, preamble, front) + format.Mark(w, format.RoleContent, preamble+front, picture) + if back := m.gap - front; back > 0 { + format.Mark(w, format.RolePadding, preamble+front+picture, back) + } - if err := writeDirectory(w, m); err != nil { + if err := writeDirectory(w, m, front); err != nil { return err } - if err := writeGap(ctx, w, m.seed, m.gap); err != nil { + if err := writeGap(ctx, w, m.seed, front); err != nil { return err } - _, err := w.Write(m.image) - return err + if _, err := w.Write(m.image); err != nil { + return err + } + return writeGap(ctx, w, m.seed, m.gap-front) +} + +// frontGap is how much of the gap stands between the directory and the +// picture: all of it, unless a damage will cut the file +// (format.Request.CutAt). Then as much as puts the cut where it keeps the +// share of the picture it keeps of the whole file, and the rest goes behind +// the picture. +// +// With all of the gap in front, a large icon cut at any share lost the whole +// picture. Bytes behind the picture are space no entry points at, as the gap +// is, and Pillow, FFmpeg, ImageMagick, the Windows Imaging Component and +// GDI+ return the same pixels either way (docs/TRUNCATE-2026-11-11.md +// section 8.3). The gap has no header, so the cut lands on the byte - except +// with too small a gap, where none of it goes in front and the cut takes more +// of the picture, the "at least" missing promises. It never needs more than +// the whole gap in front (section 8.5). +func frontGap(m memo, whole int64) int64 { + if m.cutAt <= 0 || m.cutAt >= whole { + return m.gap + } + return max(0, format.PaddingInFront(preamble, int64(len(m.image)), m.cutAt, whole)) } -func writeDirectory(w io.Writer, m memo) error { +// writeDirectory writes the directory of one entry, whose picture starts +// front bytes after it. +func writeDirectory(w io.Writer, m memo, front int64) error { var head [preamble]byte // Two reserved bytes stay zero, then the type: 1 means icon. binary.LittleEndian.PutUint16(head[2:4], 1) @@ -281,7 +316,7 @@ func writeDirectory(w io.Writer, m memo) error { binary.LittleEndian.PutUint16(e[4:6], 1) binary.LittleEndian.PutUint16(e[6:8], 32) binary.LittleEndian.PutUint32(e[8:12], uint32(len(m.image))) - binary.LittleEndian.PutUint32(e[12:16], uint32(int64(preamble)+m.gap)) + binary.LittleEndian.PutUint32(e[12:16], uint32(int64(preamble)+front)) _, err := w.Write(head[:]) return err diff --git a/internal/format/jpg/cut.go b/internal/format/jpg/cut.go new file mode 100644 index 00000000..cafc1933 --- /dev/null +++ b/internal/format/jpg/cut.go @@ -0,0 +1,129 @@ +package jpg + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// writeForACut writes a file a damage will cut after its first m.cutAt bytes, +// with the padding split around the picture so that the cut keeps the share of +// the picture it keeps of the whole file (format.Request.CutAt). +// +// With all of the padding in front of the picture, as an uncut file has it, a +// cut of any share took the whole picture. Behind the picture JPEG has one +// place for bytes that mean nothing: fill bytes, X'FF', which ITU T.81 B.1.1.2 +// lets stand in front of any marker - here the end of image marker. A comment +// behind the last scan is outside the syntax (B.2.1), and bytes after the end +// marker are outside the picture. Pillow, FFmpeg, ImageMagick, Go's own +// decoder, the Windows Imaging Component and GDI+ return the same pixels +// either way (docs/TRUNCATE-2026-11-11.md section 8.3). +// +// The fill is in no file anyone is handed: the cut falls in the picture, and +// the generator stops there. Only a whole file written for a guard has it. +func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { + pad := whole - m.body + front := frontPadding(pad, m.body-soiSize-eoiSize, m.cutAt, whole) + markAroundThePicture(w, m.body, front, pad-front) + + if _, err := w.Write([]byte{0xFF, 0xD8}); err != nil { + return err + } + if front > 0 { + segments, payload := segmentsFor(front) + if err := writeComments(ctx, w, m.seed, segments, payload); err != nil { + return err + } + } + + // The encoder's own start marker is dropped, as in Write, and its end + // marker held back, so the fill goes in front of it. + holder := &format.TailHolder{W: w, Keep: eoiSize} + skip := &headSkipper{w: holder, keep: soiSize} + if err := encode(skip, m); err != nil { + return err + } + if err := skip.check(); err != nil { + return err + } + if skip.written+soiSize != m.body { + return core.Defect(fmt.Errorf("jpg: the picture encoded to %d B where planning said %d B", skip.written+soiSize, m.body)) + } + if !bytes.Equal(holder.Tail, []byte{0xFF, 0xD9}) { + return core.Defect(fmt.Errorf("jpg: the encoded stream ends with % x, not the end of image marker", holder.Tail)) + } + if err := writeFill(ctx, w, pad-front); err != nil { + return err + } + _, err := w.Write(holder.Tail) + return err +} + +// frontPadding is how much of the padding, comment headers included, goes in +// front of the picture, so that a cut after the first cutAt bytes of a whole +// file keeps content*cutAt/whole bytes of the picture's content. +// +// A comment costs four bytes before any padding, so the front is nothing or at +// least that. Where it cannot land exactly it grows, which moves the picture +// later and cuts at most three bytes more of it - the "at least" missing +// promises. The fill behind the picture has no header, so any length is +// possible there. The one way the cut can keep more is padding too small for +// the front it should have: all of it goes in front, and the cut keeps at most +// two bytes more than the share, because the end marker counts in the file and +// not in the content (docs/TRUNCATE-2026-11-11.md section 8.5). The manifest's +// cut reports what was kept either way. +func frontPadding(pad, content, cutAt, whole int64) int64 { + front := format.PaddingInFront(soiSize, content, cutAt, whole) + switch { + case front <= 0: + return 0 + case front < comOverhead: + front = comOverhead + } + return min(front, pad) +} + +// markAroundThePicture marks the map of a file whose padding is split around +// its picture: the start marker, the comments in front, the picture, the fill +// behind and the end marker. A side with no padding has no region. +func markAroundThePicture(w io.Writer, body, front, back int64) { + format.Mark(w, format.RoleStructure, 0, soiSize) + at := int64(soiSize) + if front > 0 { + format.Mark(w, format.RolePadding, at, front) + at += front + } + format.Mark(w, format.RoleContent, at, body-soiSize-eoiSize) + at += body - soiSize - eoiSize + if back > 0 { + format.Mark(w, format.RolePadding, at, back) + at += back + } + format.Mark(w, format.RoleEnd, at, eoiSize) +} + +// writeFill writes n fill bytes, X'FF', in pieces of padChunkSize so that +// cancellation is noticed. +func writeFill(ctx context.Context, w io.Writer, n int64) error { + if n <= 0 { + return nil + } + buf := bytes.Repeat([]byte{0xFF}, int(min(n, padChunkSize))) + for remaining := n; remaining > 0; { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + k := min(remaining, int64(len(buf))) + if _, err := w.Write(buf[:k]); err != nil { + return err + } + remaining -= k + } + return nil +} diff --git a/internal/format/jpg/jpg.go b/internal/format/jpg/jpg.go index fda4cd08..37884806 100644 --- a/internal/format/jpg/jpg.go +++ b/internal/format/jpg/jpg.go @@ -9,7 +9,9 @@ // // Like PNG, the encoded size does not follow from the content, so the exact // size comes from padding. Unlike PNG, the padding goes at the FRONT and may -// need several segments, because one comment carries 65533 bytes at most. +// need several segments, because one comment carries 65533 bytes at most. A +// file a damage will cut has part of it behind the picture as well, so the cut +// takes the share of the picture asked for (cut.go). package jpg import ( @@ -104,6 +106,9 @@ func init() { // // Tk 9.0.4 is not on that list of readers and that is measured // too: it has no JPEG decoder at all. + // + // A file a damage will cut also has fill bytes in front of the + // end of image marker, measured against seven readers (cut.go). Name: "comment segment after the start of image marker", Where: format.PlacementStart, // No limit of its own. One segment carries 65533 B and this @@ -151,6 +156,10 @@ type memo struct { // this generator always writes at least one, which is what removes the // unreachable band every other format has above its floor. segments int + // cutAt is where a damage will cut the file, zero when nothing does + // (format.Request.CutAt). writeForACut splits the padding around the + // picture for it. + cutAt int64 } // checkJointLimits asks the registry's own declaration about a pair of @@ -206,6 +215,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { if err != nil { return format.Plan{}, err } + m.cutAt = r.CutAt w, h := m.width, m.height bare := m.body @@ -287,6 +297,9 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return ctx.Err() default: } + if m.cutAt > 0 && m.cutAt < p.Bytes { + return writeForACut(ctx, w, m, p.Bytes) + } // The map, all of it known from the plan: the start marker, the comments // that make up the size, the picture - its tables, frame and scan as one - diff --git a/internal/format/png/cut.go b/internal/format/png/cut.go index 28b9bc33..367559a3 100644 --- a/internal/format/png/cut.go +++ b/internal/format/png/cut.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "math/bits" "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" @@ -45,14 +44,14 @@ func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { // Both padding chunks draw from the file's seed. Nobody reads padding, and // the one behind the picture lies past any cut. split := &afterTheHead{ctx: ctx, w: w, seed: m.seed, head: headSize, front: front} - holder := &tailHolder{w: split, keep: iendSize} + holder := &format.TailHolder{W: split, Keep: iendSize} if err := encode(holder, m); err != nil { return err } - if holder.written != m.body { - return core.Defect(fmt.Errorf("png: the picture encoded to %d B where planning said %d B", holder.written, m.body)) + if holder.Written != m.body { + return core.Defect(fmt.Errorf("png: the picture encoded to %d B where planning said %d B", holder.Written, m.body)) } - if string(holder.tail[4:8]) != "IEND" { + if string(holder.Tail[4:8]) != "IEND" { return core.Defect(fmt.Errorf("png: the encoded stream does not end with IEND")) } if back := pad - front; back > 0 { @@ -60,7 +59,7 @@ func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { return err } } - _, err := w.Write(holder.tail) + _, err := w.Write(holder.Tail) return err } @@ -118,12 +117,7 @@ func frontPadding(pad, content, cutAt, whole int64) int64 { if pad == 0 || cutAt <= 0 || cutAt >= whole { return 0 } - // content*cutAt in 128 bits, because a file near the top of what one chunk - // carries times a cut near it passes an int64. The quotient fits, because - // cutAt is less than whole. - hi, lo := bits.Mul64(uint64(content), uint64(cutAt)) - kept, _ := bits.Div64(hi, lo, uint64(whole)) - front := cutAt - headSize - int64(kept) + front := format.PaddingInFront(headSize, content, cutAt, whole) switch { case front <= 0: return 0 diff --git a/internal/format/png/png.go b/internal/format/png/png.go index 935270ea..a2c070b0 100644 --- a/internal/format/png/png.go +++ b/internal/format/png/png.go @@ -300,7 +300,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // The closing chunk is always the final twelve bytes, so hold those back // and let everything before them stream straight out. Nothing is buffered // beyond that, which matters because a picture can be very large. - holder := &tailHolder{w: w, keep: iendSize} + holder := &format.TailHolder{W: w, Keep: iendSize} if err := encode(holder, m); err != nil { return err } @@ -310,7 +310,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // largest rung, so this is where the exact size arrives. The padding is // whatever is left, which is the same arithmetic planning would have // done with the same number. - m.body = holder.written + m.body = holder.Written m.padData = p.Bytes - m.body - iendSize - chunkOverhead if m.padData < 0 || m.padData > maxChunkData { // Unreachable unless ladderCeiling is wrong, and then it is better @@ -320,10 +320,10 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { m.body, m.padData, p.Bytes)) } markAfterThePicture(w, m, false) - } else if holder.written != m.body { - return core.Defect(fmt.Errorf("png: the picture encoded to %d B where planning said %d B", holder.written, m.body)) + } else if holder.Written != m.body { + return core.Defect(fmt.Errorf("png: the picture encoded to %d B where planning said %d B", holder.Written, m.body)) } - if string(holder.tail[4:8]) != "IEND" { + if string(holder.Tail[4:8]) != "IEND" { return core.Defect(fmt.Errorf("png: the encoded stream does not end with IEND")) } @@ -333,7 +333,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { } } - _, err := w.Write(holder.tail) + _, err := w.Write(holder.Tail) return err } @@ -501,11 +501,11 @@ func encode(w io.Writer, m memo) error { // couple of milliseconds. Nothing is held in memory - the bytes are counted // and dropped. func encodedBodySize(m memo) (int64, error) { - holder := &tailHolder{w: io.Discard, keep: iendSize} + holder := &format.TailHolder{W: io.Discard, Keep: iendSize} if err := encode(holder, m); err != nil { return 0, err } - return holder.written, nil + return holder.Written, nil } // minimumBytes is the smallest file this generator can produce: a one pixel @@ -581,27 +581,3 @@ func writePaddingChunk(ctx context.Context, w io.Writer, kind string, seed uint6 _, err := w.Write(tail[:]) return err } - -// tailHolder passes bytes through while keeping the last keep bytes back, so -// the caller can insert something in front of them. -type tailHolder struct { - w io.Writer - keep int - tail []byte - written int64 -} - -func (t *tailHolder) Write(p []byte) (int, error) { - n := len(p) - buf := append(t.tail, p...) - if len(buf) > t.keep { - emit := buf[:len(buf)-t.keep] - if _, err := t.w.Write(emit); err != nil { - return 0, err - } - t.written += int64(len(emit)) - buf = buf[len(buf)-t.keep:] - } - t.tail = append([]byte(nil), buf...) - return n, nil -} diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index 6095025e..aa911441 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -34,12 +34,12 @@ type cutCase struct { // A file cut short says in its manifest where the cut fell and how much of // the content is left, and gets a note when the cut missed the content. // -// The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2 and 7). A -// GIF of 1 MiB is nearly all padding after its picture, so 90% missing still -// leaves the whole picture. A PNG splits its padding around the picture for a -// cut, so the cut takes the share asked for. An icon keeps its padding BEFORE -// its picture, so a cut there takes all of it. A text is its words, so the cut -// falls in them. Two cuts in a row end the file where the second one does, and +// The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2, 7 and 8). +// A GIF of 1 MiB is nearly all padding after its picture, so 90% missing +// still leaves the whole picture. A PNG splits its padding around the picture +// for a cut, so the cut takes the share asked for. A small tar.gz keeps its +// padding BEFORE its archive, so a cut there takes all of it. A text is its +// words, so the cut falls in them. Two cuts in a row end the file where the second one does, and // the cut is said of that one. A damage that cuts nothing has no cut to report. // // Canary: the check below must refuse a cut that names the wrong role, and @@ -48,9 +48,10 @@ func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { cases := []cutCase{ {name: "behind", format: "gif", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, {name: "picture", format: "png", size: 1 << 20, missing: "90", in: "content", kept: "share"}, - // At half: the picture of 256x256 takes about 260 KB at the end of the - // file, and a tenth of a megabyte cut away falls inside it. - {name: "icon", format: "ico", size: 1 << 20, missing: "50", in: "padding", kept: "none", note: "before the content"}, + // 4 KiB on disk is 40 KiB whole, and up to 64 KiB the padding is the + // comment in the gzip header, in front of the archive. An icon or a JPEG + // was this case until it split its padding for the cut. + {name: "front", format: "targz", size: 4 << 10, missing: "90", in: "padding", kept: "none", note: "before the content"}, {name: "text", format: "txt", size: 1000, missing: "50", in: "content", kept: "some"}, {name: "twice", format: "txt", size: 1000, missing: "50", in: "content", kept: "some", cuts: 2}, } @@ -176,7 +177,7 @@ func TestTheRecordOfACutRunIsNeverLongerThanItsForecast(t *testing.T) { } opt := engine.Options{OutDir: t.TempDir(), Seed: 7, Command: "test"} planned, err := engine.Plan([]engine.Target{ - cut("picture", "png", 1<<20, "90"), cut("icon", "ico", 1<<20, "50"), cut("text", "txt", 1000, "50"), + cut("picture", "png", 1<<20, "90"), cut("behind", "gif", 1<<20, "90"), cut("text", "txt", 1000, "50"), }, opt) if err != nil { t.Fatalf("planning: %v", err) diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 81d56409..c385bbe3 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -43,18 +43,37 @@ var notYetCutAtTheShare = map[string]notYet{ "xlsx": {paddingBehind, 64 << 10, 50}, "pptx": {paddingBehind, 64 << 10, 50}, "targz": {paddingBehind, 64 << 10, 50}, - "jpg": {paddingInFront, 64 << 10, 50}, - "ico": {paddingInFront, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the // picture is more than a small share, so a cut of 1% keeps all of it. "webp": {smallBehind, 4 << 10, 1}, "tiff": {smallBehind, 4 << 10, 1}, } +// listedBeforeTheRule is every format notYetCutAtTheShare may name: the ones +// that missed the share on 2026-11-12, when the owner decided that every format +// cuts at the share before the next release and that a format added later does +// from its first commit (docs/TRUNCATE-2026-11-11.md section 3a). It never +// grows, so a new format cannot wait on the ratchet - tools/doctor.py runs this +// guard for it, and it stays red until the format splits its padding for +// format.Request.CutAt. +var listedBeforeTheRule = []string{"gif", "avif", "jxl", "mp4", "webm", "pdf", "zip", "docx", "xlsx", "pptx", "targz", "webp", "tiff"} + +// listedAfterTheRule is a problem for every format notYetAt names that +// listedBeforeTheRule does not. +func listedAfterTheRule(notYetAt map[string]notYet) []string { + var out []string + for id := range notYetAt { + if !slices.Contains(listedBeforeTheRule, id) { + out = append(out, id+": listed in notYetCutAtTheShare after 2026-11-12 - a format added since cuts at the share from its first commit, so split its padding for format.Request.CutAt instead (docs/TRUNCATE-2026-11-11.md section 3a)") + } + } + slices.Sort(out) + return out +} + const ( - paddingBehind = "its padding stands only behind its content, so a large file cut at any share keeps all of it" - paddingInFront = "its padding stands only in front of its content, so a large file cut at any share keeps none of it" - smallBehind = "in a small file the padding or directory behind its content is more than a small share" + paddingBehind = "its padding stands only behind its content, so a large file cut at any share keeps all of it" + smallBehind = "in a small file the padding or directory behind its content is more than a small share" ) // shareSlack is how far, in points of a percent, the share of the content a @@ -126,6 +145,12 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { for _, problem := range shareVerdicts(problems, notYetCutAtTheShare) { t.Error(problem) } + for _, problem := range listedAfterTheRule(notYetCutAtTheShare) { + t.Error(problem) + } + if len(listedAfterTheRule(map[string]notYet{"newformat": {paddingBehind, 64 << 10, 50}})) != 1 { + t.Errorf("canary: a format listed that was not listed on 2026-11-12 passed") + } // Canaries. whole := int64(1000) diff --git a/internal/guard/filemap_test.go b/internal/guard/filemap_test.go index 2975f9b3..bc5ca398 100644 --- a/internal/guard/filemap_test.go +++ b/internal/guard/filemap_test.go @@ -58,11 +58,19 @@ type mappedFile struct { // content from start to end, which a text is and a picture is not, so a new // binary format that forgets its marks is red here. // +// A format that splits its padding for a cut is also cut at the two ends: +// after the first byte, where the content cannot keep its share with any +// padding in front, and before the last byte, where it keeps it only with all +// of the padding in front. Those are the branches that round the front to +// nothing and to everything (docs/TRUNCATE-2026-11-11.md section 8.9), and +// each has to give the layout it promises as well as read back. +// // Canary: the first map with two regions, with the boundary between them moved // by a byte, must not be taken for the map read back. func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { dir := t.TempDir() var files []mappedFile + ends := 0 for _, f := range format.All() { // A format all of whose settings were skipped would leave the sweep // without it and the guard green, so each one has to reach a file. @@ -90,7 +98,13 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { for _, from := range cuts { cut := healthySizeFrom(t, f, setting, max(smallest, from)) name := fmt.Sprintf("%s-%d-%d-cut", f.ID, n, cut) - files = append(files, writeMapped(t, dir, f, name, setting, cut, cut/2)) + half := writeMapped(t, dir, f, name, setting, cut, cut/2) + files = append(files, half) + if n == 0 && from == cutMapSize && paddingOnBothSides(half.marked) { + at := writeTheEnds(t, dir, f, name, setting, cut) + files = append(files, at...) + ends += len(at) + } } } if len(files) == before { @@ -158,7 +172,36 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { if split == 0 { t.Errorf("no file of %d had padding on both sides of its content, so a layout for a cut was never read back", len(files)) } - t.Logf("%d files, %d with padding, %d of them on both sides of the content", len(files), padded, split) + if ends == 0 { + t.Errorf("no format split its padding at %d B, so the ends of the split were never cut", cutMapSize) + } + // Canary of the sides: a map with padding behind its content alone. + if front, back := paddingSides([][3]any{{string(format.RoleStructure), int64(0), int64(2)}, + {string(format.RoleContent), int64(2), int64(5)}, {string(format.RolePadding), int64(7), int64(3)}}); front || !back { + t.Errorf("canary: padding behind the content alone read as front %v, back %v", front, back) + } + t.Logf("%d files, %d with padding, %d of them on both sides of the content, %d cut at the ends of a split", len(files), padded, split, ends) +} + +// writeTheEnds writes a file of f that splits its padding for a cut, cut +// after its first byte and before its last, and reports each that does not +// put all of its padding behind the content and in front of it. +func writeTheEnds(t *testing.T, dir string, f format.Descriptor, name string, setting map[string]string, size int64) []mappedFile { + t.Helper() + var out []mappedFile + for _, end := range []struct { + name string + at int64 + front, back bool + }{{"first", 1, false, true}, {"last", size - 1, true, false}} { + m := writeMapped(t, dir, f, name+"-"+end.name, setting, size, end.at) + if front, back := paddingSides(m.marked); front != end.front || back != end.back { + t.Errorf("%s: cut after %d B of %d B it marked %s - padding in front %v and behind %v, where %v and %v were due", + m.name, end.at, size, mapNames(m.marked), front, back, end.front, end.back) + } + out = append(out, m) + } + return out } // writeMapped writes one healthy file of f through a map recorder, laid out @@ -270,7 +313,17 @@ func mapNames(m [][3]any) string { // paddingOnBothSides says whether a map has padding before its content and // after it, the layout a format writes for a cut (format.Request.CutAt). func paddingOnBothSides(m [][3]any) bool { + front, back := paddingSides(m) + return front && back +} + +// paddingSides says whether a map has padding in front of its content, and +// whether it has padding behind it. +func paddingSides(m [][3]any) (front, back bool) { at := slices.IndexFunc(m, func(r [3]any) bool { return r[0] == string(format.RoleContent) }) + if at < 0 { + return false, false + } isPadding := func(r [3]any) bool { return r[0] == string(format.RolePadding) } - return at > 0 && slices.ContainsFunc(m[:at], isPadding) && slices.ContainsFunc(m[at+1:], isPadding) + return slices.ContainsFunc(m[:at], isPadding), slices.ContainsFunc(m[at+1:], isPadding) } diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index c6bbf000..52c29cf8 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -465,7 +465,7 @@ ( "an icon points at its picture without counting the padding in front of it", "internal/format/ico/ico.go", - 'binary.LittleEndian.PutUint32(e[12:16], uint32(int64(preamble)+m.gap))', + 'binary.LittleEndian.PutUint32(e[12:16], uint32(int64(preamble)+front))', 'binary.LittleEndian.PutUint32(e[12:16], uint32(preamble))', "TestEveryFormatSurvivesItsReferenceTool", ), diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index 9924c060..f8a7090d 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -428,13 +428,25 @@ def layout_gif(data): def layout_jpg(data): """The start marker, the comment segments, then every segment and the scan - up to the end marker as content, and the end marker.""" + up to the end marker as content, fill bytes in front of a marker as + padding, and the end marker. + + Fill bytes are X'FF' bytes, any number of which may stand in front of any + marker (ITU T.81 B.1.1.2): every FF of a run but the last, which starts the + marker itself.""" if data[:2] != b"\xff\xd8": raise NoMap("no start of image marker") out, pos = [[STRUCTURE, 0, 2]], 2 while pos < len(data): if data[pos] != 0xFF: raise NoMap(f"no marker at {pos}") + run = pos + while run + 1 < len(data) and data[run + 1] == 0xFF: + run += 1 + if run > pos: + out.append([PADDING, pos, run - pos]) + pos = run + continue kind = data[pos + 1] if kind == 0xD9: out.append([END, pos, 2]) @@ -454,14 +466,18 @@ def layout_jpg(data): def layout_ico(data): """The directory, then whatever lies before the offset its one entry gives, - then the picture it counts.""" + then the picture it counts, then whatever lies behind it - space no entry + points at, as the space in front is.""" if data[:4] != b"\x00\x00\x01\x00" or struct.unpack(" len(data): raise NoMap(f"the picture runs from {at} for {size} B and the file is {len(data)} B") - return [[STRUCTURE, 0, head], [PADDING, head, at - head], [CONTENT, at, size]] + out = [[STRUCTURE, 0, head], [PADDING, head, at - head], [CONTENT, at, size]] + if at + size < len(data): + out.append([PADDING, at + size, len(data) - at - size]) + return out def tiff_values(data, kind, count, value_at):