From a1691c7b852ea0cc10516998f77e7d0436ceded7 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 12 Nov 2026 22:08:17 +0100 Subject: [PATCH] A GIF cut short loses the share of its picture asked for A file truncate will cut has its padding split into a comment in front of the first frame and one in front of the trailer, so the cut falls in the frames at the share asked for. Until now the whole padding stood behind the frames, so from 20 KiB up a cut at any share lost only padding. GIF89a lets a comment stand wherever a block can begin, but not in front of the colour table, so the comment goes after the head the encoder writes - screen descriptor, colour table and loop extension - and the writer reads that head back before the comment goes in. The map now counts the head as structure and the frames as content: with the colour table as content, a small picture cut at 90% kept the table and no pixel. png's insert after the head moved to format.AfterTheHead. png bytes and uncut gif bytes are unchanged. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 +- internal/format/cut.go | 39 ++++++ internal/format/gif/cut.go | 187 +++++++++++++++++++++++++++++ internal/format/gif/gif.go | 28 +++-- internal/format/png/cut.go | 49 ++------ internal/guard/cutmanifest_test.go | 9 +- internal/guard/cutshare_test.go | 6 +- internal/oracle/layout.py | 19 ++- 8 files changed, 280 insertions(+), 65 deletions(-) create mode 100644 internal/format/gif/cut.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 688fde30..6d2ae910 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,10 +80,10 @@ because it turns other people's test suites red. the length of the content is known. A cut in the padding also gets the note `cut_in_padding`. In a compressed tar.gz a cut in the padding is reported as a cut in the content. -- **`truncate` cuts the picture of a PNG, a JPEG or an icon by the - percentage you set.** `--damage truncate:missing=90` on a 1 MB PNG used to - cut only padding and leave the whole picture, and on a JPEG or a large - icon it removed the whole picture whatever percentage you set. Now it +- **`truncate` cuts the picture of a PNG, a GIF, a JPEG or an icon by the + percentage you set.** `--damage truncate:missing=90` on a 1 MB PNG or GIF + used to cut only padding and leave the whole picture, and on a JPEG or a + large icon it removed the whole picture whatever percentage you set. Now it removes 90% of the picture's content, so a viewer that shows what is left shows the part of the picture that arrived, the way an interrupted upload leaves it. Files nothing cuts are unchanged. diff --git a/internal/format/cut.go b/internal/format/cut.go index 4dbe456b..5cba050c 100644 --- a/internal/format/cut.go +++ b/internal/format/cut.go @@ -25,6 +25,45 @@ func PaddingInFront(before, content, cutAt, whole int64) int64 { return cutAt - before - int64(kept) } +// AfterTheHead passes a stream on and, once its first Head bytes have gone on, +// calls Insert before the rest: padding a format puts between the headers an +// encoder writes first and the content that follows them. Insert gets the head +// as it went out, so a format can check that the encoder wrote the head it +// counted before it trusts the length. An Insert that fails stops the stream +// with its error. +type AfterTheHead struct { + W io.Writer + Head int64 + Insert func(w io.Writer, head []byte) error + // head is what has gone on of the first Head bytes, and done says Insert + // has been called. + head []byte + done bool +} + +func (a *AfterTheHead) Write(p []byte) (int, error) { + n := 0 + if !a.done { + k := min(a.Head-int64(len(a.head)), int64(len(p))) + written, err := a.W.Write(p[:k]) + a.head = append(a.head, p[:written]...) + n = written + if err != nil || int64(len(a.head)) < a.Head { + return n, err + } + a.done = true + if err := a.Insert(a.W, a.head); err != nil { + return n, err + } + p = p[k:] + } + if len(p) == 0 { + return n, nil + } + written, err := a.W.Write(p) + return n + written, err +} + // TailHolder passes a stream on while keeping its last Keep bytes back, so a // format can write something in front of them once the stream ends - padding // in front of the end marker an encoder writes last. diff --git a/internal/format/gif/cut.go b/internal/format/gif/cut.go new file mode 100644 index 00000000..5243751e --- /dev/null +++ b/internal/format/gif/cut.go @@ -0,0 +1,187 @@ +package gif + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// loopExtensionSize is the application extension the encoder writes after the +// colour table of a picture that moves: introducer, label, block size, the +// eleven bytes of NETSCAPE2.0, a three byte sub block with the loop count, and +// the terminator. +const loopExtensionSize = 3 + 11 + 4 + 1 + +// loopExtensionStart is how that extension opens. +var loopExtensionStart = []byte("\x21\xff\x0bNETSCAPE2.0") + +// writeForACut writes a file a damage will cut after its first m.cutAt bytes, +// with the padding split into a comment in front of the first frame and one in +// front of the trailer, so that the cut keeps the share of the frames it keeps +// of the whole file (format.Request.CutAt). +// +// With all of the padding behind the frames, as an uncut file has it, a large +// file cut at any share kept the whole picture and lost only padding. GIF89a +// lets a comment stand wherever a block can begin, the beginning of the data +// stream included (section 24), but not in front of the colour table, which +// belongs to the screen descriptor (appendix B). The comment in front goes after +// the head the encoder writes - the descriptor, the colour table and, for a +// picture that moves, the loop extension - where Pillow, FFmpeg, ImageMagick, +// Go, the Windows Imaging Component, GDI+, exiftool, Tk and Chrome return the +// same frames, the same loop count and the same pixels as for the file uncut +// (docs/TRUNCATE-2026-11-11.md section 9.3). +// +// The frames stream out as the encoder writes them, so the cut stops the +// encoder where it falls. The comment in front is sized from their length, so a +// plan that did not encode the picture encodes it here first, which is what +// every GIF below the ladder ceiling pays at planning today. +func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { + if !m.bodyKnown { + body, err := encodedBodySize(m) + if err != nil { + return err + } + m.body, m.bodyKnown = body, true + } + if err := settlePadding(&m, whole, m.body+trailerSize); err != nil { + return core.Defect(fmt.Errorf("gif: %w - ladderCeiling is wrong", err)) + } + head := headBytes(m) + pad := whole - m.body - trailerSize + front := frontPadding(pad, m.body-head, head, m.cutAt, whole) + markAroundThePicture(w, m.body, head, front, pad-front) + + // Both comments draw from the file's seed. Nobody reads padding, and the + // comment behind the frames lies past any cut. + split := &format.AfterTheHead{W: w, Head: head, Insert: func(w io.Writer, got []byte) error { + if err := checkHead(got, m); err != nil { + return err + } + return writePadding(ctx, w, m.seed, front) + }} + holder := &format.TailHolder{W: split, Keep: trailerSize} + if err := encode(holder, m); err != nil { + return err + } + if holder.Written != m.body { + return core.Defect(fmt.Errorf("gif: the picture encoded to %d B where planning said %d B", holder.Written, m.body)) + } + if holder.Tail[0] != 0x3B { + return core.Defect(fmt.Errorf("gif: the encoded stream does not end with the trailer")) + } + if err := writePadding(ctx, w, m.seed, pad-front); err != nil { + return err + } + _, err := w.Write(holder.Tail) + return err +} + +// headBytes is how long the head the encoder writes in front of the first +// frame is: the signature and screen descriptor, the colour table, and the loop +// extension when the picture moves. The map counts it as structure - the colour +// table is a table the frames are read through, and the frames are what a +// reader hands back (docs/TRUNCATE-2026-11-11.md section 9.4). +func headBytes(m memo) int64 { + n := int64(screenHeader + 3*paletteSize(m.width, m.height)) + if m.frames > 1 { + n += loopExtensionSize + } + return n +} + +// checkHead is whether the head that went out before the comment is the head +// the encoder wrote: the screen descriptor, the colour table its packed field +// announces, of the size planning counted, and for a picture that moves the +// loop extension right after it - nothing more and nothing less. A head of +// another length puts the comment inside the colour table or a frame. +func checkHead(got []byte, m memo) error { + if len(got) < screenHeader { + return core.Defect(fmt.Errorf("gif: %d B went out before the comment, less than the screen descriptor", len(got))) + } + table, packed := 0, got[10] + if packed&0x80 != 0 { + table = 3 << (packed&0x07 + 1) + } + if want := 3 * paletteSize(m.width, m.height); table != want { + return core.Defect(fmt.Errorf("gif: the encoder's colour table is %d B where planning counted %d B", table, want)) + } + head := screenHeader + table + if m.frames > 1 { + head += loopExtensionSize + } + if len(got) != head { + return core.Defect(fmt.Errorf("gif: %d B went out before the comment where the encoder's head is %d B", len(got), head)) + } + if m.frames > 1 { + loop := got[screenHeader+table:] + if !bytes.HasPrefix(loop, loopExtensionStart) || loop[len(loop)-1] != 0 { + return core.Defect(fmt.Errorf("gif: the encoder did not write the loop extension right after the colour table")) + } + } + return nil +} + +// frontPadding is how much of the padding, as a comment, goes in front of the +// first frame, so that a cut after the first cutAt bytes of a whole file keeps +// frames*cutAt/whole bytes of the frames. +// +// A comment is nothing, three bytes or at least five, so both sides are +// rounded to that. Where they cannot land exactly, the front grows - at most +// four bytes - which moves the frames later and cuts more of them, the "at +// least" missing promises. The one way it can keep more is a padding too short +// for the frames the cut should take: there all of it goes in front, and the cut +// keeps at most one byte more than the share (docs/TRUNCATE-2026-11-11.md +// section 9.5). The manifest's cut reports what was kept either way. +func frontPadding(pad, frames, head, cutAt, whole int64) int64 { + if pad == 0 || cutAt <= 0 || cutAt >= whole { + return 0 + } + front := format.PaddingInFront(head, frames, cutAt, whole) + if front <= 0 { + return 0 + } + for ; front < pad; front++ { + if isComment(front) && isComment(pad-front) { + return front + } + } + return pad +} + +// isComment is whether n bytes can be spent as one comment extension, or as +// none at all. +func isComment(n int64) bool { + return n == 0 || n == emptyComment || n >= smallestCarryingComment +} + +// writePadding writes n bytes of padding as one comment, or nothing for n = 0. +func writePadding(ctx context.Context, w io.Writer, seed uint64, n int64) error { + if n == 0 { + return nil + } + blocks, payload := commentShape(n) + return writeComment(ctx, w, seed, blocks, payload) +} + +// markAroundThePicture marks the map of a file whose padding is split around +// its frames: the head, the comment in front, the frames, the comment behind +// and the trailer. A side with no padding has no comment and no region. +func markAroundThePicture(w io.Writer, body, head, front, back int64) { + format.Mark(w, format.RoleStructure, 0, head) + at := head + if front > 0 { + format.Mark(w, format.RolePadding, at, front) + at += front + } + format.Mark(w, format.RoleContent, at, body-head) + at += body - head + if back > 0 { + format.Mark(w, format.RolePadding, at, back) + at += back + } + format.Mark(w, format.RoleEnd, at, trailerSize) +} diff --git a/internal/format/gif/gif.go b/internal/format/gif/gif.go index aa21b3f6..df53da35 100644 --- a/internal/format/gif/gif.go +++ b/internal/format/gif/gif.go @@ -3,7 +3,9 @@ // A compressed format, so the size of the picture is whatever LZW decides and // the last bytes come from a padding channel - the same shape as PNG. The // channel here is the comment extension, a block the format sets aside for -// arbitrary content. +// arbitrary content. A file a damage will cut short has its padding split +// into a comment in front of the frames and one behind them, so the cut takes +// the share of the picture asked for (cut.go). // // It has one awkward property PNG does not, and it is measured rather than // guessed: an empty comment costs three bytes and a comment carrying a single @@ -166,6 +168,10 @@ type memo struct { payload int64 blocks int64 comment bool + // cutAt is where a damage will cut the file, zero when nothing does + // (format.Request.CutAt). writeForACut splits the padding around the + // frames for it. + cutAt int64 } func checkJointLimits(w, h int) error { @@ -196,6 +202,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { if err != nil { return format.Plan{}, err } + m.cutAt = r.CutAt w, h := m.width, m.height bare := m.body + trailerSize @@ -324,15 +331,19 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return ctx.Err() default: } + if m.cutAt > 0 && m.cutAt < p.Bytes { + return writeForACut(ctx, w, m, p.Bytes) + } // The map, before the bytes wherever planning measured the picture. The - // palette counts to the content, with the frames it colours: the encoder - // writes it, and a reader needs it to hand back a single pixel. - format.Mark(w, format.RoleStructure, 0, screenHeader) + // head the encoder writes in front of the first frame - the colour table + // and the loop extension with it - is structure, and the frames are the + // content (headBytes). + format.Mark(w, format.RoleStructure, 0, headBytes(m)) if m.bodyKnown { markAfterThePicture(w, m, p.Bytes, true) } else { - format.Mark(w, format.RoleContent, screenHeader, -1) + format.Mark(w, format.RoleContent, headBytes(m), -1) } // The trailer is always the final byte, so hold it back and let the rest @@ -373,12 +384,13 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // every GIF and are always this long. const screenHeader = 6 + 7 -// markAfterThePicture marks the picture when its length is known, and what -// follows it: the comment that makes up the size, if there is one, and the +// markAfterThePicture marks the frames when their length is known, and what +// follows them: the comment that makes up the size, if there is one, and the // trailer. func markAfterThePicture(w io.Writer, m memo, total int64, withPicture bool) { if withPicture { - format.Mark(w, format.RoleContent, screenHeader, m.body-screenHeader) + head := headBytes(m) + format.Mark(w, format.RoleContent, head, m.body-head) } if m.comment { format.Mark(w, format.RolePadding, m.body, total-trailerSize-m.body) diff --git a/internal/format/png/cut.go b/internal/format/png/cut.go index 367559a3..b68a6ecd 100644 --- a/internal/format/png/cut.go +++ b/internal/format/png/cut.go @@ -42,8 +42,14 @@ func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { markAroundThePicture(w, m.body, front, pad-front) // Both padding chunks draw from the file's seed. Nobody reads padding, and - // the one behind the picture lies past any cut. - split := &afterTheHead{ctx: ctx, w: w, seed: m.seed, head: headSize, front: front} + // the one behind the picture lies past any cut. The padding in front goes + // after the signature and the header chunk. + split := &format.AfterTheHead{W: w, Head: headSize, Insert: func(w io.Writer, _ []byte) error { + if front == 0 { + return nil + } + return writePaddingChunk(ctx, w, paddingChunk, m.seed, front-chunkOverhead) + }} holder := &format.TailHolder{W: split, Keep: iendSize} if err := encode(holder, m); err != nil { return err @@ -63,45 +69,6 @@ func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { return err } -// afterTheHead passes the encoded picture on, and writes the padding that goes -// in front of the picture's data once the first head bytes - the signature and -// the header chunk - are out. -type afterTheHead struct { - ctx context.Context - w io.Writer - seed uint64 - // head is how many bytes are still to pass before the padding. - head int64 - // front is the padding, its chunk's header and checksum included, and 0 - // once it is written or when there is none. - front int64 -} - -func (a *afterTheHead) Write(p []byte) (int, error) { - n := 0 - if a.head > 0 { - k := min(a.head, int64(len(p))) - written, err := a.w.Write(p[:k]) - n, a.head = written, a.head-int64(written) - if err != nil || a.head > 0 { - return n, err - } - p = p[k:] - } - if a.front > 0 { - front := a.front - a.front = 0 - if err := writePaddingChunk(a.ctx, a.w, paddingChunk, a.seed, front-chunkOverhead); err != nil { - return n, err - } - } - if len(p) == 0 { - return n, nil - } - written, err := a.w.Write(p) - return n + written, err -} - // frontPadding is how much of the padding, chunk headers included, goes in // front of the picture, so that a cut after the first cutAt bytes of a whole // file keeps content*cutAt/whole bytes of the picture's content. diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index aa911441..05bdb801 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -35,8 +35,8 @@ type cutCase struct { // the content is left, and gets a note when the cut missed the content. // // The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2, 7 and 8). -// A GIF of 1 MiB is nearly all padding after its picture, so 90% missing -// still leaves the whole picture. A PNG splits its padding around the picture +// A PDF of 1 MiB is nearly all padding after its pages, so 90% missing still +// leaves every page. A PNG splits its padding around the picture // for a cut, so the cut takes the share asked for. A small tar.gz keeps its // padding BEFORE its archive, so a cut there takes all of it. A text is its // words, so the cut falls in them. Two cuts in a row end the file where the second one does, and @@ -46,7 +46,8 @@ type cutCase struct { // one that claims content it does not have. func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { cases := []cutCase{ - {name: "behind", format: "gif", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, + // A GIF was this case until it split its padding for the cut. + {name: "behind", format: "pdf", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, {name: "picture", format: "png", size: 1 << 20, missing: "90", in: "content", kept: "share"}, // 4 KiB on disk is 40 KiB whole, and up to 64 KiB the padding is the // comment in the gzip header, in front of the archive. An icon or a JPEG @@ -177,7 +178,7 @@ func TestTheRecordOfACutRunIsNeverLongerThanItsForecast(t *testing.T) { } opt := engine.Options{OutDir: t.TempDir(), Seed: 7, Command: "test"} planned, err := engine.Plan([]engine.Target{ - cut("picture", "png", 1<<20, "90"), cut("behind", "gif", 1<<20, "90"), cut("text", "txt", 1000, "50"), + cut("picture", "png", 1<<20, "90"), cut("behind", "pdf", 1<<20, "90"), cut("text", "txt", 1000, "50"), }, opt) if err != nil { t.Fatalf("planning: %v", err) diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index c385bbe3..0ce232e6 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -32,7 +32,6 @@ type notYet struct { // that leaves is held to every size and share, and so is a new format unless // it is added here with its reason. var notYetCutAtTheShare = map[string]notYet{ - "gif": {paddingBehind, 64 << 10, 50}, "avif": {paddingBehind, 64 << 10, 50}, "jxl": {paddingBehind, 64 << 10, 50}, "mp4": {paddingBehind, 64 << 10, 50}, @@ -55,8 +54,9 @@ var notYetCutAtTheShare = map[string]notYet{ // from its first commit (docs/TRUNCATE-2026-11-11.md section 3a). It never // grows, so a new format cannot wait on the ratchet - tools/doctor.py runs this // guard for it, and it stays red until the format splits its padding for -// format.Request.CutAt. -var listedBeforeTheRule = []string{"gif", "avif", "jxl", "mp4", "webm", "pdf", "zip", "docx", "xlsx", "pptx", "targz", "webp", "tiff"} +// format.Request.CutAt. A format that leaves the ratchet leaves this list too, +// so it cannot go back. +var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "zip", "docx", "xlsx", "pptx", "targz", "webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index f8a7090d..f1717798 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -392,17 +392,19 @@ def sub_blocks_end(data, pos): def layout_gif(data): - """The signature and the screen descriptor, then everything up to the - trailer - the palette, the frames and their extensions - as content, a - comment as padding, and the trailer as the end.""" + """The signature, the screen descriptor, the global colour table and any + application extension in front of the first frame (the loop count) as + structure, the frames and their extensions as content, a comment anywhere + as padding, and the trailer as the end.""" if data[:6] not in (b"GIF87a", b"GIF89a"): raise NoMap("no GIF signature") out, pos = [[STRUCTURE, 0, 13]], 13 packed = data[10] if packed & 0x80: table = 3 * (1 << ((packed & 0x07) + 1)) - out.append([CONTENT, pos, table]) + out.append([STRUCTURE, pos, table]) pos += table + framed = False while pos < len(data): start, marker = pos, data[pos] if marker == 0x3B: @@ -412,9 +414,16 @@ def layout_gif(data): if marker == 0x21: label = data[pos + 1] pos = sub_blocks_end(data, pos + 2) - out.append([PADDING if label == 0xFE else CONTENT, start, pos - start]) + if label == 0xFE: + role = PADDING + elif label == 0xFF and not framed: + role = STRUCTURE + else: + role, framed = CONTENT, True + out.append([role, start, pos - start]) continue if marker == 0x2C: + framed = True local = data[pos + 9] pos += 10 if local & 0x80: