diff --git a/CHANGELOG.md b/CHANGELOG.md index 688fde30..6d2ae910 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,10 +80,10 @@ because it turns other people's test suites red. the length of the content is known. A cut in the padding also gets the note `cut_in_padding`. In a compressed tar.gz a cut in the padding is reported as a cut in the content. -- **`truncate` cuts the picture of a PNG, a JPEG or an icon by the - percentage you set.** `--damage truncate:missing=90` on a 1 MB PNG used to - cut only padding and leave the whole picture, and on a JPEG or a large - icon it removed the whole picture whatever percentage you set. Now it +- **`truncate` cuts the picture of a PNG, a GIF, a JPEG or an icon by the + percentage you set.** `--damage truncate:missing=90` on a 1 MB PNG or GIF + used to cut only padding and leave the whole picture, and on a JPEG or a + large icon it removed the whole picture whatever percentage you set. Now it removes 90% of the picture's content, so a viewer that shows what is left shows the part of the picture that arrived, the way an interrupted upload leaves it. Files nothing cuts are unchanged. diff --git a/internal/format/cut.go b/internal/format/cut.go index 4dbe456b..5cba050c 100644 --- a/internal/format/cut.go +++ b/internal/format/cut.go @@ -25,6 +25,45 @@ func PaddingInFront(before, content, cutAt, whole int64) int64 { return cutAt - before - int64(kept) } +// AfterTheHead passes a stream on and, once its first Head bytes have gone on, +// calls Insert before the rest: padding a format puts between the headers an +// encoder writes first and the content that follows them. Insert gets the head +// as it went out, so a format can check that the encoder wrote the head it +// counted before it trusts the length. An Insert that fails stops the stream +// with its error. +type AfterTheHead struct { + W io.Writer + Head int64 + Insert func(w io.Writer, head []byte) error + // head is what has gone on of the first Head bytes, and done says Insert + // has been called. + head []byte + done bool +} + +func (a *AfterTheHead) Write(p []byte) (int, error) { + n := 0 + if !a.done { + k := min(a.Head-int64(len(a.head)), int64(len(p))) + written, err := a.W.Write(p[:k]) + a.head = append(a.head, p[:written]...) + n = written + if err != nil || int64(len(a.head)) < a.Head { + return n, err + } + a.done = true + if err := a.Insert(a.W, a.head); err != nil { + return n, err + } + p = p[k:] + } + if len(p) == 0 { + return n, nil + } + written, err := a.W.Write(p) + return n + written, err +} + // TailHolder passes a stream on while keeping its last Keep bytes back, so a // format can write something in front of them once the stream ends - padding // in front of the end marker an encoder writes last. diff --git a/internal/format/gif/cut.go b/internal/format/gif/cut.go new file mode 100644 index 00000000..5243751e --- /dev/null +++ b/internal/format/gif/cut.go @@ -0,0 +1,187 @@ +package gif + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// loopExtensionSize is the application extension the encoder writes after the +// colour table of a picture that moves: introducer, label, block size, the +// eleven bytes of NETSCAPE2.0, a three byte sub block with the loop count, and +// the terminator. +const loopExtensionSize = 3 + 11 + 4 + 1 + +// loopExtensionStart is how that extension opens. +var loopExtensionStart = []byte("\x21\xff\x0bNETSCAPE2.0") + +// writeForACut writes a file a damage will cut after its first m.cutAt bytes, +// with the padding split into a comment in front of the first frame and one in +// front of the trailer, so that the cut keeps the share of the frames it keeps +// of the whole file (format.Request.CutAt). +// +// With all of the padding behind the frames, as an uncut file has it, a large +// file cut at any share kept the whole picture and lost only padding. GIF89a +// lets a comment stand wherever a block can begin, the beginning of the data +// stream included (section 24), but not in front of the colour table, which +// belongs to the screen descriptor (appendix B). The comment in front goes after +// the head the encoder writes - the descriptor, the colour table and, for a +// picture that moves, the loop extension - where Pillow, FFmpeg, ImageMagick, +// Go, the Windows Imaging Component, GDI+, exiftool, Tk and Chrome return the +// same frames, the same loop count and the same pixels as for the file uncut +// (docs/TRUNCATE-2026-11-11.md section 9.3). +// +// The frames stream out as the encoder writes them, so the cut stops the +// encoder where it falls. The comment in front is sized from their length, so a +// plan that did not encode the picture encodes it here first, which is what +// every GIF below the ladder ceiling pays at planning today. +func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { + if !m.bodyKnown { + body, err := encodedBodySize(m) + if err != nil { + return err + } + m.body, m.bodyKnown = body, true + } + if err := settlePadding(&m, whole, m.body+trailerSize); err != nil { + return core.Defect(fmt.Errorf("gif: %w - ladderCeiling is wrong", err)) + } + head := headBytes(m) + pad := whole - m.body - trailerSize + front := frontPadding(pad, m.body-head, head, m.cutAt, whole) + markAroundThePicture(w, m.body, head, front, pad-front) + + // Both comments draw from the file's seed. Nobody reads padding, and the + // comment behind the frames lies past any cut. + split := &format.AfterTheHead{W: w, Head: head, Insert: func(w io.Writer, got []byte) error { + if err := checkHead(got, m); err != nil { + return err + } + return writePadding(ctx, w, m.seed, front) + }} + holder := &format.TailHolder{W: split, Keep: trailerSize} + if err := encode(holder, m); err != nil { + return err + } + if holder.Written != m.body { + return core.Defect(fmt.Errorf("gif: the picture encoded to %d B where planning said %d B", holder.Written, m.body)) + } + if holder.Tail[0] != 0x3B { + return core.Defect(fmt.Errorf("gif: the encoded stream does not end with the trailer")) + } + if err := writePadding(ctx, w, m.seed, pad-front); err != nil { + return err + } + _, err := w.Write(holder.Tail) + return err +} + +// headBytes is how long the head the encoder writes in front of the first +// frame is: the signature and screen descriptor, the colour table, and the loop +// extension when the picture moves. The map counts it as structure - the colour +// table is a table the frames are read through, and the frames are what a +// reader hands back (docs/TRUNCATE-2026-11-11.md section 9.4). +func headBytes(m memo) int64 { + n := int64(screenHeader + 3*paletteSize(m.width, m.height)) + if m.frames > 1 { + n += loopExtensionSize + } + return n +} + +// checkHead is whether the head that went out before the comment is the head +// the encoder wrote: the screen descriptor, the colour table its packed field +// announces, of the size planning counted, and for a picture that moves the +// loop extension right after it - nothing more and nothing less. A head of +// another length puts the comment inside the colour table or a frame. +func checkHead(got []byte, m memo) error { + if len(got) < screenHeader { + return core.Defect(fmt.Errorf("gif: %d B went out before the comment, less than the screen descriptor", len(got))) + } + table, packed := 0, got[10] + if packed&0x80 != 0 { + table = 3 << (packed&0x07 + 1) + } + if want := 3 * paletteSize(m.width, m.height); table != want { + return core.Defect(fmt.Errorf("gif: the encoder's colour table is %d B where planning counted %d B", table, want)) + } + head := screenHeader + table + if m.frames > 1 { + head += loopExtensionSize + } + if len(got) != head { + return core.Defect(fmt.Errorf("gif: %d B went out before the comment where the encoder's head is %d B", len(got), head)) + } + if m.frames > 1 { + loop := got[screenHeader+table:] + if !bytes.HasPrefix(loop, loopExtensionStart) || loop[len(loop)-1] != 0 { + return core.Defect(fmt.Errorf("gif: the encoder did not write the loop extension right after the colour table")) + } + } + return nil +} + +// frontPadding is how much of the padding, as a comment, goes in front of the +// first frame, so that a cut after the first cutAt bytes of a whole file keeps +// frames*cutAt/whole bytes of the frames. +// +// A comment is nothing, three bytes or at least five, so both sides are +// rounded to that. Where they cannot land exactly, the front grows - at most +// four bytes - which moves the frames later and cuts more of them, the "at +// least" missing promises. The one way it can keep more is a padding too short +// for the frames the cut should take: there all of it goes in front, and the cut +// keeps at most one byte more than the share (docs/TRUNCATE-2026-11-11.md +// section 9.5). The manifest's cut reports what was kept either way. +func frontPadding(pad, frames, head, cutAt, whole int64) int64 { + if pad == 0 || cutAt <= 0 || cutAt >= whole { + return 0 + } + front := format.PaddingInFront(head, frames, cutAt, whole) + if front <= 0 { + return 0 + } + for ; front < pad; front++ { + if isComment(front) && isComment(pad-front) { + return front + } + } + return pad +} + +// isComment is whether n bytes can be spent as one comment extension, or as +// none at all. +func isComment(n int64) bool { + return n == 0 || n == emptyComment || n >= smallestCarryingComment +} + +// writePadding writes n bytes of padding as one comment, or nothing for n = 0. +func writePadding(ctx context.Context, w io.Writer, seed uint64, n int64) error { + if n == 0 { + return nil + } + blocks, payload := commentShape(n) + return writeComment(ctx, w, seed, blocks, payload) +} + +// markAroundThePicture marks the map of a file whose padding is split around +// its frames: the head, the comment in front, the frames, the comment behind +// and the trailer. A side with no padding has no comment and no region. +func markAroundThePicture(w io.Writer, body, head, front, back int64) { + format.Mark(w, format.RoleStructure, 0, head) + at := head + if front > 0 { + format.Mark(w, format.RolePadding, at, front) + at += front + } + format.Mark(w, format.RoleContent, at, body-head) + at += body - head + if back > 0 { + format.Mark(w, format.RolePadding, at, back) + at += back + } + format.Mark(w, format.RoleEnd, at, trailerSize) +} diff --git a/internal/format/gif/gif.go b/internal/format/gif/gif.go index aa21b3f6..df53da35 100644 --- a/internal/format/gif/gif.go +++ b/internal/format/gif/gif.go @@ -3,7 +3,9 @@ // A compressed format, so the size of the picture is whatever LZW decides and // the last bytes come from a padding channel - the same shape as PNG. The // channel here is the comment extension, a block the format sets aside for -// arbitrary content. +// arbitrary content. A file a damage will cut short has its padding split +// into a comment in front of the frames and one behind them, so the cut takes +// the share of the picture asked for (cut.go). // // It has one awkward property PNG does not, and it is measured rather than // guessed: an empty comment costs three bytes and a comment carrying a single @@ -166,6 +168,10 @@ type memo struct { payload int64 blocks int64 comment bool + // cutAt is where a damage will cut the file, zero when nothing does + // (format.Request.CutAt). writeForACut splits the padding around the + // frames for it. + cutAt int64 } func checkJointLimits(w, h int) error { @@ -196,6 +202,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { if err != nil { return format.Plan{}, err } + m.cutAt = r.CutAt w, h := m.width, m.height bare := m.body + trailerSize @@ -324,15 +331,19 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return ctx.Err() default: } + if m.cutAt > 0 && m.cutAt < p.Bytes { + return writeForACut(ctx, w, m, p.Bytes) + } // The map, before the bytes wherever planning measured the picture. The - // palette counts to the content, with the frames it colours: the encoder - // writes it, and a reader needs it to hand back a single pixel. - format.Mark(w, format.RoleStructure, 0, screenHeader) + // head the encoder writes in front of the first frame - the colour table + // and the loop extension with it - is structure, and the frames are the + // content (headBytes). + format.Mark(w, format.RoleStructure, 0, headBytes(m)) if m.bodyKnown { markAfterThePicture(w, m, p.Bytes, true) } else { - format.Mark(w, format.RoleContent, screenHeader, -1) + format.Mark(w, format.RoleContent, headBytes(m), -1) } // The trailer is always the final byte, so hold it back and let the rest @@ -373,12 +384,13 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // every GIF and are always this long. const screenHeader = 6 + 7 -// markAfterThePicture marks the picture when its length is known, and what -// follows it: the comment that makes up the size, if there is one, and the +// markAfterThePicture marks the frames when their length is known, and what +// follows them: the comment that makes up the size, if there is one, and the // trailer. func markAfterThePicture(w io.Writer, m memo, total int64, withPicture bool) { if withPicture { - format.Mark(w, format.RoleContent, screenHeader, m.body-screenHeader) + head := headBytes(m) + format.Mark(w, format.RoleContent, head, m.body-head) } if m.comment { format.Mark(w, format.RolePadding, m.body, total-trailerSize-m.body) diff --git a/internal/format/png/cut.go b/internal/format/png/cut.go index 367559a3..b68a6ecd 100644 --- a/internal/format/png/cut.go +++ b/internal/format/png/cut.go @@ -42,8 +42,14 @@ func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { markAroundThePicture(w, m.body, front, pad-front) // Both padding chunks draw from the file's seed. Nobody reads padding, and - // the one behind the picture lies past any cut. - split := &afterTheHead{ctx: ctx, w: w, seed: m.seed, head: headSize, front: front} + // the one behind the picture lies past any cut. The padding in front goes + // after the signature and the header chunk. + split := &format.AfterTheHead{W: w, Head: headSize, Insert: func(w io.Writer, _ []byte) error { + if front == 0 { + return nil + } + return writePaddingChunk(ctx, w, paddingChunk, m.seed, front-chunkOverhead) + }} holder := &format.TailHolder{W: split, Keep: iendSize} if err := encode(holder, m); err != nil { return err @@ -63,45 +69,6 @@ func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { return err } -// afterTheHead passes the encoded picture on, and writes the padding that goes -// in front of the picture's data once the first head bytes - the signature and -// the header chunk - are out. -type afterTheHead struct { - ctx context.Context - w io.Writer - seed uint64 - // head is how many bytes are still to pass before the padding. - head int64 - // front is the padding, its chunk's header and checksum included, and 0 - // once it is written or when there is none. - front int64 -} - -func (a *afterTheHead) Write(p []byte) (int, error) { - n := 0 - if a.head > 0 { - k := min(a.head, int64(len(p))) - written, err := a.w.Write(p[:k]) - n, a.head = written, a.head-int64(written) - if err != nil || a.head > 0 { - return n, err - } - p = p[k:] - } - if a.front > 0 { - front := a.front - a.front = 0 - if err := writePaddingChunk(a.ctx, a.w, paddingChunk, a.seed, front-chunkOverhead); err != nil { - return n, err - } - } - if len(p) == 0 { - return n, nil - } - written, err := a.w.Write(p) - return n + written, err -} - // frontPadding is how much of the padding, chunk headers included, goes in // front of the picture, so that a cut after the first cutAt bytes of a whole // file keeps content*cutAt/whole bytes of the picture's content. diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index aa911441..05bdb801 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -35,8 +35,8 @@ type cutCase struct { // the content is left, and gets a note when the cut missed the content. // // The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2, 7 and 8). -// A GIF of 1 MiB is nearly all padding after its picture, so 90% missing -// still leaves the whole picture. A PNG splits its padding around the picture +// A PDF of 1 MiB is nearly all padding after its pages, so 90% missing still +// leaves every page. A PNG splits its padding around the picture // for a cut, so the cut takes the share asked for. A small tar.gz keeps its // padding BEFORE its archive, so a cut there takes all of it. A text is its // words, so the cut falls in them. Two cuts in a row end the file where the second one does, and @@ -46,7 +46,8 @@ type cutCase struct { // one that claims content it does not have. func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { cases := []cutCase{ - {name: "behind", format: "gif", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, + // A GIF was this case until it split its padding for the cut. + {name: "behind", format: "pdf", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, {name: "picture", format: "png", size: 1 << 20, missing: "90", in: "content", kept: "share"}, // 4 KiB on disk is 40 KiB whole, and up to 64 KiB the padding is the // comment in the gzip header, in front of the archive. An icon or a JPEG @@ -177,7 +178,7 @@ func TestTheRecordOfACutRunIsNeverLongerThanItsForecast(t *testing.T) { } opt := engine.Options{OutDir: t.TempDir(), Seed: 7, Command: "test"} planned, err := engine.Plan([]engine.Target{ - cut("picture", "png", 1<<20, "90"), cut("behind", "gif", 1<<20, "90"), cut("text", "txt", 1000, "50"), + cut("picture", "png", 1<<20, "90"), cut("behind", "pdf", 1<<20, "90"), cut("text", "txt", 1000, "50"), }, opt) if err != nil { t.Fatalf("planning: %v", err) diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index c385bbe3..0ce232e6 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -32,7 +32,6 @@ type notYet struct { // that leaves is held to every size and share, and so is a new format unless // it is added here with its reason. var notYetCutAtTheShare = map[string]notYet{ - "gif": {paddingBehind, 64 << 10, 50}, "avif": {paddingBehind, 64 << 10, 50}, "jxl": {paddingBehind, 64 << 10, 50}, "mp4": {paddingBehind, 64 << 10, 50}, @@ -55,8 +54,9 @@ var notYetCutAtTheShare = map[string]notYet{ // from its first commit (docs/TRUNCATE-2026-11-11.md section 3a). It never // grows, so a new format cannot wait on the ratchet - tools/doctor.py runs this // guard for it, and it stays red until the format splits its padding for -// format.Request.CutAt. -var listedBeforeTheRule = []string{"gif", "avif", "jxl", "mp4", "webm", "pdf", "zip", "docx", "xlsx", "pptx", "targz", "webp", "tiff"} +// format.Request.CutAt. A format that leaves the ratchet leaves this list too, +// so it cannot go back. +var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "zip", "docx", "xlsx", "pptx", "targz", "webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index f8a7090d..f1717798 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -392,17 +392,19 @@ def sub_blocks_end(data, pos): def layout_gif(data): - """The signature and the screen descriptor, then everything up to the - trailer - the palette, the frames and their extensions - as content, a - comment as padding, and the trailer as the end.""" + """The signature, the screen descriptor, the global colour table and any + application extension in front of the first frame (the loop count) as + structure, the frames and their extensions as content, a comment anywhere + as padding, and the trailer as the end.""" if data[:6] not in (b"GIF87a", b"GIF89a"): raise NoMap("no GIF signature") out, pos = [[STRUCTURE, 0, 13]], 13 packed = data[10] if packed & 0x80: table = 3 * (1 << ((packed & 0x07) + 1)) - out.append([CONTENT, pos, table]) + out.append([STRUCTURE, pos, table]) pos += table + framed = False while pos < len(data): start, marker = pos, data[pos] if marker == 0x3B: @@ -412,9 +414,16 @@ def layout_gif(data): if marker == 0x21: label = data[pos + 1] pos = sub_blocks_end(data, pos + 2) - out.append([PADDING if label == 0xFE else CONTENT, start, pos - start]) + if label == 0xFE: + role = PADDING + elif label == 0xFF and not framed: + role = STRUCTURE + else: + role, framed = CONTENT, True + out.append([role, start, pos - start]) continue if marker == 0x2C: + framed = True local = data[pos + 9] pos += 10 if local & 0x80: