From f7640db5a3b773d7a27af47837acf7cd962bf278 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 08:22:48 +0100 Subject: [PATCH 1/2] An Office document cut short loses the share of its parts asked for A package truncate will cut opens with a padding part in front of every other part, and keeps the rest of its padding behind them as before, so the cut falls in the parts at the share asked for. Readers find parts through the central directory, and Python, 7-Zip, bsdtar, python-docx, openpyxl, python-pptx and LibreOffice read such a package as they read it uncut. The share guard accepts a cut in the end of a ZIP or a package when nothing but the central directory stands behind the content: at 1% of a few kilobytes the directory is longer than what the cut takes. The map guard's layout for a cut after the first byte allows the smallest padding part where the content would otherwise start the file. Uncut packages are byte for byte the same. Co-Authored-By: Claude Opus 5.5 --- internal/format/docx/docx.go | 2 +- internal/format/opc/cut.go | 111 ++++++++++++++++++++++++ internal/format/opc/opc.go | 141 +++++++++++++++++++++---------- internal/format/pptx/pptx.go | 2 +- internal/format/xlsx/xlsx.go | 2 +- internal/guard/branching_test.go | 4 +- internal/guard/cutshare_test.go | 89 +++++++++++++++++-- internal/guard/filemap_test.go | 42 +++++++-- internal/oracle/layout.py | 2 +- 9 files changed, 335 insertions(+), 60 deletions(-) create mode 100644 internal/format/opc/cut.go diff --git a/internal/format/docx/docx.go b/internal/format/docx/docx.go index cba77f24..aef0ea84 100644 --- a/internal/format/docx/docx.go +++ b/internal/format/docx/docx.go @@ -102,7 +102,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { return format.Plan{}, err } - pkg, err := opc.Settle(parts, shape, r.Bytes) + pkg, err := opc.SettleForACut(parts, shape, r.Bytes, r.CutAt) if err != nil { return format.Plan{}, refusal(err, r.Bytes, shape, paragraphs) } diff --git a/internal/format/opc/cut.go b/internal/format/opc/cut.go new file mode 100644 index 00000000..99a58358 --- /dev/null +++ b/internal/format/opc/cut.go @@ -0,0 +1,111 @@ +package opc + +import ( + "fmt" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// SettleForACut is Settle for a package a damage will cut after its first +// cutAt bytes (format.Request.CutAt): the padding is split into a part in front +// of every other part and what Settle would put behind them, so that the cut +// keeps the share of the parts it keeps of the whole package. +// +// With all of the padding behind the parts, as a package nothing cuts has it, +// a large package cut at any share kept every part and lost only padding. +// ECMA-376 part 2 names no order of parts and a reader finds them through the +// central directory: Python's zipfile, 7-Zip, bsdtar, python-docx, openpyxl, +// python-pptx and LibreOffice read a package whose padding part comes first as +// they read it today (docs/TRUNCATE-2026-11-11.md section 10.3). The map +// already counts every part as content from the first byte, so nothing of the +// content stands in front of the padding. +// +// The part in front costs its header before any padding, so the front is +// nothing or at least that, and grows where it cannot land exactly - the cut +// then takes a little more of the parts, the "at least" missing promises. +// Where the padding is too short for the parts the cut should take, all of it +// goes in front and the cut keeps more of them by the share of what stands +// behind (section 10.4). The manifest's cut reports what was kept either way. +func SettleForACut(parts []Part, shape Shape, want, cutAt int64) (Package, error) { + p, err := Settle(parts, shape, want) + if err != nil || cutAt <= 0 || cutAt >= want { + return p, err + } + c, err := measureCut(parts, shape) + if err != nil { + return Package{}, err + } + pad := want - shape.Bare + front := format.PaddingInFront(0, c.content, cutAt, want) + if front <= 0 || pad < c.frontOverhead { + return p, nil + } + front = max(front, c.frontHead) + // What the part in front costs in all: its header and padding in front, + // and its record in the central directory behind the parts. + cost := c.frontOverhead + front - c.frontHead + if cost > pad { + cost = pad + front = c.frontHead + pad - c.frontOverhead + } + q := Package{Parts: parts, Filler: NoFiller, Front: front - c.frontHead, Content: c.content} + back := pad - cost + switch { + case back == 0: + case back <= CommentLimit: + q.Comment = int(back) + case back >= c.endOverhead: + q.Filler = back - c.endOverhead + default: + return Package{}, core.Defect(fmt.Errorf("opc: %d B behind the parts is more than a comment carries and less than a padding part costs (%d B)", back, c.endOverhead)) + } + return q, nil +} + +// cutShape is what SettleForACut needs to know of a set of parts beyond Shape, +// measured by writing them rather than added up from header widths, for the +// reason Shape gives. +type cutShape struct { + // content is how long the parts come to in the archive. + content int64 + // frontHead is how many bytes an empty part in front puts before the + // parts, and frontOverhead what it costs in all, its record in the central + // directory with it. + frontHead, frontOverhead int64 + // endOverhead is what an empty padding part behind the parts costs when + // one stands in front - it carries the longer name. + endOverhead int64 +} + +func measureCut(parts []Part, shape Shape) (cutShape, error) { + var c cutShape + bare, err := size(Package{Parts: parts, Filler: NoFiller, Front: NoFiller}) + if err != nil { + return c, err + } + front, err := size(Package{Parts: parts, Filler: NoFiller, Front: 0}) + if err != nil { + return c, err + } + both, err := size(Package{Parts: parts, Filler: 0, Front: 0}) + if err != nil { + return c, err + } + c.content = bare.content.Length + c.frontHead = front.content.At + c.frontOverhead = front.size - shape.Bare + c.endOverhead = both.size - front.size + if bare.size != shape.Bare || front.content.Length != c.content { + return c, core.Defect(fmt.Errorf("opc: the parts measured %d B with %d B of them content, then %d B of content behind a padding part, where planning measured %d B", + bare.size, c.content, front.content.Length, shape.Bare)) + } + return c, nil +} + +// mapped is a package's length and the region its parts take, read off the map +// it marks as it is written (size). +type mapped struct { + size int64 + content format.Region +} diff --git a/internal/format/opc/opc.go b/internal/format/opc/opc.go index b4b17c26..a68b2266 100644 --- a/internal/format/opc/opc.go +++ b/internal/format/opc/opc.go @@ -56,6 +56,13 @@ const ( FillerName = "tfg/padding.bin" fillerExtension = "bin" + // FillerEndName is the padding part behind the others in a package laid + // out for a cut, whose first padding part, FillerName, stands in front of + // them (SettleForACut). A package may not hold two parts of one name + // (ECMA-376 part 2, M1.12), and the content types table declares the + // extension, so both are covered. + FillerEndName = "tfg/padding-end.bin" + // NoFiller means the package carries no padding part at all. NoFiller = -1 @@ -88,7 +95,15 @@ type Package struct { Comment int // Filler is how many bytes the padding part carries, or NoFiller. Filler int64 - Seed uint64 + // Front is how many bytes the padding part in front of every other part + // carries, or NoFiller - which is every package nothing cuts. With a front + // part, the one behind is FillerEndName. + Front int64 + // Content is how long the parts come to in the archive, their headers + // with them, when it is known (SettleForACut), and 0 otherwise. A cut + // that falls in them reports how much of them it kept. + Content int64 + Seed uint64 } // ContentTypes builds [Content_Types].xml. @@ -140,15 +155,15 @@ type Shape struct { // Measure works out the shape of a set of parts. func Measure(parts []Part) (Shape, error) { - bare, err := size(Package{Parts: parts, Filler: NoFiller}) + bare, err := size(Package{Parts: parts, Filler: NoFiller, Front: NoFiller}) if err != nil { return Shape{}, err } - empty, err := size(Package{Parts: parts, Filler: 0}) + empty, err := size(Package{Parts: parts, Filler: 0, Front: NoFiller}) if err != nil { return Shape{}, err } - return Shape{Bare: bare, FillerOverhead: empty - bare}, nil + return Shape{Bare: bare.size, FillerOverhead: empty.size - bare.size}, nil } // Unreachable says a size cannot be produced, and names the two that can. @@ -203,7 +218,7 @@ func Settle(parts []Part, shape Shape, want int64) (Package, error) { if want > Zip32Ceiling { return Package{}, &TooLarge{Want: want, Ceiling: Zip32Ceiling} } - p := Package{Parts: parts, Filler: NoFiller} + p := Package{Parts: parts, Filler: NoFiller, Front: NoFiller} delta := want - shape.Bare switch { case delta == 0: @@ -223,12 +238,20 @@ func Settle(parts []Part, shape Shape, want int64) (Package, error) { } } -func size(p Package) (int64, error) { +// size is a package's length and the region its parts take, read off the map +// it marks as it is written - SettleForACut asks for the region. +func size(p Package) (mapped, error) { c := &counter{} - if err := write(context.Background(), c, p, false); err != nil { - return 0, err + rec := &format.MapRecorder{W: c} + if err := write(context.Background(), rec, p, false); err != nil { + return mapped{}, err + } + for _, r := range rec.Map(c.n, true) { + if r.Role == format.RoleContent { + return mapped{size: c.n, content: r}, nil + } } - return c.n, nil + return mapped{}, core.Defect(fmt.Errorf("opc: a package of %d parts marked no content", len(p.Parts))) } // Write emits the package. @@ -241,8 +264,22 @@ func write(ctx context.Context, w io.Writer, p Package, withPadding bool) error zw := zip.NewWriter(zm.Buffer) // The map: every part is content, from the first byte. Then the filler // part, the central directory and the comment, each marked as - // archive/zip writes it (archive.StreamMap). - format.Mark(w, format.RoleContent, 0, -1) + // archive/zip writes it (archive.StreamMap). A package laid out for a cut + // opens with a padding part, and the content starts behind it. + behind := FillerName + if p.Front == NoFiller { + format.Mark(w, format.RoleContent, 0, -1) + } else { + if err := writePaddingPart(ctx, zw, zm, paddingPart{FillerName, p.Seed, p.Front, withPadding}); err != nil { + return err + } + content := p.Content + if content <= 0 { + content = -1 + } + format.Mark(w, format.RoleContent, zm.Pos(), content) + behind = FillerEndName + } if p.Comment > 0 { if p.Comment > CommentLimit { return core.Defect(fmt.Errorf("opc: a comment of %d B is past the %d B a reader accepts", p.Comment, CommentLimit)) @@ -268,45 +305,63 @@ func write(ctx context.Context, w io.Writer, p Package, withPadding bool) error } if p.Filler != NoFiller { - // Stored rather than compressed, so a byte of padding is a byte of - // file. Deflating it would make the size depend on how well random - // bytes happen to compress, which is not a thing to promise. - // - // The checksum has to be known before the header is written, and the - // padding is never held in memory, so it is generated twice: once to - // check it and once to write it. Both passes are the same arithmetic - // from the same seed, which is what makes them agree. - head := &zip.FileHeader{Name: FillerName, Method: zip.Store, Modified: fixedTime} - sum, err := fillerChecksum(ctx, p.Seed, p.Filler) - if err != nil { + if err := writePaddingPart(ctx, zw, zm, paddingPart{behind, p.Seed, p.Filler, withPadding}); err != nil { return err } - head.CRC32 = sum - head.UncompressedSize64 = uint64(p.Filler) - head.CompressedSize64 = uint64(p.Filler) - if err := zm.Hold(); err != nil { - return err - } - entry, err := zw.CreateRaw(head) - if err != nil { - return err - } - if err := zm.MarkEntry(format.RolePadding, FillerName); err != nil { - return err - } - if err := zm.Release(); err != nil { - return err - } - if withPadding { - if err := writeFiller(ctx, entry, p.Seed, p.Filler); err != nil { - return err - } - } } return zm.Close(zw) } +// paddingPart is one padding part: its name, the seed its bytes come from, +// how many it carries, and whether they are written - not while a package is +// only being measured. +type paddingPart struct { + name string + seed uint64 + n int64 + withPadding bool +} + +// writePaddingPart writes a padding part, marked as padding where its header +// starts. +// +// Stored rather than compressed, so a byte of padding is a byte of file. +// Deflating it would make the size depend on how well random bytes happen to +// compress, which is not a thing to promise. +// +// The checksum has to be known before the header is written, and the padding +// is never held in memory, so it is generated twice: once to check it and once +// to write it. Both passes are the same arithmetic from the same seed, which is +// what makes them agree. +func writePaddingPart(ctx context.Context, zw *zip.Writer, zm *archive.StreamMap, part paddingPart) error { + head := &zip.FileHeader{Name: part.name, Method: zip.Store, Modified: fixedTime} + sum, err := fillerChecksum(ctx, part.seed, part.n) + if err != nil { + return err + } + head.CRC32 = sum + head.UncompressedSize64 = uint64(part.n) + head.CompressedSize64 = uint64(part.n) + if err := zm.Hold(); err != nil { + return err + } + entry, err := zw.CreateRaw(head) + if err != nil { + return err + } + if err := zm.MarkEntry(format.RolePadding, part.name); err != nil { + return err + } + if err := zm.Release(); err != nil { + return err + } + if !part.withPadding { + return nil + } + return writeFiller(ctx, entry, part.seed, part.n) +} + // writePart writes one part with its sizes and checksum in the header. // // CreateRaw rather than CreateHeader, and this is measured rather than tidy. diff --git a/internal/format/pptx/pptx.go b/internal/format/pptx/pptx.go index 0fa32e31..208482db 100644 --- a/internal/format/pptx/pptx.go +++ b/internal/format/pptx/pptx.go @@ -106,7 +106,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { return format.Plan{}, err } - pkg, err := opc.Settle(built, shape, r.Bytes) + pkg, err := opc.SettleForACut(built, shape, r.Bytes, r.CutAt) if err != nil { return format.Plan{}, refusal(err, r.Bytes, shape, slides) } diff --git a/internal/format/xlsx/xlsx.go b/internal/format/xlsx/xlsx.go index fa88e045..bf666760 100644 --- a/internal/format/xlsx/xlsx.go +++ b/internal/format/xlsx/xlsx.go @@ -135,7 +135,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { return format.Plan{}, err } - pkg, err := opc.Settle(built, shape, r.Bytes) + pkg, err := opc.SettleForACut(built, shape, r.Bytes, r.CutAt) if err != nil { return format.Plan{}, refusal(err, r.Bytes, shape, rows, columns) } diff --git a/internal/guard/branching_test.go b/internal/guard/branching_test.go index 4a54b98c..6a530a55 100644 --- a/internal/guard/branching_test.go +++ b/internal/guard/branching_test.go @@ -64,7 +64,9 @@ const ( // of control in its own closure and went one deeper when it learnt to wrap // only once - the three became one function, chainOnce, and listen came // out of the band. - crowdedDepthFunctions = 49 + // Lowered from 49 on 2026-11-13: the office package writer's padding part + // moved out of write into writePaddingPart, so write came out of the band. + crowdedDepthFunctions = 48 // An axis this set does not watch. crowding() asks n >= band, so nothing // reaches it. diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 0ce232e6..f8c8290e 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -38,9 +38,6 @@ var notYetCutAtTheShare = map[string]notYet{ "webm": {paddingBehind, 64 << 10, 50}, "pdf": {paddingBehind, 64 << 10, 50}, "zip": {paddingBehind, 64 << 10, 50}, - "docx": {paddingBehind, 64 << 10, 50}, - "xlsx": {paddingBehind, 64 << 10, 50}, - "pptx": {paddingBehind, 64 << 10, 50}, "targz": {paddingBehind, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the // picture is more than a small share, so a cut of 1% keeps all of it. @@ -56,7 +53,7 @@ var notYetCutAtTheShare = map[string]notYet{ // guard for it, and it stays red until the format splits its padding for // format.Request.CutAt. A format that leaves the ratchet leaves this list too, // so it cannot go back. -var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "zip", "docx", "xlsx", "pptx", "targz", "webp", "tiff"} +var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "zip", "targz", "webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. @@ -133,6 +130,10 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { t.Fatalf("running: %v", err) } + plannedOf := map[string]engine.PlannedFile{} + for _, f := range planned { + plannedOf[f.Target.ID] = f + } problems := map[string]string{} for _, m := range res.Manifest.Files { missing, ok := missingOf[m.TargetID] @@ -140,7 +141,15 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { t.Errorf("%s: no file was written (%s)", m.TargetID, m.Error) continue } - problems[m.TargetID] = shareProblem(missing, m) + problem := shareProblem(missing, m) + f := plannedOf[m.TargetID] + if _, closes := endClosesTheFile[f.Desc.ID]; closes && problem == cutInTheEnd && onlyTheEndBehind(wholeMap(t, f)) { + // The end the format has to close with is longer than what the + // cut takes, and nothing else stands behind the content that the + // format could have put in front of it. + problem = "" + } + problems[m.TargetID] = problem } for _, problem := range shareVerdicts(problems, notYetCutAtTheShare) { t.Error(problem) @@ -158,6 +167,15 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { if shareProblem(50, kept) == "" { t.Errorf("canary: a cut that kept all its content passed for one that took half") } + region := func(role format.Role, at, length int64) format.Region { + return format.Region{Role: role, At: at, Length: length} + } + if onlyTheEndBehind([]format.Region{region(format.RoleContent, 0, 10), region(format.RolePadding, 10, 5), region(format.RoleEnd, 15, 5)}) { + t.Errorf("canary: padding behind the content passed for a file with nothing but its end behind it") + } + if !onlyTheEndBehind([]format.Region{region(format.RolePadding, 0, 5), region(format.RoleContent, 5, 10), region(format.RoleEnd, 15, 5)}) { + t.Errorf("canary: a file with all of its padding in front was refused for padding behind") + } listed := maps.Clone(notYetCutAtTheShare) listed["txt"] = notYet{paddingBehind, 64 << 10, 50} if !slices.ContainsFunc(shareVerdicts(problems, listed), func(s string) bool { return strings.HasPrefix(s, "txt:") }) { @@ -165,6 +183,64 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { } } +// cutInTheEnd is what shareProblem says of a cut that fell in the end a file +// closes with: a central directory, a trailer. +const cutInTheEnd = "the cut fell in the end" + +// endClosesTheFile is every format whose specification puts its end after all +// of its content, so a cut shorter than the end can only fall in it +// (onlyTheEndBehind). A format whose end may stand anywhere - a TIFF's image +// directory, which the specification lets stand before the pictures - is not +// here: it could put its end in front of the content instead. +var endClosesTheFile = map[string]string{ + "zip": "APPNOTE 4.3.6 lays a ZIP out with the central directory and its end record after every entry", + "docx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", + "xlsx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", + "pptx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", +} + +// onlyTheEndBehind is whether nothing but the end stands behind the content of +// a file's whole map - no padding there that the format could have put in +// front of the content instead. +// +// A cut that falls in the end of such a file falls there because the end is +// longer than the part the cut takes: a ZIP's central directory with a record +// for every part, cut by 1% of a few kilobytes (docs/TRUNCATE-2026-11-11.md +// section 10). No layout reaches the content then, so the share cannot be +// asked of that file. With padding behind the content the format could have +// moved it, and the cut is a miss like any other. +func onlyTheEndBehind(regions []format.Region) bool { + last := -1 + for i, r := range regions { + if r.Role == format.RoleContent { + last = i + } + } + if last < 0 { + return false + } + for _, r := range regions[last+1:] { + if r.Role != format.RoleEnd && r.Length != 0 { + return false + } + } + return true +} + +// wholeMap is the map of the whole file f is cut from: its generator written +// out through its own plan to nowhere, the damage left to the engine's +// streams, which are not here. +func wholeMap(t *testing.T, f engine.PlannedFile) []format.Region { + t.Helper() + c := &countingSink{} + rec := &format.MapRecorder{W: c} + if err := f.Desc.Generator.Write(t.Context(), rec, f.Plan); err != nil { + t.Errorf("%s: writing the whole file again: %v", f.Target.ID, err) + return nil + } + return rec.Map(c.n, true) +} + // shareCases is the files the sweep cuts of a format, each as its size and the // share cut: every size and share, or the one file named for a format listed // as not taking the share yet. @@ -231,6 +307,9 @@ func shareProblem(missing int64, f manifest.File) string { } cut := f.Damage[0].Cut if cut.In != "content" { + if cut.In == string(format.RoleEnd) { + return cutInTheEnd + } return fmt.Sprintf("the cut fell in the %s", cut.In) } if cut.ContentBytes == nil || *cut.ContentBytes <= 0 { diff --git a/internal/guard/filemap_test.go b/internal/guard/filemap_test.go index bc5ca398..7b6c72ad 100644 --- a/internal/guard/filemap_test.go +++ b/internal/guard/filemap_test.go @@ -180,6 +180,16 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { {string(format.RoleContent), int64(2), int64(5)}, {string(format.RolePadding), int64(7), int64(3)}}); front || !back { t.Errorf("canary: padding behind the content alone read as front %v, back %v", front, back) } + // Canaries of the first end: padding behind a head in front of the + // content is a miss, and so is a file with no padding behind. + region := func(role format.Role, at, length int64) [3]any { return [3]any{string(role), at, length} } + if firstEndLayout([][3]any{region(format.RoleStructure, 0, 2), region(format.RolePadding, 2, 4), + region(format.RoleContent, 6, 5), region(format.RolePadding, 11, 3)}) { + t.Errorf("canary: padding between a head and the content passed for the layout due at the first byte") + } + if firstEndLayout([][3]any{region(format.RolePadding, 0, 4), region(format.RoleContent, 4, 5), region(format.RoleEnd, 9, 3)}) { + t.Errorf("canary: a file with no padding behind its content passed for the layout due at the first byte") + } t.Logf("%d files, %d with padding, %d of them on both sides of the content, %d cut at the ends of a split", len(files), padded, split, ends) } @@ -190,20 +200,38 @@ func writeTheEnds(t *testing.T, dir string, f format.Descriptor, name string, se t.Helper() var out []mappedFile for _, end := range []struct { - name string - at int64 - front, back bool - }{{"first", 1, false, true}, {"last", size - 1, true, false}} { + name string + at int64 + due func(m [][3]any) bool + says string + }{ + {"first", 1, firstEndLayout, "padding behind the content, and in front of it none - or, where the content would start the file, one padding region that does instead"}, + {"last", size - 1, func(m [][3]any) bool { front, back := paddingSides(m); return front && !back }, "padding in front of the content only"}, + } { m := writeMapped(t, dir, f, name+"-"+end.name, setting, size, end.at) - if front, back := paddingSides(m.marked); front != end.front || back != end.back { - t.Errorf("%s: cut after %d B of %d B it marked %s - padding in front %v and behind %v, where %v and %v were due", - m.name, end.at, size, mapNames(m.marked), front, back, end.front, end.back) + if !end.due(m.marked) { + t.Errorf("%s: cut after %d B of %d B it marked %s, where %s was due", m.name, end.at, size, mapNames(m.marked), end.says) } out = append(out, m) } return out } +// firstEndLayout is the layout due for a cut after a file's first byte: its +// padding behind the content, and nothing in front of it. Unless the content +// would start the file, as a package's first part does: then a cut after one +// byte would keep a byte of it where the share is none, so the smallest +// padding region the format writes starts the file and the content follows +// it (docs/TRUNCATE-2026-11-11.md section 10). +func firstEndLayout(m [][3]any) bool { + front, back := paddingSides(m) + if !back { + return false + } + return !front || (len(m) > 1 && m[0][0] == string(format.RolePadding) && m[0][1] == int64(0) && + m[1][0] == string(format.RoleContent)) +} + // writeMapped writes one healthy file of f through a map recorder, laid out // for a cut at cutAt when that is not zero. func writeMapped(t *testing.T, dir string, f format.Descriptor, name string, setting map[string]string, size, cutAt int64) mappedFile { diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index f1717798..44e86edc 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -216,7 +216,7 @@ def layout_pdf(data): [END, xref, closed - xref], [PADDING, closed, sx - closed], [END, sx, len(data) - sx]] -ZIP_FILLERS = (b"tfg-padding.bin", b"tfg/padding.bin") +ZIP_FILLERS = (b"tfg-padding.bin", b"tfg/padding.bin", b"tfg-padding-end.bin", b"tfg/padding-end.bin") def zip_entries(data): From acead9b7eb459adcb0e4882cae24694f17938f0e Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 10:22:52 +0100 Subject: [PATCH 2/2] A ZIP cut short loses the share of its files asked for An archive a damage will cut is laid out so that the cut takes the share of the files inside it takes of the archive: an entry of padding in front of the files, behind the directories, and the rest behind them in the comment or a second entry, tfg-padding-end.bin. A squeezed archive is laid out as it is written, after a pass that squeezes its files into a counter, and the writing pass has to free the same or it stops with a defect. The files are marked with their length, so a cut inside them always says how much of them there was. The map guard allows, behind the content of a file cut before its last byte, one padding region holding the file's label and nothing else: a ZIP keeps its label in the archive comment, which closes the file. zip leaves the ratchet of formats that do not yet cut at the share. Measuring an archive moves out of zip.go into size.go, and what the compressor freed out of build into a method of its own. Three mutation entries the Office cut left without their text point at it again. Files nothing cuts are unchanged. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 + internal/format/archive/streammap.go | 20 +- internal/format/opc/opc.go | 2 +- internal/format/zip/compress.go | 26 ++- internal/format/zip/cut.go | 287 +++++++++++++++++++++++++++ internal/format/zip/filler.go | 60 ++++-- internal/format/zip/size.go | 120 +++++++++++ internal/format/zip/zip.go | 136 +++---------- internal/guard/contextroots_test.go | 2 +- internal/guard/cutshare_test.go | 3 +- internal/guard/filemap_test.go | 69 ++++++- internal/guard/mutation/entries.py | 14 +- 12 files changed, 597 insertions(+), 149 deletions(-) create mode 100644 internal/format/zip/cut.go create mode 100644 internal/format/zip/size.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d2ae910..d62c6d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,13 @@ because it turns other people's test suites red. removes 90% of the picture's content, so a viewer that shows what is left shows the part of the picture that arrived, the way an interrupted upload leaves it. Files nothing cuts are unchanged. +- **`truncate` cuts the files inside a ZIP, and the parts of a Word, Excel or + PowerPoint document, by the percentage you set.** `--damage + truncate:missing=50` on a 1 MB ZIP used to cut only padding. Now it removes + half of the files' content, so `tar -xf` gets back the first half of the + file inside. In a small file a cut of a few percent can fall in the index at + its end, and then every file inside is kept - the manifest says `in: end`. + Files nothing cuts are unchanged. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/format/archive/streammap.go b/internal/format/archive/streammap.go index 8a547d06..0af9116b 100644 --- a/internal/format/archive/streammap.go +++ b/internal/format/archive/streammap.go @@ -59,6 +59,12 @@ func (m *StreamMap) Pos() int64 { return m.out + int64(len(m.held)) + int64(m.Buffer.Buffered()) } +// MarkNext marks the region of role, length bytes long or -1 for up to the +// next mark, that starts with the next byte archive/zip writes. +func (m *StreamMap) MarkNext(role format.Role, length int64) { + format.Mark(m.w, role, m.Pos(), length) +} + func (m *StreamMap) mapped() bool { _, ok := m.w.(format.Mapper) return ok @@ -121,11 +127,13 @@ func (m *StreamMap) MarkLast(role format.Role, n int64) error { return nil } -// MarkEntry marks the region of role that starts with the local header of the -// entry named name, which archive/zip has just written: the last thing held. -// The header is found by its own fields rather than worked out, because -// archive/zip adds an extra field of its own to a header that names a time. -func (m *StreamMap) MarkEntry(role format.Role, name string) error { +// MarkEntry marks the region of role, length bytes long or -1 for up to the +// next mark, that starts with the local header of the entry named name, which +// archive/zip has just written: the last thing held. The header is found by its +// own fields rather than worked out, because archive/zip adds an extra field of +// its own to a header that names a time, and finishes the entry before it with +// a data descriptor only as this one opens. +func (m *StreamMap) MarkEntry(role format.Role, name string, length int64) error { if !m.holding { return nil } @@ -138,7 +146,7 @@ func (m *StreamMap) MarkEntry(role format.Role, name string) error { // the guard of the map reads back as a defect. return nil } - format.Mark(m.w, role, m.out+int64(at), -1) + format.Mark(m.w, role, m.out+int64(at), length) return nil } diff --git a/internal/format/opc/opc.go b/internal/format/opc/opc.go index a68b2266..ccb0f565 100644 --- a/internal/format/opc/opc.go +++ b/internal/format/opc/opc.go @@ -350,7 +350,7 @@ func writePaddingPart(ctx context.Context, zw *zip.Writer, zm *archive.StreamMap if err != nil { return err } - if err := zm.MarkEntry(format.RolePadding, part.name); err != nil { + if err := zm.MarkEntry(format.RolePadding, part.name, -1); err != nil { return err } if err := zm.Release(); err != nil { diff --git a/internal/format/zip/compress.go b/internal/format/zip/compress.go index ead06580..a14df5ff 100644 --- a/internal/format/zip/compress.go +++ b/internal/format/zip/compress.go @@ -32,10 +32,11 @@ import ( // to predict. func padCompressed(m *memo, p *format.Plan, r format.Request, groups []format.Content) error { m.withFiller, m.target = true, r.Bytes - withFiller, err := archiveSize(*m) + measured, err := archiveSize(*m) if err != nil { return err } + withFiller := measured.size m.fillerSize = r.Bytes - withFiller if m.fillerSize < 0 { return &format.BelowMinimumError{ @@ -87,6 +88,29 @@ func (m memo) methodFor(e entryPlan) uint16 { return stdzip.Store } +// freedBy is what the compressor freed of the files, given back through the +// filler so the archive still lands on the ordered size: their stored length +// less squeezed, what the compressor wrote of them. Nought on the counting +// pass, where no contents are written and the plan's own arithmetic is the +// answer. +// +// Asked as a question rather than worked out where the files are written, +// because the answer does not exist yet: the last entry's compressed bytes +// only reach the writer when its entry closes, and what closes it is the +// filler's own header. +func (m memo) freedBy(withContents bool, squeezed *int64) func() int64 { + return func() int64 { + if !withContents || !m.squeeze.On() { + return 0 + } + var plain int64 + for _, c := range m.children { + plain += c.plan.Bytes + } + return plain - *squeezed + } +} + // tally counts the bytes a compressor emits, so the writer can learn exactly // how much the compressor freed. // diff --git a/internal/format/zip/cut.go b/internal/format/zip/cut.go new file mode 100644 index 00000000..0724c49f --- /dev/null +++ b/internal/format/zip/cut.go @@ -0,0 +1,287 @@ +package zip + +import ( + stdzip "archive/zip" + "context" + "fmt" + "io" + "strings" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/archive" +) + +// fillerEndName is the entry of padding behind the files of an archive laid out +// for a cut, where fillerName stands in front of them. Two names, because two +// entries of one name are one file overwritten when the archive is unpacked. +const fillerEndName = "tfg-padding-end.bin" + +// split is how an archive's padding is laid out around its files for a cut +// (settleForACut), and its zero value is the archive nothing cuts. +type split struct { + // front says an entry of padding stands in front of the files, behind the + // directories, and frontBytes is how much padding it holds. + front bool + frontBytes int64 + // sized says content is how long the files' entries come to. The files are + // marked with it, so that a cut inside them says how much of them there + // was: a cut stops the writing before the mark that would end them. + sized bool + content int64 + // freed is how much shorter the compressor makes the files of a squeezed + // archive, measured before it is written. + freed int64 +} + +// contentLength is the length the files are marked with, -1 for up to the next +// mark. +func (s split) contentLength() int64 { + if !s.sized { + return -1 + } + return s.content +} + +// settleForACut lays the padding of an archive a damage will cut after its +// first cutAt bytes (format.Request.CutAt) out around its files, so that the +// cut keeps the share of them it keeps of the whole archive. +// +// With all of its padding behind the files, as an archive nothing cuts has it, +// a large archive cut at any share kept every file and lost only padding. +// APPNOTE names no order of entries and a reader finds them through the central +// directory: Python's zipfile, 7-Zip and bsdtar read an archive whose padding +// entry comes first as they read it today (docs/TRUNCATE-2026-11-11.md section +// 10.3). So an entry of padding goes in front of the files and behind the +// directories, which stay structure. +// +// A stored archive is settled here. A squeezed one is settled as it is written +// (settleSqueezedForACut), because how long its files come to is known only +// once they are squeezed, and planning does not squeeze. +func settleForACut(m *memo, p *format.Plan, cutAt, target int64, label string) error { + m.cutAt = cutAt + if !m.cuts(target) || m.squeeze.On() { + return nil + } + c, err := measureCut(*m, label) + if err != nil { + return err + } + if err := m.splitPadding(c, target, 0, label); err != nil { + return err + } + if m.split.front { + p.Properties["padding_entry"] = fillerName + } + return nil +} + +// settleSqueezedForACut is settleForACut for a squeezed archive, as it is +// written. Its files are squeezed into a counter first, to learn how long they +// come to, and the writing pass then has to come to the same +// (writeCompressedFiller). That generates every file inside twice, and only for +// a squeezed archive a damage cuts. Nothing for any other archive. +func (m *memo) settleSqueezedForACut(ctx context.Context, target int64) error { + if !m.squeeze.On() || !m.cuts(target) { + return nil + } + // padCompressed leaves the comment the label alone. + label := m.comment + c, err := measureCut(*m, label) + if err != nil { + return err + } + squeezed, err := squeezedFiles(ctx, *m) + if err != nil { + return err + } + return m.splitPadding(c, target, c.content-squeezed, label) +} + +// cuts says the archive is laid out for a cut: one falls inside it, and there +// are files for it to keep a share of. +func (m memo) cuts(target int64) bool { + return m.cutAt > 0 && m.cutAt < target && len(m.children) > 0 +} + +// splitPadding lays the padding out around the files for the cut at m.cutAt, +// in an archive of target bytes whose files the compressor makes freed bytes +// shorter, nought when they are stored (docs/TRUNCATE-2026-11-11.md section +// 10.4). +// +// The entry in front costs its header before any padding, so the front is +// nothing or at least that, and grows where it cannot land exactly - the cut +// then takes a little more of the files, the "at least" missing promises. +// Where the padding is too short for the files the cut should take, all of it +// goes in front, and the cut keeps more of them by the share of what stands +// behind. The files are marked with their length whether or not anything goes +// in front, so a cut inside them says how much of them there was. +func (m *memo) splitPadding(c cutShape, target, freed int64, label string) error { + content := c.content - freed + m.split = split{sized: true, content: content, freed: freed} + pad := target - (c.bare - freed) + front := format.PaddingInFront(c.before, content, m.cutAt, target) + // A squeezed archive keeps its entry of padding behind the files. It takes + // back what the compressor frees, and the plan names it. + var behind int64 + if m.squeeze.On() { + behind = c.endOverhead + } + if front <= 0 || pad < c.frontOverhead+behind { + return nil + } + cost := min(c.frontOverhead+max(front, c.frontHead)-c.frontHead, pad-behind) + m.split.front, m.split.frontBytes = true, cost-c.frontOverhead + if err := m.padBehind(c, pad-cost, freed, label); err != nil { + return err + } + got, err := archiveSize(*m) + if err != nil { + return err + } + if got.size != target || got.content.Length != c.content || got.content.At != c.before+c.frontHead+m.split.frontBytes { + return core.Defect(fmt.Errorf("zip: laid out for a cut, the archive measures %d B with its files at %d for %d B, where %d B with them at %d for %d B was settled", + got.size, got.content.At, got.content.Length, target, c.before+c.frontHead+m.split.frontBytes, c.content)) + } + return nil +} + +// padBehind puts back bytes of padding behind the files: into the comment +// while they fit beside the label, and into an entry of padding above that. A +// squeezed archive puts them all into the entry, which takes back what the +// compressor frees as it is written. +func (m *memo) padBehind(c cutShape, back, freed int64, label string) error { + m.comment, m.withFiller, m.fillerSize = label, false, 0 + switch { + case m.squeeze.On(): + m.withFiller, m.fillerSize = true, back-c.endOverhead-freed + case back <= int64(commentPaddingLimit-len(label)): + m.comment = label + strings.Repeat(" ", int(back)) + case back >= c.endOverhead: + m.withFiller, m.fillerSize = true, back-c.endOverhead + default: + return core.Defect(fmt.Errorf("zip: %d B behind the files is more than the comment carries and less than an entry of padding costs (%d B)", back, c.endOverhead)) + } + return nil +} + +// cutShape is what splitPadding needs to know of an archive, measured by +// writing its structure rather than added up from header widths, for the +// reason archiveSize gives. +type cutShape struct { + // bare is how long the archive comes to with no padding at all: no entry + // of it, and the comment the label alone. + bare int64 + // before is how many bytes stand in front of the files, the directories, + // and content how long the files' entries come to when they are stored. + before, content int64 + // frontHead is how many bytes an empty entry in front puts before the + // files, and frontOverhead what it costs in all, its record in the central + // directory with it. + frontHead, frontOverhead int64 + // endOverhead is what an empty entry of padding behind the files costs + // when one stands in front - it carries the longer name. + endOverhead int64 +} + +func measureCut(m memo, label string) (cutShape, error) { + m.comment, m.withFiller, m.fillerSize, m.split = label, false, 0, split{} + bare, err := archiveSize(m) + if err != nil { + return cutShape{}, err + } + m.split.front = true + front, err := archiveSize(m) + if err != nil { + return cutShape{}, err + } + m.withFiller = true + both, err := archiveSize(m) + if err != nil { + return cutShape{}, err + } + c := cutShape{ + bare: bare.size, before: bare.content.At, content: bare.content.Length, + frontHead: front.content.At - bare.content.At, frontOverhead: front.size - bare.size, + endOverhead: both.size - front.size, + } + if front.content.Length != c.content || both.content.Length != c.content { + return c, core.Defect(fmt.Errorf("zip: the files measured %d B, then %d B behind an entry of padding and %d B between two", + c.content, front.content.Length, both.content.Length)) + } + return c, nil +} + +// squeezedFiles is how long the files' entries come to once squeezed: the pass +// that measures, squeezing every file into a counter. +func squeezedFiles(ctx context.Context, m memo) (int64, error) { + m.withFiller, m.split = false, split{} + c := &counter{} + rec := &format.MapRecorder{W: c} + if err := build(ctx, rec, m, true); err != nil { + return 0, err + } + content, err := contentOf(rec.Map(c.n, true)) + if err != nil { + return 0, err + } + return content.Length, nil +} + +// startTheFiles marks where the files start, once the directories are +// written: at the next byte, or, in an archive laid out for a cut, behind an +// entry of padding it writes first - where openFile marks them. +// +// The entry is named fillerName, the name of the one entry of padding an +// archive nothing cuts has, and stored and locked like it. Its index follows +// that of the one behind the files, so the two take different salts. +func startTheFiles(ctx context.Context, zw *stdzip.Writer, zm *archive.StreamMap, m memo, withContents bool) error { + if !m.split.front { + zm.MarkNext(format.RoleContent, m.split.contentLength()) + return nil + } + if err := zm.Hold(); err != nil { + return err + } + return writeStoredFiller(ctx, zw, zm, m, withContents, storedFiller{ + name: fillerName, size: m.split.frontBytes, index: len(m.children) + 1, + }) +} + +// openFile is openEntry for a file inside, which marks where the files start +// when an entry of padding stands in front of them: at the first file's local +// header, which archive/zip writes only once it has finished the entry in +// front with its data descriptor. +func openFile(zw *stdzip.Writer, zm *archive.StreamMap, m memo, e entryPlan) (io.Writer, func() error, error) { + if e.index != 0 || !m.split.front { + return openEntry(zw, m, e) + } + if err := zm.Hold(); err != nil { + return nil, nil, err + } + entry, shut, err := openEntry(zw, m, e) + if err != nil { + return nil, nil, err + } + if err := zm.MarkEntry(format.RoleContent, e.name, m.split.contentLength()); err != nil { + return nil, nil, err + } + return entry, shut, zm.Release() +} + +// mapped is an archive's length and the region its files take, read off the +// map a counting pass marks (archiveSize). +type mapped struct { + size int64 + content format.Region +} + +// contentOf is the region of content in a map. +func contentOf(regions []format.Region) (format.Region, error) { + for _, r := range regions { + if r.Role == format.RoleContent { + return r, nil + } + } + return format.Region{}, core.Defect(fmt.Errorf("zip: writing the archive marked no files")) +} diff --git a/internal/format/zip/filler.go b/internal/format/zip/filler.go index 7bdddcad..c1d763cf 100644 --- a/internal/format/zip/filler.go +++ b/internal/format/zip/filler.go @@ -3,6 +3,7 @@ package zip import ( stdzip "archive/zip" "context" + "fmt" "io" "github.com/donislawdev/TestingFilesGenerator/internal/format" @@ -41,48 +42,70 @@ func writeFillerEntry(ctx context.Context, zw *stdzip.Writer, zm *archive.Stream if m.squeeze.On() { return writeCompressedFiller(ctx, zw, zm, m, withContents, freed) } + return writeStoredFiller(ctx, zw, zm, m, withContents, storedFiller{ + name: m.fillerEntryName(), size: m.fillerSize, index: len(m.children), + }) +} - // The stored path, unchanged. It settles the size before the header - // because a locked entry states its length there, and a locked archive - // takes this path. - size := m.fillerSize +// storedFiller is one entry of padding whose length is settled before its +// header: the one behind the files of a stored archive, and the one in front of +// them in an archive laid out for a cut (writeFrontEntry). +type storedFiller struct { + name string + size int64 + index int +} + +// writeStoredFiller writes f, with what follows its header held (zm.Hold) +// until it is marked. It settles the size before the header because a locked +// entry states its length there, and a locked archive takes this path. +func writeStoredFiller(ctx context.Context, zw *stdzip.Writer, zm *archive.StreamMap, m memo, withContents bool, f storedFiller) error { // The filler is locked with everything else. An archive where one entry // opens without the password and the rest do not is a file nobody // asked for, and the arithmetic is the same either way. crc, err := plaintextCRC(ctx, m, withContents, func(w io.Writer) error { - return writeFiller(ctx, w, m.seed, size) + return writeFiller(ctx, w, m.seed, f.size) }) if err != nil { return err } entry, shut, err := openEntry(zw, m, entryPlan{ - name: fillerName, plain: size, index: len(m.children), stored: true, + name: f.name, plain: f.size, index: f.index, stored: true, withContents: withContents, crc: crc, }) if err != nil { return err } - if err := markFiller(zm); err != nil { + if err := markFiller(zm, f.name); err != nil { return err } if !withContents { return nil } - if err := writeFiller(ctx, entry, m.seed, size); err != nil { + if err := writeFiller(ctx, entry, m.seed, f.size); err != nil { return err } return shut() } -// markFiller marks the filler entry as padding and sends on what was held -// while its header was written. -func markFiller(zm *archive.StreamMap) error { - if err := zm.MarkEntry(format.RolePadding, fillerName); err != nil { +// markFiller marks the filler entry named name as padding and sends on what +// was held while its header was written. +func markFiller(zm *archive.StreamMap, name string) error { + if err := zm.MarkEntry(format.RolePadding, name, -1); err != nil { return err } return zm.Release() } +// fillerEntryName is the name of the entry of padding behind the files: +// fillerName, unless an entry in front of them has it (split). +func (m memo) fillerEntryName() string { + if m.split.front { + return fillerEndName + } + return fillerName +} + // writeCompressedFiller is the same entry for an archive whose others are // squeezed, with the header written before the length is known. // @@ -91,13 +114,13 @@ func markFiller(zm *archive.StreamMap) error { // archive is never a locked one. func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, zm *archive.StreamMap, m memo, withContents bool, freed func() int64) error { entry, shut, err := openEntry(zw, m, entryPlan{ - name: fillerName, index: len(m.children), stored: true, + name: m.fillerEntryName(), index: len(m.children), stored: true, withContents: withContents, }) if err != nil { return err } - if err := markFiller(zm); err != nil { + if err := markFiller(zm, m.fillerEntryName()); err != nil { return err } if !withContents { @@ -105,6 +128,13 @@ func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, zm *archive.S // writing its bytes here would count them twice. return nil } + // An archive laid out for a cut was settled on what a pass that only + // measured found the compressor frees, and its files were marked with the + // length that gave. Squeezing the same files again has to free the same. + got := freed() + if m.split.sized && got != m.split.freed { + return core.Defect(fmt.Errorf("zip: the files inside came to %d B less once squeezed while measuring and %d B less while writing", m.split.freed, got)) + } // What the compressor freed can be NEGATIVE, and that is the case this // arithmetic was written without. Deflate grows data it cannot shrink - // about five bytes for every 65535 - so an archive holding already @@ -122,7 +152,7 @@ func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, zm *archive.S // knowable without squeezing them, and planning is not allowed to. The // floor reported is measured rather than derived - it is what this // archive came to with no padding at all. - size := m.fillerSize + freed() + size := m.fillerSize + got if size < 0 { return &format.BelowMinimumError{ Format: "ZIP", diff --git a/internal/format/zip/size.go b/internal/format/zip/size.go new file mode 100644 index 00000000..c1e3c398 --- /dev/null +++ b/internal/format/zip/size.go @@ -0,0 +1,120 @@ +package zip + +import ( + "context" + "fmt" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/archive" +) + +// How long an archive comes to, measured by writing its structure: what +// planning, the minimum and the layout for a cut all ask. + +// archiveSize measures the archive without generating anything it holds. +// +// The structure is built for real - every header, every name, the comment, the +// central directory - and only the file contents are left out and added on as +// numbers. Everything inside is stored rather than compressed, so that is exact +// rather than close. +// +// Measured on the standard library on 2026-08-03, because a document about a +// format has been wrong here five times out of five and this one is arithmetic +// standing in for a measurement: +// +// an archive of 3 entries with no contents 424 B +// the same 3 entries holding 4096 B each 12 712 B +// 424 + 3*4096 12 712 B exact +// +// Checked at 0, 1, 1000, 4096 and 100 000 bytes an entry, and the comment moves +// the total by exactly its own length at 0, 1, 2, 50, 4096 and 65 535. +// +// Why it matters: planning used to call this two or three times and each call +// generated every file inside. That made --dry-run cost about what the real run +// costs - measured at 960 ms against a 56 ms baseline for a 256 MB archive, +// while writing it for real took 1585 ms. A dry run is the step this project +// tells people to take before anything large, so it has to be the cheap one. +// +// If the arithmetic is ever wrong the size guard says so immediately: the +// engine refuses any file whose written length differs from its plan, and the +// property test walks about 120 sizes for this format. +// +// It reads the region the files take off the map the pass marks, with the +// contents added on the same way, for laying the padding out around them +// (cut.go). +func archiveSize(m memo) (mapped, error) { + // What this pass writes of the files is their headers alone, so the + // length the writing pass marks them with is not theirs here. + m.split.sized = false + c := &counter{} + rec := &format.MapRecorder{W: c} + if err := build(context.Background(), rec, m, false); err != nil { + return mapped{}, err + } + content, err := contentOf(rec.Map(c.n, true)) + if err != nil { + return mapped{}, err + } + var files int64 + for _, ch := range m.children { + files += ch.plan.Bytes + m.lock.EntryOverhead() + } + total := c.n + files + content.Length += files + if m.split.front { + front := m.split.frontBytes + m.lock.EntryOverhead() + total += front + content.At += front + } + if m.withFiller { + total += m.fillerSize + m.lock.EntryOverhead() + } + return mapped{size: total, content: content}, nil +} + +type counter struct{ n int64 } + +func (c *counter) Write(p []byte) (int, error) { c.n += int64(len(p)); return len(p), nil } + +// minimumBytes is the smallest archive this generator can produce: one empty +// entry, no label, no padding. +// It panics rather than returning a number nobody measured. +// +// This used to answer 1<<62 on any failure, on the reasoning that refusing +// every size is safer than declaring a wrong minimum. It is not safer, it is +// quieter: ZIP would refuse every request ever made with a message about a +// minimum of 4611686018427387904 B, and nothing would say why. That is rule 6 +// broken in the way this project keeps finding. +// +// The condition is a programming mistake rather than a runtime one. It needs +// the default entry format to be unregistered when this runs, and the language +// says when that can happen: packages are initialised in the order of their +// import paths, so the txt package is registered before this one by rule rather +// than by luck. This paragraph said "true by accident" until 2026-08-25, when +// an outside review built an argument on the opposite claim - that the +// specification guarantees nothing here. Both were wrong, and the comment being +// wrong about its own mechanism is the worse of the two. +// +// What the rule does not cover is a rename that moves the entry format after +// this package in that order, or a second entry point linking this package +// without that one. The first is what a guard asks, since a format has been +// renamed in this tree before. format.Register already panics on the same class +// of mistake, so this matches it: a build that cannot state its own minimum +// fails at start rather than at every use. +func minimumBytes() int64 { + desc, err := format.Get(archive.DefaultFormat) + if err != nil { + panic(fmt.Sprintf("zip: the default entry format %q is not registered yet, so the minimum size of an archive cannot be worked out. Check the import order in internal/format/all", archive.DefaultFormat)) + } + cp, err := desc.Generator.Plan(format.Request{Bytes: 0, Seed: 0, Label: false}) + if err != nil { + panic(fmt.Sprintf("zip: the default entry format %q cannot produce an empty file, so the minimum size of an archive cannot be worked out: %v", archive.DefaultFormat, err)) + } + n, err := archiveSize(memo{children: []child{{ + name: "txt_0001.txt", desc: desc, plan: cp, + }}}) + if err != nil { + panic(fmt.Sprintf("zip: the smallest archive cannot be measured: %v", err)) + } + return n.size +} diff --git a/internal/format/zip/zip.go b/internal/format/zip/zip.go index cd27e0df..defbaaeb 100644 --- a/internal/format/zip/zip.go +++ b/internal/format/zip/zip.go @@ -126,6 +126,12 @@ type memo struct { // made. Only the compressed path needs it: a stored archive settles its // padding while planning, so a size it cannot reach is refused there. target int64 + // cutAt is where a damage will cut the archive (format.Request.CutAt), 0 + // when nothing does, and split is how its padding is laid out around the + // files for that cut (cut.go). The zero value of split is the archive + // nothing cuts. + cutAt int64 + split split } func (g generator) Plan(r format.Request) (format.Plan, error) { @@ -168,6 +174,9 @@ func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan if err := pad(&m, &p, r, target, bare, label, groups); err != nil { return format.Plan{}, err } + if err := settleForACut(&m, &p, r.CutAt, target, label); err != nil { + return format.Plan{}, err + } if err := withinZip32(target); err != nil { return format.Plan{}, err @@ -251,9 +260,11 @@ func settleSize(m *memo, r format.Request) (target, bare int64, label string, er // Measure the archive with no padding at all. Every entry is stored rather // than compressed, so the size follows from the parts exactly and nothing // has to be guessed. - if bare, err = archiveSize(*m); err != nil { + measured, err := archiveSize(*m) + if err != nil { return 0, 0, "", err } + bare = measured.size if !r.SizeFromContents { return r.Bytes, bare, label, nil @@ -275,8 +286,8 @@ func settleSize(m *memo, r format.Request) (target, bare int64, label string, er if err != nil { return 0, 0, "", err } - settled = measured == size - size = measured + settled = measured.size == size + size = measured.size } if !settled { return 0, 0, "", core.Defect(fmt.Errorf( @@ -380,10 +391,11 @@ func pad(m *memo, p *format.Plan, r format.Request, target, bare int64, label st // in Tier 1 with a ceiling. m.comment = label + strings.Repeat(" ", int(room)) m.withFiller = true - withFiller, err := archiveSize(*m) + measured, err := archiveSize(*m) if err != nil { return err } + withFiller := measured.size m.fillerSize = r.Bytes - withFiller if m.fillerSize < 0 { return &format.BelowMinimumError{ @@ -413,6 +425,9 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { if !ok { return core.Defect(fmt.Errorf("zip: the plan was not produced by this generator")) } + if err := m.settleSqueezedForACut(ctx, p.Bytes); err != nil { + return err + } return build(ctx, w, m, true) } @@ -558,7 +573,9 @@ func build(ctx context.Context, w io.Writer, m memo, withContents bool) error { if err := writeDirectories(ctx, zw, m.layout); err != nil { return err } - format.Mark(w, format.RoleContent, zm.Pos(), -1) + if err := startTheFiles(ctx, zw, zm, m, withContents); err != nil { + return err + } for i, c := range m.children { select { @@ -573,7 +590,7 @@ func build(ctx context.Context, w io.Writer, m memo, withContents bool) error { if err != nil { return core.Defect(fmt.Errorf("zip: the %s file inside could not be checksummed: %w", c.desc.ID, err)) } - entry, shut, err := openEntry(zw, m, entryPlan{ + entry, shut, err := openFile(zw, zm, m, entryPlan{ name: c.name, plain: c.plan.Bytes, index: i, withContents: withContents, crc: crc, }) if err != nil { @@ -589,114 +606,9 @@ func build(ctx context.Context, w io.Writer, m memo, withContents bool) error { } } - // What the compressor freed, given back through the filler so the archive - // still lands on the ordered size. Nought on the counting pass, where no - // contents are written and the plan's own arithmetic is the answer. - // Asked as a question rather than worked out here, because the answer does - // not exist yet: the last entry's compressed bytes only reach the writer - // when its entry closes, and what closes it is the filler's own header. - freed := func() int64 { - if !withContents || !m.squeeze.On() { - return 0 - } - var plain int64 - for _, c := range m.children { - plain += c.plan.Bytes - } - return plain - squeezed - } - if err := writeFillerEntry(ctx, zw, zm, m, withContents, freed); err != nil { + if err := writeFillerEntry(ctx, zw, zm, m, withContents, m.freedBy(withContents, &squeezed)); err != nil { return err } return zm.Close(zw) } - -// archiveSize measures the archive without generating anything it holds. -// -// The structure is built for real - every header, every name, the comment, the -// central directory - and only the file contents are left out and added on as -// numbers. Everything inside is stored rather than compressed, so that is exact -// rather than close. -// -// Measured on the standard library on 2026-08-03, because a document about a -// format has been wrong here five times out of five and this one is arithmetic -// standing in for a measurement: -// -// an archive of 3 entries with no contents 424 B -// the same 3 entries holding 4096 B each 12 712 B -// 424 + 3*4096 12 712 B exact -// -// Checked at 0, 1, 1000, 4096 and 100 000 bytes an entry, and the comment moves -// the total by exactly its own length at 0, 1, 2, 50, 4096 and 65 535. -// -// Why it matters: planning used to call this two or three times and each call -// generated every file inside. That made --dry-run cost about what the real run -// costs - measured at 960 ms against a 56 ms baseline for a 256 MB archive, -// while writing it for real took 1585 ms. A dry run is the step this project -// tells people to take before anything large, so it has to be the cheap one. -// -// If the arithmetic is ever wrong the size guard says so immediately: the -// engine refuses any file whose written length differs from its plan, and the -// property test walks about 120 sizes for this format. -func archiveSize(m memo) (int64, error) { - c := &counter{} - if err := build(context.Background(), c, m, false); err != nil { - return 0, err - } - total := c.n - for _, ch := range m.children { - total += ch.plan.Bytes + m.lock.EntryOverhead() - } - if m.withFiller { - total += m.fillerSize + m.lock.EntryOverhead() - } - return total, nil -} - -type counter struct{ n int64 } - -func (c *counter) Write(p []byte) (int, error) { c.n += int64(len(p)); return len(p), nil } - -// minimumBytes is the smallest archive this generator can produce: one empty -// entry, no label, no padding. -// It panics rather than returning a number nobody measured. -// -// This used to answer 1<<62 on any failure, on the reasoning that refusing -// every size is safer than declaring a wrong minimum. It is not safer, it is -// quieter: ZIP would refuse every request ever made with a message about a -// minimum of 4611686018427387904 B, and nothing would say why. That is rule 6 -// broken in the way this project keeps finding. -// -// The condition is a programming mistake rather than a runtime one. It needs -// the default entry format to be unregistered when this runs, and the language -// says when that can happen: packages are initialised in the order of their -// import paths, so the txt package is registered before this one by rule rather -// than by luck. This paragraph said "true by accident" until 2026-08-25, when -// an outside review built an argument on the opposite claim - that the -// specification guarantees nothing here. Both were wrong, and the comment being -// wrong about its own mechanism is the worse of the two. -// -// What the rule does not cover is a rename that moves the entry format after -// this package in that order, or a second entry point linking this package -// without that one. The first is what a guard asks, since a format has been -// renamed in this tree before. format.Register already panics on the same class -// of mistake, so this matches it: a build that cannot state its own minimum -// fails at start rather than at every use. -func minimumBytes() int64 { - desc, err := format.Get(archive.DefaultFormat) - if err != nil { - panic(fmt.Sprintf("zip: the default entry format %q is not registered yet, so the minimum size of an archive cannot be worked out. Check the import order in internal/format/all", archive.DefaultFormat)) - } - cp, err := desc.Generator.Plan(format.Request{Bytes: 0, Seed: 0, Label: false}) - if err != nil { - panic(fmt.Sprintf("zip: the default entry format %q cannot produce an empty file, so the minimum size of an archive cannot be worked out: %v", archive.DefaultFormat, err)) - } - n, err := archiveSize(memo{children: []child{{ - name: "txt_0001.txt", desc: desc, plan: cp, - }}}) - if err != nil { - panic(fmt.Sprintf("zip: the smallest archive cannot be measured: %v", err)) - } - return n -} diff --git a/internal/guard/contextroots_test.go b/internal/guard/contextroots_test.go index 455d64c4..86197e94 100644 --- a/internal/guard/contextroots_test.go +++ b/internal/guard/contextroots_test.go @@ -40,7 +40,7 @@ var contextRoots = map[string]string{ "internal/format/zip/zip.go generator.Plan": "the Generator interface's Plan for callers without a context - the engine asks PlanContext", "internal/engine/damagedformat.go damagedGenerator.Plan": "the Generator interface's Plan for callers without a context, such as SmallestAccepted - the engine asks PlanContext", "internal/format/opc/opc.go size": "measures a package with the padding left out, so there is nothing long to stop", - "internal/format/zip/zip.go archiveSize": "measures an archive with the contents left out, so there is nothing long to stop", + "internal/format/zip/size.go archiveSize": "measures an archive with the contents left out, so there is nothing long to stop", } func TestAContextIsMadeFromNothingOnlyWhereItIsMeantToStart(t *testing.T) { diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index f8c8290e..9a095bba 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -37,7 +37,6 @@ var notYetCutAtTheShare = map[string]notYet{ "mp4": {paddingBehind, 64 << 10, 50}, "webm": {paddingBehind, 64 << 10, 50}, "pdf": {paddingBehind, 64 << 10, 50}, - "zip": {paddingBehind, 64 << 10, 50}, "targz": {paddingBehind, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the // picture is more than a small share, so a cut of 1% keeps all of it. @@ -53,7 +52,7 @@ var notYetCutAtTheShare = map[string]notYet{ // guard for it, and it stays red until the format splits its padding for // format.Request.CutAt. A format that leaves the ratchet leaves this list too, // so it cannot go back. -var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "zip", "targz", "webp", "tiff"} +var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "targz", "webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. diff --git a/internal/guard/filemap_test.go b/internal/guard/filemap_test.go index 7b6c72ad..6cb7840e 100644 --- a/internal/guard/filemap_test.go +++ b/internal/guard/filemap_test.go @@ -190,6 +190,7 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { if firstEndLayout([][3]any{region(format.RolePadding, 0, 4), region(format.RoleContent, 4, 5), region(format.RoleEnd, 9, 3)}) { t.Errorf("canary: a file with no padding behind its content passed for the layout due at the first byte") } + askTheLastEndCanaries(t, dir) t.Logf("%d files, %d with padding, %d of them on both sides of the content, %d cut at the ends of a split", len(files), padded, split, ends) } @@ -199,17 +200,18 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { func writeTheEnds(t *testing.T, dir string, f format.Descriptor, name string, setting map[string]string, size int64) []mappedFile { t.Helper() var out []mappedFile + label := format.Request{Bytes: size, Label: true, Seed: 7}.LabelText(f.ID) for _, end := range []struct { name string at int64 - due func(m [][3]any) bool + due func(m mappedFile) bool says string }{ - {"first", 1, firstEndLayout, "padding behind the content, and in front of it none - or, where the content would start the file, one padding region that does instead"}, - {"last", size - 1, func(m [][3]any) bool { front, back := paddingSides(m); return front && !back }, "padding in front of the content only"}, + {"first", 1, func(m mappedFile) bool { return firstEndLayout(m.marked) }, "padding behind the content, and in front of it none - or, where the content would start the file, one padding region that does instead"}, + {"last", size - 1, func(m mappedFile) bool { return lastEndLayout(t, m, label) }, "padding in front of the content only - or, behind it, the label alone where the format keeps it in a field that closes the file"}, } { m := writeMapped(t, dir, f, name+"-"+end.name, setting, size, end.at) - if !end.due(m.marked) { + if !end.due(m) { t.Errorf("%s: cut after %d B of %d B it marked %s, where %s was due", m.name, end.at, size, mapNames(m.marked), end.says) } out = append(out, m) @@ -232,6 +234,65 @@ func firstEndLayout(m [][3]any) bool { m[1][0] == string(format.RoleContent)) } +// lastEndLayout is the layout due for a cut before a file's last byte: all of +// its padding in front of the content. Behind the content it allows one padding +// region, and only when that holds the file's label and nothing else. A ZIP +// keeps its label in the archive comment, which closes the file, so no layout +// can move it in front (docs/TRUNCATE-2026-11-11.md section 10.4). +func lastEndLayout(t *testing.T, m mappedFile, label string) bool { + t.Helper() + front, back := paddingSides(m.marked) + return front && (!back || onlyTheLabelBehind(t, m, label)) +} + +// onlyTheLabelBehind says the padding behind the content of m is one region, +// and its bytes are label. +func onlyTheLabelBehind(t *testing.T, m mappedFile, label string) bool { + t.Helper() + at := slices.IndexFunc(m.marked, func(r [3]any) bool { return r[0] == string(format.RoleContent) }) + var behind [][3]any + for _, r := range m.marked[at+1:] { + if r[0] == string(format.RolePadding) { + behind = append(behind, r) + } + } + if len(behind) != 1 { + return false + } + data, err := os.ReadFile(m.path) + if err != nil { + t.Fatalf("%s: %v", m.name, err) + } + start, n := behind[0][1].(int64), behind[0][2].(int64) + return start+n <= int64(len(data)) && string(data[start:start+n]) == label +} + +// askTheLastEndCanaries asks lastEndLayout about files made for it: the label +// alone behind the content passes, and padding of its length that is not it, +// or the label with padding beside it, does not. +func askTheLastEndCanaries(t *testing.T, dir string) { + t.Helper() + file := func(tail string) mappedFile { + path := filepath.Join(dir, "canary-last-end") + if err := os.WriteFile(path, []byte("ppccccc"+tail), 0o600); err != nil { + t.Fatal(err) + } + return mappedFile{name: "canary", path: path, marked: [][3]any{ + {string(format.RolePadding), int64(0), int64(2)}, {string(format.RoleContent), int64(2), int64(5)}, + {string(format.RolePadding), int64(7), int64(len(tail))}, + }} + } + if !lastEndLayout(t, file("tfg"), "tfg") { + t.Errorf("canary: the label alone behind the content failed the layout due at the last byte") + } + if lastEndLayout(t, file("tfx"), "tfg") { + t.Errorf("canary: padding of the label's length that is not the label passed for the layout due at the last byte") + } + if lastEndLayout(t, file("tfg "), "tfg") { + t.Errorf("canary: the label with padding beside it passed for the layout due at the last byte") + } +} + // writeMapped writes one healthy file of f through a map recorder, laid out // for a cut at cutAt when that is not zero. func writeMapped(t *testing.T, dir string, f format.Descriptor, name string, setting map[string]string, size, cutAt int64) mappedFile { diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 52c29cf8..1ee94990 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -360,8 +360,8 @@ ( "an Office package goes back to writing its sizes after the data instead of in the header", "internal/format/opc/opc.go", - "entry, err := zw.CreateRaw(head)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := zm.MarkEntry(", - "entry, err := zw.CreateHeader(head)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := zm.MarkEntry(", + "entry, err := zw.CreateRaw(head)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif err := zm.MarkEntry(", + "entry, err := zw.CreateHeader(head)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif err := zm.MarkEntry(", "TestEveryFormatSurvivesItsReferenceTool", ), ( @@ -6752,8 +6752,8 @@ "that matters most here - a function sits one level under a ceiling " "that was already exact, and no guard could see it", "internal/guard/branching_test.go", + "\tcrowdedDepthFunctions = 48\n", "\tcrowdedDepthFunctions = 49\n", - "\tcrowdedDepthFunctions = 50\n", "TestTheBranchingCeilingsAreTodaysMeasurementAndNotALooserNumber", ), ( @@ -7602,8 +7602,8 @@ ( "a compressed archive stops giving back what the compressor freed, so it comes out short", "internal/format/zip/filler.go", - "size := m.fillerSize + freed()", - "size := m.fillerSize + freed()*0", + "size := m.fillerSize + got", + "size := m.fillerSize + got*0", "TestACompressedArchiveStillHitsTheSizeToTheByte", ), ( @@ -15870,8 +15870,8 @@ # stop during it waits for four gigabytes of padding to be hashed. "an Office package checksums its padding without looking at the stop", "internal/format/opc/opc.go", - "sum, err := fillerChecksum(ctx, p.Seed, p.Filler)", - "sum, err := fillerChecksum(context.Background(), p.Seed, p.Filler)", + "sum, err := fillerChecksum(ctx, part.seed, part.n)", + "sum, err := fillerChecksum(context.Background(), part.seed, part.n)", "TestAnOfficePackageStopsWhileItChecksumsItsPadding", ), (