diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d2ae910..d62c6d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,13 @@ because it turns other people's test suites red. removes 90% of the picture's content, so a viewer that shows what is left shows the part of the picture that arrived, the way an interrupted upload leaves it. Files nothing cuts are unchanged. +- **`truncate` cuts the files inside a ZIP, and the parts of a Word, Excel or + PowerPoint document, by the percentage you set.** `--damage + truncate:missing=50` on a 1 MB ZIP used to cut only padding. Now it removes + half of the files' content, so `tar -xf` gets back the first half of the + file inside. In a small file a cut of a few percent can fall in the index at + its end, and then every file inside is kept - the manifest says `in: end`. + Files nothing cuts are unchanged. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/format/archive/streammap.go b/internal/format/archive/streammap.go index 8a547d06..0af9116b 100644 --- a/internal/format/archive/streammap.go +++ b/internal/format/archive/streammap.go @@ -59,6 +59,12 @@ func (m *StreamMap) Pos() int64 { return m.out + int64(len(m.held)) + int64(m.Buffer.Buffered()) } +// MarkNext marks the region of role, length bytes long or -1 for up to the +// next mark, that starts with the next byte archive/zip writes. +func (m *StreamMap) MarkNext(role format.Role, length int64) { + format.Mark(m.w, role, m.Pos(), length) +} + func (m *StreamMap) mapped() bool { _, ok := m.w.(format.Mapper) return ok @@ -121,11 +127,13 @@ func (m *StreamMap) MarkLast(role format.Role, n int64) error { return nil } -// MarkEntry marks the region of role that starts with the local header of the -// entry named name, which archive/zip has just written: the last thing held. -// The header is found by its own fields rather than worked out, because -// archive/zip adds an extra field of its own to a header that names a time. -func (m *StreamMap) MarkEntry(role format.Role, name string) error { +// MarkEntry marks the region of role, length bytes long or -1 for up to the +// next mark, that starts with the local header of the entry named name, which +// archive/zip has just written: the last thing held. The header is found by its +// own fields rather than worked out, because archive/zip adds an extra field of +// its own to a header that names a time, and finishes the entry before it with +// a data descriptor only as this one opens. +func (m *StreamMap) MarkEntry(role format.Role, name string, length int64) error { if !m.holding { return nil } @@ -138,7 +146,7 @@ func (m *StreamMap) MarkEntry(role format.Role, name string) error { // the guard of the map reads back as a defect. return nil } - format.Mark(m.w, role, m.out+int64(at), -1) + format.Mark(m.w, role, m.out+int64(at), length) return nil } diff --git a/internal/format/docx/docx.go b/internal/format/docx/docx.go index cba77f24..aef0ea84 100644 --- a/internal/format/docx/docx.go +++ b/internal/format/docx/docx.go @@ -102,7 +102,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { return format.Plan{}, err } - pkg, err := opc.Settle(parts, shape, r.Bytes) + pkg, err := opc.SettleForACut(parts, shape, r.Bytes, r.CutAt) if err != nil { return format.Plan{}, refusal(err, r.Bytes, shape, paragraphs) } diff --git a/internal/format/opc/cut.go b/internal/format/opc/cut.go new file mode 100644 index 00000000..99a58358 --- /dev/null +++ b/internal/format/opc/cut.go @@ -0,0 +1,111 @@ +package opc + +import ( + "fmt" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// SettleForACut is Settle for a package a damage will cut after its first +// cutAt bytes (format.Request.CutAt): the padding is split into a part in front +// of every other part and what Settle would put behind them, so that the cut +// keeps the share of the parts it keeps of the whole package. +// +// With all of the padding behind the parts, as a package nothing cuts has it, +// a large package cut at any share kept every part and lost only padding. +// ECMA-376 part 2 names no order of parts and a reader finds them through the +// central directory: Python's zipfile, 7-Zip, bsdtar, python-docx, openpyxl, +// python-pptx and LibreOffice read a package whose padding part comes first as +// they read it today (docs/TRUNCATE-2026-11-11.md section 10.3). The map +// already counts every part as content from the first byte, so nothing of the +// content stands in front of the padding. +// +// The part in front costs its header before any padding, so the front is +// nothing or at least that, and grows where it cannot land exactly - the cut +// then takes a little more of the parts, the "at least" missing promises. +// Where the padding is too short for the parts the cut should take, all of it +// goes in front and the cut keeps more of them by the share of what stands +// behind (section 10.4). The manifest's cut reports what was kept either way. +func SettleForACut(parts []Part, shape Shape, want, cutAt int64) (Package, error) { + p, err := Settle(parts, shape, want) + if err != nil || cutAt <= 0 || cutAt >= want { + return p, err + } + c, err := measureCut(parts, shape) + if err != nil { + return Package{}, err + } + pad := want - shape.Bare + front := format.PaddingInFront(0, c.content, cutAt, want) + if front <= 0 || pad < c.frontOverhead { + return p, nil + } + front = max(front, c.frontHead) + // What the part in front costs in all: its header and padding in front, + // and its record in the central directory behind the parts. + cost := c.frontOverhead + front - c.frontHead + if cost > pad { + cost = pad + front = c.frontHead + pad - c.frontOverhead + } + q := Package{Parts: parts, Filler: NoFiller, Front: front - c.frontHead, Content: c.content} + back := pad - cost + switch { + case back == 0: + case back <= CommentLimit: + q.Comment = int(back) + case back >= c.endOverhead: + q.Filler = back - c.endOverhead + default: + return Package{}, core.Defect(fmt.Errorf("opc: %d B behind the parts is more than a comment carries and less than a padding part costs (%d B)", back, c.endOverhead)) + } + return q, nil +} + +// cutShape is what SettleForACut needs to know of a set of parts beyond Shape, +// measured by writing them rather than added up from header widths, for the +// reason Shape gives. +type cutShape struct { + // content is how long the parts come to in the archive. + content int64 + // frontHead is how many bytes an empty part in front puts before the + // parts, and frontOverhead what it costs in all, its record in the central + // directory with it. + frontHead, frontOverhead int64 + // endOverhead is what an empty padding part behind the parts costs when + // one stands in front - it carries the longer name. + endOverhead int64 +} + +func measureCut(parts []Part, shape Shape) (cutShape, error) { + var c cutShape + bare, err := size(Package{Parts: parts, Filler: NoFiller, Front: NoFiller}) + if err != nil { + return c, err + } + front, err := size(Package{Parts: parts, Filler: NoFiller, Front: 0}) + if err != nil { + return c, err + } + both, err := size(Package{Parts: parts, Filler: 0, Front: 0}) + if err != nil { + return c, err + } + c.content = bare.content.Length + c.frontHead = front.content.At + c.frontOverhead = front.size - shape.Bare + c.endOverhead = both.size - front.size + if bare.size != shape.Bare || front.content.Length != c.content { + return c, core.Defect(fmt.Errorf("opc: the parts measured %d B with %d B of them content, then %d B of content behind a padding part, where planning measured %d B", + bare.size, c.content, front.content.Length, shape.Bare)) + } + return c, nil +} + +// mapped is a package's length and the region its parts take, read off the map +// it marks as it is written (size). +type mapped struct { + size int64 + content format.Region +} diff --git a/internal/format/opc/opc.go b/internal/format/opc/opc.go index b4b17c26..ccb0f565 100644 --- a/internal/format/opc/opc.go +++ b/internal/format/opc/opc.go @@ -56,6 +56,13 @@ const ( FillerName = "tfg/padding.bin" fillerExtension = "bin" + // FillerEndName is the padding part behind the others in a package laid + // out for a cut, whose first padding part, FillerName, stands in front of + // them (SettleForACut). A package may not hold two parts of one name + // (ECMA-376 part 2, M1.12), and the content types table declares the + // extension, so both are covered. + FillerEndName = "tfg/padding-end.bin" + // NoFiller means the package carries no padding part at all. NoFiller = -1 @@ -88,7 +95,15 @@ type Package struct { Comment int // Filler is how many bytes the padding part carries, or NoFiller. Filler int64 - Seed uint64 + // Front is how many bytes the padding part in front of every other part + // carries, or NoFiller - which is every package nothing cuts. With a front + // part, the one behind is FillerEndName. + Front int64 + // Content is how long the parts come to in the archive, their headers + // with them, when it is known (SettleForACut), and 0 otherwise. A cut + // that falls in them reports how much of them it kept. + Content int64 + Seed uint64 } // ContentTypes builds [Content_Types].xml. @@ -140,15 +155,15 @@ type Shape struct { // Measure works out the shape of a set of parts. func Measure(parts []Part) (Shape, error) { - bare, err := size(Package{Parts: parts, Filler: NoFiller}) + bare, err := size(Package{Parts: parts, Filler: NoFiller, Front: NoFiller}) if err != nil { return Shape{}, err } - empty, err := size(Package{Parts: parts, Filler: 0}) + empty, err := size(Package{Parts: parts, Filler: 0, Front: NoFiller}) if err != nil { return Shape{}, err } - return Shape{Bare: bare, FillerOverhead: empty - bare}, nil + return Shape{Bare: bare.size, FillerOverhead: empty.size - bare.size}, nil } // Unreachable says a size cannot be produced, and names the two that can. @@ -203,7 +218,7 @@ func Settle(parts []Part, shape Shape, want int64) (Package, error) { if want > Zip32Ceiling { return Package{}, &TooLarge{Want: want, Ceiling: Zip32Ceiling} } - p := Package{Parts: parts, Filler: NoFiller} + p := Package{Parts: parts, Filler: NoFiller, Front: NoFiller} delta := want - shape.Bare switch { case delta == 0: @@ -223,12 +238,20 @@ func Settle(parts []Part, shape Shape, want int64) (Package, error) { } } -func size(p Package) (int64, error) { +// size is a package's length and the region its parts take, read off the map +// it marks as it is written - SettleForACut asks for the region. +func size(p Package) (mapped, error) { c := &counter{} - if err := write(context.Background(), c, p, false); err != nil { - return 0, err + rec := &format.MapRecorder{W: c} + if err := write(context.Background(), rec, p, false); err != nil { + return mapped{}, err + } + for _, r := range rec.Map(c.n, true) { + if r.Role == format.RoleContent { + return mapped{size: c.n, content: r}, nil + } } - return c.n, nil + return mapped{}, core.Defect(fmt.Errorf("opc: a package of %d parts marked no content", len(p.Parts))) } // Write emits the package. @@ -241,8 +264,22 @@ func write(ctx context.Context, w io.Writer, p Package, withPadding bool) error zw := zip.NewWriter(zm.Buffer) // The map: every part is content, from the first byte. Then the filler // part, the central directory and the comment, each marked as - // archive/zip writes it (archive.StreamMap). - format.Mark(w, format.RoleContent, 0, -1) + // archive/zip writes it (archive.StreamMap). A package laid out for a cut + // opens with a padding part, and the content starts behind it. + behind := FillerName + if p.Front == NoFiller { + format.Mark(w, format.RoleContent, 0, -1) + } else { + if err := writePaddingPart(ctx, zw, zm, paddingPart{FillerName, p.Seed, p.Front, withPadding}); err != nil { + return err + } + content := p.Content + if content <= 0 { + content = -1 + } + format.Mark(w, format.RoleContent, zm.Pos(), content) + behind = FillerEndName + } if p.Comment > 0 { if p.Comment > CommentLimit { return core.Defect(fmt.Errorf("opc: a comment of %d B is past the %d B a reader accepts", p.Comment, CommentLimit)) @@ -268,45 +305,63 @@ func write(ctx context.Context, w io.Writer, p Package, withPadding bool) error } if p.Filler != NoFiller { - // Stored rather than compressed, so a byte of padding is a byte of - // file. Deflating it would make the size depend on how well random - // bytes happen to compress, which is not a thing to promise. - // - // The checksum has to be known before the header is written, and the - // padding is never held in memory, so it is generated twice: once to - // check it and once to write it. Both passes are the same arithmetic - // from the same seed, which is what makes them agree. - head := &zip.FileHeader{Name: FillerName, Method: zip.Store, Modified: fixedTime} - sum, err := fillerChecksum(ctx, p.Seed, p.Filler) - if err != nil { + if err := writePaddingPart(ctx, zw, zm, paddingPart{behind, p.Seed, p.Filler, withPadding}); err != nil { return err } - head.CRC32 = sum - head.UncompressedSize64 = uint64(p.Filler) - head.CompressedSize64 = uint64(p.Filler) - if err := zm.Hold(); err != nil { - return err - } - entry, err := zw.CreateRaw(head) - if err != nil { - return err - } - if err := zm.MarkEntry(format.RolePadding, FillerName); err != nil { - return err - } - if err := zm.Release(); err != nil { - return err - } - if withPadding { - if err := writeFiller(ctx, entry, p.Seed, p.Filler); err != nil { - return err - } - } } return zm.Close(zw) } +// paddingPart is one padding part: its name, the seed its bytes come from, +// how many it carries, and whether they are written - not while a package is +// only being measured. +type paddingPart struct { + name string + seed uint64 + n int64 + withPadding bool +} + +// writePaddingPart writes a padding part, marked as padding where its header +// starts. +// +// Stored rather than compressed, so a byte of padding is a byte of file. +// Deflating it would make the size depend on how well random bytes happen to +// compress, which is not a thing to promise. +// +// The checksum has to be known before the header is written, and the padding +// is never held in memory, so it is generated twice: once to check it and once +// to write it. Both passes are the same arithmetic from the same seed, which is +// what makes them agree. +func writePaddingPart(ctx context.Context, zw *zip.Writer, zm *archive.StreamMap, part paddingPart) error { + head := &zip.FileHeader{Name: part.name, Method: zip.Store, Modified: fixedTime} + sum, err := fillerChecksum(ctx, part.seed, part.n) + if err != nil { + return err + } + head.CRC32 = sum + head.UncompressedSize64 = uint64(part.n) + head.CompressedSize64 = uint64(part.n) + if err := zm.Hold(); err != nil { + return err + } + entry, err := zw.CreateRaw(head) + if err != nil { + return err + } + if err := zm.MarkEntry(format.RolePadding, part.name, -1); err != nil { + return err + } + if err := zm.Release(); err != nil { + return err + } + if !part.withPadding { + return nil + } + return writeFiller(ctx, entry, part.seed, part.n) +} + // writePart writes one part with its sizes and checksum in the header. // // CreateRaw rather than CreateHeader, and this is measured rather than tidy. diff --git a/internal/format/pptx/pptx.go b/internal/format/pptx/pptx.go index 0fa32e31..208482db 100644 --- a/internal/format/pptx/pptx.go +++ b/internal/format/pptx/pptx.go @@ -106,7 +106,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { return format.Plan{}, err } - pkg, err := opc.Settle(built, shape, r.Bytes) + pkg, err := opc.SettleForACut(built, shape, r.Bytes, r.CutAt) if err != nil { return format.Plan{}, refusal(err, r.Bytes, shape, slides) } diff --git a/internal/format/xlsx/xlsx.go b/internal/format/xlsx/xlsx.go index fa88e045..bf666760 100644 --- a/internal/format/xlsx/xlsx.go +++ b/internal/format/xlsx/xlsx.go @@ -135,7 +135,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { return format.Plan{}, err } - pkg, err := opc.Settle(built, shape, r.Bytes) + pkg, err := opc.SettleForACut(built, shape, r.Bytes, r.CutAt) if err != nil { return format.Plan{}, refusal(err, r.Bytes, shape, rows, columns) } diff --git a/internal/format/zip/compress.go b/internal/format/zip/compress.go index ead06580..a14df5ff 100644 --- a/internal/format/zip/compress.go +++ b/internal/format/zip/compress.go @@ -32,10 +32,11 @@ import ( // to predict. func padCompressed(m *memo, p *format.Plan, r format.Request, groups []format.Content) error { m.withFiller, m.target = true, r.Bytes - withFiller, err := archiveSize(*m) + measured, err := archiveSize(*m) if err != nil { return err } + withFiller := measured.size m.fillerSize = r.Bytes - withFiller if m.fillerSize < 0 { return &format.BelowMinimumError{ @@ -87,6 +88,29 @@ func (m memo) methodFor(e entryPlan) uint16 { return stdzip.Store } +// freedBy is what the compressor freed of the files, given back through the +// filler so the archive still lands on the ordered size: their stored length +// less squeezed, what the compressor wrote of them. Nought on the counting +// pass, where no contents are written and the plan's own arithmetic is the +// answer. +// +// Asked as a question rather than worked out where the files are written, +// because the answer does not exist yet: the last entry's compressed bytes +// only reach the writer when its entry closes, and what closes it is the +// filler's own header. +func (m memo) freedBy(withContents bool, squeezed *int64) func() int64 { + return func() int64 { + if !withContents || !m.squeeze.On() { + return 0 + } + var plain int64 + for _, c := range m.children { + plain += c.plan.Bytes + } + return plain - *squeezed + } +} + // tally counts the bytes a compressor emits, so the writer can learn exactly // how much the compressor freed. // diff --git a/internal/format/zip/cut.go b/internal/format/zip/cut.go new file mode 100644 index 00000000..0724c49f --- /dev/null +++ b/internal/format/zip/cut.go @@ -0,0 +1,287 @@ +package zip + +import ( + stdzip "archive/zip" + "context" + "fmt" + "io" + "strings" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/archive" +) + +// fillerEndName is the entry of padding behind the files of an archive laid out +// for a cut, where fillerName stands in front of them. Two names, because two +// entries of one name are one file overwritten when the archive is unpacked. +const fillerEndName = "tfg-padding-end.bin" + +// split is how an archive's padding is laid out around its files for a cut +// (settleForACut), and its zero value is the archive nothing cuts. +type split struct { + // front says an entry of padding stands in front of the files, behind the + // directories, and frontBytes is how much padding it holds. + front bool + frontBytes int64 + // sized says content is how long the files' entries come to. The files are + // marked with it, so that a cut inside them says how much of them there + // was: a cut stops the writing before the mark that would end them. + sized bool + content int64 + // freed is how much shorter the compressor makes the files of a squeezed + // archive, measured before it is written. + freed int64 +} + +// contentLength is the length the files are marked with, -1 for up to the next +// mark. +func (s split) contentLength() int64 { + if !s.sized { + return -1 + } + return s.content +} + +// settleForACut lays the padding of an archive a damage will cut after its +// first cutAt bytes (format.Request.CutAt) out around its files, so that the +// cut keeps the share of them it keeps of the whole archive. +// +// With all of its padding behind the files, as an archive nothing cuts has it, +// a large archive cut at any share kept every file and lost only padding. +// APPNOTE names no order of entries and a reader finds them through the central +// directory: Python's zipfile, 7-Zip and bsdtar read an archive whose padding +// entry comes first as they read it today (docs/TRUNCATE-2026-11-11.md section +// 10.3). So an entry of padding goes in front of the files and behind the +// directories, which stay structure. +// +// A stored archive is settled here. A squeezed one is settled as it is written +// (settleSqueezedForACut), because how long its files come to is known only +// once they are squeezed, and planning does not squeeze. +func settleForACut(m *memo, p *format.Plan, cutAt, target int64, label string) error { + m.cutAt = cutAt + if !m.cuts(target) || m.squeeze.On() { + return nil + } + c, err := measureCut(*m, label) + if err != nil { + return err + } + if err := m.splitPadding(c, target, 0, label); err != nil { + return err + } + if m.split.front { + p.Properties["padding_entry"] = fillerName + } + return nil +} + +// settleSqueezedForACut is settleForACut for a squeezed archive, as it is +// written. Its files are squeezed into a counter first, to learn how long they +// come to, and the writing pass then has to come to the same +// (writeCompressedFiller). That generates every file inside twice, and only for +// a squeezed archive a damage cuts. Nothing for any other archive. +func (m *memo) settleSqueezedForACut(ctx context.Context, target int64) error { + if !m.squeeze.On() || !m.cuts(target) { + return nil + } + // padCompressed leaves the comment the label alone. + label := m.comment + c, err := measureCut(*m, label) + if err != nil { + return err + } + squeezed, err := squeezedFiles(ctx, *m) + if err != nil { + return err + } + return m.splitPadding(c, target, c.content-squeezed, label) +} + +// cuts says the archive is laid out for a cut: one falls inside it, and there +// are files for it to keep a share of. +func (m memo) cuts(target int64) bool { + return m.cutAt > 0 && m.cutAt < target && len(m.children) > 0 +} + +// splitPadding lays the padding out around the files for the cut at m.cutAt, +// in an archive of target bytes whose files the compressor makes freed bytes +// shorter, nought when they are stored (docs/TRUNCATE-2026-11-11.md section +// 10.4). +// +// The entry in front costs its header before any padding, so the front is +// nothing or at least that, and grows where it cannot land exactly - the cut +// then takes a little more of the files, the "at least" missing promises. +// Where the padding is too short for the files the cut should take, all of it +// goes in front, and the cut keeps more of them by the share of what stands +// behind. The files are marked with their length whether or not anything goes +// in front, so a cut inside them says how much of them there was. +func (m *memo) splitPadding(c cutShape, target, freed int64, label string) error { + content := c.content - freed + m.split = split{sized: true, content: content, freed: freed} + pad := target - (c.bare - freed) + front := format.PaddingInFront(c.before, content, m.cutAt, target) + // A squeezed archive keeps its entry of padding behind the files. It takes + // back what the compressor frees, and the plan names it. + var behind int64 + if m.squeeze.On() { + behind = c.endOverhead + } + if front <= 0 || pad < c.frontOverhead+behind { + return nil + } + cost := min(c.frontOverhead+max(front, c.frontHead)-c.frontHead, pad-behind) + m.split.front, m.split.frontBytes = true, cost-c.frontOverhead + if err := m.padBehind(c, pad-cost, freed, label); err != nil { + return err + } + got, err := archiveSize(*m) + if err != nil { + return err + } + if got.size != target || got.content.Length != c.content || got.content.At != c.before+c.frontHead+m.split.frontBytes { + return core.Defect(fmt.Errorf("zip: laid out for a cut, the archive measures %d B with its files at %d for %d B, where %d B with them at %d for %d B was settled", + got.size, got.content.At, got.content.Length, target, c.before+c.frontHead+m.split.frontBytes, c.content)) + } + return nil +} + +// padBehind puts back bytes of padding behind the files: into the comment +// while they fit beside the label, and into an entry of padding above that. A +// squeezed archive puts them all into the entry, which takes back what the +// compressor frees as it is written. +func (m *memo) padBehind(c cutShape, back, freed int64, label string) error { + m.comment, m.withFiller, m.fillerSize = label, false, 0 + switch { + case m.squeeze.On(): + m.withFiller, m.fillerSize = true, back-c.endOverhead-freed + case back <= int64(commentPaddingLimit-len(label)): + m.comment = label + strings.Repeat(" ", int(back)) + case back >= c.endOverhead: + m.withFiller, m.fillerSize = true, back-c.endOverhead + default: + return core.Defect(fmt.Errorf("zip: %d B behind the files is more than the comment carries and less than an entry of padding costs (%d B)", back, c.endOverhead)) + } + return nil +} + +// cutShape is what splitPadding needs to know of an archive, measured by +// writing its structure rather than added up from header widths, for the +// reason archiveSize gives. +type cutShape struct { + // bare is how long the archive comes to with no padding at all: no entry + // of it, and the comment the label alone. + bare int64 + // before is how many bytes stand in front of the files, the directories, + // and content how long the files' entries come to when they are stored. + before, content int64 + // frontHead is how many bytes an empty entry in front puts before the + // files, and frontOverhead what it costs in all, its record in the central + // directory with it. + frontHead, frontOverhead int64 + // endOverhead is what an empty entry of padding behind the files costs + // when one stands in front - it carries the longer name. + endOverhead int64 +} + +func measureCut(m memo, label string) (cutShape, error) { + m.comment, m.withFiller, m.fillerSize, m.split = label, false, 0, split{} + bare, err := archiveSize(m) + if err != nil { + return cutShape{}, err + } + m.split.front = true + front, err := archiveSize(m) + if err != nil { + return cutShape{}, err + } + m.withFiller = true + both, err := archiveSize(m) + if err != nil { + return cutShape{}, err + } + c := cutShape{ + bare: bare.size, before: bare.content.At, content: bare.content.Length, + frontHead: front.content.At - bare.content.At, frontOverhead: front.size - bare.size, + endOverhead: both.size - front.size, + } + if front.content.Length != c.content || both.content.Length != c.content { + return c, core.Defect(fmt.Errorf("zip: the files measured %d B, then %d B behind an entry of padding and %d B between two", + c.content, front.content.Length, both.content.Length)) + } + return c, nil +} + +// squeezedFiles is how long the files' entries come to once squeezed: the pass +// that measures, squeezing every file into a counter. +func squeezedFiles(ctx context.Context, m memo) (int64, error) { + m.withFiller, m.split = false, split{} + c := &counter{} + rec := &format.MapRecorder{W: c} + if err := build(ctx, rec, m, true); err != nil { + return 0, err + } + content, err := contentOf(rec.Map(c.n, true)) + if err != nil { + return 0, err + } + return content.Length, nil +} + +// startTheFiles marks where the files start, once the directories are +// written: at the next byte, or, in an archive laid out for a cut, behind an +// entry of padding it writes first - where openFile marks them. +// +// The entry is named fillerName, the name of the one entry of padding an +// archive nothing cuts has, and stored and locked like it. Its index follows +// that of the one behind the files, so the two take different salts. +func startTheFiles(ctx context.Context, zw *stdzip.Writer, zm *archive.StreamMap, m memo, withContents bool) error { + if !m.split.front { + zm.MarkNext(format.RoleContent, m.split.contentLength()) + return nil + } + if err := zm.Hold(); err != nil { + return err + } + return writeStoredFiller(ctx, zw, zm, m, withContents, storedFiller{ + name: fillerName, size: m.split.frontBytes, index: len(m.children) + 1, + }) +} + +// openFile is openEntry for a file inside, which marks where the files start +// when an entry of padding stands in front of them: at the first file's local +// header, which archive/zip writes only once it has finished the entry in +// front with its data descriptor. +func openFile(zw *stdzip.Writer, zm *archive.StreamMap, m memo, e entryPlan) (io.Writer, func() error, error) { + if e.index != 0 || !m.split.front { + return openEntry(zw, m, e) + } + if err := zm.Hold(); err != nil { + return nil, nil, err + } + entry, shut, err := openEntry(zw, m, e) + if err != nil { + return nil, nil, err + } + if err := zm.MarkEntry(format.RoleContent, e.name, m.split.contentLength()); err != nil { + return nil, nil, err + } + return entry, shut, zm.Release() +} + +// mapped is an archive's length and the region its files take, read off the +// map a counting pass marks (archiveSize). +type mapped struct { + size int64 + content format.Region +} + +// contentOf is the region of content in a map. +func contentOf(regions []format.Region) (format.Region, error) { + for _, r := range regions { + if r.Role == format.RoleContent { + return r, nil + } + } + return format.Region{}, core.Defect(fmt.Errorf("zip: writing the archive marked no files")) +} diff --git a/internal/format/zip/filler.go b/internal/format/zip/filler.go index 7bdddcad..c1d763cf 100644 --- a/internal/format/zip/filler.go +++ b/internal/format/zip/filler.go @@ -3,6 +3,7 @@ package zip import ( stdzip "archive/zip" "context" + "fmt" "io" "github.com/donislawdev/TestingFilesGenerator/internal/format" @@ -41,48 +42,70 @@ func writeFillerEntry(ctx context.Context, zw *stdzip.Writer, zm *archive.Stream if m.squeeze.On() { return writeCompressedFiller(ctx, zw, zm, m, withContents, freed) } + return writeStoredFiller(ctx, zw, zm, m, withContents, storedFiller{ + name: m.fillerEntryName(), size: m.fillerSize, index: len(m.children), + }) +} - // The stored path, unchanged. It settles the size before the header - // because a locked entry states its length there, and a locked archive - // takes this path. - size := m.fillerSize +// storedFiller is one entry of padding whose length is settled before its +// header: the one behind the files of a stored archive, and the one in front of +// them in an archive laid out for a cut (writeFrontEntry). +type storedFiller struct { + name string + size int64 + index int +} + +// writeStoredFiller writes f, with what follows its header held (zm.Hold) +// until it is marked. It settles the size before the header because a locked +// entry states its length there, and a locked archive takes this path. +func writeStoredFiller(ctx context.Context, zw *stdzip.Writer, zm *archive.StreamMap, m memo, withContents bool, f storedFiller) error { // The filler is locked with everything else. An archive where one entry // opens without the password and the rest do not is a file nobody // asked for, and the arithmetic is the same either way. crc, err := plaintextCRC(ctx, m, withContents, func(w io.Writer) error { - return writeFiller(ctx, w, m.seed, size) + return writeFiller(ctx, w, m.seed, f.size) }) if err != nil { return err } entry, shut, err := openEntry(zw, m, entryPlan{ - name: fillerName, plain: size, index: len(m.children), stored: true, + name: f.name, plain: f.size, index: f.index, stored: true, withContents: withContents, crc: crc, }) if err != nil { return err } - if err := markFiller(zm); err != nil { + if err := markFiller(zm, f.name); err != nil { return err } if !withContents { return nil } - if err := writeFiller(ctx, entry, m.seed, size); err != nil { + if err := writeFiller(ctx, entry, m.seed, f.size); err != nil { return err } return shut() } -// markFiller marks the filler entry as padding and sends on what was held -// while its header was written. -func markFiller(zm *archive.StreamMap) error { - if err := zm.MarkEntry(format.RolePadding, fillerName); err != nil { +// markFiller marks the filler entry named name as padding and sends on what +// was held while its header was written. +func markFiller(zm *archive.StreamMap, name string) error { + if err := zm.MarkEntry(format.RolePadding, name, -1); err != nil { return err } return zm.Release() } +// fillerEntryName is the name of the entry of padding behind the files: +// fillerName, unless an entry in front of them has it (split). +func (m memo) fillerEntryName() string { + if m.split.front { + return fillerEndName + } + return fillerName +} + // writeCompressedFiller is the same entry for an archive whose others are // squeezed, with the header written before the length is known. // @@ -91,13 +114,13 @@ func markFiller(zm *archive.StreamMap) error { // archive is never a locked one. func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, zm *archive.StreamMap, m memo, withContents bool, freed func() int64) error { entry, shut, err := openEntry(zw, m, entryPlan{ - name: fillerName, index: len(m.children), stored: true, + name: m.fillerEntryName(), index: len(m.children), stored: true, withContents: withContents, }) if err != nil { return err } - if err := markFiller(zm); err != nil { + if err := markFiller(zm, m.fillerEntryName()); err != nil { return err } if !withContents { @@ -105,6 +128,13 @@ func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, zm *archive.S // writing its bytes here would count them twice. return nil } + // An archive laid out for a cut was settled on what a pass that only + // measured found the compressor frees, and its files were marked with the + // length that gave. Squeezing the same files again has to free the same. + got := freed() + if m.split.sized && got != m.split.freed { + return core.Defect(fmt.Errorf("zip: the files inside came to %d B less once squeezed while measuring and %d B less while writing", m.split.freed, got)) + } // What the compressor freed can be NEGATIVE, and that is the case this // arithmetic was written without. Deflate grows data it cannot shrink - // about five bytes for every 65535 - so an archive holding already @@ -122,7 +152,7 @@ func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, zm *archive.S // knowable without squeezing them, and planning is not allowed to. The // floor reported is measured rather than derived - it is what this // archive came to with no padding at all. - size := m.fillerSize + freed() + size := m.fillerSize + got if size < 0 { return &format.BelowMinimumError{ Format: "ZIP", diff --git a/internal/format/zip/size.go b/internal/format/zip/size.go new file mode 100644 index 00000000..c1e3c398 --- /dev/null +++ b/internal/format/zip/size.go @@ -0,0 +1,120 @@ +package zip + +import ( + "context" + "fmt" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/archive" +) + +// How long an archive comes to, measured by writing its structure: what +// planning, the minimum and the layout for a cut all ask. + +// archiveSize measures the archive without generating anything it holds. +// +// The structure is built for real - every header, every name, the comment, the +// central directory - and only the file contents are left out and added on as +// numbers. Everything inside is stored rather than compressed, so that is exact +// rather than close. +// +// Measured on the standard library on 2026-08-03, because a document about a +// format has been wrong here five times out of five and this one is arithmetic +// standing in for a measurement: +// +// an archive of 3 entries with no contents 424 B +// the same 3 entries holding 4096 B each 12 712 B +// 424 + 3*4096 12 712 B exact +// +// Checked at 0, 1, 1000, 4096 and 100 000 bytes an entry, and the comment moves +// the total by exactly its own length at 0, 1, 2, 50, 4096 and 65 535. +// +// Why it matters: planning used to call this two or three times and each call +// generated every file inside. That made --dry-run cost about what the real run +// costs - measured at 960 ms against a 56 ms baseline for a 256 MB archive, +// while writing it for real took 1585 ms. A dry run is the step this project +// tells people to take before anything large, so it has to be the cheap one. +// +// If the arithmetic is ever wrong the size guard says so immediately: the +// engine refuses any file whose written length differs from its plan, and the +// property test walks about 120 sizes for this format. +// +// It reads the region the files take off the map the pass marks, with the +// contents added on the same way, for laying the padding out around them +// (cut.go). +func archiveSize(m memo) (mapped, error) { + // What this pass writes of the files is their headers alone, so the + // length the writing pass marks them with is not theirs here. + m.split.sized = false + c := &counter{} + rec := &format.MapRecorder{W: c} + if err := build(context.Background(), rec, m, false); err != nil { + return mapped{}, err + } + content, err := contentOf(rec.Map(c.n, true)) + if err != nil { + return mapped{}, err + } + var files int64 + for _, ch := range m.children { + files += ch.plan.Bytes + m.lock.EntryOverhead() + } + total := c.n + files + content.Length += files + if m.split.front { + front := m.split.frontBytes + m.lock.EntryOverhead() + total += front + content.At += front + } + if m.withFiller { + total += m.fillerSize + m.lock.EntryOverhead() + } + return mapped{size: total, content: content}, nil +} + +type counter struct{ n int64 } + +func (c *counter) Write(p []byte) (int, error) { c.n += int64(len(p)); return len(p), nil } + +// minimumBytes is the smallest archive this generator can produce: one empty +// entry, no label, no padding. +// It panics rather than returning a number nobody measured. +// +// This used to answer 1<<62 on any failure, on the reasoning that refusing +// every size is safer than declaring a wrong minimum. It is not safer, it is +// quieter: ZIP would refuse every request ever made with a message about a +// minimum of 4611686018427387904 B, and nothing would say why. That is rule 6 +// broken in the way this project keeps finding. +// +// The condition is a programming mistake rather than a runtime one. It needs +// the default entry format to be unregistered when this runs, and the language +// says when that can happen: packages are initialised in the order of their +// import paths, so the txt package is registered before this one by rule rather +// than by luck. This paragraph said "true by accident" until 2026-08-25, when +// an outside review built an argument on the opposite claim - that the +// specification guarantees nothing here. Both were wrong, and the comment being +// wrong about its own mechanism is the worse of the two. +// +// What the rule does not cover is a rename that moves the entry format after +// this package in that order, or a second entry point linking this package +// without that one. The first is what a guard asks, since a format has been +// renamed in this tree before. format.Register already panics on the same class +// of mistake, so this matches it: a build that cannot state its own minimum +// fails at start rather than at every use. +func minimumBytes() int64 { + desc, err := format.Get(archive.DefaultFormat) + if err != nil { + panic(fmt.Sprintf("zip: the default entry format %q is not registered yet, so the minimum size of an archive cannot be worked out. Check the import order in internal/format/all", archive.DefaultFormat)) + } + cp, err := desc.Generator.Plan(format.Request{Bytes: 0, Seed: 0, Label: false}) + if err != nil { + panic(fmt.Sprintf("zip: the default entry format %q cannot produce an empty file, so the minimum size of an archive cannot be worked out: %v", archive.DefaultFormat, err)) + } + n, err := archiveSize(memo{children: []child{{ + name: "txt_0001.txt", desc: desc, plan: cp, + }}}) + if err != nil { + panic(fmt.Sprintf("zip: the smallest archive cannot be measured: %v", err)) + } + return n.size +} diff --git a/internal/format/zip/zip.go b/internal/format/zip/zip.go index cd27e0df..defbaaeb 100644 --- a/internal/format/zip/zip.go +++ b/internal/format/zip/zip.go @@ -126,6 +126,12 @@ type memo struct { // made. Only the compressed path needs it: a stored archive settles its // padding while planning, so a size it cannot reach is refused there. target int64 + // cutAt is where a damage will cut the archive (format.Request.CutAt), 0 + // when nothing does, and split is how its padding is laid out around the + // files for that cut (cut.go). The zero value of split is the archive + // nothing cuts. + cutAt int64 + split split } func (g generator) Plan(r format.Request) (format.Plan, error) { @@ -168,6 +174,9 @@ func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan if err := pad(&m, &p, r, target, bare, label, groups); err != nil { return format.Plan{}, err } + if err := settleForACut(&m, &p, r.CutAt, target, label); err != nil { + return format.Plan{}, err + } if err := withinZip32(target); err != nil { return format.Plan{}, err @@ -251,9 +260,11 @@ func settleSize(m *memo, r format.Request) (target, bare int64, label string, er // Measure the archive with no padding at all. Every entry is stored rather // than compressed, so the size follows from the parts exactly and nothing // has to be guessed. - if bare, err = archiveSize(*m); err != nil { + measured, err := archiveSize(*m) + if err != nil { return 0, 0, "", err } + bare = measured.size if !r.SizeFromContents { return r.Bytes, bare, label, nil @@ -275,8 +286,8 @@ func settleSize(m *memo, r format.Request) (target, bare int64, label string, er if err != nil { return 0, 0, "", err } - settled = measured == size - size = measured + settled = measured.size == size + size = measured.size } if !settled { return 0, 0, "", core.Defect(fmt.Errorf( @@ -380,10 +391,11 @@ func pad(m *memo, p *format.Plan, r format.Request, target, bare int64, label st // in Tier 1 with a ceiling. m.comment = label + strings.Repeat(" ", int(room)) m.withFiller = true - withFiller, err := archiveSize(*m) + measured, err := archiveSize(*m) if err != nil { return err } + withFiller := measured.size m.fillerSize = r.Bytes - withFiller if m.fillerSize < 0 { return &format.BelowMinimumError{ @@ -413,6 +425,9 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { if !ok { return core.Defect(fmt.Errorf("zip: the plan was not produced by this generator")) } + if err := m.settleSqueezedForACut(ctx, p.Bytes); err != nil { + return err + } return build(ctx, w, m, true) } @@ -558,7 +573,9 @@ func build(ctx context.Context, w io.Writer, m memo, withContents bool) error { if err := writeDirectories(ctx, zw, m.layout); err != nil { return err } - format.Mark(w, format.RoleContent, zm.Pos(), -1) + if err := startTheFiles(ctx, zw, zm, m, withContents); err != nil { + return err + } for i, c := range m.children { select { @@ -573,7 +590,7 @@ func build(ctx context.Context, w io.Writer, m memo, withContents bool) error { if err != nil { return core.Defect(fmt.Errorf("zip: the %s file inside could not be checksummed: %w", c.desc.ID, err)) } - entry, shut, err := openEntry(zw, m, entryPlan{ + entry, shut, err := openFile(zw, zm, m, entryPlan{ name: c.name, plain: c.plan.Bytes, index: i, withContents: withContents, crc: crc, }) if err != nil { @@ -589,114 +606,9 @@ func build(ctx context.Context, w io.Writer, m memo, withContents bool) error { } } - // What the compressor freed, given back through the filler so the archive - // still lands on the ordered size. Nought on the counting pass, where no - // contents are written and the plan's own arithmetic is the answer. - // Asked as a question rather than worked out here, because the answer does - // not exist yet: the last entry's compressed bytes only reach the writer - // when its entry closes, and what closes it is the filler's own header. - freed := func() int64 { - if !withContents || !m.squeeze.On() { - return 0 - } - var plain int64 - for _, c := range m.children { - plain += c.plan.Bytes - } - return plain - squeezed - } - if err := writeFillerEntry(ctx, zw, zm, m, withContents, freed); err != nil { + if err := writeFillerEntry(ctx, zw, zm, m, withContents, m.freedBy(withContents, &squeezed)); err != nil { return err } return zm.Close(zw) } - -// archiveSize measures the archive without generating anything it holds. -// -// The structure is built for real - every header, every name, the comment, the -// central directory - and only the file contents are left out and added on as -// numbers. Everything inside is stored rather than compressed, so that is exact -// rather than close. -// -// Measured on the standard library on 2026-08-03, because a document about a -// format has been wrong here five times out of five and this one is arithmetic -// standing in for a measurement: -// -// an archive of 3 entries with no contents 424 B -// the same 3 entries holding 4096 B each 12 712 B -// 424 + 3*4096 12 712 B exact -// -// Checked at 0, 1, 1000, 4096 and 100 000 bytes an entry, and the comment moves -// the total by exactly its own length at 0, 1, 2, 50, 4096 and 65 535. -// -// Why it matters: planning used to call this two or three times and each call -// generated every file inside. That made --dry-run cost about what the real run -// costs - measured at 960 ms against a 56 ms baseline for a 256 MB archive, -// while writing it for real took 1585 ms. A dry run is the step this project -// tells people to take before anything large, so it has to be the cheap one. -// -// If the arithmetic is ever wrong the size guard says so immediately: the -// engine refuses any file whose written length differs from its plan, and the -// property test walks about 120 sizes for this format. -func archiveSize(m memo) (int64, error) { - c := &counter{} - if err := build(context.Background(), c, m, false); err != nil { - return 0, err - } - total := c.n - for _, ch := range m.children { - total += ch.plan.Bytes + m.lock.EntryOverhead() - } - if m.withFiller { - total += m.fillerSize + m.lock.EntryOverhead() - } - return total, nil -} - -type counter struct{ n int64 } - -func (c *counter) Write(p []byte) (int, error) { c.n += int64(len(p)); return len(p), nil } - -// minimumBytes is the smallest archive this generator can produce: one empty -// entry, no label, no padding. -// It panics rather than returning a number nobody measured. -// -// This used to answer 1<<62 on any failure, on the reasoning that refusing -// every size is safer than declaring a wrong minimum. It is not safer, it is -// quieter: ZIP would refuse every request ever made with a message about a -// minimum of 4611686018427387904 B, and nothing would say why. That is rule 6 -// broken in the way this project keeps finding. -// -// The condition is a programming mistake rather than a runtime one. It needs -// the default entry format to be unregistered when this runs, and the language -// says when that can happen: packages are initialised in the order of their -// import paths, so the txt package is registered before this one by rule rather -// than by luck. This paragraph said "true by accident" until 2026-08-25, when -// an outside review built an argument on the opposite claim - that the -// specification guarantees nothing here. Both were wrong, and the comment being -// wrong about its own mechanism is the worse of the two. -// -// What the rule does not cover is a rename that moves the entry format after -// this package in that order, or a second entry point linking this package -// without that one. The first is what a guard asks, since a format has been -// renamed in this tree before. format.Register already panics on the same class -// of mistake, so this matches it: a build that cannot state its own minimum -// fails at start rather than at every use. -func minimumBytes() int64 { - desc, err := format.Get(archive.DefaultFormat) - if err != nil { - panic(fmt.Sprintf("zip: the default entry format %q is not registered yet, so the minimum size of an archive cannot be worked out. Check the import order in internal/format/all", archive.DefaultFormat)) - } - cp, err := desc.Generator.Plan(format.Request{Bytes: 0, Seed: 0, Label: false}) - if err != nil { - panic(fmt.Sprintf("zip: the default entry format %q cannot produce an empty file, so the minimum size of an archive cannot be worked out: %v", archive.DefaultFormat, err)) - } - n, err := archiveSize(memo{children: []child{{ - name: "txt_0001.txt", desc: desc, plan: cp, - }}}) - if err != nil { - panic(fmt.Sprintf("zip: the smallest archive cannot be measured: %v", err)) - } - return n -} diff --git a/internal/guard/branching_test.go b/internal/guard/branching_test.go index 4a54b98c..6a530a55 100644 --- a/internal/guard/branching_test.go +++ b/internal/guard/branching_test.go @@ -64,7 +64,9 @@ const ( // of control in its own closure and went one deeper when it learnt to wrap // only once - the three became one function, chainOnce, and listen came // out of the band. - crowdedDepthFunctions = 49 + // Lowered from 49 on 2026-11-13: the office package writer's padding part + // moved out of write into writePaddingPart, so write came out of the band. + crowdedDepthFunctions = 48 // An axis this set does not watch. crowding() asks n >= band, so nothing // reaches it. diff --git a/internal/guard/contextroots_test.go b/internal/guard/contextroots_test.go index 455d64c4..86197e94 100644 --- a/internal/guard/contextroots_test.go +++ b/internal/guard/contextroots_test.go @@ -40,7 +40,7 @@ var contextRoots = map[string]string{ "internal/format/zip/zip.go generator.Plan": "the Generator interface's Plan for callers without a context - the engine asks PlanContext", "internal/engine/damagedformat.go damagedGenerator.Plan": "the Generator interface's Plan for callers without a context, such as SmallestAccepted - the engine asks PlanContext", "internal/format/opc/opc.go size": "measures a package with the padding left out, so there is nothing long to stop", - "internal/format/zip/zip.go archiveSize": "measures an archive with the contents left out, so there is nothing long to stop", + "internal/format/zip/size.go archiveSize": "measures an archive with the contents left out, so there is nothing long to stop", } func TestAContextIsMadeFromNothingOnlyWhereItIsMeantToStart(t *testing.T) { diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 0ce232e6..9a095bba 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -37,10 +37,6 @@ var notYetCutAtTheShare = map[string]notYet{ "mp4": {paddingBehind, 64 << 10, 50}, "webm": {paddingBehind, 64 << 10, 50}, "pdf": {paddingBehind, 64 << 10, 50}, - "zip": {paddingBehind, 64 << 10, 50}, - "docx": {paddingBehind, 64 << 10, 50}, - "xlsx": {paddingBehind, 64 << 10, 50}, - "pptx": {paddingBehind, 64 << 10, 50}, "targz": {paddingBehind, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the // picture is more than a small share, so a cut of 1% keeps all of it. @@ -56,7 +52,7 @@ var notYetCutAtTheShare = map[string]notYet{ // guard for it, and it stays red until the format splits its padding for // format.Request.CutAt. A format that leaves the ratchet leaves this list too, // so it cannot go back. -var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "zip", "docx", "xlsx", "pptx", "targz", "webp", "tiff"} +var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "targz", "webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. @@ -133,6 +129,10 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { t.Fatalf("running: %v", err) } + plannedOf := map[string]engine.PlannedFile{} + for _, f := range planned { + plannedOf[f.Target.ID] = f + } problems := map[string]string{} for _, m := range res.Manifest.Files { missing, ok := missingOf[m.TargetID] @@ -140,7 +140,15 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { t.Errorf("%s: no file was written (%s)", m.TargetID, m.Error) continue } - problems[m.TargetID] = shareProblem(missing, m) + problem := shareProblem(missing, m) + f := plannedOf[m.TargetID] + if _, closes := endClosesTheFile[f.Desc.ID]; closes && problem == cutInTheEnd && onlyTheEndBehind(wholeMap(t, f)) { + // The end the format has to close with is longer than what the + // cut takes, and nothing else stands behind the content that the + // format could have put in front of it. + problem = "" + } + problems[m.TargetID] = problem } for _, problem := range shareVerdicts(problems, notYetCutAtTheShare) { t.Error(problem) @@ -158,6 +166,15 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { if shareProblem(50, kept) == "" { t.Errorf("canary: a cut that kept all its content passed for one that took half") } + region := func(role format.Role, at, length int64) format.Region { + return format.Region{Role: role, At: at, Length: length} + } + if onlyTheEndBehind([]format.Region{region(format.RoleContent, 0, 10), region(format.RolePadding, 10, 5), region(format.RoleEnd, 15, 5)}) { + t.Errorf("canary: padding behind the content passed for a file with nothing but its end behind it") + } + if !onlyTheEndBehind([]format.Region{region(format.RolePadding, 0, 5), region(format.RoleContent, 5, 10), region(format.RoleEnd, 15, 5)}) { + t.Errorf("canary: a file with all of its padding in front was refused for padding behind") + } listed := maps.Clone(notYetCutAtTheShare) listed["txt"] = notYet{paddingBehind, 64 << 10, 50} if !slices.ContainsFunc(shareVerdicts(problems, listed), func(s string) bool { return strings.HasPrefix(s, "txt:") }) { @@ -165,6 +182,64 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { } } +// cutInTheEnd is what shareProblem says of a cut that fell in the end a file +// closes with: a central directory, a trailer. +const cutInTheEnd = "the cut fell in the end" + +// endClosesTheFile is every format whose specification puts its end after all +// of its content, so a cut shorter than the end can only fall in it +// (onlyTheEndBehind). A format whose end may stand anywhere - a TIFF's image +// directory, which the specification lets stand before the pictures - is not +// here: it could put its end in front of the content instead. +var endClosesTheFile = map[string]string{ + "zip": "APPNOTE 4.3.6 lays a ZIP out with the central directory and its end record after every entry", + "docx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", + "xlsx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", + "pptx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", +} + +// onlyTheEndBehind is whether nothing but the end stands behind the content of +// a file's whole map - no padding there that the format could have put in +// front of the content instead. +// +// A cut that falls in the end of such a file falls there because the end is +// longer than the part the cut takes: a ZIP's central directory with a record +// for every part, cut by 1% of a few kilobytes (docs/TRUNCATE-2026-11-11.md +// section 10). No layout reaches the content then, so the share cannot be +// asked of that file. With padding behind the content the format could have +// moved it, and the cut is a miss like any other. +func onlyTheEndBehind(regions []format.Region) bool { + last := -1 + for i, r := range regions { + if r.Role == format.RoleContent { + last = i + } + } + if last < 0 { + return false + } + for _, r := range regions[last+1:] { + if r.Role != format.RoleEnd && r.Length != 0 { + return false + } + } + return true +} + +// wholeMap is the map of the whole file f is cut from: its generator written +// out through its own plan to nowhere, the damage left to the engine's +// streams, which are not here. +func wholeMap(t *testing.T, f engine.PlannedFile) []format.Region { + t.Helper() + c := &countingSink{} + rec := &format.MapRecorder{W: c} + if err := f.Desc.Generator.Write(t.Context(), rec, f.Plan); err != nil { + t.Errorf("%s: writing the whole file again: %v", f.Target.ID, err) + return nil + } + return rec.Map(c.n, true) +} + // shareCases is the files the sweep cuts of a format, each as its size and the // share cut: every size and share, or the one file named for a format listed // as not taking the share yet. @@ -231,6 +306,9 @@ func shareProblem(missing int64, f manifest.File) string { } cut := f.Damage[0].Cut if cut.In != "content" { + if cut.In == string(format.RoleEnd) { + return cutInTheEnd + } return fmt.Sprintf("the cut fell in the %s", cut.In) } if cut.ContentBytes == nil || *cut.ContentBytes <= 0 { diff --git a/internal/guard/filemap_test.go b/internal/guard/filemap_test.go index bc5ca398..6cb7840e 100644 --- a/internal/guard/filemap_test.go +++ b/internal/guard/filemap_test.go @@ -180,6 +180,17 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { {string(format.RoleContent), int64(2), int64(5)}, {string(format.RolePadding), int64(7), int64(3)}}); front || !back { t.Errorf("canary: padding behind the content alone read as front %v, back %v", front, back) } + // Canaries of the first end: padding behind a head in front of the + // content is a miss, and so is a file with no padding behind. + region := func(role format.Role, at, length int64) [3]any { return [3]any{string(role), at, length} } + if firstEndLayout([][3]any{region(format.RoleStructure, 0, 2), region(format.RolePadding, 2, 4), + region(format.RoleContent, 6, 5), region(format.RolePadding, 11, 3)}) { + t.Errorf("canary: padding between a head and the content passed for the layout due at the first byte") + } + if firstEndLayout([][3]any{region(format.RolePadding, 0, 4), region(format.RoleContent, 4, 5), region(format.RoleEnd, 9, 3)}) { + t.Errorf("canary: a file with no padding behind its content passed for the layout due at the first byte") + } + askTheLastEndCanaries(t, dir) t.Logf("%d files, %d with padding, %d of them on both sides of the content, %d cut at the ends of a split", len(files), padded, split, ends) } @@ -189,21 +200,99 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { func writeTheEnds(t *testing.T, dir string, f format.Descriptor, name string, setting map[string]string, size int64) []mappedFile { t.Helper() var out []mappedFile + label := format.Request{Bytes: size, Label: true, Seed: 7}.LabelText(f.ID) for _, end := range []struct { - name string - at int64 - front, back bool - }{{"first", 1, false, true}, {"last", size - 1, true, false}} { + name string + at int64 + due func(m mappedFile) bool + says string + }{ + {"first", 1, func(m mappedFile) bool { return firstEndLayout(m.marked) }, "padding behind the content, and in front of it none - or, where the content would start the file, one padding region that does instead"}, + {"last", size - 1, func(m mappedFile) bool { return lastEndLayout(t, m, label) }, "padding in front of the content only - or, behind it, the label alone where the format keeps it in a field that closes the file"}, + } { m := writeMapped(t, dir, f, name+"-"+end.name, setting, size, end.at) - if front, back := paddingSides(m.marked); front != end.front || back != end.back { - t.Errorf("%s: cut after %d B of %d B it marked %s - padding in front %v and behind %v, where %v and %v were due", - m.name, end.at, size, mapNames(m.marked), front, back, end.front, end.back) + if !end.due(m) { + t.Errorf("%s: cut after %d B of %d B it marked %s, where %s was due", m.name, end.at, size, mapNames(m.marked), end.says) } out = append(out, m) } return out } +// firstEndLayout is the layout due for a cut after a file's first byte: its +// padding behind the content, and nothing in front of it. Unless the content +// would start the file, as a package's first part does: then a cut after one +// byte would keep a byte of it where the share is none, so the smallest +// padding region the format writes starts the file and the content follows +// it (docs/TRUNCATE-2026-11-11.md section 10). +func firstEndLayout(m [][3]any) bool { + front, back := paddingSides(m) + if !back { + return false + } + return !front || (len(m) > 1 && m[0][0] == string(format.RolePadding) && m[0][1] == int64(0) && + m[1][0] == string(format.RoleContent)) +} + +// lastEndLayout is the layout due for a cut before a file's last byte: all of +// its padding in front of the content. Behind the content it allows one padding +// region, and only when that holds the file's label and nothing else. A ZIP +// keeps its label in the archive comment, which closes the file, so no layout +// can move it in front (docs/TRUNCATE-2026-11-11.md section 10.4). +func lastEndLayout(t *testing.T, m mappedFile, label string) bool { + t.Helper() + front, back := paddingSides(m.marked) + return front && (!back || onlyTheLabelBehind(t, m, label)) +} + +// onlyTheLabelBehind says the padding behind the content of m is one region, +// and its bytes are label. +func onlyTheLabelBehind(t *testing.T, m mappedFile, label string) bool { + t.Helper() + at := slices.IndexFunc(m.marked, func(r [3]any) bool { return r[0] == string(format.RoleContent) }) + var behind [][3]any + for _, r := range m.marked[at+1:] { + if r[0] == string(format.RolePadding) { + behind = append(behind, r) + } + } + if len(behind) != 1 { + return false + } + data, err := os.ReadFile(m.path) + if err != nil { + t.Fatalf("%s: %v", m.name, err) + } + start, n := behind[0][1].(int64), behind[0][2].(int64) + return start+n <= int64(len(data)) && string(data[start:start+n]) == label +} + +// askTheLastEndCanaries asks lastEndLayout about files made for it: the label +// alone behind the content passes, and padding of its length that is not it, +// or the label with padding beside it, does not. +func askTheLastEndCanaries(t *testing.T, dir string) { + t.Helper() + file := func(tail string) mappedFile { + path := filepath.Join(dir, "canary-last-end") + if err := os.WriteFile(path, []byte("ppccccc"+tail), 0o600); err != nil { + t.Fatal(err) + } + return mappedFile{name: "canary", path: path, marked: [][3]any{ + {string(format.RolePadding), int64(0), int64(2)}, {string(format.RoleContent), int64(2), int64(5)}, + {string(format.RolePadding), int64(7), int64(len(tail))}, + }} + } + if !lastEndLayout(t, file("tfg"), "tfg") { + t.Errorf("canary: the label alone behind the content failed the layout due at the last byte") + } + if lastEndLayout(t, file("tfx"), "tfg") { + t.Errorf("canary: padding of the label's length that is not the label passed for the layout due at the last byte") + } + if lastEndLayout(t, file("tfg "), "tfg") { + t.Errorf("canary: the label with padding beside it passed for the layout due at the last byte") + } +} + // writeMapped writes one healthy file of f through a map recorder, laid out // for a cut at cutAt when that is not zero. func writeMapped(t *testing.T, dir string, f format.Descriptor, name string, setting map[string]string, size, cutAt int64) mappedFile { diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 52c29cf8..1ee94990 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -360,8 +360,8 @@ ( "an Office package goes back to writing its sizes after the data instead of in the header", "internal/format/opc/opc.go", - "entry, err := zw.CreateRaw(head)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := zm.MarkEntry(", - "entry, err := zw.CreateHeader(head)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := zm.MarkEntry(", + "entry, err := zw.CreateRaw(head)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif err := zm.MarkEntry(", + "entry, err := zw.CreateHeader(head)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif err := zm.MarkEntry(", "TestEveryFormatSurvivesItsReferenceTool", ), ( @@ -6752,8 +6752,8 @@ "that matters most here - a function sits one level under a ceiling " "that was already exact, and no guard could see it", "internal/guard/branching_test.go", + "\tcrowdedDepthFunctions = 48\n", "\tcrowdedDepthFunctions = 49\n", - "\tcrowdedDepthFunctions = 50\n", "TestTheBranchingCeilingsAreTodaysMeasurementAndNotALooserNumber", ), ( @@ -7602,8 +7602,8 @@ ( "a compressed archive stops giving back what the compressor freed, so it comes out short", "internal/format/zip/filler.go", - "size := m.fillerSize + freed()", - "size := m.fillerSize + freed()*0", + "size := m.fillerSize + got", + "size := m.fillerSize + got*0", "TestACompressedArchiveStillHitsTheSizeToTheByte", ), ( @@ -15870,8 +15870,8 @@ # stop during it waits for four gigabytes of padding to be hashed. "an Office package checksums its padding without looking at the stop", "internal/format/opc/opc.go", - "sum, err := fillerChecksum(ctx, p.Seed, p.Filler)", - "sum, err := fillerChecksum(context.Background(), p.Seed, p.Filler)", + "sum, err := fillerChecksum(ctx, part.seed, part.n)", + "sum, err := fillerChecksum(context.Background(), part.seed, part.n)", "TestAnOfficePackageStopsWhileItChecksumsItsPadding", ), ( diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index f1717798..44e86edc 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -216,7 +216,7 @@ def layout_pdf(data): [END, xref, closed - xref], [PADDING, closed, sx - closed], [END, sx, len(data) - sx]] -ZIP_FILLERS = (b"tfg-padding.bin", b"tfg/padding.bin") +ZIP_FILLERS = (b"tfg-padding.bin", b"tfg/padding.bin", b"tfg-padding-end.bin", b"tfg/padding-end.bin") def zip_entries(data):