From 6430acfd077fc9b22e26ad104a7a0b7ae1194788 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 11:34:52 +0100 Subject: [PATCH 1/5] An Office document cut at its smallest says how long its parts were A Word, Excel or PowerPoint document too small for a padding part in front of its parts was written as one nothing cuts, and its parts were marked with no length. The writer stops at the cut, before the mark that would end them, so the manifest of a docx cut by half at 610 to 660 B stated where the cut fell and how much it kept, but not how long the content was. The parts are now marked with their length whether or not a padding part stands in front. The bytes written are unchanged. A new guard cuts every format that takes the share by half at its smallest size and requires the content's length, equal to how far the whole file's map runs the content. It asserts that the ZIP packages reach the layout with no padding part in front. The share guard's exception for a cut in the end of a ZIP package now asks the whole file's map whether the cut offset lies in an end region, rather than trusting where the manifest says the cut fell. Co-Authored-By: Claude Opus 5.5 --- internal/format/opc/cut.go | 4 + internal/format/opc/opc.go | 10 +- internal/guard/cutmanifest_test.go | 116 ++++++++++++++++++++++++ internal/guard/cutshare_test.go | 31 ++++++- internal/guard/mutationcoverage_test.go | 4 + 5 files changed, 157 insertions(+), 8 deletions(-) diff --git a/internal/format/opc/cut.go b/internal/format/opc/cut.go index 99a58358..564e1c6b 100644 --- a/internal/format/opc/cut.go +++ b/internal/format/opc/cut.go @@ -38,6 +38,10 @@ func SettleForACut(parts []Part, shape Shape, want, cutAt int64) (Package, error } pad := want - shape.Bare front := format.PaddingInFront(0, c.content, cutAt, want) + // The parts are marked with their length whether or not a part goes in + // front: the cut stops the writing before the mark behind them, so a cut + // inside them says how much of them there was only from this. + p.Content = c.content if front <= 0 || pad < c.frontOverhead { return p, nil } diff --git a/internal/format/opc/opc.go b/internal/format/opc/opc.go index ccb0f565..83944e2a 100644 --- a/internal/format/opc/opc.go +++ b/internal/format/opc/opc.go @@ -267,16 +267,16 @@ func write(ctx context.Context, w io.Writer, p Package, withPadding bool) error // archive/zip writes it (archive.StreamMap). A package laid out for a cut // opens with a padding part, and the content starts behind it. behind := FillerName + content := p.Content + if content <= 0 { + content = -1 + } if p.Front == NoFiller { - format.Mark(w, format.RoleContent, 0, -1) + format.Mark(w, format.RoleContent, 0, content) } else { if err := writePaddingPart(ctx, zw, zm, paddingPart{FillerName, p.Seed, p.Front, withPadding}); err != nil { return err } - content := p.Content - if content <= 0 { - content = -1 - } format.Mark(w, format.RoleContent, zm.Pos(), content) behind = FillerEndName } diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index 05bdb801..e771583c 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -9,6 +9,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/damage" "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/manifest" ) @@ -212,3 +213,118 @@ func TestTheRecordOfACutRunIsNeverLongerThanItsForecast(t *testing.T) { t.Errorf("the forecast is %d B over a record of %d B", slack, written) } } + +// A file cut at the smallest size it is cut at says how long its content was, +// as a larger one does: the share a cut took is measured against that length. +// +// At its smallest a file has the least padding, too little for the layout a +// cut gets in a larger one. A Word document's padding part in front of the +// others costs more than the padding there is, so the package is written as +// one nothing cuts, and its content is marked in a branch the share guard's +// larger files never reach (docs/REVIEW-239-2026-11-13.md). What it reaches, +// asserted rather than assumed: the ZIP packages, cut at their smallest, have +// no padding in front of their content and are cut inside it. +// +// Every format that takes the share (notYetCutAtTheShare lists the rest) is cut +// by half at its smallest. Where the cut falls in the content, its length has +// to be stated and to be how far the whole file's map lets the content run - +// up to the region behind it, not the length the generator marked it with. +// +// Canary: a cut whose content length is missing, and one whose length is a +// byte short, must not pass. +func TestACutAtTheSmallestSizeSaysHowLongTheContentWas(t *testing.T) { + chain := damage.Chain{{ID: damage.Truncate, Values: damage.Values{damage.SettingMissing: "50"}}} + var targets []engine.Target + for _, f := range format.All() { + if _, listed := notYetCutAtTheShare[f.ID]; listed { + continue + } + size := engine.SmallestDamaged(f, chain, format.Request{Label: true}) + targets = append(targets, engine.Target{ID: f.ID, Format: f.ID, Sizes: engine.Uniform(1, size), Label: true, Damage: chain}) + } + opt := engine.Options{OutDir: t.TempDir(), Seed: 7, ManifestName: engine.DefaultManifestName} + planned, err := engine.Plan(targets, opt) + if err != nil { + t.Fatalf("planning: %v", err) + } + res, err := engine.Run(t.Context(), planned, opt) + if err != nil { + t.Fatalf("running: %v", err) + } + plannedOf := map[string]engine.PlannedFile{} + for _, f := range planned { + plannedOf[f.Target.ID] = f + } + inContent := 0 + for _, m := range res.Manifest.Files { + if m.Failed || len(m.Damage) != 1 || m.Damage[0].Cut == nil { + t.Errorf("%s: no file with one cut was written (%s, %+v)", m.TargetID, m.Error, m.Damage) + continue + } + cut := *m.Damage[0].Cut + whole := wholeMap(t, plannedOf[m.TargetID]) + if _, zipped := endClosesTheFile[m.TargetID]; zipped && (paddingInFront(whole) || cut.In != string(format.RoleContent)) { + t.Errorf("%s: at %d B the cut fell in the %s with padding in front of the content: %v - the layout too short for a padding part in front is not reached", + m.TargetID, m.Bytes, cut.In, paddingInFront(whole)) + } + if cut.In != string(format.RoleContent) { + continue + } + inContent++ + if problem := contentLengthProblem(cut, whole); problem != "" { + t.Errorf("%s: cut at %d B: %s", m.TargetID, m.Bytes, problem) + } + } + if inContent == 0 { + t.Errorf("no cut fell in the content, so no length was asked for") + } + + // Canaries. + regions := []format.Region{{Role: format.RoleContent, At: 0, Length: 10}, {Role: format.RolePadding, At: 10, Length: 5}} + if contentLengthProblem(manifest.Cut{In: "content"}, regions) == "" { + t.Errorf("canary: a cut that does not state the content's length passed") + } + short := int64(9) + if contentLengthProblem(manifest.Cut{In: "content", ContentBytes: &short}, regions) == "" { + t.Errorf("canary: content said to be a byte shorter than the map runs passed") + } +} + +// contentLengthProblem says how the content length cut states is not how far +// the content runs in the whole file's map, or nothing when it is. Each +// content region runs to the start of the region behind it, or to the end of +// the file - the length it was marked with is the claim being checked. +func contentLengthProblem(cut manifest.Cut, whole []format.Region) string { + if cut.ContentBytes == nil { + return "the length of the content is not stated" + } + var runs int64 + for i, r := range whole { + if r.Role != format.RoleContent { + continue + } + end := r.At + r.Length + if i+1 < len(whole) { + end = whole[i+1].At + } + runs += end - r.At + } + if *cut.ContentBytes != runs { + return fmt.Sprintf("the content is said to be %d B long, where the whole file's map runs it for %d B", *cut.ContentBytes, runs) + } + return "" +} + +// paddingInFront is whether padding of any length stands in front of the +// first content region of a map. +func paddingInFront(regions []format.Region) bool { + for _, r := range regions { + switch { + case r.Role == format.RoleContent: + return false + case r.Role == format.RolePadding && r.Length > 0: + return true + } + } + return false +} diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 9a095bba..9468c502 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -142,11 +142,16 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { } problem := shareProblem(missing, m) f := plannedOf[m.TargetID] - if _, closes := endClosesTheFile[f.Desc.ID]; closes && problem == cutInTheEnd && onlyTheEndBehind(wholeMap(t, f)) { + if _, closes := endClosesTheFile[f.Desc.ID]; closes && problem == cutInTheEnd { // The end the format has to close with is longer than what the // cut takes, and nothing else stands behind the content that the - // format could have put in front of it. - problem = "" + // format could have put in front of it. Both read off the whole + // file's map, not the manifest the cut file's map wrote. + whole := wholeMap(t, f) + _, at, cuts := f.Target.Damage.Cut(f.Plan.Bytes) + if cuts && cutFallsInTheEnd(whole, at) && onlyTheEndBehind(whole) { + problem = "" + } } problems[m.TargetID] = problem } @@ -175,6 +180,16 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { if !onlyTheEndBehind([]format.Region{region(format.RolePadding, 0, 5), region(format.RoleContent, 5, 10), region(format.RoleEnd, 15, 5)}) { t.Errorf("canary: a file with all of its padding in front was refused for padding behind") } + ends := []format.Region{region(format.RoleContent, 0, 10), region(format.RoleEnd, 10, 0), region(format.RoleEnd, 10, 5)} + if cutFallsInTheEnd(ends, 5) { + t.Errorf("canary: a cut in the content, reported as one in the end, passed for a cut in the end") + } + if !cutFallsInTheEnd(ends, 12) { + t.Errorf("canary: a cut in the end was refused for one outside it") + } + if cutFallsInTheEnd(ends[:2], 10) { + t.Errorf("canary: a cut at an end of no bytes passed for a cut in the end") + } listed := maps.Clone(notYetCutAtTheShare) listed["txt"] = notYet{paddingBehind, 64 << 10, 50} if !slices.ContainsFunc(shareVerdicts(problems, listed), func(s string) bool { return strings.HasPrefix(s, "txt:") }) { @@ -226,6 +241,16 @@ func onlyTheEndBehind(regions []format.Region) bool { return true } +// cutFallsInTheEnd is whether the byte at, the first a cut drops, lies in an +// end of a file's whole map - asked of the map, because the manifest says +// where the cut fell from the map the cut file wrote, which is what the +// exception for a cut in the end must not take on trust. +func cutFallsInTheEnd(regions []format.Region, at int64) bool { + return slices.ContainsFunc(regions, func(r format.Region) bool { + return r.Role == format.RoleEnd && r.At <= at && at < r.At+r.Length + }) +} + // wholeMap is the map of the whole file f is cut from: its generator written // out through its own plan to nowhere, the damage left to the engine's // streams, which are not here. diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index ac5250c8..0c26b607 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -155,6 +155,10 @@ var provenByProbe = map[string]string{ "padding a byte late: red, and by marking the picture of a png laid out for a cut a byte late: red at 8 KiB and at 64 KiB.", "TestACutFileSaysWhereTheCutFellAndWhatIsLeft": "its own canary - a cut named in the wrong role, and content claimed gone that is there, must not pass - " + "and broken on 2026-11-12 by reporting every cut in the content: red for the picture and the icon.", + "TestACutAtTheSmallestSizeSaysHowLongTheContentWas": "its own canaries - a cut that does not state the content's length, and one stating a " + + "byte less than the map runs, must not pass - its reach check - the ZIP packages at their smallest have no padding in front and are cut " + + "in the content - and red on 2026-11-13 before the Office packages marked their parts with a length without a padding part in front: " + + "docx at 610 B, xlsx at 863 B, pptx at 2405 B.", "TestACutTakesTheShareOfTheContentItIsAskedFor": "its own canaries - a cut that kept all its content must not pass for one that took half, and txt listed as " + "not cutting at the share must be reported - and broken on 2026-11-12 by never telling png where its cut falls: red in all six of its files.", "TestAGuardThatDrawsAfterALongPauseStillDraws": "its own canary - the box drawn after test.NewApp alone has to come out blank, or the pause " + From 507162c0d6ab5403e2d29aa51afeb39f52eeebea Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 13:35:13 +0100 Subject: [PATCH 2/5] A tar.gz cut short loses the share of its files asked for A tar.gz kept its padding in the gzip header, in front of everything, and above what the header holds in a filler entry behind the files. Cut at any share, a small archive lost all of its files and a large one lost padding and none of them. An archive a damage cuts is now laid out for the cut (format.Request.CutAt): padding in front of the files in the extra field and, above it, an entry of its own, and the rest behind them, so the cut keeps the share of the files it keeps of the whole. gzip.Writer compresses the tar as one stream, so this archive is framed by hand to RFC 1951 and RFC 1952: one member, one deflate stream, each part in stored blocks of its own and the files, when squeezed, through a fresh compressor ended with a sync flush. Each part ends on a byte, so the map gives the files' length, a squeezed archive's included, and the files are squeezed once to measure them rather than settled over rounds. The tar stays whole blocks with nothing after its end marker, as POSIX and strict.py have it. The last bytes behind land in the deflate frame: the marker in as many stored blocks as take the rest in steps of five, and one to three empty blocks of fixed codes for the remainder. The few rests no frame takes go in front, so the cut takes up to four bytes more. The map's reader in layout.py now decodes deflate blocks, to find where each part starts in a squeezed stream rather than reading it all as content. tar.gz leaves the list of formats not yet cut at the share and joins those whose end closes the file. Files nothing cuts write the same bytes as before, stored and squeezed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 +- internal/format/targz/cut.go | 580 +++++++++++++++++++++++++++++ internal/format/targz/filemap.go | 6 +- internal/format/targz/size.go | 96 +++-- internal/format/targz/targz.go | 16 +- internal/guard/cutmanifest_test.go | 13 +- internal/guard/cutshare_test.go | 22 +- internal/oracle/layout.py | 236 +++++++++--- 8 files changed, 882 insertions(+), 97 deletions(-) create mode 100644 internal/format/targz/cut.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d62c6d33..118f4522 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,8 +78,7 @@ because it turns other people's test suites red. the manifest's `damage` now carries `cut`: `in` (`content`, `padding`, `structure` or `end`) and `content_kept_bytes`, and `content_bytes` where the length of the content is known. A cut in the padding also gets the - note `cut_in_padding`. In a compressed tar.gz a cut in the padding is - reported as a cut in the content. + note `cut_in_padding`. - **`truncate` cuts the picture of a PNG, a GIF, a JPEG or an icon by the percentage you set.** `--damage truncate:missing=90` on a 1 MB PNG or GIF used to cut only padding and leave the whole picture, and on a JPEG or a @@ -94,6 +93,13 @@ because it turns other people's test suites red. file inside. In a small file a cut of a few percent can fall in the index at its end, and then every file inside is kept - the manifest says `in: end`. Files nothing cuts are unchanged. +- **`truncate` cuts the files inside a tar.gz by the percentage you set, + compressed or not.** `--damage truncate:missing=50` on a 1 MB tar.gz used + to cut only padding. Now it removes half of the files' content, so + `tar -xzf` gets back the first part of the file inside and then stops at + the unexpected end. When the cut is shorter than the end of the archive - + 1% of 64 KiB - it falls there and every file is kept, and the manifest says + `in: end`. Files nothing cuts are unchanged. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/format/targz/cut.go b/internal/format/targz/cut.go new file mode 100644 index 00000000..d0befaab --- /dev/null +++ b/internal/format/targz/cut.go @@ -0,0 +1,580 @@ +package targz + +import ( + "archive/tar" + "compress/flate" + "context" + "encoding/binary" + "fmt" + "hash" + "hash/crc32" + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// An archive laid out for a cut (format.Request.CutAt, +// docs/TRUNCATE-2026-11-11.md section 11). +// +// An archive nothing cuts keeps its padding in the gzip header, in front of +// everything, and what the header cannot hold in a filler entry behind the +// files. Cut at any share, a large one lost padding and no file, or every file +// and no padding. So an archive a damage cuts has its padding on both sides of +// its files, split so the cut keeps the share of them it keeps of the whole: +// +// gzip header - the extra field, padding in front to the byte, and the +// comment, which carries the label +// directories - stored +// front entry - fillerName, stored, when the front is more than the extra +// field holds +// files - stored, or squeezed by a compressor of their own +// back entry - stored: fillerName, or fillerEndName when one stands in +// front +// marker - the end of archive marker, stored, its last block final +// trailer - the CRC-32 and the length of the tar +// +// One gzip member and one deflate stream, every part in blocks of its own, so +// each part ends on a byte and the map says where. gzip.Writer compresses the +// whole tar as one stream and cannot write this, so the frame is written here, +// to RFC 1951 and RFC 1952. +// +// The files of a squeezed archive go through a compressor that starts with an +// empty window and ends on a byte with a sync flush, so how long they come to +// does not depend on what stands in front of them. One pass into a counter +// measures it, where an archive nothing cuts settles its padding over several +// (compress.go). +// +// The padding behind lands on the byte in the deflate stream, because the tar +// cannot take it: POSIX makes a tar whole blocks of 512 with nothing after the +// marker, and strict.py holds this format to that. The back entry takes whole +// blocks, the marker as many stored blocks as take the rest in steps of five +// bytes, and one to three empty blocks of fixed codes - ten bits each, the block +// compress/flate itself closes a stored stream with - the last few bytes. A +// rest of 1 to 4 or 8 bytes behind the marker's floor has no such frame, and +// goes in front. +// +// Rejected, after the readers were measured (section 11.3). A second gzip +// member behind the first, which 7-Zip reports as a tar of 0 B and which would +// make the squeezed files depend on the padding in front of them. One member +// for each part, which a reader that reads only the first member takes for a +// whole archive however it is cut. And zeros after the marker, which every +// reader takes and which leave the tar a part of a block long. + +const ( + // fillerEndName is the entry of padding behind the files when one stands + // in front of them with fillerName. Two names, because two entries of one + // name are one file overwritten when the archive is unpacked. + fillerEndName = "tfg-padding-end.bin" + + // storedHeader is what a stored deflate block that starts on a byte costs + // in front of its bytes: the byte holding BFINAL and BTYPE, then LEN and + // NLEN (RFC 1951 section 3.2.4). + storedHeader = 5 + + // markerLength is the end of archive marker, two blocks of zeros, and + // tailFloor the shortest tail: the marker in one stored block. + markerLength = 2 * tarBlock + tailFloor = markerLength + storedHeader +) + +// frame is how an archive laid out for a cut is framed (layOut), and its zero +// value is the archive nothing cuts, framed by gzip.Writer. +type frame struct { + on bool + // comment is the label, with one space after it when the front is one + // byte, which the extra field cannot cost. + comment string + withExtra bool + extraLen int64 + // front and back are how many bytes of the tar the entries of padding in + // front of the files and behind them take, header and data, 0 for none. + front, back int64 + // content is how long the files' part comes to in the file. + content int64 + // markerBlocks is how many stored blocks the end of archive marker takes, + // and empties how many empty blocks of fixed codes stand in front of its + // second block and its third. + markerBlocks int64 + empties [2]int +} + +// cuts says the archive is laid out for a cut: one falls inside it, and there +// are files for it to keep a share of. +func (m memo) cuts(target int64) bool { + return m.cutAt > 0 && m.cutAt < target && len(m.children) > 0 +} + +// settleForACut lays out a stored archive a damage will cut, while planning. +// A squeezed one is laid out as it is written (squeezeForACut), because how +// long its files come to is known only once they are squeezed. An archive too +// small for the frame keeps the layout of one nothing cuts, with the few bytes +// of padding pad put in its header, in front of the files. +func (m *memo) settleForACut(p *format.Plan, target int64, label string) error { + if m.squeeze.On() || !m.cuts(target) { + return nil + } + f, fits, err := layOut(*m, label, fewestBlocks(filesLength(*m)).length(), target) + if err != nil || !fits { + return err + } + m.frame = f + delete(p.Properties, "padding_entry") + if f.front > 0 || f.back > 0 { + p.Properties["padding_entry"] = fillerName + } + return nil +} + +// squeezeForACut lays out a squeezed archive a damage will cut, as it is +// written: its files are squeezed into a counter to learn how long they come +// to, and the pass that writes them has to come to the same (writeForACut). +// That generates every file inside twice, and only for a squeezed archive a +// damage cuts. +func (m *memo) squeezeForACut(ctx context.Context) error { + if !m.squeeze.On() || !m.cuts(m.target) { + return nil + } + c := &counter{} + if err := writeFiles(ctx, c, io.Discard, *m); err != nil { + return err + } + f, _, err := layOut(*m, m.comment, c.n, m.target) + m.frame = f + return err +} + +// layOut frames an archive of target bytes whose files' part comes to content +// bytes, with the padding in front of the files that keeps the share +// (format.PaddingInFront) and the rest behind them. It says false when the +// frame does not fit: it costs a few bytes more than the layout of an archive +// nothing cuts, because each part closes its own blocks. +// +// The front is the extra field alone up to what it holds, and above that an +// entry of padding in front of the files with the extra field taking the last +// bytes the entry's 512 byte steps cannot. One byte, which no field costs, +// lengthens the comment, and without a label the front is two instead. That, +// and a rest behind that no frame of the marker takes, put a few bytes more in +// front, and the cut then takes as many more of the files - the "at least" +// missing promises. Where the padding is too short for the files the cut +// should take, all of it goes in front. +func layOut(m memo, label string, content, target int64) (frame, bool, error) { + head := gzipHead + commentCost(label) + dirs := fewestBlocks(dirsLength(m)).length() + spare := target - (head + dirs + content + tailFloor + gzipTrailer) + if spare < 0 { + return frame{}, false, nil + } + want := min(max(format.PaddingInFront(head+dirs, content, m.cutAt, target), 0), spare) + // Each round puts in front what the tail of the one before left over, so + // the second lands, or the third when the second met the one byte only a + // label can cost. + for range 3 { + f := frame{on: true, comment: label, content: content} + front := f.placeFront(want, spare, label) + left := f.placeTail(spare - front) + if left == 0 { + return f, true, f.agrees(m, target) + } + want = front + left + } + return frame{}, false, core.Defect(fmt.Errorf("targz: no frame lays out %d B of padding around the files for a cut at %d", spare, m.cutAt)) +} + +// placeFront puts front bytes of padding in front of the files, and returns +// how many it put there. +func (f *frame) placeFront(front, spare int64, label string) int64 { + switch { + case front == 0: + case front == 1 && label != "": + f.comment = label + " " + case front == 1 && spare < 2: + return 0 + case front == 1: + f.withExtra = true + return 2 + case front <= extraPaddingLimit+2: + f.withExtra, f.extraLen = true, front-2 + default: + f.front = entryIn(front - 2) + f.withExtra, f.extraLen = true, front-fewestBlocks(f.front).length()-2 + } + return front +} + +// placeTail lays out rest bytes behind the files beyond tailFloor: the back +// entry in whole tar blocks, as many as leave the marker a rest it can be +// framed to, and the marker's frame. It returns what no frame took, which is +// at most four bytes. +func (f *frame) placeTail(rest int64) int64 { + f.back = entryIn(rest) + over := rest - fewestBlocks(f.back).length() + if !markerFrames(over) && f.back > 0 { + f.back -= tarBlock + over = rest - fewestBlocks(f.back).length() + } + left := int64(0) + for !markerFrames(over) { + over-- + left++ + } + f.frameMarker(over) + return left +} + +// markerFrames says whether over bytes beyond one stored block can frame the +// end of archive marker: five for each block more, and one, two or four for +// one to three empty blocks of fixed codes in front of a block after the +// first (emptiesCost). Three takes two such places, and so two blocks more. +func markerFrames(over int64) bool { + return over == 0 || over >= 5 && over != 8 +} + +// frameMarker frames the end of archive marker in over bytes beyond one +// stored block, which markerFrames allows. +func (f *frame) frameMarker(over int64) { + f.markerBlocks = 1 + over/5 + switch over % 5 { + case 1: + f.empties = [2]int{1, 0} + case 2: + f.empties = [2]int{2, 0} + case 3: + f.empties = [2]int{1, 2} + case 4: + f.empties = [2]int{3, 0} + } +} + +// emptiesCost is what n empty blocks of fixed codes cost in front of a stored +// block that would otherwise start on a byte: ten bits each, and the stored +// block's own three bits then end in a later byte. +func emptiesCost(n int) int64 { + return int64((10*n+3+7)/8 - 1) +} + +// entryIn is the longest entry of padding, header and data in whole tar +// blocks, whose stored blocks come to no more than budget bytes - 0 when not +// even a header fits. +func entryIn(budget int64) int64 { + if budget < tarBlock+storedHeader { + return 0 + } + blocks := (budget + storeBlock - 1) / storeBlock + n := max((budget-storedHeader*blocks)/tarBlock*tarBlock, tarBlock) + for fewestBlocks(n+tarBlock).length() <= budget { + n += tarBlock + } + return n +} + +// marker is the end of archive marker in its stored blocks. +func (f frame) marker() blocks { + return blocks{n: markerLength, k: f.markerBlocks} +} + +// agrees says whether the archive f frames comes to target bytes, added up +// part by part as markForACut and writeForACut lay them down - a defect +// rather than an archive the engine refuses for its length. +func (f frame) agrees(m memo, target int64) error { + got := gzipHead + extraCost(f.withExtra, f.extraLen) + commentCost(f.comment) + + fewestBlocks(dirsLength(m)).length() + fewestBlocks(f.front).length() + f.content + + fewestBlocks(f.back).length() + f.marker().length() + + emptiesCost(f.empties[0]) + emptiesCost(f.empties[1]) + gzipTrailer + if got != target { + return core.Defect(fmt.Errorf("targz: laid out for a cut, the archive comes to %d B where %d B was asked for", got, target)) + } + return nil +} + +// behindName is the name of the entry of padding behind the files. +func (f frame) behindName() string { + if f.front > 0 { + return fillerEndName + } + return fillerName +} + +// markForACut marks the map of an archive laid out for a cut, before its first +// byte. Every part ends on a byte, so each has a length, the files' included - +// a cut inside them says how long they were. +func markForACut(w io.Writer, m memo) { + f := m.frame + extra, comment := extraCost(f.withExtra, f.extraLen), commentCost(f.comment) + head := gzipHead + extra + comment + format.Mark(w, format.RoleStructure, 0, gzipHead) + format.Mark(w, format.RolePadding, gzipHead, extra) + format.Mark(w, format.RoleStructure, gzipHead+extra, comment) + dirs := fewestBlocks(dirsLength(m)).length() + format.Mark(w, format.RoleStructure, head, dirs) + front := fewestBlocks(f.front).length() + format.Mark(w, format.RolePadding, head+dirs, front) + at := head + dirs + front + format.Mark(w, format.RoleContent, at, f.content) + at += f.content + back := fewestBlocks(f.back).length() + format.Mark(w, format.RolePadding, at, back) + format.Mark(w, format.RoleEnd, at+back, -1) +} + +// writeForACut writes an archive laid out for a cut. +func writeForACut(ctx context.Context, w io.Writer, m memo) error { + f := m.frame + markForACut(w, m) + if _, err := w.Write(gzipHeader(m.squeeze.Level, f)); err != nil { + return err + } + sum := &tarSum{crc: crc32.NewIEEE()} + if err := storedPart(w, sum, fewestBlocks(dirsLength(m)), func(dst io.Writer) error { + return flushed(tar.NewWriter(dst), func(tw *tar.Writer) error { return writeDirectories(ctx, tw, m) }) + }); err != nil { + return err + } + if err := writeFront(ctx, w, sum, m); err != nil { + return err + } + files := &counter{} + if err := writeFiles(ctx, io.MultiWriter(w, files), sum, m); err != nil { + return err + } + if files.n != f.content { + return core.Defect(fmt.Errorf("targz: the files inside came to %d B where the pass that measured them made %d B", files.n, f.content)) + } + if err := writeTail(ctx, w, sum, m); err != nil { + return err + } + var trailer [gzipTrailer]byte + binary.LittleEndian.PutUint32(trailer[:4], sum.crc.Sum32()) + binary.LittleEndian.PutUint32(trailer[4:], uint32(sum.n)) + _, err := w.Write(trailer[:]) + return err +} + +// writeFront writes the entry of padding in front of the files, if there is +// one. Its bytes come from the seed of an index no file inside has, so the two +// entries of padding do not open with the same bytes. +func writeFront(ctx context.Context, w, sum io.Writer, m memo) error { + f := m.frame + if f.front == 0 { + return nil + } + return storedPart(w, sum, fewestBlocks(f.front), func(dst io.Writer) error { + return flushed(tar.NewWriter(dst), func(tw *tar.Writer) error { + return writeFillerEntry(ctx, tw, m, fillerName, f.front-tarBlock, core.FileSeed(m.seed, len(m.children))) + }) + }) +} + +// writeFiles writes the files' part into w, stored or squeezed as the archive +// is, and the tar's bytes into sum. Squeezed, the compressor ends with a sync +// flush rather than closing: that ends the files on a byte with a block that +// is not the last, and the tail carries the stream on. +func writeFiles(ctx context.Context, w, sum io.Writer, m memo) error { + entries := func(tw *tar.Writer) error { return writeFileEntries(ctx, tw, m) } + if !m.squeeze.On() { + return storedPart(w, sum, fewestBlocks(filesLength(m)), func(dst io.Writer) error { + return flushed(tar.NewWriter(dst), entries) + }) + } + zw, err := flate.NewWriter(w, m.squeeze.Level) + if err != nil { + return core.Defect(fmt.Errorf("targz: the files inside could not be squeezed: %w", err)) + } + if err := flushed(tar.NewWriter(io.MultiWriter(zw, sum)), entries); err != nil { + return err + } + return zw.Flush() +} + +// writeTail writes the back entry and the end of archive marker, each in its +// own stored blocks. One tar writer writes both, so the marker is the one +// archive/tar closes an archive with, and only where its bytes go changes. +func writeTail(ctx context.Context, w, sum io.Writer, m memo) error { + f := m.frame + to := &switchWriter{} + tw := tar.NewWriter(to) + if err := storedPart(w, sum, fewestBlocks(f.back), func(dst io.Writer) error { + to.w = dst + if f.back == 0 { + return nil + } + return flushed(tw, func(tw *tar.Writer) error { + return writeFillerEntry(ctx, tw, m, f.behindName(), f.back-tarBlock, m.seed) + }) + }); err != nil { + return err + } + s := &storedSegment{w: w, b: f.marker(), last: true, empties: f.empties} + to.w = io.MultiWriter(s, sum) + if err := tw.Close(); err != nil { + return err + } + return s.done() +} + +// switchWriter passes what it is given to w, which can change between writes. +type switchWriter struct{ w io.Writer } + +func (s *switchWriter) Write(p []byte) (int, error) { return s.w.Write(p) } + +// flushed runs body on tw and then pads its last entry to a whole block, +// without the end of archive marker Close would add. +func flushed(tw *tar.Writer, body func(*tar.Writer) error) error { + if err := body(tw); err != nil { + return err + } + return tw.Flush() +} + +// gzipHeader is the gzip header of an archive laid out for a cut, as +// gzip.Writer writes that of one nothing cuts (RFC 1952 section 2.3): no name +// and no time, the extra field and the comment when there are any, XFL as the +// level says, and the system unknown. +func gzipHeader(level int, f frame) []byte { + h := []byte{0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 255} + switch level { + case flate.BestCompression: + h[8] = 2 + case flate.BestSpeed: + h[8] = 4 + } + if f.withExtra { + h[3] |= 4 + h = binary.LittleEndian.AppendUint16(h, uint16(f.extraLen)) + h = append(h, make([]byte, f.extraLen)...) + } + if f.comment != "" { + h[3] |= 16 + h = append(h, f.comment...) + h = append(h, 0) + } + return h +} + +// tarSum is the CRC-32 and the length of the tar, which the gzip trailer +// carries. +type tarSum struct { + crc hash.Hash32 + n int64 +} + +func (s *tarSum) Write(p []byte) (int, error) { + s.n += int64(len(p)) + return s.crc.Write(p) +} + +// blocks is n bytes cut into k stored blocks, as evenly as they go: the first +// n%k blocks a byte longer than the rest. k is no more than n, so no block is +// empty, and at least n/storeBlock rounded up, so none is longer than a stored +// block can be. +type blocks struct{ n, k int64 } + +// fewestBlocks is n bytes in as few stored blocks as hold them. +func fewestBlocks(n int64) blocks { + return blocks{n: n, k: (n + storeBlock - 1) / storeBlock} +} + +// length is how long the blocks come to in the file, headers and all. +func (b blocks) length() int64 { return b.n + storedHeader*b.k } + +// size is how many bytes block i carries. +func (b blocks) size(i int64) int64 { + if i < b.n%b.k { + return b.n/b.k + 1 + } + return b.n / b.k +} + +// storedPart writes the bytes body gives it into w as the stored blocks b +// lays out, and into sum as well. A body that gives more or fewer than b.n +// bytes is a defect. +func storedPart(w, sum io.Writer, b blocks, body func(io.Writer) error) error { + s := &storedSegment{w: w, b: b} + if err := body(io.MultiWriter(s, sum)); err != nil { + return err + } + return s.done() +} + +// storedSegment writes what it is given as the stored blocks b lays out, +// opening each with its header as the first of its bytes arrives. The last +// block is final when last is set, and empties are the empty blocks of fixed +// codes in front of the second block and the third. +type storedSegment struct { + w io.Writer + b blocks + last bool + empties [2]int + // next is the block to open next, and left how many bytes the open one + // still takes. + next, left int64 +} + +func (s *storedSegment) Write(p []byte) (int, error) { + n := 0 + for len(p) > 0 { + if err := s.openWhenFull(); err != nil { + return n, err + } + k := min(s.left, int64(len(p))) + written, err := s.w.Write(p[:k]) + n += written + s.left -= int64(written) + if err != nil { + return n, err + } + p = p[k:] + } + return n, nil +} + +// openWhenFull opens the next block once the open one has all its bytes. +func (s *storedSegment) openWhenFull() error { + if s.left > 0 { + return nil + } + if s.next == s.b.k { + return core.Defect(fmt.Errorf("targz: a part of the archive came to more than the %d B it was laid out for", s.b.n)) + } + empties := 0 + if s.next == 1 || s.next == 2 { + empties = s.empties[s.next-1] + } + size := s.b.size(s.next) + s.next++ + s.left = size + _, err := s.w.Write(blockHeader(empties, s.last && s.next == s.b.k, size)) + return err +} + +// done says whether every block got all its bytes. +func (s *storedSegment) done() error { + if s.next != s.b.k || s.left != 0 { + return core.Defect(fmt.Errorf("targz: a part of the archive came to less than the %d B it was laid out for", s.b.n)) + } + return nil +} + +// blockHeader is what stands in front of a stored block's bytes: empties +// empty blocks of fixed codes - BFINAL 0, BTYPE 01 and the end of block code, +// seven zero bits - then the block's own BFINAL and BTYPE 00, the bits to the +// next byte, LEN and NLEN (RFC 1951 sections 3.2.3, 3.2.4 and 3.2.6). Bits go +// in from the least significant end of each byte. +func blockHeader(empties int, final bool, size int64) []byte { + var bits uint64 + n := 0 + for range empties { + bits |= 1 << (n + 1) + n += 10 + } + if final { + bits |= 1 << n + } + n += 3 + h := make([]byte, 0, (n+7)/8+4) + for i := 0; i < n; i += 8 { + h = append(h, byte(bits>>i)) + } + h = binary.LittleEndian.AppendUint16(h, uint16(size)) + return binary.LittleEndian.AppendUint16(h, ^uint16(size)) +} diff --git a/internal/format/targz/filemap.go b/internal/format/targz/filemap.go index 9195b546..52717aa8 100644 --- a/internal/format/targz/filemap.go +++ b/internal/format/targz/filemap.go @@ -19,7 +19,11 @@ import ( // A compressed stream is content from its first byte to the trailer, filler // entry included. Where the filler's bytes land inside deflate's blocks is // not knowable without decoding them, so the map cannot part it from the -// files, and a cut there reads as a cut in the content (O336). +// files (O336). An archive a damage cuts is laid out with every part in blocks +// of its own and marked by markForACut (cut.go), unless it is too small for +// that frame. The frame costs a few bytes more than the smallest stored +// archive, and a compressed one is never smaller than that, so an archive +// written this way and cut has no room for a filler entry to stand in. const ( // gzipHead is the fixed part of a gzip header. This writer gives no name diff --git a/internal/format/targz/size.go b/internal/format/targz/size.go index d8c4c186..2f45c9b7 100644 --- a/internal/format/targz/size.go +++ b/internal/format/targz/size.go @@ -68,16 +68,28 @@ func tarLength(m memo) int64 { // splits a path across a 155 byte prefix and a 100 byte name, so a header // is the same 512 bytes at every depth it accepts - flat all the way to // the refusal, with no step. maxDepth is what keeps it on the near side. - total += int64(len(m.layout.Directories())) * tarBlock - for _, c := range m.children { - total += tarBlock + roundUpBlock(c.plan.Bytes) - } + total += dirsLength(m) + filesLength(m) if m.withFiller { total += tarBlock + roundUpBlock(m.fillerSize) } return total } +// dirsLength is how many bytes of the tar the directories take, a block each. +func dirsLength(m memo) int64 { + return int64(len(m.layout.Directories())) * tarBlock +} + +// filesLength is how many bytes of the tar the files inside take, a header and +// their data in whole blocks each. +func filesLength(m memo) int64 { + var total int64 + for _, c := range m.children { + total += tarBlock + roundUpBlock(c.plan.Bytes) + } + return total +} + // gzipFixed is the whole file except the comment. // // The framing comes from framing.go, which asks the gzip that is linked rather @@ -301,37 +313,14 @@ func build(ctx context.Context, w io.Writer, m memo) error { } tw := tar.NewWriter(zw) - // Directories first, outermost first, and only when asked for. They are - // not in m.children on purpose: a child's seed is FileSeed(seed, index) - // over a running index, so a directory in that list would shift the seed - // of every file after it and rewrite its contents. That is untouchable - // rule 2 - an edit in one place moving the bytes in another. - for _, dir := range m.layout.Directories() { - if err := ctx.Err(); err != nil { - return err - } - if err := writeDirectory(tw, dir, m.own); err != nil { - return core.Defect(fmt.Errorf("targz: the directory %q could not be named: %w", dir, err)) - } + if err := writeDirectories(ctx, tw, m); err != nil { + return err } - - for _, c := range m.children { - if err := ctx.Err(); err != nil { - return err - } - if err := writeEntry(ctx, tw, tarEntry{name: c.name, size: c.plan.Bytes, own: m.own}, - func(dst io.Writer) error { - return c.desc.Generator.Write(ctx, dst, c.plan) - }); err != nil { - return core.Defect(fmt.Errorf("targz: the %s file inside could not be written: %w", c.desc.ID, err)) - } + if err := writeFileEntries(ctx, tw, m); err != nil { + return err } - if m.withFiller { - if err := writeEntry(ctx, tw, tarEntry{name: fillerName, size: m.fillerSize, own: m.own}, - func(dst io.Writer) error { - return writeFiller(ctx, dst, m.seed, m.fillerSize) - }); err != nil { + if err := writeFillerEntry(ctx, tw, m, fillerName, m.fillerSize, m.seed); err != nil { return err } } @@ -353,6 +342,49 @@ func build(ctx context.Context, w io.Writer, m memo) error { return sm.Release() } +// writeDirectories names the directories the files sit in: first, outermost +// first, and only when asked for. They are not in m.children on purpose: a +// child's seed is FileSeed(seed, index) over a running index, so a directory +// in that list would shift the seed of every file after it and rewrite its +// contents. That is untouchable rule 2 - an edit in one place moving the bytes +// in another. +func writeDirectories(ctx context.Context, tw *tar.Writer, m memo) error { + for _, dir := range m.layout.Directories() { + if err := ctx.Err(); err != nil { + return err + } + if err := writeDirectory(tw, dir, m.own); err != nil { + return core.Defect(fmt.Errorf("targz: the directory %q could not be named: %w", dir, err)) + } + } + return nil +} + +// writeFileEntries writes an entry for every file inside, each generated as +// it goes in. +func writeFileEntries(ctx context.Context, tw *tar.Writer, m memo) error { + for _, c := range m.children { + if err := ctx.Err(); err != nil { + return err + } + if err := writeEntry(ctx, tw, tarEntry{name: c.name, size: c.plan.Bytes, own: m.own}, + func(dst io.Writer) error { + return c.desc.Generator.Write(ctx, dst, c.plan) + }); err != nil { + return core.Defect(fmt.Errorf("targz: the %s file inside could not be written: %w", c.desc.ID, err)) + } + } + return nil +} + +// writeFillerEntry writes an entry of padding, size bytes of random data drawn +// from seed. +func writeFillerEntry(ctx context.Context, tw *tar.Writer, m memo, name string, size int64, seed uint64) error { + return writeEntry(ctx, tw, tarEntry{name: name, size: size, own: m.own}, func(dst io.Writer) error { + return writeFiller(ctx, dst, seed, size) + }) +} + // writeEntry emits one tar entry. // // USTAR is asked for by name rather than left to the library. The library diff --git a/internal/format/targz/targz.go b/internal/format/targz/targz.go index ba751886..3273a97b 100644 --- a/internal/format/targz/targz.go +++ b/internal/format/targz/targz.go @@ -189,6 +189,11 @@ type memo struct { // thing minimumBytes met on the first try. withExtra bool extraLen int64 + // cutAt is where a damage will cut the archive (format.Request.CutAt), 0 + // when nothing does, and frame is how the archive is laid out for that cut + // (cut.go). The zero value of frame is the archive nothing cuts. + cutAt int64 + frame frame } func (g generator) Plan(r format.Request) (format.Plan, error) { @@ -224,7 +229,7 @@ func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan return format.Plan{}, err } - m := memo{seed: r.Seed, own: own, layout: layout, squeeze: squeeze} + m := memo{seed: r.Seed, own: own, layout: layout, squeeze: squeeze, cutAt: r.CutAt} if m.children, err = planChildren(ctx, r, groups, layout); err != nil { return format.Plan{}, err } @@ -248,6 +253,9 @@ func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan } else if err := pad(&m, &p, target, label, groups); err != nil { return format.Plan{}, err } + if err := m.settleForACut(&p, target, label); err != nil { + return format.Plan{}, err + } p.Memo = m p.Holds = format.OneAtATime(m.children, func(c child) format.Plan { return c.plan }) @@ -388,6 +396,12 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { if !ok { return core.Defect(fmt.Errorf("targz: the plan was not produced by this generator")) } + if err := m.squeezeForACut(ctx); err != nil { + return err + } + if m.frame.on { + return writeForACut(ctx, w, m) + } if m.squeeze.On() { return writeCompressed(ctx, w, m) } diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index e771583c..53b2ebc0 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -38,9 +38,8 @@ type cutCase struct { // The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2, 7 and 8). // A PDF of 1 MiB is nearly all padding after its pages, so 90% missing still // leaves every page. A PNG splits its padding around the picture -// for a cut, so the cut takes the share asked for. A small tar.gz keeps its -// padding BEFORE its archive, so a cut there takes all of it. A text is its -// words, so the cut falls in them. Two cuts in a row end the file where the second one does, and +// for a cut, so the cut takes the share asked for, and so does a tar.gz. A +// text is its words, so the cut falls in them. Two cuts in a row end the file where the second one does, and // the cut is said of that one. A damage that cuts nothing has no cut to report. // // Canary: the check below must refuse a cut that names the wrong role, and @@ -50,10 +49,10 @@ func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { // A GIF was this case until it split its padding for the cut. {name: "behind", format: "pdf", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, {name: "picture", format: "png", size: 1 << 20, missing: "90", in: "content", kept: "share"}, - // 4 KiB on disk is 40 KiB whole, and up to 64 KiB the padding is the - // comment in the gzip header, in front of the archive. An icon or a JPEG - // was this case until it split its padding for the cut. - {name: "front", format: "targz", size: 4 << 10, missing: "90", in: "padding", kept: "none", note: "before the content"}, + // 4 KiB on disk is 40 KiB whole. It was the case of a cut in the padding + // in front of the content, with an icon and a JPEG before it, until it + // split its padding for the cut. + {name: "front", format: "targz", size: 4 << 10, missing: "90", in: "content", kept: "share"}, {name: "text", format: "txt", size: 1000, missing: "50", in: "content", kept: "some"}, {name: "twice", format: "txt", size: 1000, missing: "50", in: "content", kept: "some", cuts: 2}, } diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 9468c502..35be63d4 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -32,12 +32,11 @@ type notYet struct { // that leaves is held to every size and share, and so is a new format unless // it is added here with its reason. var notYetCutAtTheShare = map[string]notYet{ - "avif": {paddingBehind, 64 << 10, 50}, - "jxl": {paddingBehind, 64 << 10, 50}, - "mp4": {paddingBehind, 64 << 10, 50}, - "webm": {paddingBehind, 64 << 10, 50}, - "pdf": {paddingBehind, 64 << 10, 50}, - "targz": {paddingBehind, 64 << 10, 50}, + "avif": {paddingBehind, 64 << 10, 50}, + "jxl": {paddingBehind, 64 << 10, 50}, + "mp4": {paddingBehind, 64 << 10, 50}, + "webm": {paddingBehind, 64 << 10, 50}, + "pdf": {paddingBehind, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the // picture is more than a small share, so a cut of 1% keeps all of it. "webp": {smallBehind, 4 << 10, 1}, @@ -52,7 +51,7 @@ var notYetCutAtTheShare = map[string]notYet{ // guard for it, and it stays red until the format splits its padding for // format.Request.CutAt. A format that leaves the ratchet leaves this list too, // so it cannot go back. -var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "targz", "webp", "tiff"} +var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. @@ -207,10 +206,11 @@ const cutInTheEnd = "the cut fell in the end" // directory, which the specification lets stand before the pictures - is not // here: it could put its end in front of the content instead. var endClosesTheFile = map[string]string{ - "zip": "APPNOTE 4.3.6 lays a ZIP out with the central directory and its end record after every entry", - "docx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", - "xlsx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", - "pptx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", + "zip": "APPNOTE 4.3.6 lays a ZIP out with the central directory and its end record after every entry", + "docx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", + "xlsx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", + "pptx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", + "targz": "POSIX tar puts the end of archive marker after every entry, and RFC 1952 closes a gzip member with its trailer", } // onlyTheEndBehind is whether nothing but the end stands behind the content of diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index 44e86edc..8b3c86ef 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -24,6 +24,7 @@ import re import struct import sys +import zlib STRUCTURE, CONTENT, PADDING, END = "structure", "content", "padding", "end" @@ -305,65 +306,214 @@ def gzip_header_length(data): return out, pos -def stored_blocks(data, pos): - """The stored deflate blocks from pos, as (file offset of the block, - offset of its data in the stream, length), and where they end.""" - blocks, t = [], 0 +# DEFLATE's lengths and distances (RFC 1951 section 3.2.5): the base of each +# length code from 257 and how many extra bits it reads, how many each distance +# code reads, and the order the lengths of the code length code come in. +LENGTH_BASE = (3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, + 131, 163, 195, 227, 258) +LENGTH_EXTRA = (0,) * 8 + (1,) * 4 + (2,) * 4 + (3,) * 4 + (4,) * 4 + (5,) * 4 + (0,) +DISTANCE_EXTRA = (0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13) +CODE_LENGTH_ORDER = (16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15) + + +class Bits: + """A deflate stream read bit by bit, the least significant bit of each + byte first (RFC 1951 section 3.1.1).""" + + def __init__(self, data, pos): + self.data, self.at = data, pos * 8 + + def peek(self, n): + i = self.at >> 3 + return (int.from_bytes(self.data[i:i + 4], "little") >> (self.at & 7)) & ((1 << n) - 1) + + def take(self, n): + value = self.peek(n) + self.at += n + return value + + +def huffman(lengths): + """The decoding table of the canonical Huffman code these lengths give + (RFC 1951 section 3.2.2): for every value of the next maxbits bits, the + symbol they open with and its length, or None for a value no code opens.""" + maxbits = max(lengths, default=0) + table = [(None, 0)] * (1 << maxbits) + count = [0] * (maxbits + 2) + for n in lengths: + count[n] += 1 + count[0] = 0 + code, first = 0, [0] * (maxbits + 1) + for bits in range(1, maxbits + 1): + code = (code + count[bits - 1]) << 1 + first[bits] = code + for symbol, n in enumerate(lengths): + if n: + reversed_code = int(format(first[n], f"0{n}b")[::-1], 2) + first[n] += 1 + for value in range(reversed_code, 1 << maxbits, 1 << n): + table[value] = (symbol, n) + return table, maxbits + + +def decode(bits, code): + table, maxbits = code + symbol, n = table[bits.peek(maxbits)] + if symbol is None: + raise NoMap(f"no Huffman code opens the bits at {bits.at}") + bits.at += n + return symbol + + +FIXED = (huffman([8] * 144 + [9] * 112 + [7] * 24 + [8] * 8), huffman([5] * 32)) + + +def dynamic_codes(bits): + """The two codes a block of dynamic Huffman codes opens with (RFC 1951 + section 3.2.7).""" + literals, distances, lengths = bits.take(5) + 257, bits.take(5) + 1, bits.take(4) + 4 + order = [0] * 19 + for i in range(lengths): + order[CODE_LENGTH_ORDER[i]] = bits.take(3) + code = huffman(order) + out = [] + while len(out) < literals + distances: + symbol = decode(bits, code) + if symbol < 16: + out.append(symbol) + elif symbol == 16 and out: + out += [out[-1]] * (3 + bits.take(2)) + elif symbol == 17: + out += [0] * (3 + bits.take(3)) + elif symbol == 18: + out += [0] * (11 + bits.take(7)) + else: + raise NoMap(f"a repeat with nothing to repeat at {bits.at}") + if len(out) != literals + distances: + raise NoMap(f"the code lengths run past the {literals + distances} the block declares") + return huffman(out[:literals]), huffman(out[literals:]) + + +def inflated_length(bits, literals, distances): + """How many bytes a block of Huffman codes carries, read up to its end of + block code. Nothing of the bytes is kept: a match's length counts, and its + distance is read past.""" + n = 0 + while True: + symbol = decode(bits, literals) + if symbol < 256: + n += 1 + continue + if symbol == 256: + return n + if symbol > 285: + raise NoMap(f"the length code {symbol} at {bits.at} is not one") + n += LENGTH_BASE[symbol - 257] + bits.take(LENGTH_EXTRA[symbol - 257]) + distance = decode(bits, distances) + if distance > 29: + raise NoMap(f"the distance code {distance} at {bits.at} is not one") + bits.at += DISTANCE_EXTRA[distance] + + +def deflate_blocks(data, pos): + """Every block of the deflate stream from pos (RFC 1951 section 3.2.3), as + (the bit its header starts at, the byte its data starts at - a stored + block's, None for the rest -, where its bytes start in what the stream + inflates to, how many there are, whether it is stored), and the byte after + the stream.""" + bits, t, blocks = Bits(data, pos), 0, [] while True: - header = data[pos] - if data[pos:pos + 2] == b"\x03\x00": - # An empty last block of fixed codes - BFINAL, BTYPE 01 and the - # end of block code - which is how compress/flate closes a stream. - return blocks, pos + 2 - if header & 0b110: - raise NoMap(f"the block at {pos} is not stored") - n = struct.unpack("> 3 + n, check = struct.unpack(" len(data) * 8: + raise NoMap("the deflate stream runs off the end of the file") + if final: + return blocks, (bits.at + 7) >> 3 + + +def stream_at(blocks, t): + """Where the stream holds the tar's byte t: at the header of the block it + opens, when that block starts on a byte, or in its place in a stored block. + None when it lies in a compressed block, which has no byte for it.""" + for start, data_at, first, n, stored in blocks: + if n == 0: + continue + if t == first: + return start >> 3 if start % 8 == 0 else None + if first < t < first + n: + return data_at + t - first if stored else None + return None -def layout_targz(data, settings): - """The gzip header, then - in a stored stream - the tar's directories, - files, filler entry and end of archive marker where their bytes land in the - file, and the end marker with the closing block and the trailer as the end. - A compressed stream is content from its first byte to the trailer.""" - out, pos = gzip_header_length(data) - if settings.get("compression", "none") != "none": - return out + [[CONTENT, pos, len(data) - 8 - pos], [END, len(data) - 8, 8]] - blocks, stop = stored_blocks(data, pos) - tar = b"".join(data[at + 5:at + 5 + n] for at, _, n in blocks) - - def where(t): - for at, start, n in blocks: - if t == start: - return at - if start < t < start + n: - return at + 5 + t - start - raise NoMap(f"the tar's byte {t} is in no block") - - t, files_seen, regions = 0, False, [] +def tar_parts(tar): + """The tar's parts, as (role, where it starts): directories in front of the + files as structure, the files as content, the entries tfg-padding.bin and + tfg-padding-end.bin as padding, and the end of archive marker as the end. + Neighbours of one role are one.""" + t, files_seen, parts = 0, False, [] + + def add(role, at): + if not parts or parts[-1][0] != role: + parts.append((role, at)) + while t + 512 <= len(tar): head = tar[t:t + 512] if head == bytes(512): - regions.append((END, t)) - break + add(END, t) + return parts name = head[:100].rstrip(b"\x00") size = int(head[124:136].rstrip(b"\x00 ") or b"0", 8) kind = head[156:157] - if name == b"tfg-padding.bin": + if name in (b"tfg-padding.bin", b"tfg-padding-end.bin"): role = PADDING elif kind == b"5" and not files_seen: role = STRUCTURE else: role, files_seen = CONTENT, True - regions.append((role, t)) + add(role, t) t += 512 + (size + 511) // 512 * 512 - for i, (role, start) in enumerate(regions): - end = where(regions[i + 1][1]) if i + 1 < len(regions) else len(data) - out.append([role, where(start), end - where(start)]) + raise NoMap("the tar has no end of archive marker") + + +def layout_targz(data, settings): + """The gzip header, then the tar's parts (tar_parts) from where the deflate + stream holds their first bytes, the end of archive marker to the end of the + file with whatever closes the stream and the trailer. + + A part starts inside a stored block, or at the header of the block it opens + when that block starts on a byte - as each part of an archive laid out for a + cut does, in blocks of its own. A part that starts inside a compressed block + has no byte to start at. A squeezed archive nothing cuts is one compressed + stream, so its parts do, and then the stream from its first byte to the + trailer is content.""" + out, pos = gzip_header_length(data) + blocks, stop = deflate_blocks(data, pos) + if stop != len(data) - 8: + raise NoMap(f"the deflate stream ends at {stop}, where the trailer starts at {len(data) - 8}") + squeezed = settings.get("compression", "none") != "none" + if not squeezed and not all(stored for _, _, _, n, stored in blocks if n): + raise NoMap("a block of a stored archive is compressed") + parts = tar_parts(zlib.decompress(data[pos:stop], -15)) + starts = [stream_at(blocks, t) for _, t in parts] + if None in starts: + if not squeezed: + raise NoMap(f"a part of the tar starts on no byte of the stream: {starts}") + return out + [[CONTENT, pos, len(data) - 8 - pos], [END, len(data) - 8, 8]] + for (role, _), start, end in zip(parts, starts, starts[1:] + [len(data)]): + out.append([role, start, end - start]) return out From 62248e0f60bfff3b89a7d86905d6863e18689c9a Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 13:39:51 +0100 Subject: [PATCH 3/5] A tar.gz with 16 KiB to 64 KiB of padding opens in busybox busybox 1.37, the gzip and tar of Alpine, reads a gzip header into a buffer of 16 KiB and refuses a longer extra field as corrupted data: 16 384 B opens and 16 385 B does not. The extra field carried up to 65 531 B of padding, so every tar.gz whose header carried 16 385 to 65 533 B - the default 64 KiB archive among them - could not be opened in an Alpine container. The field now carries at most 16 384 B, and padding past that goes to the filler entry behind the files, as it already did above the old limit. Those archives change bytes, which is a breaking change with its entry in the changelog and the one golden case it moves re-measured. Every other tar.gz writes the same bytes, compressed ones included. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +++++++ internal/format/targz/compress.go | 6 +++--- internal/format/targz/size.go | 4 ++-- internal/format/targz/targz.go | 19 ++++++++++++------- internal/guard/testdata/generator-golden.json | 11 +++++++++-- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 118f4522..440f06e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,13 @@ because it turns other people's test suites red. 5493 B, to 320x240 at 7732 B and to 640x480 at 18722 B, about a tenth above where it did. A file between the old step and the new one is not byte for byte what 0.5.0 wrote. +- **A tar.gz 16 KiB to 64 KiB larger than the files inside it now opens in + Alpine Linux.** busybox, the `gzip` and `tar` of Alpine and many small + container images, refused these archives as corrupted data - the default + 64 KiB one among them. The gzip header now carries at most 16 KiB of the + padding, which busybox reads, and the padding file inside the archive takes + the rest, as it did for larger archives. Those archives are not byte for + byte what 0.5.0 wrote. Every other tar.gz is unchanged. ### Added diff --git a/internal/format/targz/compress.go b/internal/format/targz/compress.go index 5c743543..7ae59af9 100644 --- a/internal/format/targz/compress.go +++ b/internal/format/targz/compress.go @@ -31,8 +31,8 @@ import ( // // Measured 2026-09-01 across levels 1, 6 and 9 and targets from 64 KB to // 10 MB: every one landed on the ordered size, and the remainder left for the -// extra field came out between 567 and 3759 bytes - far inside the 65 531 the -// field holds (O163). +// extra field came out between 567 and 3759 bytes - far inside the 16 384 the +// field holds (extraPaddingLimit). // // This costs passes, and the cost is real: one pass over a 10 MB archive is // 25 ms at level 1 and 140 ms at level 6, against 8 ms stored. It buys the one @@ -143,7 +143,7 @@ func nextFiller(target, got, filler int64) (next, extra int64, useExtra, done bo // compiler broke it. Probe: tools/probes/targzsettle. // // Undershooting is safe and overshooting is not: the extra field adds - // exactly what it is given, up to 65 531 B, so a round that lands under + // exactly what it is given, up to extraPaddingLimit, so a round that lands under // the target finishes on the next pass. A whole block is well inside // that. next := filler - (2 - deficit) - tarBlock diff --git a/internal/format/targz/size.go b/internal/format/targz/size.go index 2f45c9b7..7dc9a822 100644 --- a/internal/format/targz/size.go +++ b/internal/format/targz/size.go @@ -149,8 +149,8 @@ func maxCost() int64 { return commentPaddingLimit + extraPaddingLimit + 2 } // place works out where a given number of padding bytes goes. // // Two channels, and which one is used is decided by size rather than by -// preference. The extra field is where bulk goes: it holds 65 531 bytes, it -// is byte granular, and Go reads all of it - which the comment does not, and +// preference. The extra field is where bulk goes: it holds extraPaddingLimit +// bytes, it is byte granular, and Go reads all of it - which the comment does not, and // that is O163. But it cannot cost one byte, because its own length field // costs two, so a single byte of padding has nowhere to go there. // diff --git a/internal/format/targz/targz.go b/internal/format/targz/targz.go index 3273a97b..b3e35b83 100644 --- a/internal/format/targz/targz.go +++ b/internal/format/targz/targz.go @@ -44,20 +44,25 @@ const ( // extraPaddingLimit is how many bytes the gzip extra field may carry. // - // The field's own limit is 65 535, since XLEN is two bytes wide. This build - // stops at 65 531, which is the number measured across every reader on - // 2026-08-04 and re-measured against Go on 2026-09-01 - accepted to the - // last byte, beside a comment, with the cost exactly two bytes more than - // what is carried. + // The field's own limit is 65 535, since XLEN is two bytes wide, and 7-Zip, + // GNU gzip and tar, bsdtar, Python, node and Go take 65 531 to the last + // byte (measured 2026-08-04 and 2026-09-01). busybox does not. Its gzip + // reads the header into a buffer of 16 KiB and refuses a longer field as + // corrupted data - measured 2026-11-13 on busybox 1.37 in Alpine 3.22, by + // bisection to the byte: 16 384 opens and 16 385 does not + // (tools/probes/damagecheck/gzmembers.py, O339). Alpine is the base of a + // great many CI containers, so the limit is busybox's. Padding past it + // goes to a filler entry, as it always has above the limit. // - // That cost is what makes this the right channel rather than a bigger one: + // n bytes carried cost n+2 in the file, and that cost is what makes this + // the right channel rather than a bigger one: // it is byte granular. The comment was too, but it had to hold the label as // well, and a tar filler entry is aligned to 512 bytes so it delivers bulk // and never the last few. Capping the comment at what Go reads and leaving // the rest to the filler was measured and rejected: 831 sizes in the same // sweep stopped being reachable at all, because nothing could bridge the // gap under one block. - extraPaddingLimit = 65531 + extraPaddingLimit = 16384 // fillerName is the entry that carries padding the comment cannot hold. // Named plainly, because somebody opening the archive should see what it diff --git a/internal/guard/testdata/generator-golden.json b/internal/guard/testdata/generator-golden.json index 01fd83f6..1ec1b63f 100644 --- a/internal/guard/testdata/generator-golden.json +++ b/internal/guard/testdata/generator-golden.json @@ -233,8 +233,8 @@ }, "targz_with_three_pdfs": { "bytes": 65536, - "sha256": "e13105e912e3ebef8f5a41bb30be6160e6afd8611cec245aa781fc26d21d0d9c", - "measured_on": "2026-09-01" + "sha256": "328332cd43f3d1854fd78ac461ad74b2ba8454721f1206158463d084c30d03e1", + "measured_on": "2026-11-13" }, "tiff_100kib_sized_to_fit": { "bytes": 102400, @@ -876,6 +876,13 @@ "mp4_wide", "webm_wide" ] + }, + { + "on": "2026-11-13", + "why": "The gzip extra field of a tar.gz carries at most 16 384 B of padding, where it carried 65 531. busybox 1.37 (Alpine 3.22) refuses a longer field as corrupted data - bisected to the byte, 16 384 opens and 16 385 does not - so every tar.gz whose header carried 16 385 to 65 533 B of padding, the default 64 KiB one among them, could not be opened in an Alpine container (O339). Padding past the limit goes to the filler entry behind the files, as it did above the old limit. Only those archives change, and the archive of three PDFs at 64 KiB is one of them. This is a breaking change by the owner's decision of 2026-11-13, docs/TRUNCATE-2026-11-11.md section 11.3.", + "files": [ + "targz_with_three_pdfs" + ] } ] } From 237d11e32deb9b3a2d0dd70560049843604be9a5 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 13:42:13 +0100 Subject: [PATCH 4/5] A tar.gz laid out for a cut is held to every frame of its end marker The last bytes behind the files of an archive laid out for a cut land in the deflate frame around the end of archive marker: stored blocks for steps of five bytes, and one to three empty blocks of fixed codes for the rest. Which frame an archive takes follows its size, and the share and map guards ask for too few sizes to meet every one - a frame a byte short passed both. The new guard writes 64 consecutive sizes cut half way, holds each to its size and to compress/gzip and archive/tar, and asserts it met the marker in plain stored blocks and behind one, two, three, and one and then two empty blocks. Its canary is a stream built by hand that compress/flate inflates to the bytes it holds. Co-Authored-By: Claude Opus 5.5 --- internal/guard/mutationcoverage_test.go | 4 + internal/guard/targzcut_test.go | 176 ++++++++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 internal/guard/targzcut_test.go diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index 0c26b607..c301cbea 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -155,6 +155,10 @@ var provenByProbe = map[string]string{ "padding a byte late: red, and by marking the picture of a png laid out for a cut a byte late: red at 8 KiB and at 64 KiB.", "TestACutFileSaysWhereTheCutFellAndWhatIsLeft": "its own canary - a cut named in the wrong role, and content claimed gone that is there, must not pass - " + "and broken on 2026-11-12 by reporting every cut in the content: red for the picture and the icon.", + "TestATarGzLaidOutForACutComesToEverySizeAndReads": "its own canary - a stream built by hand, one and then two empty blocks of " + + "fixed codes, must inflate to its bytes and read as those runs - its reach check, every frame of the marker met - and red on " + + "2026-11-13 for a frame a byte short (the plan refuses at 65541 B) and for an empty block's BTYPE bit one place off " + + "(compress/flate refuses at 65537 B), where the first passed the share and the map guards.", "TestACutAtTheSmallestSizeSaysHowLongTheContentWas": "its own canaries - a cut that does not state the content's length, and one stating a " + "byte less than the map runs, must not pass - its reach check - the ZIP packages at their smallest have no padding in front and are cut " + "in the content - and red on 2026-11-13 before the Office packages marked their parts with a length without a padding part in front: " + diff --git a/internal/guard/targzcut_test.go b/internal/guard/targzcut_test.go new file mode 100644 index 00000000..503fd9a0 --- /dev/null +++ b/internal/guard/targzcut_test.go @@ -0,0 +1,176 @@ +package guard + +import ( + "archive/tar" + "bytes" + "compress/flate" + "compress/gzip" + "encoding/binary" + "errors" + "fmt" + "io" + "slices" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" +) + +// A tar.gz laid out for a cut comes to every size asked for and reads as the +// archive it is, whatever frame its end of archive marker takes. +// +// The last bytes behind its files land in the deflate frame around the marker +// (internal/format/targz/cut.go): stored blocks for steps of five bytes, and +// one to three empty blocks of fixed codes in front of a block of the marker +// for the rest. Which frame an archive takes follows its size, and no other +// guard asks for enough sizes in a row to meet every one - a frame that adds a +// byte too few passed the share and the map guards (2026-11-13). So this +// writes consecutive sizes, cut half way, holds each to its size and to +// archive/tar, and asserts that it met the marker in plain stored blocks and +// behind one, two, three, and one and then two empty blocks. +// +// Canary: the reader of the frame counts the empty blocks of a stream built by +// hand, which compress/flate inflates to the bytes it holds. +func TestATarGzLaidOutForACutComesToEverySizeAndReads(t *testing.T) { + desc, err := format.Get("targz") + if err != nil { + t.Fatal(err) + } + met := map[string]bool{} + for size := int64(64 << 10); size < 64<<10+64; size++ { + data := writeWhole(t, desc, size) + if int64(len(data)) != size { + t.Errorf("%d B: laid out for a cut, the archive came to %d B", size, len(data)) + continue + } + if err := readsEveryEntry(data); err != nil { + t.Errorf("%d B: %v", size, err) + continue + } + runs, err := emptyBlockRuns(data[gzipStreamStart(data):]) + if err != nil { + t.Errorf("%d B: %v", size, err) + continue + } + met[fmt.Sprint(runs)] = true + } + for _, frame := range []string{"[]", "[1]", "[2]", "[3]", "[1 2]"} { + if !met[frame] { + t.Errorf("no size met the marker framed behind empty blocks %s - met %v", frame, met) + } + } + + // Canary: two bytes stored, one empty block and a stored byte, two empty + // blocks and the last stored byte. + stream := []byte{0x00, 2, 0, 0xfd, 0xff, 'a', 'b', 0x02, 0x00, 1, 0, 0xfe, 0xff, 'c', 0x02, 0x08, 0x10, 1, 0, 0xfe, 0xff, 'd'} + inflated, err := io.ReadAll(flate.NewReader(bytes.NewReader(stream))) + if err != nil || string(inflated) != "abcd" { + t.Errorf("canary: the stream built by hand inflates to %q, %v", inflated, err) + } + if runs, err := emptyBlockRuns(stream); err != nil || !slices.Equal(runs, []int{1, 2}) { + t.Errorf("canary: the stream built by hand read as runs %v, %v", runs, err) + } +} + +// writeWhole writes a tar.gz of size bytes laid out for a cut half way, whole. +func writeWhole(t *testing.T, desc format.Descriptor, size int64) []byte { + t.Helper() + p, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, CutAt: size / 2}) + if err != nil { + t.Fatalf("%d B: planning: %v", size, err) + } + var out bytes.Buffer + if err := desc.Generator.Write(t.Context(), &out, p); err != nil { + t.Fatalf("%d B: writing: %v", size, err) + } + return out.Bytes() +} + +// readsEveryEntry reads every entry of a tar.gz to its end, through +// compress/gzip, which checks the trailer, and archive/tar. +func readsEveryEntry(data []byte) error { + zr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return err + } + tr := tar.NewReader(zr) + files := 0 + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + if _, err := io.Copy(io.Discard, tr); err != nil { + return fmt.Errorf("%s: %w", h.Name, err) + } + files++ + } + if _, err := io.Copy(io.Discard, zr); err != nil { + return fmt.Errorf("after the tar: %w", err) + } + if files < 2 { + return fmt.Errorf("%d entries, where a file and padding beside it were due", files) + } + return nil +} + +// gzipStreamStart is where the deflate stream of a gzip member starts: after +// the fixed header, the extra field and the comment (RFC 1952 section 2.3) - +// the only fields this tool writes. +func gzipStreamStart(data []byte) int { + at := 10 + if data[3]&4 != 0 { + at += 2 + int(binary.LittleEndian.Uint16(data[10:12])) + } + if data[3]&16 != 0 { + at += bytes.IndexByte(data[at:], 0) + 1 + } + return at +} + +// emptyBlockRuns reads a deflate stream of stored blocks and empty blocks of +// fixed codes, and says how many empty blocks stand together in front of each +// stored block that has any. Any other block is an error. +func emptyBlockRuns(s []byte) ([]int, error) { + var runs []int + run, bit := 0, 0 + for bit/8 < len(s) { + final, kind := bitsAt(s, bit, 1), bitsAt(s, bit+1, 2) + switch { + case kind == 0: + at := (bit + 3 + 7) / 8 + if at+4 > len(s) { + return nil, fmt.Errorf("the stored block at bit %d runs off the end", bit) + } + if run > 0 { + runs = append(runs, run) + run = 0 + } + bit = (at + 4 + int(binary.LittleEndian.Uint16(s[at:]))) * 8 + case kind == 1 && bitsAt(s, bit+3, 7) == 0: + run++ + bit += 10 + default: + return nil, fmt.Errorf("the block at bit %d is neither stored nor an empty block of fixed codes", bit) + } + if final == 1 { + return runs, nil + } + } + return nil, errors.New("the stream has no final block") +} + +// bitsAt is n bits of s from bit at, the least significant first. +func bitsAt(s []byte, at, n int) int { + v := 0 + for i := range n { + b := at + i + if b/8 < len(s) && s[b/8]>>(b%8)&1 == 1 { + v |= 1 << i + } + } + return v +} From 9eda6152ce1a50039f556df72c18d688313d2872 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 14:32:17 +0100 Subject: [PATCH 5/5] A tar.gz one byte over its cut frame's floor keeps the uncut layout One byte of spare goes to the comment, as a space behind the label. Without a label no part of the frame takes it, and planning ended in an internal defect - 9757 B with no label, cut half way. Such a size now keeps the layout of an archive nothing cuts, as sizes below the floor do, and a new guard writes the first sizes from the smallest with a label and without. The reader of maps stops on a deflate stream that ends inside a code, where the bits past the end read as zeros and a code of zeros decoded for ever, and on a tar header with a size below zero, which held the walk in place. The map guard hands it one of each. Co-Authored-By: Claude Opus 5.5 --- internal/format/targz/cut.go | 6 +- internal/guard/filemap_test.go | 68 +++++++++++++++++++ internal/guard/mutationcoverage_test.go | 3 + internal/guard/targzcut_test.go | 88 +++++++++++++++++++++---- internal/oracle/layout.py | 19 +++++- 5 files changed, 167 insertions(+), 17 deletions(-) diff --git a/internal/format/targz/cut.go b/internal/format/targz/cut.go index d0befaab..01ef82a0 100644 --- a/internal/format/targz/cut.go +++ b/internal/format/targz/cut.go @@ -162,7 +162,9 @@ func layOut(m memo, label string, content, target int64) (frame, bool, error) { head := gzipHead + commentCost(label) dirs := fewestBlocks(dirsLength(m)).length() spare := target - (head + dirs + content + tailFloor + gzipTrailer) - if spare < 0 { + if spare < 0 || spare == 1 && label == "" { + // One byte over the frame's floor can go only to the comment, as a + // space behind the label, so without a label it does not fit either. return frame{}, false, nil } want := min(max(format.PaddingInFront(head+dirs, content, m.cutAt, target), 0), spare) @@ -345,6 +347,8 @@ func writeForACut(ctx context.Context, w io.Writer, m memo) error { } var trailer [gzipTrailer]byte binary.LittleEndian.PutUint32(trailer[:4], sum.crc.Sum32()) + // ISIZE is the length modulo 2^32 (RFC 1952 section 2.3.1), so the + // conversion drops what it should. binary.LittleEndian.PutUint32(trailer[4:], uint32(sum.n)) _, err := w.Write(trailer[:]) return err diff --git a/internal/guard/filemap_test.go b/internal/guard/filemap_test.go index 6cb7840e..7d491e94 100644 --- a/internal/guard/filemap_test.go +++ b/internal/guard/filemap_test.go @@ -1,6 +1,8 @@ package guard import ( + "bytes" + "compress/gzip" "errors" "fmt" "io" @@ -191,6 +193,7 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { t.Errorf("canary: a file with no padding behind its content passed for the layout due at the first byte") } askTheLastEndCanaries(t, dir) + askTheReaderOfMapsToStop(t, dir) t.Logf("%d files, %d with padding, %d of them on both sides of the content, %d cut at the ends of a split", len(files), padded, split, ends) } @@ -293,6 +296,71 @@ func askTheLastEndCanaries(t *testing.T, dir string) { } } +// askTheReaderOfMapsToStop hands the reader of maps two tar.gz files it must +// refuse rather than read for ever: a squeezed one cut half way, whose last +// Huffman code runs off the end, where the bits read as zeros and a code of +// zeros for a byte decoded again and again, and one whose tar header gives a +// size below zero, which held the walk through the tar in place +// (docs/REVIEW-240-2026-11-13.md). A loop comes back as the guard's time +// running out, a refusal of another kind as the words it gives. +func askTheReaderOfMapsToStop(t *testing.T, dir string) { + t.Helper() + // Letters drawn unevenly and without a period, so the stream codes mostly + // single bytes in a block of its own codes. + var text bytes.Buffer + x := uint32(1) + for range 4000 { + x = x*1103515245 + 12345 + text.WriteByte("aaaaaaaabbbcd"[x>>16%13]) + } + squeezed := gzipped(t, text.Bytes(), gzip.BestCompression) + header := make([]byte, 512) + copy(header, "a.txt") + copy(header[124:], "-0000001000") + cases := []struct { + name, says string + data []byte + setting map[string]string + }{ + {"cut", "runs off the end of the file at bit", squeezed[:len(squeezed)/2], map[string]string{"compression": "best"}}, + {"below-zero", "gives a size below zero", gzipped(t, append(header, make([]byte, 1024)...), gzip.NoCompression), nil}, + } + reqs := make([]oracle.LayoutRequest, len(cases)) + for i, c := range cases { + path := filepath.Join(dir, "canary-"+c.name+".targz") + if err := os.WriteFile(path, c.data, 0o600); err != nil { + t.Fatal(err) + } + reqs[i] = oracle.LayoutRequest{Format: "targz", Path: path, Settings: c.setting} + } + got, _, err := oracle.LayoutBatch(t.Context(), reqs) + if err != nil { + t.Fatalf("canary: reading the maps back: %v", err) + } + for i, c := range cases { + if !strings.Contains(got[i].Error, c.says) { + t.Errorf("canary: the reader of maps answered a tar.gz %s with %q, where %q was due", c.name, got[i].Error, c.says) + } + } +} + +// gzipped is data in one gzip member, at level. +func gzipped(t *testing.T, data []byte, level int) []byte { + t.Helper() + var out bytes.Buffer + zw, err := gzip.NewWriterLevel(&out, level) + if err != nil { + t.Fatal(err) + } + if _, err := zw.Write(data); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + // writeMapped writes one healthy file of f through a map recorder, laid out // for a cut at cutAt when that is not zero. func writeMapped(t *testing.T, dir string, f format.Descriptor, name string, setting map[string]string, size, cutAt int64) mappedFile { diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index c301cbea..e86b7f0a 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -159,6 +159,9 @@ var provenByProbe = map[string]string{ "fixed codes, must inflate to its bytes and read as those runs - its reach check, every frame of the marker met - and red on " + "2026-11-13 for a frame a byte short (the plan refuses at 65541 B) and for an empty block's BTYPE bit one place off " + "(compress/flate refuses at 65537 B), where the first passed the share and the map guards.", + "TestATarGzCutNearItsSmallestComesToEverySize": "its reach check - the frame met, and without a label the size after its floor " + + "that keeps the layout of an archive nothing cuts, which with a label is due never - and red on 2026-11-13 without the " + + "case of one byte over the floor (an internal defect at 9757 B) and with that case taken for a label too (label true).", "TestACutAtTheSmallestSizeSaysHowLongTheContentWas": "its own canaries - a cut that does not state the content's length, and one stating a " + "byte less than the map runs, must not pass - its reach check - the ZIP packages at their smallest have no padding in front and are cut " + "in the content - and red on 2026-11-13 before the Office packages marked their parts with a length without a padding part in front: " + diff --git a/internal/guard/targzcut_test.go b/internal/guard/targzcut_test.go index 503fd9a0..b8e4fb6c 100644 --- a/internal/guard/targzcut_test.go +++ b/internal/guard/targzcut_test.go @@ -72,26 +72,91 @@ func TestATarGzLaidOutForACutComesToEverySizeAndReads(t *testing.T) { } } +// A tar.gz a damage cuts comes to every size from its smallest, with a label +// and without. +// +// The frame for a cut costs a few bytes more than the smallest archive, so +// near the smallest size an archive keeps the layout of one nothing cuts until +// the frame fits. One byte over the frame's floor goes to the comment, as a +// space behind the label. Without a label no field takes it, and that size +// stopped the run with an internal defect (docs/REVIEW-240-2026-11-13.md). So +// this writes the first sizes from the smallest, cut half way and not, holds +// each to its size and to compress/gzip and archive/tar, and asserts that it +// met the frame - and, only without a label, the size after its floor that +// keeps the layout of an archive nothing cuts. +func TestATarGzCutNearItsSmallestComesToEverySize(t *testing.T) { + desc, err := format.Get("targz") + if err != nil { + t.Fatal(err) + } + for _, label := range []bool{false, true} { + smallest := desc.SmallestAccepted(format.Request{Label: label}) + framed, plainAfter := false, false + for size := smallest; size < smallest+16; size++ { + req := format.Request{Bytes: size, Label: label, Seed: 7} + if _, err := desc.Generator.Plan(req); err != nil { + // The one size in the gap just above the smallest archive. + continue + } + plain := writeRequest(t, desc, req) + req.CutAt = size / 2 + cut := writeRequest(t, desc, req) + if int64(len(cut)) != size { + t.Errorf("label %v, %d B: a cut half way came to %d B", label, size, len(cut)) + continue + } + if _, err := entriesRead(cut); err != nil { + t.Errorf("label %v, %d B: %v", label, size, err) + } + laid := !bytes.Equal(cut, plain) + plainAfter = plainAfter || framed && !laid + framed = framed || laid + } + if !framed { + t.Errorf("label %v: none of the 16 sizes from %d B was laid out for a cut", label, smallest) + } + if plainAfter != !label { + t.Errorf("label %v: a size after the frame's floor kept the layout of an archive nothing cuts: %v - due only without a label", label, plainAfter) + } + } +} + // writeWhole writes a tar.gz of size bytes laid out for a cut half way, whole. func writeWhole(t *testing.T, desc format.Descriptor, size int64) []byte { t.Helper() - p, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, CutAt: size / 2}) + return writeRequest(t, desc, format.Request{Bytes: size, Label: true, Seed: 7, CutAt: size / 2}) +} + +// writeRequest plans and writes req, whole. +func writeRequest(t *testing.T, desc format.Descriptor, req format.Request) []byte { + t.Helper() + p, err := desc.Generator.Plan(req) if err != nil { - t.Fatalf("%d B: planning: %v", size, err) + t.Fatalf("%d B: planning: %v", req.Bytes, err) } var out bytes.Buffer if err := desc.Generator.Write(t.Context(), &out, p); err != nil { - t.Fatalf("%d B: writing: %v", size, err) + t.Fatalf("%d B: writing: %v", req.Bytes, err) } return out.Bytes() } -// readsEveryEntry reads every entry of a tar.gz to its end, through -// compress/gzip, which checks the trailer, and archive/tar. +// readsEveryEntry reads every entry of a tar.gz laid out for a cut to its end, +// and holds it to a file and padding beside it. func readsEveryEntry(data []byte) error { + files, err := entriesRead(data) + if err == nil && files < 2 { + err = fmt.Errorf("%d entries, where a file and padding beside it were due", files) + } + return err +} + +// entriesRead reads every entry of a tar.gz to its end, through +// compress/gzip, which checks the trailer, and archive/tar, and counts them. +func entriesRead(data []byte) (int, error) { zr, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { - return err + return 0, err } tr := tar.NewReader(zr) files := 0 @@ -101,20 +166,17 @@ func readsEveryEntry(data []byte) error { break } if err != nil { - return err + return files, err } if _, err := io.Copy(io.Discard, tr); err != nil { - return fmt.Errorf("%s: %w", h.Name, err) + return files, fmt.Errorf("%s: %w", h.Name, err) } files++ } if _, err := io.Copy(io.Discard, zr); err != nil { - return fmt.Errorf("after the tar: %w", err) - } - if files < 2 { - return fmt.Errorf("%d entries, where a file and padding beside it were due", files) + return files, fmt.Errorf("after the tar: %w", err) } - return nil + return files, nil } // gzipStreamStart is where the deflate stream of a gzip member starts: after diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index 8b3c86ef..9569770d 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -324,12 +324,22 @@ def __init__(self, data, pos): self.data, self.at = data, pos * 8 def peek(self, n): + """The next n bits, as zeros past the end of the data, so a short code + at the very end still decodes.""" i = self.at >> 3 return (int.from_bytes(self.data[i:i + 4], "little") >> (self.at & 7)) & ((1 << n) - 1) + def skip(self, n): + """Moves past n bits, and stops a stream that runs off the end of the + data: there peek reads zeros, and a code of zeros for a byte would + otherwise decode for ever.""" + self.at += n + if self.at > len(self.data) * 8: + raise NoMap(f"the deflate stream runs off the end of the file at bit {self.at - n}") + def take(self, n): value = self.peek(n) - self.at += n + self.skip(n) return value @@ -361,7 +371,7 @@ def decode(bits, code): symbol, n = table[bits.peek(maxbits)] if symbol is None: raise NoMap(f"no Huffman code opens the bits at {bits.at}") - bits.at += n + bits.skip(n) return symbol @@ -412,7 +422,7 @@ def inflated_length(bits, literals, distances): distance = decode(bits, distances) if distance > 29: raise NoMap(f"the distance code {distance} at {bits.at} is not one") - bits.at += DISTANCE_EXTRA[distance] + bits.skip(DISTANCE_EXTRA[distance]) def deflate_blocks(data, pos): @@ -476,6 +486,9 @@ def add(role, at): return parts name = head[:100].rstrip(b"\x00") size = int(head[124:136].rstrip(b"\x00 ") or b"0", 8) + if size < 0: + # int() takes a sign, and a size below zero would hold t in place. + raise NoMap(f"the tar header at {t} gives a size below zero") kind = head[156:157] if name in (b"tfg-padding.bin", b"tfg-padding-end.bin"): role = PADDING