From 645485d33c16100b5cfd3b1c56f77b2f0861af60 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 12 Nov 2026 11:39:43 +0100 Subject: [PATCH 1/2] A file cut short says where the cut fell and how much of its content is left In many formats a large file is mostly padding after its content, so truncate:missing=90 on a 1 MB PNG cut only padding and every viewer showed the whole picture. The manifest now says so per file: the truncate entry in damage carries cut - in (structure, content, padding or end), content_bytes and content_kept_bytes - and a cut in the padding gets the note cut_in_padding. The map behind it is marked by each generator while it writes (format.Mark), with offsets of its own: six formats choose their picture without coding it when planning, so a map in the plan would code every damaged file twice. A text marks nothing and is content from start to end. archive/zip buffers what it writes and finishes an entry only when the next one opens, so archive.StreamMap hands it its buffer and holds bytes back while the filler entry and the central directory are marked. A compressed tar.gz is content to its trailer, because where deflate puts the filler is not known without decoding it. The engine records the map only when a damage cuts the tail (damage.Chain.Cut), and forecasts the longest cut for the record's size. Guards: every format's map, at every setting that picks a shape and two sizes, is the one internal/oracle/layout.py reads back from its structure, and every region is marked before its first byte is written. The manifest says where the cut fell for a picture, an icon and a text, and a run that cuts is never told less than the record it writes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 + internal/damage/apply.go | 28 ++ internal/damage/truncate.go | 9 +- internal/engine/cut.go | 146 ++++++ internal/engine/damage.go | 14 +- internal/engine/engine.go | 26 +- internal/engine/entrynotes.go | 22 + internal/engine/parallel.go | 29 +- internal/engine/recordsize.go | 15 +- internal/format/archive/streammap.go | 228 +++++++++ internal/format/avif/avif.go | 5 + internal/format/avif/boxes.go | 39 ++ internal/format/bmp/bmp.go | 6 + internal/format/gif/gif.go | 28 ++ internal/format/ico/ico.go | 8 + internal/format/jpg/jpg.go | 12 + internal/format/jxl/jxl.go | 8 + internal/format/layout.go | 119 +++++ internal/format/mp4/mp4.go | 9 + internal/format/opc/opc.go | 21 +- internal/format/pdf/document.go | 11 +- internal/format/pdf/minimum.go | 2 +- internal/format/pdf/pdf.go | 15 +- internal/format/png/png.go | 28 ++ internal/format/targz/filemap.go | 65 +++ internal/format/targz/size.go | 20 +- internal/format/tiff/tiff.go | 8 + internal/format/wav/filemap.go | 19 + internal/format/wav/label.go | 28 ++ internal/format/wav/wav.go | 25 +- internal/format/webm/webm.go | 9 + internal/format/webp/webp.go | 8 + internal/format/zip/filler.go | 52 ++- internal/format/zip/zip.go | 37 +- internal/guard/codeshape_test.go | 6 +- internal/guard/cutmanifest_test.go | 186 ++++++++ internal/guard/filemap_test.go | 231 +++++++++ internal/guard/layers_test.go | 2 +- internal/guard/mutationcoverage_test.go | 6 + internal/gui/text/locale/registry/en.json | 4 +- internal/gui/text/locale/registry/pl.json | 2 +- internal/gui/text/locale/said/en.json | 10 + internal/gui/text/locale/said/pl.json | 2 + internal/manifest/manifest.go | 23 + internal/oracle/layout.py | 541 ++++++++++++++++++++++ internal/oracle/layoutbatch.go | 61 +++ internal/oracle/strictbatch.go | 56 ++- 47 files changed, 2113 insertions(+), 124 deletions(-) create mode 100644 internal/engine/cut.go create mode 100644 internal/engine/entrynotes.go create mode 100644 internal/format/archive/streammap.go create mode 100644 internal/format/avif/boxes.go create mode 100644 internal/format/layout.go create mode 100644 internal/format/targz/filemap.go create mode 100644 internal/format/wav/filemap.go create mode 100644 internal/format/wav/label.go create mode 100644 internal/guard/cutmanifest_test.go create mode 100644 internal/guard/filemap_test.go create mode 100644 internal/oracle/layout.py create mode 100644 internal/oracle/layoutbatch.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d30e7d6..9e6de164 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,14 @@ because it turns other people's test suites red. readers differ - a text cut short is still text - the manifest says the outcome is `unspecified`, offers both answers in the new `policy_options` field and says why. `tfg damage ` lists those formats. +- **A file cut short says where the cut fell and how much of its content is + left.** In many formats a large file is mostly padding after its content, + so `--damage truncate:missing=90` on a 1 MB PNG still leaves the whole + picture. The `truncate` entry in the manifest's `damage` now carries `cut`: + `in` (`content`, `padding`, `structure` or `end`), `content_bytes` and + `content_kept_bytes`. A cut in the padding also gets the note + `cut_in_padding`. In a compressed tar.gz a cut in the padding is reported as + a cut in the content. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/damage/apply.go b/internal/damage/apply.go index 03752ff7..4055b241 100644 --- a/internal/damage/apply.go +++ b/internal/damage/apply.go @@ -295,6 +295,34 @@ func (c Chain) Stretches() bool { return false } +// Cut is where the damage that cuts the tail cuts what the generator writes, +// for a file of size bytes on disk: its place in the list, and how many of the +// generator's bytes it keeps, which is the offset of the first byte it drops. +// +// False when no damage cuts the tail, and when one before it changes the +// length - the bytes it keeps would then not be the generator's, and an offset +// in them says nothing about the map of the file the generator wrote. The same +// condition as Stretches, for the same reason. +func (c Chain) Cut(size int64) (int, int64, bool) { + steps, err := c.steps() + if err != nil { + return 0, 0, false + } + outs, _, ok := lengths(steps, size) + if !ok { + return 0, 0, false + } + for i, s := range steps { + if s.d.CutsTail { + return i, outs[i], true + } + if s.d.Length != nil { + return 0, 0, false + } + } + return 0, 0, false +} + // Open builds the chain as one writer for a file of size bytes on disk, with // the first damage of the list outermost so that the bytes meet them in the // order written. seed is the file's own seed, and each damage is handed one diff --git a/internal/damage/truncate.go b/internal/damage/truncate.go index 02c7c5d7..a9dc1383 100644 --- a/internal/damage/truncate.go +++ b/internal/damage/truncate.go @@ -48,8 +48,15 @@ func init() { // anyone says it, and the sentence below names the percent. Min: fewestMissing, Max: mostMissing, Default: strconv.Itoa(defaultMissing), + // The last sentence is the limit a person cannot see from the + // outside: in 13 formats of 32 the padding stands after the + // content, so a large file cut at any share keeps its whole + // picture or sound (docs/TRUNCATE-2026-11-11.md). The manifest + // says per file where the cut fell. Detail: "How much of the whole file is gone, as a percentage. " + - "At least this much is cut - a few bytes more where the format cannot end exactly there.", + "At least this much is cut - a few bytes more where the format cannot end exactly there. " + + "A large file is often mostly padding after its content, so the content can stay whole - " + + "the manifest says what each cut left.", }, }, Floor: func(v Values) int64 { return smallestKept(missingShare(v)) }, diff --git a/internal/engine/cut.go b/internal/engine/cut.go new file mode 100644 index 00000000..debaafa1 --- /dev/null +++ b/internal/engine/cut.go @@ -0,0 +1,146 @@ +package engine + +import ( + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/manifest" +) + +// Where a cut fell in the file it was made in, for the manifest +// (damage[].cut, docs/TRUNCATE-2026-11-11.md section 6.9). +// +// Found out while writing rather than planned. The generator marks the map of +// the file as it writes it (format.Mark), and the map is all a cut needs: the +// role of the first byte it drops, and how much of the content lies before it. + +// wrote is what writing a file found out that its manifest entry carries. +type wrote struct { + sha string + // cut is where the damage that cut the file fell, nil for a file nothing + // cut. + cut *manifest.Cut +} + +// cutMap is the map a file's generator marks while it writes, and the offset +// the cut falls at. +type cutMap struct { + format.MapRecorder + at int64 +} + +// cutMapper is sink with a map recorder around it, and the recorder - or sink +// and nil when nothing cuts the file, which leaves every other file written +// exactly as before. +func cutMapper(f PlannedFile, sink io.Writer) (io.Writer, *cutMap) { + if !f.Damaged() { + return sink, nil + } + _, at, ok := f.Target.Damage.Cut(f.Plan.Bytes) + if !ok { + return sink, nil + } + m := &cutMap{MapRecorder: format.MapRecorder{W: sink}, at: at} + return m, m +} + +// cut is where the cut fell, nil for a file nothing cut. The map is not +// complete: the generator stopped at the cut, so a region it had not ended by +// then has no end it can be given. +func (m *cutMap) cut(f PlannedFile) *manifest.Cut { + if m == nil { + return nil + } + return cutIn(m.Map(wholeOf(f), false), m.at) +} + +// wholeOf is how long the file the generator was asked for is. A file cut +// short is planned through the damage, which asks the generator for more than +// reaches the disk. +func wholeOf(f PlannedFile) int64 { + if m, ok := f.Plan.Memo.(damagedMemo); ok { + return m.plan.Bytes + } + return f.Plan.Bytes +} + +// cutIn is where a cut at offset at fell in the map of a file. +// +// The role is that of the region the first dropped byte lies in, and a gap the +// map left is counted to the region before it - a map with gaps is a defect the +// guard TestEveryFormatsMapIsTheOneItsStructureReadsBack catches, and here it +// costs a word rather than a file. The content's length is stated only when +// every region of content has a known end: a generator stopped at the cut never +// marks what lies after it, and a total missing those would be a guess. +func cutIn(regions []format.Region, at int64) *manifest.Cut { + c := &manifest.Cut{In: string(format.RoleContent)} + var total int64 + known, any := true, false + for _, r := range regions { + if r.At <= at { + c.In = string(r.Role) + } + if r.Role != format.RoleContent { + continue + } + any = true + known = known && r.Length >= 0 + total += max(0, r.Length) + c.ContentKeptBytes += max(0, min(at, endOf(r, at))-r.At) + } + if known && any { + c.ContentBytes = &total + } + return c +} + +// endOf is where r ends, or at for a region whose end is not known: the last +// one marked, so the cut lies inside it. +func endOf(r format.Region, at int64) int64 { + if r.Length < 0 { + return at + } + return r.At + r.Length +} + +// withCutNote is notes with the note a cut that missed the content gets. +func (res *Result) withCutNote(notes []manifest.Note, c *manifest.Cut) []manifest.Note { + if code, said, ok := cutNote(c); ok { + return append(notes, res.note(code, said)) + } + return notes +} + +// cutNote is the note a cut that missed the content gets, or false. +// +// Only for a cut in the padding, the case that looks like a cut and is not one: +// a picture cut at 90% that every viewer shows whole. Said in two ways, because +// padding before the content takes all of it with any share. +func cutNote(c *manifest.Cut) (string, core.Said, bool) { + if c == nil || c.In != string(format.RolePadding) { + return "", core.Said{}, false + } + if c.ContentKeptBytes == 0 { + return cutInPadding, core.Says("engine.CutBeforeContent", + "The cut fell in the padding before the content, so none of the content is left."), true + } + return cutInPadding, core.Says("engine.CutAfterContent", + "The cut fell in the padding after the content, so the content is whole and only the end of the file is missing."), true +} + +// cutInPadding is the code of that note. A public name under untouchable rule +// 10 (owner's call, 2026-11-12). +const cutInPadding = "cut_in_padding" + +// forecastCut is the longest cut entry a file can be given, for the record a +// run is told before its first byte (PredictRecord). +// +// The real one is known only after writing, so this is the most it can come +// to: the padding, whose note is the longer of the two, and both numbers as +// long as the whole file, which neither can pass. More than the run writes, +// which is the safe side for the free space it is held against. +func forecastCut(f PlannedFile) *manifest.Cut { + whole := wholeOf(f) + return &manifest.Cut{In: string(format.RolePadding), ContentBytes: &whole, ContentKeptBytes: whole} +} diff --git a/internal/engine/damage.go b/internal/engine/damage.go index 153caba0..e2f0f76f 100644 --- a/internal/engine/damage.go +++ b/internal/engine/damage.go @@ -184,13 +184,21 @@ func damagedExpectation(f PlannedFile) manifest.Expected { // // Nil for a file nothing damaged, so the key is absent rather than empty and // every manifest written before this existed is unchanged. -func damageFor(f PlannedFile) []manifest.Damage { +// +// cut is where the damage that cut the file fell, and goes to that damage's +// entry alone. +func damageFor(f PlannedFile, cut *manifest.Cut) []manifest.Damage { if !f.Damaged() { return nil } + at, _, cuts := f.Target.Damage.Cut(f.Plan.Bytes) out := make([]manifest.Damage, 0, len(f.Target.Damage)) - for _, s := range f.Target.Damage { - out = append(out, manifest.Damage{Type: s.ID, Settings: s.Resolved()}) + for i, s := range f.Target.Damage { + d := manifest.Damage{Type: s.ID, Settings: s.Resolved()} + if cuts && i == at { + d.Cut = cut + } + out = append(out, d) } return out } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 812dc1e1..eb9d2185 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -530,7 +530,7 @@ func Run(ctx context.Context, files []PlannedFile, opt Options) (*Result, error) // separate one that can drift away from it. res.Started = true for _, f := range files { - m.Add(res.entryFor(f, "", false, nil)) + m.Add(res.entryFor(f, wrote{}, false, nil)) } m.Run.Complete = true return res, nil @@ -610,7 +610,7 @@ func Run(ctx context.Context, files []PlannedFile, opt Options) (*Result, error) for i, r := range written { switch { case r.ok: - m.Add(res.entryFor(files[i], r.sha, true, nil)) + m.Add(res.entryFor(files[i], r.wrote, true, nil)) case r.err == nil: // Never started. A cancelled run leaves these behind and they are // neither a success nor a failure, so they get no entry - which is @@ -625,7 +625,7 @@ func Run(ctx context.Context, files []PlannedFile, opt Options) (*Result, error) // One file failing does not end the run. Nine thousand good // files are worth keeping, and the entry says what went wrong. res.Failures++ - m.Add(res.entryFor(files[i], "", false, r.err)) + m.Add(res.entryFor(files[i], wrote{}, false, r.err)) } } @@ -723,7 +723,7 @@ func releaseRunLock(path string, own os.FileInfo) { _ = core.RemoveOwn(path, own) } -func (res *Result) entryFor(f PlannedFile, sha string, materialized bool, failure error) manifest.File { +func (res *Result) entryFor(f PlannedFile, w wrote, materialized bool, failure error) manifest.File { var notes []manifest.Note for _, n := range f.Plan.Notes { notes = append(notes, res.note(n.Code, n.Detail)) @@ -745,6 +745,7 @@ func (res *Result) entryFor(f PlannedFile, sha string, materialized bool, failur notes = append(notes, res.note("label_not_guaranteed", core.Says("engine.LabelNotGuaranteed", "The label was written before the file was damaged, so the damage may have overwritten it."))) } + notes = res.withCutNote(notes, w.cut) e := manifest.File{ ID: f.ID, @@ -755,13 +756,13 @@ func (res *Result) entryFor(f PlannedFile, sha string, materialized bool, failur Bytes: f.Plan.Bytes, Format: f.Desc.ID, Fidelity: string(f.Desc.Fidelity), - Hashes: manifest.Hashes{SHA256: sha}, + Hashes: manifest.Hashes{SHA256: w.sha}, Seed: core.SeedLabel(f.Seed), Generator: manifest.GeneratorRef{Name: f.Desc.ID, Version: f.Desc.GeneratorVersion}, Determinism: string(f.Plan.Determinism), Properties: props, LabelEmbedded: label, - Damage: damageFor(f), + Damage: damageFor(f, w.cut), Notes: notes, Expected: expectationFor(f), Group: f.Target.Group, @@ -779,19 +780,6 @@ func (res *Result) entryFor(f PlannedFile, sha string, materialized bool, failur return e } -// note is one note for the manifest, remembered as a sentence once for each -// different thing it says. -func (res *Result) note(code string, said core.Said) manifest.Note { - detail := said.String() - if res.NotesSaid == nil { - res.NotesSaid = map[string]core.Said{} - } - if _, have := res.NotesSaid[detail]; !have { - res.NotesSaid[detail] = said - } - return manifest.Note{Code: code, Detail: detail} -} - func expectationFor(f PlannedFile) manifest.Expected { switch f.Target.Expected { case "": diff --git a/internal/engine/entrynotes.go b/internal/engine/entrynotes.go new file mode 100644 index 00000000..990533ca --- /dev/null +++ b/internal/engine/entrynotes.go @@ -0,0 +1,22 @@ +package engine + +import ( + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/manifest" +) + +// The notes a manifest entry carries, each sentence remembered once for the +// window to say in its own language. + +// note is one note for the manifest, remembered as a sentence once for each +// different thing it says. +func (res *Result) note(code string, said core.Said) manifest.Note { + detail := said.String() + if res.NotesSaid == nil { + res.NotesSaid = map[string]core.Said{} + } + if _, have := res.NotesSaid[detail]; !have { + res.NotesSaid[detail] = said + } + return manifest.Note{Code: code, Detail: detail} +} diff --git a/internal/engine/parallel.go b/internal/engine/parallel.go index 015782d9..d62ce88c 100644 --- a/internal/engine/parallel.go +++ b/internal/engine/parallel.go @@ -90,7 +90,7 @@ func widthFor(n int) int { // started, which a cancelled run leaves behind and which is neither a success // nor a failure - it gets no manifest entry, exactly as it gets none today. type fileResult struct { - sha string + wrote ok bool err error } @@ -179,9 +179,9 @@ func (w *writers) drain(ctx context.Context) { if !w.queue.take(ctx, i) { return } - sum, err := writeOne(ctx, w.files[i], w.outDir, &p) + got, err := writeOne(ctx, w.files[i], w.outDir, &p) w.queue.give(i) - w.out[i] = fileResult{sha: sum, ok: err == nil, err: err} + w.out[i] = fileResult{wrote: got, ok: err == nil, err: err} p.finished(w.files[i].Plan, err == nil) } } @@ -504,7 +504,7 @@ func (p *fileProgress) finished(plan format.Plan, ok bool) { // // It lives here rather than in engine.go because it is what a worker does, and // this file is meant to be the whole answer to "what runs beside what". -func writeOne(ctx context.Context, f PlannedFile, outDir string, p *fileProgress) (string, error) { +func writeOne(ctx context.Context, f PlannedFile, outDir string, p *fileProgress) (wrote, error) { final := filepath.Join(outDir, f.Name) // The process id is in the name because two runs writing into one directory // used to meet on it. Measured on 2026-08-03: two runs of the same target @@ -549,9 +549,9 @@ func writeOne(ctx context.Context, f PlannedFile, outDir string, p *fileProgress if errors.Is(err, fs.ErrExist) { // In our own words. The fault is the one preflight names, arriving // later than preflight can look. - return "", &CollisionError{Path: tmp} + return wrote{}, &CollisionError{Path: tmp} } - return "", err + return wrote{}, err } h := sha256.New() @@ -577,10 +577,13 @@ func writeOne(ctx context.Context, f PlannedFile, outDir string, p *fileProgress if err != nil { _ = fh.Close() _ = os.Remove(tmp) - return "", err + return wrote{}, err } sink, streams = damaged, opened } + // Outermost, so the generator marks its map on the writer it is handed + // and the offsets are those of the file it writes, before the cut. + sink, mapped := cutMapper(f, sink) writeErr := writeDamaged(ctx, f, sink, streams) if writeErr == nil { @@ -599,11 +602,11 @@ func writeOne(ctx context.Context, f PlannedFile, outDir string, p *fileProgress if writeErr != nil { _ = os.Remove(tmp) - return "", writeErr + return wrote{}, writeErr } if closeErr != nil { _ = os.Remove(tmp) - return "", closeErr + return wrote{}, closeErr } // The size is the promise. A generator that missed it by a byte is a bug @@ -611,7 +614,7 @@ func writeOne(ctx context.Context, f PlannedFile, outDir string, p *fileProgress // never reaches its final name. if counter.n != f.Plan.Bytes { _ = os.Remove(tmp) - return "", core.Defect(fmt.Errorf("generator for %s produced %d B where the plan said %d B", + return wrote{}, core.Defect(fmt.Errorf("generator for %s produced %d B where the plan said %d B", f.Desc.ID, counter.n, f.Plan.Bytes)) } @@ -624,11 +627,11 @@ func writeOne(ctx context.Context, f PlannedFile, outDir string, p *fileProgress if err := core.Publish(tmp, final); err != nil { _ = os.Remove(tmp) if errors.Is(err, fs.ErrExist) { - return "", &CollisionError{Path: final} + return wrote{}, &CollisionError{Path: final} } - return "", err + return wrote{}, err } - return hex.EncodeToString(h.Sum(nil)), nil + return wrote{sha: hex.EncodeToString(h.Sum(nil)), cut: mapped.cut(f)}, nil } type countingWriter struct { diff --git a/internal/engine/recordsize.go b/internal/engine/recordsize.go index ac504751..0d419a1a 100644 --- a/internal/engine/recordsize.go +++ b/internal/engine/recordsize.go @@ -36,6 +36,17 @@ func (r RecordSize) Bytes() int64 { return r.Manifest + r.Instructions } // records is this long, so the document is as long as the one the run writes. var hashPlaceholder = strings.Repeat("0", 64) +// forecastOf is what writing f is counted to find out before it is written: +// a hash as long as every hash, and the longest cut a file cut short can be +// given (forecastCut). +func forecastOf(f PlannedFile) wrote { + w := wrote{sha: hashPlaceholder} + if _, _, ok := f.Target.Damage.Cut(f.Plan.Bytes); ok && f.Damaged() { + w.cut = forecastCut(f) + } + return w +} + // PredictRecord is the record a run of these files writes when every file is // produced. A file that fails gains a note in its entry, so a run with // failures writes a little more than this says. @@ -60,7 +71,7 @@ func PredictRecord(ctx context.Context, files []PlannedFile, opt Options) (Recor if err := askedToStop(ctx, i); err != nil { return RecordSize{}, err } - e := scratch.entryFor(f, hashPlaceholder, true, nil) + e := scratch.entryFor(f, forecastOf(f), true, nil) explains = explains || e.Purpose != "" counted.Add(e) } @@ -84,7 +95,7 @@ func predictInstructions(ctx context.Context, files []PlannedFile, opt Options, if err := askedToStop(ctx, i); err != nil { return 0, err } - whole.Add(scratch.entryFor(f, hashPlaceholder, true, nil)) + whole.Add(scratch.entryFor(f, forecastOf(f), true, nil)) } return int64(len(whole.Instructions(filepath.Base(ManifestPath(opt))))), nil } diff --git a/internal/format/archive/streammap.go b/internal/format/archive/streammap.go new file mode 100644 index 00000000..8a547d06 --- /dev/null +++ b/internal/format/archive/streammap.go @@ -0,0 +1,228 @@ +package archive + +import ( + "bufio" + "bytes" + "encoding/binary" + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// StreamMap is what a writer of an archive needs to mark the map of the file +// (format.Mark) when a library writes it: where the stream stands at any +// moment, and a way to hold bytes back until the marks that describe them are +// made. ZIP and the office packages hand it to archive/zip as Buffer, and +// TAR.GZ hands it to compress/gzip itself. +// +// archive/zip buffers what it writes, finishes an entry only when the next +// one opens - the end of a squeezed entry and its data descriptor go out +// then - and flushes its buffer itself when it closes. So the offset of the +// filler entry is known only after its header was written, and the central +// directory's only after the archive closed (docs/TRUNCATE-2026-11-11.md +// section 6.9). Buffer is handed to zip.NewWriter, which writes through it +// rather than through one of its own (bufio.NewWriterSize returns a Writer it +// is handed), so Pos is exact. Hold keeps what follows from reaching the +// generator's writer until Release, and the mark is made in between. +// +// Holding happens only when somebody keeps a map, so an archive nothing cuts +// is written exactly as before. +type StreamMap struct { + // Buffer is what zip.NewWriter is to be given. + Buffer *bufio.Writer + w io.Writer + out int64 + held []byte + holding bool +} + +// NewStreamMap is a StreamMap writing to w, the writer the generator was handed. +func NewStreamMap(w io.Writer) *StreamMap { + m := &StreamMap{w: w} + // The size archive/zip gives its own buffer. + m.Buffer = bufio.NewWriterSize(m, 4096) + return m +} + +func (m *StreamMap) Write(p []byte) (int, error) { + if m.holding { + m.held = append(m.held, p...) + return len(p), nil + } + n, err := m.w.Write(p) + m.out += int64(n) + return n, err +} + +// Pos is the offset the next byte archive/zip writes will have. +func (m *StreamMap) Pos() int64 { + return m.out + int64(len(m.held)) + int64(m.Buffer.Buffered()) +} + +func (m *StreamMap) mapped() bool { + _, ok := m.w.(format.Mapper) + return ok +} + +// Hold keeps every byte written from here on until Release, when somebody +// keeps a map. What was buffered before goes out first. +func (m *StreamMap) Hold() error { + if !m.mapped() { + return nil + } + if err := m.Buffer.Flush(); err != nil { + return err + } + m.holding = true + return nil +} + +// Release sends on what was held, after the marks that describe it. +func (m *StreamMap) Release() error { + if !m.holding { + return nil + } + if err := m.Buffer.Flush(); err != nil { + return err + } + m.holding = false + held := m.held + m.held = nil + _, err := m.Write(held) + return err +} + +// Close closes c - the archive's writer - with what it writes as it closes +// held back until it is marked (MarkTail). +func (m *StreamMap) Close(c io.Closer) error { + if err := m.Hold(); err != nil { + return err + } + if err := c.Close(); err != nil { + return err + } + if err := m.MarkTail(); err != nil { + return err + } + return m.Release() +} + +// MarkLast marks the last n bytes held as role: what a library wrote last, +// whose start is known only once it is all there - the trailer of a gzip +// stream, which its writer adds as it closes. +func (m *StreamMap) MarkLast(role format.Role, n int64) error { + if !m.holding { + return nil + } + if err := m.Buffer.Flush(); err != nil { + return err + } + format.Mark(m.w, role, m.out+int64(len(m.held))-n, n) + return nil +} + +// MarkEntry marks the region of role that starts with the local header of the +// entry named name, which archive/zip has just written: the last thing held. +// The header is found by its own fields rather than worked out, because +// archive/zip adds an extra field of its own to a header that names a time. +func (m *StreamMap) MarkEntry(role format.Role, name string) error { + if !m.holding { + return nil + } + if err := m.Buffer.Flush(); err != nil { + return err + } + at, ok := lastLocalHeader(m.held, name) + if !ok { + // A header that cannot be found leaves the region unmarked, which + // the guard of the map reads back as a defect. + return nil + } + format.Mark(m.w, role, m.out+int64(at), -1) + return nil +} + +// MarkTail marks what archive/zip wrote while it closed, all of it held: the +// central directory and the end of central directory record as the end, and +// the archive comment after them as padding. +func (m *StreamMap) MarkTail() error { + if !m.holding { + return nil + } + if err := m.Buffer.Flush(); err != nil { + return err + } + eocd, comment, ok := endRecord(m.held) + if !ok { + return nil + } + directory := int64(binary.LittleEndian.Uint32(m.held[eocd+16:])) + if directory == 0xFFFFFFFF { + var found bool + if directory, found = zip64Directory(m.held, m.out, eocd); !found { + return nil + } + } + end := m.out + int64(eocd) + 22 + format.Mark(m.w, format.RoleEnd, directory, end-directory) + format.Mark(m.w, format.RolePadding, end, int64(comment)) + return nil +} + +// lastLocalHeader is where the last local header of name starts in b. Not +// always the last thing in b: a locked entry writes its salt straight after +// its header. +func lastLocalHeader(b []byte, name string) (int, bool) { + for at := len(b) - 30 - len(name); at >= 0; at-- { + if !bytes.HasPrefix(b[at:], []byte("PK\x03\x04")) { + continue + } + n := int(binary.LittleEndian.Uint16(b[at+26:])) + extra := int(binary.LittleEndian.Uint16(b[at+28:])) + if n == len(name) && at+30+n+extra <= len(b) && string(b[at+30:at+30+n]) == name { + return at, true + } + } + return 0, false +} + +// endRecord is where the end of central directory record starts in b, when it +// and its comment end b, and the comment's length. +func endRecord(b []byte) (int, int, bool) { + for at := len(b) - 22; at >= 0; at-- { + if !bytes.HasPrefix(b[at:], []byte("PK\x05\x06")) { + continue + } + comment := int(binary.LittleEndian.Uint16(b[at+20:])) + if at+22+comment == len(b) { + return at, comment, true + } + } + return 0, 0, false +} + +// zip64Directory is where the central directory starts, from the ZIP64 end +// record the locator before the end record points at. out is the offset of +// b's first byte in the file. +func zip64Directory(b []byte, out int64, eocd int) (int64, bool) { + locator := eocd - 20 + if locator < 0 || !bytes.HasPrefix(b[locator:], []byte("PK\x06\x07")) { + return 0, false + } + record := le64(b[locator+8:]) - out + if record < 0 || record+56 > int64(len(b)) { + return 0, false + } + return le64(b[record+48:]), true +} + +// le64 is the eight byte little-endian number at the start of b. Put +// together by hand, because the call that reads one has the name of the bulk +// random draw a guard keeps in one place. +func le64(b []byte) int64 { + var v int64 + for i := 7; i >= 0; i-- { + v = v<<8 | int64(b[i]) + } + return v +} diff --git a/internal/format/avif/avif.go b/internal/format/avif/avif.go index 36ad4e8f..814131b1 100644 --- a/internal/format/avif/avif.go +++ b/internal/format/avif/avif.go @@ -491,6 +491,11 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { m.width, m.height, len(coded), m.total)) } + // The map: the coder's boxes, its media data the content, and the free + // boxes after them that make up the size. + markBoxes(w, coded) + format.Mark(w, format.RolePadding, int64(len(coded)), pad) + if _, err := w.Write(coded); err != nil { return err } diff --git a/internal/format/avif/boxes.go b/internal/format/avif/boxes.go new file mode 100644 index 00000000..dcec1a31 --- /dev/null +++ b/internal/format/avif/boxes.go @@ -0,0 +1,39 @@ +package avif + +import ( + "encoding/binary" + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/isobmff" +) + +// markBoxes marks the map of the top level boxes the coder handed back in b, +// written from the first byte of the file w writes: the media data as the +// content, every other box as structure. +// +// The boxes are the coder's, so they are read rather than assumed: by the four +// byte length, the eight byte one a length of one announces, and the rest of b +// a length of nought means. +func markBoxes(w io.Writer, b []byte) { + for pos := int64(0); pos+isobmff.BoxHeader <= int64(len(b)); { + size := int64(binary.BigEndian.Uint32(b[pos:])) + switch { + case size == 0: + size = int64(len(b)) - pos + case size == 1 && pos+2*isobmff.BoxHeader <= int64(len(b)): + size = int64(binary.BigEndian.Uint32(b[pos+8:]))<<32 | int64(binary.BigEndian.Uint32(b[pos+12:])) + } + if size < isobmff.BoxHeader { + // Not a box this could walk past. What is left stays unmarked, + // and the guard of the map says so. + return + } + role := format.RoleStructure + if string(b[pos+4:pos+isobmff.BoxHeader]) == "mdat" { + role = format.RoleContent + } + format.Mark(w, role, pos, size) + pos += size + } +} diff --git a/internal/format/bmp/bmp.go b/internal/format/bmp/bmp.go index 485fae13..6ab69d10 100644 --- a/internal/format/bmp/bmp.go +++ b/internal/format/bmp/bmp.go @@ -289,6 +289,12 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } + // The map, all of it known from the plan: the two headers, the gap that + // makes up the size, and the pixels. The gap stands BEFORE the pixels. + format.Mark(w, format.RoleStructure, 0, headers) + format.Mark(w, format.RolePadding, headers, m.gap) + format.Mark(w, format.RoleContent, headers+m.gap, pixelBytes(m.width, m.height)) + if err := writeHeaders(w, m, p.Bytes); err != nil { return err } diff --git a/internal/format/gif/gif.go b/internal/format/gif/gif.go index ca4f099d..fe992a2f 100644 --- a/internal/format/gif/gif.go +++ b/internal/format/gif/gif.go @@ -325,6 +325,16 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } + // The map, before the bytes wherever planning measured the picture. The + // palette counts to the content, with the frames it colours: the encoder + // writes it, and a reader needs it to hand back a single pixel. + format.Mark(w, format.RoleStructure, 0, screenHeader) + if m.bodyKnown { + markAfterThePicture(w, m, p.Bytes, true) + } else { + format.Mark(w, format.RoleContent, screenHeader, -1) + } + // The trailer is always the final byte, so hold it back and let the rest // stream out. Nothing else is buffered. holder := &tailHolder{w: w, keep: trailerSize} @@ -341,6 +351,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { // beats writing a file of the wrong length. return core.Defect(fmt.Errorf("gif: %w - ladderCeiling is wrong", err)) } + markAfterThePicture(w, m, p.Bytes, false) } else if holder.written != m.body { return core.Defect(fmt.Errorf("gif: the picture encoded to %d B where planning said %d B", holder.written, m.body)) } @@ -358,6 +369,23 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return err } +// screenHeader is the signature and the logical screen descriptor, which open +// every GIF and are always this long. +const screenHeader = 6 + 7 + +// markAfterThePicture marks the picture when its length is known, and what +// follows it: the comment that makes up the size, if there is one, and the +// trailer. +func markAfterThePicture(w io.Writer, m memo, total int64, withPicture bool) { + if withPicture { + format.Mark(w, format.RoleContent, screenHeader, m.body-screenHeader) + } + if m.comment { + format.Mark(w, format.RolePadding, m.body, total-trailerSize-m.body) + } + format.Mark(w, format.RoleEnd, total-trailerSize, trailerSize) +} + // writeComment emits the comment extension without ever holding it in memory. func writeComment(ctx context.Context, w io.Writer, seed uint64, blocks, payload int64) error { if _, err := w.Write([]byte{0x21, 0xFE}); err != nil { diff --git a/internal/format/ico/ico.go b/internal/format/ico/ico.go index 3fcca382..65f66054 100644 --- a/internal/format/ico/ico.go +++ b/internal/format/ico/ico.go @@ -250,6 +250,14 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } + // The map, all of it known from the plan: the directory, the gap that + // makes up the size, and the picture. The gap stands BEFORE the picture, + // so a cut takes the whole picture whatever share is asked for + // (docs/TRUNCATE-2026-11-11.md section 2). + format.Mark(w, format.RoleStructure, 0, preamble) + format.Mark(w, format.RolePadding, preamble, m.gap) + format.Mark(w, format.RoleContent, preamble+m.gap, int64(len(m.image))) + if err := writeDirectory(w, m); err != nil { return err } diff --git a/internal/format/jpg/jpg.go b/internal/format/jpg/jpg.go index c38e69f5..fda4cd08 100644 --- a/internal/format/jpg/jpg.go +++ b/internal/format/jpg/jpg.go @@ -34,6 +34,9 @@ const ( // bytes and is what the padding is inserted after. soiSize = 2 + // eoiSize is the end of image marker, which the encoder writes last. + eoiSize = 2 + // comOverhead is what one comment segment costs beyond its payload: the // marker FF FE and a two byte length. The length field counts itself, // which is why the payload stops four bytes short of 65535 rather than @@ -285,6 +288,15 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } + // The map, all of it known from the plan: the start marker, the comments + // that make up the size, the picture - its tables, frame and scan as one - + // and the end marker. + pad := p.Bytes - m.body + format.Mark(w, format.RoleStructure, 0, soiSize) + format.Mark(w, format.RolePadding, soiSize, pad) + format.Mark(w, format.RoleContent, soiSize+pad, m.body-soiSize-eoiSize) + format.Mark(w, format.RoleEnd, p.Bytes-eoiSize, eoiSize) + // The start marker goes out first, then the padding, then the rest of the // encoded picture. Nothing is buffered: the encoder streams straight // through once its own two byte marker has been dropped. diff --git a/internal/format/jxl/jxl.go b/internal/format/jxl/jxl.go index d2a7d919..6d7ca328 100644 --- a/internal/format/jxl/jxl.go +++ b/internal/format/jxl/jxl.go @@ -513,6 +513,14 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { m.width, m.height, len(coded), containerOverhead, m.total)) } + // The map, all of it known once the picture is coded: the signature and + // file type boxes, the box of the code stream, and the free boxes that + // make up the size. + head := int64(len(signature) + len(fileType)) + format.Mark(w, format.RoleStructure, 0, head) + format.Mark(w, format.RoleContent, head, boxHeader+int64(len(coded))) + format.Mark(w, format.RolePadding, head+boxHeader+int64(len(coded)), pad) + if _, err := w.Write(signature[:]); err != nil { return err } diff --git a/internal/format/layout.go b/internal/format/layout.go new file mode 100644 index 00000000..3f08f9cb --- /dev/null +++ b/internal/format/layout.go @@ -0,0 +1,119 @@ +package format + +import "io" + +// The map of a file: which of its bytes are the content a reader hands back, +// and which are the structure around it, the padding that makes up the size +// asked for, and the end of the file. +// +// A damage aims at a ROLE of the bytes and is handed an offset. truncate cut +// "the end of the content" and hit the padding instead, because the padding +// is most of a large file and sits after the content in 13 formats of 32 - +// missing=90 and missing=1 gave the same file with the whole picture in it +// (docs/TRUNCATE-2026-11-11.md sections 1 and 2). The map is what lets the +// manifest say where a cut fell, and the next damages aim where they mean to. +// +// Marked by the generator WHILE it writes, never worked out in the plan. Six +// formats choose their picture without coding it when planning, so the length +// of their content is not known before the writer has it, and a map in the plan +// would code every damaged file of them twice (section 6.2). The offsets are +// the generator's own rather than counted by the writer it is handed: a PNG's +// end is held back in a buffer while the picture streams out, and archive/zip +// buffers everything it writes, so a count taken on the engine's side sees the +// bytes later than they were written (section 6.9). Whether the offsets are +// true is measured by a second reading of the file's structure, the guard +// TestEveryFormatsMapIsTheOneItsStructureReadsBack. + +// Role is what a region of a file is for. The words reach the manifest +// (damage[].cut.in), so each is a public name under untouchable rule 10. +type Role string + +const ( + // RoleStructure is the bytes a reader needs to find the content: a + // signature, headers, a table of what is where. + RoleStructure Role = "structure" + // RoleContent is what a reader hands back: the pixels, the samples, the + // pages, the records, the entries of an archive. + RoleContent Role = "content" + // RolePadding is the bytes that make up the size asked for, which no + // reader hands back. + RolePadding Role = "padding" + // RoleEnd is what closes the file: an end marker, a trailer, a central + // directory. + RoleEnd Role = "end" +) + +// Region is one stretch of the bytes a generator writes. +type Region struct { + Role Role + // At is the offset of the region's first byte in the file the generator + // writes - before any damage, so a cut file has the map of its whole. + At int64 + // Length is how many bytes the region takes, or -1 when the generator does + // not know yet. The region then runs to the next one marked, or to the end + // of the file. + Length int64 +} + +// Mapper is a writer that keeps the map of the file written through it. The +// engine hands one to a generator when a damage needs the map, and a guard +// hands one when it reads the map back. +type Mapper interface { + MarkRegion(Region) +} + +// Mark says that a region of role starts at offset at of the file written to +// w, length bytes long or -1 when that is not known yet. Marked before the +// region's first byte where its extent is known, and as soon as it is known +// otherwise, because a generator stops at a cut and a mark after it never +// arrives. +// +// w is the writer the generator was handed. A file written into another one - +// a picture inside an archive - writes to a writer of the archive's own, so +// its marks go nowhere instead of into the archive's map. Nothing at all when +// nobody keeps a map, which is every file not damaged. +func Mark(w io.Writer, role Role, at, length int64) { + if m, ok := w.(Mapper); ok { + m.MarkRegion(Region{Role: role, At: at, Length: length}) + } +} + +// MapRecorder passes every byte to W and keeps the regions marked on it. +type MapRecorder struct { + W io.Writer + regions []Region +} + +func (m *MapRecorder) Write(p []byte) (int, error) { return m.W.Write(p) } + +// MarkRegion keeps r. +func (m *MapRecorder) MarkRegion(r Region) { m.regions = append(m.regions, r) } + +// Map is the regions marked, with each length not stated filled in from the +// next region, or from whole for the last one. +// +// whole is how long the file written would have been, whether or not it was +// all written. A file whose generator marked nothing is content from start to +// end, which is what a text is: its words are the file. A region whose end is +// not known - the last one marked, of a file cut before the next mark - keeps +// -1 when complete is false, so a cut inside it says "not known" rather than +// stretching the region to the end of a whole that was never written. +func (m *MapRecorder) Map(whole int64, complete bool) []Region { + if len(m.regions) == 0 { + return []Region{{Role: RoleContent, At: 0, Length: whole}} + } + out := make([]Region, len(m.regions)) + copy(out, m.regions) + for i := range out { + if out[i].Length >= 0 { + continue + } + switch { + case i+1 < len(out): + out[i].Length = out[i+1].At - out[i].At + case complete: + out[i].Length = whole - out[i].At + } + } + return out +} diff --git a/internal/format/mp4/mp4.go b/internal/format/mp4/mp4.go index 2862b580..3aa217fd 100644 --- a/internal/format/mp4/mp4.go +++ b/internal/format/mp4/mp4.go @@ -138,8 +138,17 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { } defer pics.Close() + // The map: the file type and the movie box, then the media data box with + // the pictures, then the free boxes that make up the size. The movie box + // is the same length whatever the pictures come to, so it is marked before + // a byte of it goes out, and the rest once the first pass has the length. + mdatAt := int64(len(l.ftyp)) + l.moovLen() + format.Mark(w, format.RoleStructure, 0, mdatAt) + o := &out{w: w, buf: make([]byte, 0, 64<<10)} content := l.writeMovie(o, z) + format.Mark(w, format.RoleContent, mdatAt, l.front()-mdatAt+content) + format.Mark(w, format.RolePadding, l.front()+content, p.Bytes-l.front()-content) o.mediaHeader(l.bigData, content) if err := writeSamples(ctx, o, l, pics); err != nil { return err diff --git a/internal/format/opc/opc.go b/internal/format/opc/opc.go index 1b7b7ab0..b4b17c26 100644 --- a/internal/format/opc/opc.go +++ b/internal/format/opc/opc.go @@ -35,6 +35,9 @@ import ( "strings" "time" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/archive" + "github.com/donislawdev/TestingFilesGenerator/internal/core" ) @@ -234,7 +237,12 @@ func Write(ctx context.Context, w io.Writer, p Package) error { } func write(ctx context.Context, w io.Writer, p Package, withPadding bool) error { - zw := zip.NewWriter(w) + zm := archive.NewStreamMap(w) + zw := zip.NewWriter(zm.Buffer) + // The map: every part is content, from the first byte. Then the filler + // part, the central directory and the comment, each marked as + // archive/zip writes it (archive.StreamMap). + format.Mark(w, format.RoleContent, 0, -1) if p.Comment > 0 { if p.Comment > CommentLimit { return core.Defect(fmt.Errorf("opc: a comment of %d B is past the %d B a reader accepts", p.Comment, CommentLimit)) @@ -276,10 +284,19 @@ func write(ctx context.Context, w io.Writer, p Package, withPadding bool) error head.CRC32 = sum head.UncompressedSize64 = uint64(p.Filler) head.CompressedSize64 = uint64(p.Filler) + if err := zm.Hold(); err != nil { + return err + } entry, err := zw.CreateRaw(head) if err != nil { return err } + if err := zm.MarkEntry(format.RolePadding, FillerName); err != nil { + return err + } + if err := zm.Release(); err != nil { + return err + } if withPadding { if err := writeFiller(ctx, entry, p.Seed, p.Filler); err != nil { return err @@ -287,7 +304,7 @@ func write(ctx context.Context, w io.Writer, p Package, withPadding bool) error } } - return zw.Close() + return zm.Close(zw) } // writePart writes one part with its sizes and checksum in the header. diff --git a/internal/format/pdf/document.go b/internal/format/pdf/document.go index 67682bf9..6c58a75f 100644 --- a/internal/format/pdf/document.go +++ b/internal/format/pdf/document.go @@ -8,12 +8,18 @@ import ( "strings" ) +// places is where the parts of a document start, for its map: the first +// object, and the cross reference table the trailer follows. +type places struct { + objects, xref int64 +} + // document builds everything up to the trailer, and the closing lines. // // The two are returned apart because the padding goes between them. The // closing lines carry the offset of the cross reference table, which sits in // the first part and therefore does not move. -func document(m memo) (prefix, suffix []byte) { +func document(m memo) (prefix, suffix []byte, at places) { var body bytes.Buffer // The version is the one asked for and nothing else follows from it: the // file uses nothing newer than 1.4, so every version offered here is true @@ -63,6 +69,7 @@ func document(m memo) (prefix, suffix []byte) { } xrefPos := body.Len() + at = places{objects: int64(offsets[0]), xref: int64(xrefPos)} fmt.Fprintf(&body, "xref\n0 %d\n", len(objects)+1) body.WriteString("0000000000 65535 f \n") for _, off := range offsets { @@ -71,5 +78,5 @@ func document(m memo) (prefix, suffix []byte) { fmt.Fprintf(&body, "trailer\n<>\n", len(objects)+1) suffix = []byte(fmt.Sprintf("startxref\n%d\n%%%%EOF\n", xrefPos)) - return body.Bytes(), suffix + return body.Bytes(), suffix, at } diff --git a/internal/format/pdf/minimum.go b/internal/format/pdf/minimum.go index 42ebb1d4..e9967762 100644 --- a/internal/format/pdf/minimum.go +++ b/internal/format/pdf/minimum.go @@ -63,6 +63,6 @@ func minimumBytes() int64 { if err != nil { panic("pdf: the default settings are refused: " + err.Error()) } - prefix, suffix := document(memo{opts: opts}) + prefix, suffix, _ := document(memo{opts: opts}) return int64(len(prefix) + len(suffix)) } diff --git a/internal/format/pdf/pdf.go b/internal/format/pdf/pdf.go index c73dd125..d1e132b2 100644 --- a/internal/format/pdf/pdf.go +++ b/internal/format/pdf/pdf.go @@ -51,6 +51,8 @@ type memo struct { // not depend on the padding, because the offset it carries points at the // cross reference table, which sits before the padding. suffix []byte + // at is where the objects and the cross reference table start. + at places padLen int64 } @@ -66,7 +68,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } m := memo{opts: opts, seed: r.Seed, label: label} - m.prefix, m.suffix = document(m) + m.prefix, m.suffix, m.at = document(m) // One number answers both questions: how much padding this file needs, and // what is too small to make. Every line of body text is the same width @@ -168,6 +170,17 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } + // The map, all of it known from the plan: the header lines, the objects - + // the pages and what they draw with - the cross reference table and the + // trailer, the comment that makes up the size, and startxref with the + // closing marker. The padding stands between two parts of the end. + head, end := int64(len(m.prefix)), int64(len(m.prefix))+m.padLen + format.Mark(w, format.RoleStructure, 0, m.at.objects) + format.Mark(w, format.RoleContent, m.at.objects, m.at.xref-m.at.objects) + format.Mark(w, format.RoleEnd, m.at.xref, head-m.at.xref) + format.Mark(w, format.RolePadding, head, m.padLen) + format.Mark(w, format.RoleEnd, end, int64(len(m.suffix))) + if _, err := w.Write(m.prefix); err != nil { return err } diff --git a/internal/format/png/png.go b/internal/format/png/png.go index 09fdb501..357e7987 100644 --- a/internal/format/png/png.go +++ b/internal/format/png/png.go @@ -32,6 +32,10 @@ const ( // always empty. iendSize = 12 + // headSize is the signature and the header chunk, which are always first + // and always this long: eight bytes, and a chunk of thirteen. + headSize = 8 + chunkOverhead + 13 + // paddingChunk is a private ancillary chunk. // // The naming rules carry meaning. Lower case first letter means ancillary, @@ -276,6 +280,15 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } + // The map, before the bytes wherever planning measured the picture. The + // picture's chunks are the content, their headers and checksums with them. + format.Mark(w, format.RoleStructure, 0, headSize) + if m.bodyKnown { + markAfterThePicture(w, m, true) + } else { + format.Mark(w, format.RoleContent, headSize, -1) + } + // The closing chunk is always the final twelve bytes, so hold those back // and let everything before them stream straight out. Nothing is buffered // beyond that, which matters because a picture can be very large. @@ -298,6 +311,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { "png: the picture encoded to %d B, which leaves %d B of padding for a %d B file - ladderCeiling is wrong", m.body, m.padData, p.Bytes)) } + markAfterThePicture(w, m, false) } else if holder.written != m.body { return core.Defect(fmt.Errorf("png: the picture encoded to %d B where planning said %d B", holder.written, m.body)) } @@ -315,6 +329,20 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return err } +// markAfterThePicture marks the picture when its length is known, and what +// follows it: the padding chunk, if there is one, and the closing chunk. +func markAfterThePicture(w io.Writer, m memo, withPicture bool) { + if withPicture { + format.Mark(w, format.RoleContent, headSize, m.body-headSize) + } + at := m.body + if m.withPad { + format.Mark(w, format.RolePadding, at, chunkOverhead+m.padData) + at += chunkOverhead + m.padData + } + format.Mark(w, format.RoleEnd, at, iendSize) +} + // ladderCeiling is the most the largest rung has ever been seen to encode to, // with room to spare. It is only ever used to decide that a request is far // enough above the ladder that no search is needed, so being generous costs a diff --git a/internal/format/targz/filemap.go b/internal/format/targz/filemap.go new file mode 100644 index 00000000..9195b546 --- /dev/null +++ b/internal/format/targz/filemap.go @@ -0,0 +1,65 @@ +package targz + +import ( + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// The map of a tar.gz (format.Mark, docs/TRUNCATE-2026-11-11.md section 6.9). +// +// The gzip header first: its fixed part, the extra field that carries padding, +// and the comment that carries the label. Then the stream. A stored stream - +// the default - carries the tar unchanged in blocks of storeBlock bytes, each +// behind a header of its own, so where each entry lands in the file is +// arithmetic: the directories, the files as the content, the filler entry as +// padding, and the end of archive marker with the stream's closing block and +// the gzip trailer as the end. +// +// A compressed stream is content from its first byte to the trailer, filler +// entry included. Where the filler's bytes land inside deflate's blocks is +// not knowable without decoding them, so the map cannot part it from the +// files, and a cut there reads as a cut in the content (O336). + +const ( + // gzipHead is the fixed part of a gzip header. This writer gives no name + // and asks for no header checksum, so nothing else is in it. + gzipHead = 10 + // gzipTrailer is the checksum and the length that close a gzip stream. + gzipTrailer = 8 +) + +// markArchive marks the map of the archive m describes, before its first byte. +func markArchive(w io.Writer, m memo) { + extra, comment := extraCost(m.withExtra, m.extraLen), commentCost(m.comment) + head := gzipHead + extra + comment + format.Mark(w, format.RoleStructure, 0, gzipHead) + format.Mark(w, format.RolePadding, gzipHead, extra) + format.Mark(w, format.RoleStructure, gzipHead+extra, comment) + if m.squeeze.On() { + format.Mark(w, format.RoleContent, head, -1) + return + } + + f, _ := measuredFraming() + // at is where the tar's byte t lands in the file. A region that starts + // at the start of a block takes the block's header with it. + at := func(t int64) int64 { + k, r := t/storeBlock, t%storeBlock + out := head + k*(storeBlock+f.perBlock) + if r != 0 { + out += f.perBlock + r + } + return out + } + dirs := int64(len(m.layout.Directories())) * tarBlock + files := dirs + for _, c := range m.children { + files += tarBlock + roundUpBlock(c.plan.Bytes) + } + end := tarLength(m) - 2*tarBlock + format.Mark(w, format.RoleStructure, head, at(dirs)-head) + format.Mark(w, format.RoleContent, at(dirs), at(files)-at(dirs)) + format.Mark(w, format.RolePadding, at(files), at(end)-at(files)) + format.Mark(w, format.RoleEnd, at(end), -1) +} diff --git a/internal/format/targz/size.go b/internal/format/targz/size.go index 472a88fc..d8c4c186 100644 --- a/internal/format/targz/size.go +++ b/internal/format/targz/size.go @@ -283,10 +283,15 @@ func solveFiller(base, target int64, label string) (size int64, header headerPad // the stream measured before the comment can be sized, so it is two passes and // a separate decision. func build(ctx context.Context, w io.Writer, m memo) error { - zw, err := gzip.NewWriterLevel(w, m.squeeze.Level) + // The map goes before the first byte, and the trailer of a compressed + // stream - whose start only its last byte tells - is held back while it is + // marked (filemap.go). + sm := archive.NewStreamMap(w) + zw, err := gzip.NewWriterLevel(sm, m.squeeze.Level) if err != nil { return core.Defect(fmt.Errorf("targz: the archive could not be started: %w", err)) } + markArchive(w, m) zw.Comment = m.comment // The extra field is where padding rides. It precedes everything, like // the comment, so how long it is has to be settled before the first byte @@ -334,7 +339,18 @@ func build(ctx context.Context, w io.Writer, m memo) error { if err := tw.Close(); err != nil { return err } - return zw.Close() + if m.squeeze.On() { + if err := sm.Hold(); err != nil { + return err + } + } + if err := zw.Close(); err != nil { + return err + } + if err := sm.MarkLast(format.RoleEnd, gzipTrailer); err != nil { + return err + } + return sm.Release() } // writeEntry emits one tar entry. diff --git a/internal/format/tiff/tiff.go b/internal/format/tiff/tiff.go index b6a1eaf5..e0e25fcc 100644 --- a/internal/format/tiff/tiff.go +++ b/internal/format/tiff/tiff.go @@ -328,6 +328,14 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } + // The map, all of it known from the plan: the header, the gap that makes + // up the size, the pixels, and the directory last, which says where they + // are - the end, as a ZIP's central directory is. + format.Mark(w, format.RoleStructure, 0, header) + format.Mark(w, format.RolePadding, header, m.gap) + format.Mark(w, format.RoleContent, pixelsAt(m), pixelBytes(m.width, m.height)) + format.Mark(w, format.RoleEnd, directoryAt(m), p.Bytes-directoryAt(m)) + if err := writeHeader(w, m); err != nil { return err } diff --git a/internal/format/wav/filemap.go b/internal/format/wav/filemap.go new file mode 100644 index 00000000..ae547df9 --- /dev/null +++ b/internal/format/wav/filemap.go @@ -0,0 +1,19 @@ +package wav + +import ( + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// mark marks the map of the file (format.Mark), all of it known from the +// plan: the headers and the label, the data chunk with its samples, and the +// padding chunk after it. +func (m memo) mark(w io.Writer) { + data := int64(baseSize) + int64(len(m.info)) - chunkHeader + format.Mark(w, format.RoleStructure, 0, data) + format.Mark(w, format.RoleContent, data, chunkHeader+m.dataLen) + if m.withPad { + format.Mark(w, format.RolePadding, data+chunkHeader+m.dataLen, chunkHeader+m.junkLen) + } +} diff --git a/internal/format/wav/label.go b/internal/format/wav/label.go new file mode 100644 index 00000000..a6673373 --- /dev/null +++ b/internal/format/wav/label.go @@ -0,0 +1,28 @@ +package wav + +import "encoding/binary" + +// The label rides in a LIST INFO chunk between the format and the data. + +// infoChunk builds a LIST INFO chunk carrying the label in the software +// field, which is where a player looks for it. +func infoChunk(label string) []byte { + text := append([]byte(label), 0) + if len(text)%2 == 1 { + text = append(text, 0) + } + inner := make([]byte, 0, 4+chunkHeader+len(text)) + inner = append(inner, []byte("INFO")...) + inner = append(inner, []byte("ISFT")...) + var n [4]byte + binary.LittleEndian.PutUint32(n[:], uint32(len(text))) + inner = append(inner, n[:]...) + inner = append(inner, text...) + + out := make([]byte, 0, chunkHeader+len(inner)) + out = append(out, []byte("LIST")...) + binary.LittleEndian.PutUint32(n[:], uint32(len(inner))) + out = append(out, n[:]...) + out = append(out, inner...) + return out +} diff --git a/internal/format/wav/wav.go b/internal/format/wav/wav.go index 79f5dcb0..fb6e2aec 100644 --- a/internal/format/wav/wav.go +++ b/internal/format/wav/wav.go @@ -293,6 +293,8 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { } total := p.Bytes + m.mark(w) + // The RIFF size field counts everything after itself. if err := writeAll(w, []byte("RIFF")); err != nil { return err @@ -425,29 +427,6 @@ func appendSample(buf []byte, v float64, width int) []byte { } } -// infoChunk builds a LIST INFO chunk carrying the label in the software -// field, which is where a player looks for it. -func infoChunk(label string) []byte { - text := append([]byte(label), 0) - if len(text)%2 == 1 { - text = append(text, 0) - } - inner := make([]byte, 0, 4+chunkHeader+len(text)) - inner = append(inner, []byte("INFO")...) - inner = append(inner, []byte("ISFT")...) - var n [4]byte - binary.LittleEndian.PutUint32(n[:], uint32(len(text))) - inner = append(inner, n[:]...) - inner = append(inner, text...) - - out := make([]byte, 0, chunkHeader+len(inner)) - out = append(out, []byte("LIST")...) - binary.LittleEndian.PutUint32(n[:], uint32(len(inner))) - out = append(out, n[:]...) - out = append(out, inner...) - return out -} - func fill(ctx context.Context, w io.Writer, seed uint64, n int64) error { if n <= 0 { return nil diff --git a/internal/format/webm/webm.go b/internal/format/webm/webm.go index 0d3f42f2..636674a6 100644 --- a/internal/format/webm/webm.go +++ b/internal/format/webm/webm.go @@ -148,6 +148,13 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { body := uint64(p.Bytes - l.outside()) cuesAt := body - l.cuesLen() + // The map: the headers, the seek head, the info and the tracks, then the + // clusters - coded as they go, so their end is marked when they reach it - + // then the Void that makes up the size and the Cues at the end. + seg := l.outside() + format.Mark(w, format.RoleStructure, 0, seg+int64(l.front())) + format.Mark(w, format.RoleContent, seg+int64(l.front()), -1) + out := &sticky{w: w} out.write(l.head) out.write(idBytes(idSegment)) @@ -166,6 +173,8 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return core.Defect(fmt.Errorf("webm: the clusters came to %d B and the Cues are to start at %d B, which leaves no room for the Void every one of these carries", end, cuesAt)) } + format.Mark(w, format.RolePadding, seg+int64(end), void) + format.Mark(w, format.RoleEnd, seg+int64(cuesAt), int64(l.cuesLen())) if err := writeVoid(ctx, w, void, f.Choice.Seed); err != nil { return err } diff --git a/internal/format/webp/webp.go b/internal/format/webp/webp.go index 91259b14..0c6ee09f 100644 --- a/internal/format/webp/webp.go +++ b/internal/format/webp/webp.go @@ -330,6 +330,14 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { body += chunkHeader + 4 + m.chunk } + // The map, all of it known from the plan: the container's header, the + // picture's chunk, and what makes up the size after it - the private + // chunk and the bytes past the container's end. + picture := chunkHeader + 4 + stream + stream%2 + format.Mark(w, format.RoleStructure, 0, riffHeader) + format.Mark(w, format.RoleContent, riffHeader, picture) + format.Mark(w, format.RolePadding, riffHeader+picture, p.Bytes-riffHeader-picture) + if err := writeContainer(w, body, stream); err != nil { return err } diff --git a/internal/format/zip/filler.go b/internal/format/zip/filler.go index 7e1bde98..7bdddcad 100644 --- a/internal/format/zip/filler.go +++ b/internal/format/zip/filler.go @@ -6,6 +6,7 @@ import ( "io" "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/archive" "github.com/donislawdev/TestingFilesGenerator/internal/core" ) @@ -27,12 +28,18 @@ import ( // header - Flush does not. Measured while writing this: reading the total // before the filler's header went down gave nought, and the archive overshot // by exactly the compressed length of its entries. -func writeFillerEntry(ctx context.Context, zw *stdzip.Writer, m memo, withContents bool, freed func() int64) error { +// +// zm marks where the entry starts, which is known only once its header is +// written: the entry before it ends as this one opens. +func writeFillerEntry(ctx context.Context, zw *stdzip.Writer, zm *archive.StreamMap, m memo, withContents bool, freed func() int64) error { if !m.withFiller { return nil } + if err := zm.Hold(); err != nil { + return err + } if m.squeeze.On() { - return writeCompressedFiller(ctx, zw, m, withContents, freed) + return writeCompressedFiller(ctx, zw, zm, m, withContents, freed) } // The stored path, unchanged. It settles the size before the header @@ -55,6 +62,9 @@ func writeFillerEntry(ctx context.Context, zw *stdzip.Writer, m memo, withConten if err != nil { return err } + if err := markFiller(zm); err != nil { + return err + } if !withContents { return nil } @@ -64,13 +74,22 @@ func writeFillerEntry(ctx context.Context, zw *stdzip.Writer, m memo, withConten return shut() } +// markFiller marks the filler entry as padding and sends on what was held +// while its header was written. +func markFiller(zm *archive.StreamMap) error { + if err := zm.MarkEntry(format.RolePadding, fillerName); err != nil { + return err + } + return zm.Release() +} + // writeCompressedFiller is the same entry for an archive whose others are // squeezed, with the header written before the length is known. // // It carries no checksum and takes no lock, and neither is an omission: // ReadCompression refuses compression together with a password, so a squeezed // archive is never a locked one. -func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, m memo, withContents bool, freed func() int64) error { +func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, zm *archive.StreamMap, m memo, withContents bool, freed func() int64) error { entry, shut, err := openEntry(zw, m, entryPlan{ name: fillerName, index: len(m.children), stored: true, withContents: withContents, @@ -78,6 +97,9 @@ func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, m memo, withC if err != nil { return err } + if err := markFiller(zm); err != nil { + return err + } if !withContents { // The counting pass adds the filler's length arithmetically, so // writing its bytes here would count them twice. @@ -116,3 +138,27 @@ func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, m memo, withC } return shut() } + +// writeFiller emits the padding entry without holding it in memory. +func writeFiller(ctx context.Context, w io.Writer, seed uint64, n int64) error { + rng := core.NewRand(seed) + buf := make([]byte, writeChunk) + for remaining := n; remaining > 0; { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + size := int64(len(buf)) + if remaining < size { + size = remaining + } + chunk := buf[:size] + core.FillRandomBE(chunk, rng) + if _, err := w.Write(chunk); err != nil { + return err + } + remaining -= size + } + return nil +} diff --git a/internal/format/zip/zip.go b/internal/format/zip/zip.go index 7bfb4cf0..cd27e0df 100644 --- a/internal/format/zip/zip.go +++ b/internal/format/zip/zip.go @@ -530,7 +530,13 @@ func plaintextCRC(ctx context.Context, m memo, withContents bool, write func(io. } func build(ctx context.Context, w io.Writer, m memo, withContents bool) error { - zw := stdzip.NewWriter(w) + zm := archive.NewStreamMap(w) + zw := stdzip.NewWriter(zm.Buffer) + // The map: the directories' entries, the files' entries as the content, + // the filler entry, the central directory, and the comment. What only the + // writing tells - where the filler and the directory start - is marked as + // archive/zip writes it (archive.StreamMap). + format.Mark(w, format.RoleStructure, 0, -1) if err := zw.SetComment(m.comment); err != nil { return core.Defect(fmt.Errorf("zip: the archive comment was refused: %w", err)) } @@ -552,6 +558,7 @@ func build(ctx context.Context, w io.Writer, m memo, withContents bool) error { if err := writeDirectories(ctx, zw, m.layout); err != nil { return err } + format.Mark(w, format.RoleContent, zm.Pos(), -1) for i, c := range m.children { select { @@ -598,35 +605,11 @@ func build(ctx context.Context, w io.Writer, m memo, withContents bool) error { } return plain - squeezed } - if err := writeFillerEntry(ctx, zw, m, withContents, freed); err != nil { + if err := writeFillerEntry(ctx, zw, zm, m, withContents, freed); err != nil { return err } - return zw.Close() -} - -// writeFiller emits the padding entry without holding it in memory. -func writeFiller(ctx context.Context, w io.Writer, seed uint64, n int64) error { - rng := core.NewRand(seed) - buf := make([]byte, writeChunk) - for remaining := n; remaining > 0; { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - size := int64(len(buf)) - if remaining < size { - size = remaining - } - chunk := buf[:size] - core.FillRandomBE(chunk, rng) - if _, err := w.Write(chunk); err != nil { - return err - } - remaining -= size - } - return nil + return zm.Close(zw) } // archiveSize measures the archive without generating anything it holds. diff --git a/internal/guard/codeshape_test.go b/internal/guard/codeshape_test.go index ba7dc0d1..42ca323c 100644 --- a/internal/guard/codeshape_test.go +++ b/internal/guard/codeshape_test.go @@ -72,7 +72,11 @@ const ( // moved out of oracle/oracle.go into strict.go when the format it // learned took the file past the ceiling. The longest file is // engine/engine.go now. - longestFile = 392 + // Lowered from 392 on 2026-11-12: the notes of an entry moved out of + // engine/engine.go, the label out of format/wav/wav.go and the filler + // writer out of format/zip/zip.go, when the map of a file took all three + // past the ceiling. The longest file is recipe/target.go now. + longestFile = 390 // Depth answers a different question than length, and it is the better // question of the two. A hundred line function that is flat reads top to diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go new file mode 100644 index 00000000..854d18c0 --- /dev/null +++ b/internal/guard/cutmanifest_test.go @@ -0,0 +1,186 @@ +package guard + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/damage" + "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/manifest" +) + +// cutCase is one file cut short and what its manifest has to say about it. +type cutCase struct { + name string + format string + size int64 + missing string + // in is the role of the first byte cut away. + in string + // kept says how much of the content is left: "all", "none" or "some". + kept string + // note is the word that tells the two notes of a cut in the padding + // apart, empty when the cut gets none. + note string +} + +// A file cut short says in its manifest where the cut fell and how much of +// the content is left, and gets a note when the cut missed the content. +// +// The three shapes a cut takes (docs/TRUNCATE-2026-11-11.md section 2). A PNG +// of 1 MiB is nearly all padding after its picture, so 90% missing still +// leaves the whole picture. An icon keeps its padding BEFORE its picture, so a +// cut there takes all of it. A text is its words, so the cut falls in them. A +// damage that cuts nothing has no cut to report. +// +// Canary: the check below must refuse a cut that names the wrong role, and +// one that claims content it does not have. +func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { + cases := []cutCase{ + {name: "picture", format: "png", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, + // At half: the picture of 256x256 takes about 260 KB at the end of the + // file, and a tenth of a megabyte cut away falls inside it. + {name: "icon", format: "ico", size: 1 << 20, missing: "50", in: "padding", kept: "none", note: "before the content"}, + {name: "text", format: "txt", size: 1000, missing: "50", in: "content", kept: "some"}, + } + targets := []engine.Target{{ID: "untouched", Format: "txt", Sizes: engine.Uniform(1, 1000), Label: true, + Damage: damage.Chain{{ID: "zero-head"}}}} + for _, c := range cases { + targets = append(targets, engine.Target{ID: c.name, Format: c.format, Sizes: engine.Uniform(1, c.size), Label: true, + Damage: damage.Chain{{ID: damage.Truncate, Values: damage.Values{damage.SettingMissing: c.missing}}}}) + } + opt := engine.Options{OutDir: t.TempDir(), Seed: 7, ManifestName: engine.DefaultManifestName} + planned, err := engine.Plan(targets, opt) + if err != nil { + t.Fatalf("planning: %v", err) + } + res, err := engine.Run(t.Context(), planned, opt) + if err != nil { + t.Fatalf("running: %v", err) + } + byTarget := map[string]manifest.File{} + for _, f := range res.Manifest.Files { + byTarget[f.TargetID] = f + } + + if f := byTarget["untouched"]; len(f.Damage) != 1 || f.Damage[0].Cut != nil { + t.Errorf("a damage that cuts nothing reported a cut: %+v", f.Damage) + } + for _, c := range cases { + f, ok := byTarget[c.name] + if !ok || f.Failed { + t.Errorf("%s: no file was written (%s)", c.name, f.Error) + continue + } + for _, problem := range cutProblems(c, f) { + t.Errorf("%s: %s", c.name, problem) + } + } + + // Canary: the picture's entry, told the cut fell in its content, and the + // text's, told none of its content is left. + picture := byTarget["picture"] + if picture.Damage[0].Cut != nil { + wrong := cases[0] + wrong.in = "content" + if len(cutProblems(wrong, picture)) == 0 { + t.Errorf("canary: a cut in the padding passed for a cut in the content") + } + } + text := cases[2] + text.kept = "none" + if len(cutProblems(text, byTarget["text"])) == 0 { + t.Errorf("canary: a text that kept half its words passed for one that kept none") + } +} + +// cutProblems is everything about f's cut that is not what c says it is. +func cutProblems(c cutCase, f manifest.File) []string { + if len(f.Damage) != 1 || f.Damage[0].Cut == nil { + return []string{fmt.Sprintf("the damage entry has no cut: %+v", f.Damage)} + } + cut := f.Damage[0].Cut + var out []string + if cut.In != c.in { + out = append(out, fmt.Sprintf("the cut fell in %q, not in %q", cut.In, c.in)) + } + if cut.ContentBytes == nil { + return append(out, "the content's length is not stated") + } + whole, kept := *cut.ContentBytes, cut.ContentKeptBytes + switch { + case c.kept == "all" && kept != whole: + out = append(out, fmt.Sprintf("%d B of %d B of content kept, not all of it", kept, whole)) + case c.kept == "none" && kept != 0: + out = append(out, fmt.Sprintf("%d B of content kept, not none", kept)) + case c.kept == "some" && (kept != f.Bytes || whole <= kept): + out = append(out, fmt.Sprintf("%d B of %d B of content kept, where a %d B file of nothing but content keeps all of it and loses the rest", kept, whole, f.Bytes)) + } + var said []string + for _, n := range f.Notes { + if n.Code == "cut_in_padding" { + said = append(said, n.Detail) + } + } + switch { + case c.note == "" && len(said) > 0: + out = append(out, fmt.Sprintf("a note about the padding where the cut fell in %s: %q", c.in, said)) + case c.note != "" && (len(said) != 1 || !strings.Contains(said[0], c.note)): + out = append(out, fmt.Sprintf("the note about the padding says %q, which should say %q", said, c.note)) + } + return out +} + +// The record a run that cuts its files is told beforehand is never shorter +// than the record it writes, and not much longer. +// +// The cut is known only once a file is written, so the forecast counts the +// longest one a file can be given (engine forecastCut). A forecast under the +// record lets the free space check pass a disk the record does not fit on, +// and one far over refuses a disk that had room. +// +// What it reaches, asserted rather than assumed: the record carries the note +// a cut in the padding gets, the longest thing the forecast allows for. +func TestTheRecordOfACutRunIsNeverLongerThanItsForecast(t *testing.T) { + cut := func(id, format string, size int64, missing string) engine.Target { + return engine.Target{ID: id, Format: format, Sizes: engine.Uniform(2, size), Label: true, + Damage: damage.Chain{{ID: damage.Truncate, Values: damage.Values{damage.SettingMissing: missing}}}} + } + opt := engine.Options{OutDir: t.TempDir(), Seed: 7, Command: "test"} + planned, err := engine.Plan([]engine.Target{ + cut("picture", "png", 1<<20, "90"), cut("icon", "ico", 1<<20, "50"), cut("text", "txt", 1000, "50"), + }, opt) + if err != nil { + t.Fatalf("planning: %v", err) + } + forecast, err := engine.PredictRecord(t.Context(), planned, opt) + if err != nil { + t.Fatalf("forecasting: %v", err) + } + res, err := engine.Run(t.Context(), planned, opt) + if err != nil { + t.Fatalf("running: %v", err) + } + rec, err := engine.SaveRecord(res, opt) + if err != nil { + t.Fatalf("saving the record: %v", err) + } + body, err := os.ReadFile(rec.Manifest) + if err != nil { + t.Fatalf("reading the record: %v", err) + } + if !strings.Contains(string(body), `"cut_in_padding"`) { + t.Fatalf("no file of the run was cut in its padding, so the longest entry was never written") + } + written := int64(len(body)) + if written > forecast.Manifest { + t.Errorf("the record is %d B and the run was told %d B", written, forecast.Manifest) + } + // Each of the six files may be forecast a role word, two numbers and a + // note it does not get - a few hundred bytes, not kilobytes. + if slack := forecast.Manifest - written; slack > 6*256 { + t.Errorf("the forecast is %d B over a record of %d B", slack, written) + } +} diff --git a/internal/guard/filemap_test.go b/internal/guard/filemap_test.go new file mode 100644 index 00000000..7a421db5 --- /dev/null +++ b/internal/guard/filemap_test.go @@ -0,0 +1,231 @@ +package guard + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/oracle" +) + +// mapSize is the large size every format's map is also read at, beside the +// smallest one it writes. Above it the picture formats take the path that +// codes the picture only while writing, where the generator learns the length +// of its content late (docs/TRUNCATE-2026-11-11.md section 6.2): 16 KiB is the +// line for png, and avif, jxl and the films take it at every size. +const mapSize = 1 << 20 + +// mappedFile is one file written through a map recorder, with the map its +// generator marked. +type mappedFile struct { + name string + format string + path string + // setting is the one setting moved from its default, empty for none. + setting map[string]string + marked [][3]any +} + +// The map a generator marks while it writes is the map its file's structure +// reads back, for every format, every value of every setting that picks a +// shape, at the smallest size and a large one. +// +// The map is what the manifest's damage[].cut is worked out from, and the +// offsets in it are the generator's own word (internal/format/layout.go). The +// second reading is internal/oracle/layout.py, written to each format's rules +// rather than to what the generator meant. A generator that marks nothing is +// content from start to end, which a text is and a picture is not, so a new +// binary format that forgets its marks is red here. +// +// Canary: the first map with two regions, with the boundary between them moved +// by a byte, must not be taken for the map read back. +func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { + dir := t.TempDir() + var files []mappedFile + for _, f := range format.All() { + variants := append([]map[string]string{{}}, settingVariants(f)...) + for n, setting := range variants { + // A setting the format refuses without another beside it is + // skipped, as the witness guard does, rather than taking the + // whole sweep down. + _, err := f.Generator.Plan(format.Request{Bytes: mapSize, Label: true, Properties: setting}) + var below *format.BelowMinimumError + if err != nil && !errors.As(err, &below) { + continue + } + smallest := f.SmallestAccepted(format.Request{Label: true, Properties: setting}) + for _, from := range []int64{smallest, max(smallest, mapSize)} { + size := healthySizeFrom(t, f, setting, from) + name := fmt.Sprintf("%s-%d-%d", f.ID, n, size) + files = append(files, writeMapped(t, dir, f, name, setting, size)) + } + } + } + + reqs := make([]oracle.LayoutRequest, len(files)) + for i, m := range files { + reqs[i] = oracle.LayoutRequest{Format: m.format, Path: m.path, Settings: m.setting} + } + got, available, err := oracle.LayoutBatch(t.Context(), reqs) + if !available { + t.Skip("the reader of maps needs python, and there is none on this machine") + } + if err != nil { + t.Fatalf("reading the maps back: %v", err) + } + + padded, canaried := 0, false + for i, m := range files { + if got[i].Error != "" { + t.Errorf("%s: the structure gives no map: %s", m.name, got[i].Error) + continue + } + read := normalRegions(got[i].Regions) + if !sameMap(m.marked, read) { + t.Errorf("%s: the generator marked %s and the structure reads %s", m.name, mapNames(m.marked), mapNames(read)) + continue + } + if contents := countRole(m.marked, format.RoleContent); contents > 1 { + t.Errorf("%s: %d regions of content - a cut counts the content as one, and the regions after a cut are never marked", m.name, contents) + } + if countRole(m.marked, format.RolePadding) > 0 { + padded++ + } + if !canaried && len(m.marked) >= 2 { + canaried = true + moved := slices.Clone(m.marked) + moved[0] = [3]any{moved[0][0], moved[0][1], moved[0][2].(int64) + 1} + moved[1] = [3]any{moved[1][0], moved[1][1].(int64) + 1, moved[1][2].(int64) - 1} + if sameMap(moved, read) { + t.Errorf("canary: %s with its first boundary moved by a byte passed for the map read back", m.name) + } + } + } + // Canary of the order: a mark for bytes already written is reported. + late := &orderedRecorder{MapRecorder: format.MapRecorder{W: io.Discard}} + _, _ = late.Write(make([]byte, 10)) + format.Mark(late, format.RoleContent, 5, -1) + if len(late.late) != 1 { + t.Errorf("canary: a mark at 5 after 10 B were written was not reported as late") + } + + // What this asserts, so a sweep that reached no padding cannot pass. + if !canaried { + t.Errorf("no file had two regions, so the canary asked nothing") + } + if padded == 0 { + t.Errorf("no file of %d had padding in its map, so the role a cut misses most was never asked", len(files)) + } + t.Logf("%d files, %d with padding", len(files), padded) +} + +// writeMapped writes one healthy file of f through a map recorder. +func writeMapped(t *testing.T, dir string, f format.Descriptor, name string, setting map[string]string, size int64) mappedFile { + t.Helper() + p, err := f.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, Properties: setting}) + if err != nil { + t.Fatalf("%s: planning: %v", name, err) + } + path := filepath.Join(dir, name+"."+f.ID) + fh, err := os.Create(path) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + rec := &orderedRecorder{MapRecorder: format.MapRecorder{W: fh}} + werr := f.Generator.Write(t.Context(), rec, p) + if cerr := fh.Close(); werr == nil { + werr = cerr + } + if werr != nil { + t.Fatalf("%s: writing: %v", name, werr) + } + for _, late := range rec.late { + t.Errorf("%s: %s - a cut there stops the generator before the mark, and the map says nothing of the bytes it cut", name, late) + } + return mappedFile{name: name, format: f.ID, path: path, setting: setting, marked: mergedRegions(rec.Map(size, true))} +} + +// orderedRecorder is a map recorder that also counts the bytes reaching it, +// and keeps every mark that came after a byte of its own region had already +// arrived. +// +// That is the order a cut needs. A generator stops at the cut, so a region +// marked after its bytes went out is never marked when the cut falls in it - +// and a file with no marks at all reads as content from start to end. +type orderedRecorder struct { + format.MapRecorder + n int64 + late []string +} + +func (o *orderedRecorder) Write(p []byte) (int, error) { + n, err := o.MapRecorder.Write(p) + o.n += int64(n) + return n, err +} + +func (o *orderedRecorder) MarkRegion(r format.Region) { + if o.n > r.At { + o.late = append(o.late, fmt.Sprintf("%s at %d was marked after %d B had been written", r.Role, r.At, o.n)) + } + o.MapRecorder.MarkRegion(r) +} + +// mergedRegions is a map as [role, at, length], neighbours of one role as one +// and empty regions dropped - the shape layout.py answers in. +func mergedRegions(regions []format.Region) [][3]any { + var out [][3]any + for _, r := range regions { + if r.Length == 0 { + continue + } + if n := len(out); n > 0 && out[n-1][0] == string(r.Role) && out[n-1][1].(int64)+out[n-1][2].(int64) == r.At { + out[n-1][2] = out[n-1][2].(int64) + r.Length + continue + } + out = append(out, [3]any{string(r.Role), r.At, r.Length}) + } + return out +} + +// normalRegions is a map read back, with its numbers as int64 rather than the +// float64 JSON gives. +func normalRegions(in [][3]any) [][3]any { + out := make([][3]any, len(in)) + for i, r := range in { + role, _ := r[0].(string) + at, _ := r[1].(float64) + length, _ := r[2].(float64) + out[i] = [3]any{role, int64(at), int64(length)} + } + return out +} + +func sameMap(a, b [][3]any) bool { + return slices.Equal(a, b) +} + +func countRole(m [][3]any, role format.Role) int { + n := 0 + for _, r := range m { + if r[0] == string(role) { + n++ + } + } + return n +} + +// mapNames is the map as words, for a message. +func mapNames(m [][3]any) string { + var parts []string + for _, r := range m { + parts = append(parts, fmt.Sprintf("%s@%d+%d", r[0], r[1], r[2])) + } + return strings.Join(parts, " ") +} diff --git a/internal/guard/layers_test.go b/internal/guard/layers_test.go index e13d2f93..90ed4afb 100644 --- a/internal/guard/layers_test.go +++ b/internal/guard/layers_test.go @@ -182,7 +182,7 @@ var sameLayerAllowed = map[string][]string{ "internal/format/htmlfile": {"internal/format", "internal/format/picture", "internal/format/textenc"}, "internal/format/svgfile": {"internal/format", "internal/format/imagedim"}, "internal/format/bmp": {"internal/format", "internal/format/imagelabel", "internal/format/imagedim"}, - "internal/format/opc": {"internal/format"}, + "internal/format/opc": {"internal/format", "internal/format/archive"}, "internal/format/docx": {"internal/format", "internal/format/opc"}, "internal/format/xlsx": {"internal/format", "internal/format/opc"}, "internal/format/pptx": {"internal/format", "internal/format/opc"}, diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index feca6ddf..a7025293 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -150,6 +150,12 @@ var provenByProbe = map[string]string{ "TestEveryAvifRungStillFitsInsideItsDeclaredCeilings": "its own reach check - no damaged label drawn is a failure - and broken on 2026-11-12 twice, with " + "the heaviest seeds: the first ceiling of 640x480 set back to 16389 (red, 17314 B) and the second to 18722 (red, 20024 B). " + "Without those seeds both stayed green (docs/REVIEW-234-2026-11-12.md).", + "TestEveryFormatsMapIsTheOneItsStructureReadsBack": "its own canaries - the first boundary moved by a byte must not pass, a mark for bytes already written must be " + + "reported, a sweep with no padding is a failure - and broken on 2026-11-12 by marking the png padding a byte late: red.", + "TestACutFileSaysWhereTheCutFellAndWhatIsLeft": "its own canary - a cut named in the wrong role, and content claimed gone that is there, must not pass - " + + "and broken on 2026-11-12 by reporting every cut in the content: red for the picture and the icon.", + "TestTheRecordOfACutRunIsNeverLongerThanItsForecast": "its own reach check - a run with no cut in the padding is a failure - and broken on 2026-11-12 " + + "by leaving the cut out of the forecast: red, a record of 9638 B told 8130 B.", "TestADamageLabelIsBoundedAndSaysHowManyItLeftOut": "its own canary - no chain up to 200 damages shortened is a failure - and broken on 2026-11-12 by letting damage.Chain.Label return the whole chain: red.", "TestADamagedMailCarriesItsLabelWholeInEveryTextEncoding": "broken on 2026-11-12 twice: eml writing the label raw in quoted-printable (red, the one damage label read back as bytesd), " + "and eml leaving a long label on one line (red, a line of 92 characters).", diff --git a/internal/gui/text/locale/registry/en.json b/internal/gui/text/locale/registry/en.json index 6f88d8a3..0b810f0d 100644 --- a/internal/gui/text/locale/registry/en.json +++ b/internal/gui/text/locale/registry/en.json @@ -1296,8 +1296,8 @@ }, "Detail.damage/truncate.missing": { "description": "The sentence under the missing setting of the truncate damage, after what it takes.", - "hash": "sha256-0b344bd34150", - "other": "How much of the whole file is gone, as a percentage. At least this much is cut - a few bytes more where the format cannot end exactly there." + "hash": "sha256-11e2c87619ac", + "other": "How much of the whole file is gone, as a percentage. At least this much is cut - a few bytes more where the format cannot end exactly there. A large file is often mostly padding after its content, so the content can stay whole - the manifest says what each cut left." }, "Detail.damage/zero-head.bytes": { "description": "The sentence under the bytes setting of the zero-head damage, after what it takes.", diff --git a/internal/gui/text/locale/registry/pl.json b/internal/gui/text/locale/registry/pl.json index bb522725..d601b950 100644 --- a/internal/gui/text/locale/registry/pl.json +++ b/internal/gui/text/locale/registry/pl.json @@ -258,7 +258,7 @@ "Choice.winding.reversed": { "hash": "sha256-bc64f64fb479", "other": "odwrócony" }, "Choice.winding.rfc7946": { "hash": "sha256-38755035559d", "other": "rfc7946" }, "Detail.damage/append-junk.bytes": { "hash": "sha256-4403339e8eb1", "other": "Ile losowych bajtów trafia za koniec pliku. Poniżej 64 niektóre formaty wychodzą z uszkodzeniem, na które żaden czytnik nie narzeka." }, - "Detail.damage/truncate.missing": { "hash": "sha256-0b344bd34150", "other": "Jaka część całego pliku znika, w procentach. Ucięte jest co najmniej tyle - o kilka bajtów więcej tam, gdzie format nie może się skończyć dokładnie w tym miejscu." }, + "Detail.damage/truncate.missing": { "hash": "sha256-11e2c87619ac", "other": "Jaka część całego pliku znika, w procentach. Ucięte jest co najmniej tyle - o kilka bajtów więcej tam, gdzie format nie może się skończyć dokładnie w tym miejscu. Duży plik to często głównie wypełnienie za treścią, więc treść może zostać cała - manifest mówi, co zostawiło każde cięcie." }, "Detail.damage/zero-head.bytes": { "hash": "sha256-8700126f6712", "other": "Ile bajtów na początku zostaje wyzerowanych. Poniżej czterech niektóre formaty wychodzą z uszkodzeniem, na które żaden czytnik nie narzeka." }, "Detail.format/avif.height": { "hash": "sha256-4b336c279584", "other": "Jak wysoki jest obraz. Gdy pole zostaje puste, wybierany jest rozmiar, który mieści się w zamówionych bajtach." }, "Detail.format/avif.quality": { "hash": "sha256-f6e1c3e1af83", "other": "Ile szczegółów zachowuje obraz. Wyższa wartość wygląda lepiej i zajmuje więcej pliku, zostawiając mniej miejsca na dopełnienie." }, diff --git a/internal/gui/text/locale/said/en.json b/internal/gui/text/locale/said/en.json index 00637d7d..e9d67579 100644 --- a/internal/gui/text/locale/said/en.json +++ b/internal/gui/text/locale/said/en.json @@ -1018,6 +1018,16 @@ "hash": "sha256-1e90995301d1", "other": "writing {{.File}}" }, + "engine.CutAfterContent": { + "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run.", + "hash": "sha256-34a36096782c", + "other": "The cut fell in the padding after the content, so the content is whole and only the end of the file is missing." + }, + "engine.CutBeforeContent": { + "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run.", + "hash": "sha256-ce3c81abf2d2", + "other": "The cut fell in the padding before the content, so none of the content is left." + }, "engine.DamagedAboveWhy": { "description": "Said by internal/engine - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Damage}}, {{.Format}}, {{.Maximum}}, written the way the program writes them - keep each spelled exactly that way.", "hash": "sha256-59fb1ae064c2", diff --git a/internal/gui/text/locale/said/pl.json b/internal/gui/text/locale/said/pl.json index a34943b7..67adb0c7 100644 --- a/internal/gui/text/locale/said/pl.json +++ b/internal/gui/text/locale/said/pl.json @@ -202,6 +202,8 @@ "engine.CrashPlanning": { "hash": "sha256-90f7d5da615f", "other": "planowania pliku" }, "engine.CrashWhile": { "hash": "sha256-071bf3d7823e", "other": "{{.While}}" }, "engine.CrashWriting": { "hash": "sha256-1e90995301d1", "other": "zapisu {{.File}}" }, + "engine.CutAfterContent": { "hash": "sha256-34a36096782c", "other": "Cięcie trafiło w wypełnienie za treścią, więc treść jest cała, a brakuje tylko końca pliku." }, + "engine.CutBeforeContent": { "hash": "sha256-ce3c81abf2d2", "other": "Cięcie trafiło w wypełnienie przed treścią, więc z treści nic nie zostało." }, "engine.DamagedAboveWhy": { "hash": "sha256-59fb1ae064c2", "other": "{{.Damage}} musi wyjść od pliku {{.Format}} o rozmiarze najwyżej {{.Maximum}}" }, "engine.DamagedBelowWhy": { "hash": "sha256-5fe7c6a1ed13", "other": "{{.Damage}} musi wyjść od pliku {{.Format}} o rozmiarze co najmniej {{.Minimum}}" }, "engine.DamagedFloorWhy": { "hash": "sha256-7829143089a2", "other": "{{.Damage}} potrzebuje co najmniej {{.Floor}}" }, diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go index af6d7936..9fa38b06 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifest/manifest.go @@ -275,6 +275,29 @@ type Failure struct { type Damage struct { Type string `json:"type"` Settings map[string]string `json:"settings,omitempty"` + // Cut is where a damage that cuts the end off the file fell. Left out for + // every other damage, so the entries written before it existed keep their + // bytes, and manifest_version stays for the reason File.Damage gives. + Cut *Cut `json:"cut,omitempty"` +} + +// Cut is where a cut fell in the file it was made in, and how much of the +// content it left. +// +// Asked because the cut does not always fall where the setting suggests. In a +// format that keeps its padding after the content, a large file is nearly all +// padding, so the cut falls there and the content stays whole whatever share +// was asked for (docs/TRUNCATE-2026-11-11.md). A test that means "half of the +// picture is gone" can read here whether it is. +type Cut struct { + // In is what the first byte cut away was for: structure, content, + // padding or end. + In string `json:"in"` + // ContentBytes is how long the content of the whole file was. Absent when + // the generator did not know when it stopped, rather than a guess. + ContentBytes *int64 `json:"content_bytes,omitempty"` + // ContentKeptBytes is how much of the content is still in the file. + ContentKeptBytes int64 `json:"content_kept_bytes"` } // Hashes identify the bytes. diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py new file mode 100644 index 00000000..75d3aad2 --- /dev/null +++ b/internal/oracle/layout.py @@ -0,0 +1,541 @@ +"""The map of a file, read back from its structure. + +A generator marks the map of the file it writes - which bytes are structure, +content, padding and the end (internal/format/layout.go). This reads the same +map out of the finished file, from the format's own rules, so the guard +TestEveryFormatsMapIsTheOneItsStructureReadsBack can hold one against the +other. The same bargain as strict.py: a second implementation, in another +language, written to the specification rather than to what the generator +meant. Its own walkers rather than strict.py's, which end the process on the +first refusal - right for a verdict, and fatal to a batch of maps. + +Coarse on purpose. Every role is one or a few regions, never one per record: +an archive's entries are content with their local headers, a picture's chunks +are content with their checksums. A damage aims at a role, and a role is all +the map has to answer. + +Usage: python layout.py --batch +Reads one JSON request per line, {"format": ..., "path": ...}, and writes one +JSON answer per line in the order asked, {"path": ..., "regions": [[role, at, +length], ...]} or {"path": ..., "error": ...}. +""" + +import json +import re +import struct +import sys + +STRUCTURE, CONTENT, PADDING, END = "structure", "content", "padding", "end" + + +class NoMap(Exception): + """The file does not have the shape its map is read from.""" + + +def merged(regions): + """Neighbouring regions of one role as one, and empty ones dropped.""" + out = [] + for role, at, length in regions: + if length == 0: + continue + if out and out[-1][0] == role and out[-1][1] + out[-1][2] == at: + out[-1][2] += length + else: + out.append([role, at, length]) + return out + + +def layout_text(data): + """A text is its words. The label, the records and the padding note of + the last record are all words a reader hands back.""" + return [[CONTENT, 0, len(data)]] + + +PNG_ROLES = {b"IHDR": STRUCTURE, b"IDAT": CONTENT, b"tfGp": PADDING, b"IEND": END} + + +def layout_png(data): + if data[:8] != b"\x89PNG\r\n\x1a\n": + raise NoMap("no PNG signature") + out, pos = [[STRUCTURE, 0, 8]], 8 + while pos < len(data): + if pos + 8 > len(data): + raise NoMap(f"a chunk header runs past the end at {pos}") + length = struct.unpack(">I", data[pos:pos + 4])[0] + kind = data[pos + 4:pos + 8] + out.append([PNG_ROLES.get(kind, STRUCTURE), pos, 12 + length]) + pos += 12 + length + return out + + +RIFF_ROLES = {b"data": CONTENT, b"JUNK": PADDING} + + +def riff_chunks(data, start, roles): + """The chunks of a RIFF list from start, each as a region, an alignment + byte after an odd length counted to its chunk unless it is the last one + (the walk strict.check_wav makes).""" + out, pos = [], start + while pos + 8 <= len(data): + kind = data[pos:pos + 4] + length = struct.unpack(" len(data): + raise NoMap(f"the container declares {declared} B and the file is {len(data)} B") + roles = {b"VP8L": CONTENT, b"VP8 ": CONTENT} + out = [[STRUCTURE, 0, 12]] + out += [[r if r != STRUCTURE else PADDING, at, n] for r, at, n in riff_chunks(data[:declared], 12, roles)] + out.append([PADDING, declared, len(data) - declared]) + return out + + +def iso_boxes(data, roles): + """The top level boxes of an ISO base media file, each by the role its name + has in roles, structure when it has none.""" + out, pos = [], 0 + while pos + 8 <= len(data): + size = struct.unpack(">I", data[pos:pos + 4])[0] + kind = data[pos + 4:pos + 8] + if size == 1: + size = struct.unpack(">Q", data[pos + 8:pos + 16])[0] + elif size == 0: + size = len(data) - pos + if size < 8 or pos + size > len(data): + raise NoMap(f"box {kind!r} at {pos} declares {size} B") + out.append([roles.get(kind, STRUCTURE), pos, size]) + pos += size + if pos != len(data): + raise NoMap(f"the boxes end at {pos} and the file is {len(data)} B") + return out + + +def layout_avif(data): + return iso_boxes(data, {b"mdat": CONTENT, b"free": PADDING, b"skip": PADDING}) + + +def layout_jxl(data): + return iso_boxes(data, {b"jxlc": CONTENT, b"jxlp": CONTENT, b"free": PADDING, b"skip": PADDING}) + + +def ebml_id(data, at): + """An element ID at at, with its marker bits kept, and its length.""" + first = data[at] + width = 1 + while width <= 4 and not first & (0x80 >> (width - 1)): + width += 1 + if width > 4: + raise NoMap(f"no element ID at {at}") + return int.from_bytes(data[at:at + width], "big"), width + + +def ebml_size(data, at): + """An element's data size at at, and the length of the field.""" + first = data[at] + width = 1 + while width <= 8 and not first & (0x80 >> (width - 1)): + width += 1 + if width > 8: + raise NoMap(f"no size at {at}") + value = first & (0xFF >> width) + for b in data[at + 1:at + width]: + value = value << 8 | b + return value, width + + +WEBM_ROLES = {0x1F43B675: CONTENT, 0xEC: PADDING, 0x1C53BB6B: END} + + +def layout_webm(data): + """The EBML header and the segment's headers, the seek head, the info and + the tracks as structure, the clusters as content, a Void as padding and the + Cues as the end.""" + out, pos = [], 0 + while pos < len(data): + ident, w = ebml_id(data, pos) + size, s = ebml_size(data, pos + w) + if ident != 0x18538067: + out.append([STRUCTURE, pos, w + s + size]) + pos += w + s + size + continue + out.append([STRUCTURE, pos, w + s]) + inner, stop = pos + w + s, pos + w + s + size + while inner < stop: + child, cw = ebml_id(data, inner) + csize, cs = ebml_size(data, inner + cw) + out.append([WEBM_ROLES.get(child, STRUCTURE), inner, cw + cs + csize]) + inner += cw + cs + csize + pos = stop + if pos != len(data): + raise NoMap(f"the elements end at {pos} and the file is {len(data)} B") + return out + + +def layout_mp4(data): + return iso_boxes(data, {b"mdat": CONTENT, b"free": PADDING, b"skip": PADDING}) + + +def layout_pdf(data): + """The header lines, the objects up to the cross reference table startxref + points at, the table with the trailer, any comment after the trailer as + padding, and startxref with the closing marker - the end in two parts.""" + if not data.startswith(b"%PDF-"): + raise NoMap("no PDF header") + first = re.search(rb"\d+ 0 obj\n", data) + sx = data.rfind(b"startxref\n") + if first is None or sx < 0: + raise NoMap("no object or no startxref") + xref = int(data[sx + 10:data.index(b"\n", sx + 10)]) + trailer = data.find(b"trailer\n", xref) + if data[xref:xref + 5] != b"xref\n" or trailer < 0: + raise NoMap(f"startxref points at {xref}, where no cross reference table starts") + closed = data.index(b">>\n", trailer) + 3 + return [[STRUCTURE, 0, first.start()], [CONTENT, first.start(), xref - first.start()], + [END, xref, closed - xref], [PADDING, closed, sx - closed], [END, sx, len(data) - sx]] + + +ZIP_FILLERS = (b"tfg-padding.bin", b"tfg/padding.bin") + + +def zip_entries(data): + """The end of central directory record, where the directory starts, and + each entry the directory lists as (local header offset, name).""" + eocd = -1 + for at in range(len(data) - 22, -1, -1): + if data[at:at + 4] == b"PK\x05\x06" and at + 22 + struct.unpack("= len(data): + raise NoMap("a chain of sub blocks runs off the end") + size = data[pos] + pos += 1 + size + if size == 0: + return pos + + +def layout_gif(data): + """The signature and the screen descriptor, then everything up to the + trailer - the palette, the frames and their extensions - as content, a + comment as padding, and the trailer as the end.""" + if data[:6] not in (b"GIF87a", b"GIF89a"): + raise NoMap("no GIF signature") + out, pos = [[STRUCTURE, 0, 13]], 13 + packed = data[10] + if packed & 0x80: + table = 3 * (1 << ((packed & 0x07) + 1)) + out.append([CONTENT, pos, table]) + pos += table + while pos < len(data): + start, marker = pos, data[pos] + if marker == 0x3B: + out.append([END, pos, 1]) + pos += 1 + continue + if marker == 0x21: + label = data[pos + 1] + pos = sub_blocks_end(data, pos + 2) + out.append([PADDING if label == 0xFE else CONTENT, start, pos - start]) + continue + if marker == 0x2C: + local = data[pos + 9] + pos += 10 + if local & 0x80: + pos += 3 * (1 << ((local & 0x07) + 1)) + pos = sub_blocks_end(data, pos + 1) + out.append([CONTENT, start, pos - start]) + continue + raise NoMap(f"the byte {marker:#04x} at {pos} is not a block marker") + return out + + +def layout_jpg(data): + """The start marker, the comment segments, then every segment and the scan + up to the end marker as content, and the end marker.""" + if data[:2] != b"\xff\xd8": + raise NoMap("no start of image marker") + out, pos = [[STRUCTURE, 0, 2]], 2 + while pos < len(data): + if data[pos] != 0xFF: + raise NoMap(f"no marker at {pos}") + kind = data[pos + 1] + if kind == 0xD9: + out.append([END, pos, 2]) + pos += 2 + continue + length = struct.unpack(">H", data[pos + 2:pos + 4])[0] + end = pos + 2 + length + if kind == 0xDA: + # The scan runs to the next marker that is not a stuffed byte or + # a restart marker. + while end + 1 < len(data) and not (data[end] == 0xFF and data[end + 1] not in (0x00, *range(0xD0, 0xD8))): + end += 1 + out.append([PADDING if kind == 0xFE else CONTENT, pos, end - pos]) + pos = end + return out + + +def layout_ico(data): + """The directory, then whatever lies before the offset its one entry gives, + then the picture it counts.""" + if data[:4] != b"\x00\x00\x01\x00" or struct.unpack(" Date: Thu, 12 Nov 2026 12:16:42 +0100 Subject: [PATCH 2/2] Two cuts in a row are reported where the file ends, and the map guards ask more damage.Chain.Cut returned the first damage that cuts the tail. Two truncates of half a 1000 B text leave 1000 B, and the cut said 2000 B were kept, on the first entry. Each cut keeps the start of what the one before it kept, so the last one before anything else changes the length is where the file ends, and the cut is reported on its entry. The guards ask more: a cut of two truncates, a content said to be empty, and a format whose settings all went unswept are failures, and the PNG map reader refuses a chunk that runs past the end of the file. The Polish notes say dopelnienie for padding, as the rest of the window does, and README says what the cut entry of the manifest holds. Two tests that took over 60 s under the race detector are named in its weights. Co-Authored-By: Claude Opus 5.5 --- .github/race-weights | 2 ++ README.md | 9 ++++++- internal/damage/apply.go | 18 ++++++++----- internal/guard/cutmanifest_test.go | 31 ++++++++++++++++++----- internal/guard/filemap_test.go | 6 +++++ internal/gui/text/locale/registry/pl.json | 2 +- internal/gui/text/locale/said/pl.json | 4 +-- internal/oracle/layout.py | 2 ++ 8 files changed, 57 insertions(+), 17 deletions(-) diff --git a/.github/race-weights b/.github/race-weights index 55d4494f..79e2a3e1 100644 --- a/.github/race-weights +++ b/.github/race-weights @@ -24,7 +24,9 @@ unlisted 0.58 99.9 TestEveryBase64TextSizeIsExact 93.8 TestAFilmPictureLargerThanAnyAV1LevelIsRefusedNotBroken 73.6 TestEveryAvifRungStillFitsInsideItsDeclaredCeilings +68.4 TestEveryFormatsMapIsTheOneItsStructureReadsBack 67.9 TestADeclaredDefaultIsTheOneTheFormatUses +65.6 TestADamagedFileIsExactlyTheSizeAskedFor 64.3 TestNoTrackedFileCarriesPrivateContent 63.8 TestNoGeneratorHoldsTheWholeFileInMemory 60.8 TestEveryDamageIsRefusedByTheStructuralCheckInEveryFormat diff --git a/README.md b/README.md index fce3f3b2..5860a58d 100644 --- a/README.md +++ b/README.md @@ -522,6 +522,13 @@ how many more there are, and the manifest lists every one. The label is still written before the damage, so the damage can overwrite it, and the manifest does not promise it is there. +A cut falls on whatever the file holds at that point. A large picture, sound, +document or archive is mostly padding after its content, so `truncate` can leave +the content whole. The manifest says where each cut fell: the `truncate` entry +carries `cut`, with `in` (`content`, `padding`, `structure` or `end`), +`content_bytes` and `content_kept_bytes`. A cut in the padding also gets the note +`cut_in_padding`. + The smallest file a damage can be given follows its settings, so the column is measured with the defaults. Ask for less and the run is refused before anything is written, naming a size that would work. A setting outside what the damage takes, @@ -988,7 +995,7 @@ Yes. Add `--damage zero-head` and the file comes out at exactly the size you asked for with its first bytes overwritten by zeros, so a reader turns it away, and the manifest says your system should reject it. `--damage truncate` cuts the end off and `--damage append-junk` writes bytes after it, still at the size you -asked for. [`tfg damage`](#tfg-damage) lists what this build can do, what each +asked for, and the manifest says where a cut fell. [`tfg damage`](#tfg-damage) lists what this build can do, what each damage takes, and where the outcome depends on your system. ``` diff --git a/internal/damage/apply.go b/internal/damage/apply.go index 4055b241..62384420 100644 --- a/internal/damage/apply.go +++ b/internal/damage/apply.go @@ -299,6 +299,11 @@ func (c Chain) Stretches() bool { // for a file of size bytes on disk: its place in the list, and how many of the // generator's bytes it keeps, which is the offset of the first byte it drops. // +// The LAST of the damages that cut before anything changes the length in +// another way: each keeps the start of what the one before it kept, so the +// last keeps the least, and that is where the file ends (two truncates of half +// a 1000 B text keep 1000 B, not the 2000 B the first one passes on). +// // False when no damage cuts the tail, and when one before it changes the // length - the bytes it keeps would then not be the generator's, and an offset // in them says nothing about the map of the file the generator wrote. The same @@ -312,15 +317,16 @@ func (c Chain) Cut(size int64) (int, int64, bool) { if !ok { return 0, 0, false } + at, kept, found := 0, int64(0), false for i, s := range steps { - if s.d.CutsTail { - return i, outs[i], true - } - if s.d.Length != nil { - return 0, 0, false + switch { + case s.d.CutsTail: + at, kept, found = i, outs[i], true + case s.d.Length != nil: + return at, kept, found } } - return 0, 0, false + return at, kept, found } // Open builds the chain as one writer for a file of size bytes on disk, with diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index 854d18c0..cb4af7a2 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -24,6 +24,9 @@ type cutCase struct { // note is the word that tells the two notes of a cut in the padding // apart, empty when the cut gets none. note string + // cuts is how many truncates the chain holds, one when not said. The + // last of them keeps the least and carries the cut. + cuts int } // A file cut short says in its manifest where the cut fell and how much of @@ -32,8 +35,9 @@ type cutCase struct { // The three shapes a cut takes (docs/TRUNCATE-2026-11-11.md section 2). A PNG // of 1 MiB is nearly all padding after its picture, so 90% missing still // leaves the whole picture. An icon keeps its padding BEFORE its picture, so a -// cut there takes all of it. A text is its words, so the cut falls in them. A -// damage that cuts nothing has no cut to report. +// cut there takes all of it. A text is its words, so the cut falls in them. +// Two cuts in a row end the file where the second one does, and the cut is +// said of that one. A damage that cuts nothing has no cut to report. // // Canary: the check below must refuse a cut that names the wrong role, and // one that claims content it does not have. @@ -44,12 +48,17 @@ func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { // file, and a tenth of a megabyte cut away falls inside it. {name: "icon", format: "ico", size: 1 << 20, missing: "50", in: "padding", kept: "none", note: "before the content"}, {name: "text", format: "txt", size: 1000, missing: "50", in: "content", kept: "some"}, + {name: "twice", format: "txt", size: 1000, missing: "50", in: "content", kept: "some", cuts: 2}, } targets := []engine.Target{{ID: "untouched", Format: "txt", Sizes: engine.Uniform(1, 1000), Label: true, Damage: damage.Chain{{ID: "zero-head"}}}} for _, c := range cases { + var chain damage.Chain + for range max(1, c.cuts) { + chain = append(chain, damage.Spec{ID: damage.Truncate, Values: damage.Values{damage.SettingMissing: c.missing}}) + } targets = append(targets, engine.Target{ID: c.name, Format: c.format, Sizes: engine.Uniform(1, c.size), Label: true, - Damage: damage.Chain{{ID: damage.Truncate, Values: damage.Values{damage.SettingMissing: c.missing}}}}) + Damage: chain}) } opt := engine.Options{OutDir: t.TempDir(), Seed: 7, ManifestName: engine.DefaultManifestName} planned, err := engine.Plan(targets, opt) @@ -82,7 +91,7 @@ func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { // Canary: the picture's entry, told the cut fell in its content, and the // text's, told none of its content is left. picture := byTarget["picture"] - if picture.Damage[0].Cut != nil { + if len(picture.Damage) == 1 && picture.Damage[0].Cut != nil { wrong := cases[0] wrong.in = "content" if len(cutProblems(wrong, picture)) == 0 { @@ -98,10 +107,16 @@ func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { // cutProblems is everything about f's cut that is not what c says it is. func cutProblems(c cutCase, f manifest.File) []string { - if len(f.Damage) != 1 || f.Damage[0].Cut == nil { - return []string{fmt.Sprintf("the damage entry has no cut: %+v", f.Damage)} + last := max(1, c.cuts) - 1 + if len(f.Damage) != last+1 || f.Damage[last].Cut == nil { + return []string{fmt.Sprintf("the last damage entry has no cut: %+v", f.Damage)} + } + for _, d := range f.Damage[:last] { + if d.Cut != nil { + return []string{fmt.Sprintf("a cut that does not end the file reported one: %+v", f.Damage)} + } } - cut := f.Damage[0].Cut + cut := f.Damage[last].Cut var out []string if cut.In != c.in { out = append(out, fmt.Sprintf("the cut fell in %q, not in %q", cut.In, c.in)) @@ -111,6 +126,8 @@ func cutProblems(c cutCase, f manifest.File) []string { } whole, kept := *cut.ContentBytes, cut.ContentKeptBytes switch { + case whole <= 0: + out = append(out, fmt.Sprintf("the content is said to be %d B long, and every case here cuts beside content", whole)) case c.kept == "all" && kept != whole: out = append(out, fmt.Sprintf("%d B of %d B of content kept, not all of it", kept, whole)) case c.kept == "none" && kept != 0: diff --git a/internal/guard/filemap_test.go b/internal/guard/filemap_test.go index 7a421db5..60babd81 100644 --- a/internal/guard/filemap_test.go +++ b/internal/guard/filemap_test.go @@ -49,6 +49,9 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { dir := t.TempDir() var files []mappedFile for _, f := range format.All() { + // A format all of whose settings were skipped would leave the sweep + // without it and the guard green, so each one has to reach a file. + before := len(files) variants := append([]map[string]string{{}}, settingVariants(f)...) for n, setting := range variants { // A setting the format refuses without another beside it is @@ -66,6 +69,9 @@ func TestEveryFormatsMapIsTheOneItsStructureReadsBack(t *testing.T) { files = append(files, writeMapped(t, dir, f, name, setting, size)) } } + if len(files) == before { + t.Errorf("%s: no setting of it was written, so its map was never read back", f.ID) + } } reqs := make([]oracle.LayoutRequest, len(files)) diff --git a/internal/gui/text/locale/registry/pl.json b/internal/gui/text/locale/registry/pl.json index d601b950..6702c8f7 100644 --- a/internal/gui/text/locale/registry/pl.json +++ b/internal/gui/text/locale/registry/pl.json @@ -258,7 +258,7 @@ "Choice.winding.reversed": { "hash": "sha256-bc64f64fb479", "other": "odwrócony" }, "Choice.winding.rfc7946": { "hash": "sha256-38755035559d", "other": "rfc7946" }, "Detail.damage/append-junk.bytes": { "hash": "sha256-4403339e8eb1", "other": "Ile losowych bajtów trafia za koniec pliku. Poniżej 64 niektóre formaty wychodzą z uszkodzeniem, na które żaden czytnik nie narzeka." }, - "Detail.damage/truncate.missing": { "hash": "sha256-11e2c87619ac", "other": "Jaka część całego pliku znika, w procentach. Ucięte jest co najmniej tyle - o kilka bajtów więcej tam, gdzie format nie może się skończyć dokładnie w tym miejscu. Duży plik to często głównie wypełnienie za treścią, więc treść może zostać cała - manifest mówi, co zostawiło każde cięcie." }, + "Detail.damage/truncate.missing": { "hash": "sha256-11e2c87619ac", "other": "Jaka część całego pliku znika, w procentach. Ucięte jest co najmniej tyle - o kilka bajtów więcej tam, gdzie format nie może się skończyć dokładnie w tym miejscu. Duży plik to często głównie dopełnienie za treścią, więc treść może zostać cała - manifest mówi, co zostawiło każde cięcie." }, "Detail.damage/zero-head.bytes": { "hash": "sha256-8700126f6712", "other": "Ile bajtów na początku zostaje wyzerowanych. Poniżej czterech niektóre formaty wychodzą z uszkodzeniem, na które żaden czytnik nie narzeka." }, "Detail.format/avif.height": { "hash": "sha256-4b336c279584", "other": "Jak wysoki jest obraz. Gdy pole zostaje puste, wybierany jest rozmiar, który mieści się w zamówionych bajtach." }, "Detail.format/avif.quality": { "hash": "sha256-f6e1c3e1af83", "other": "Ile szczegółów zachowuje obraz. Wyższa wartość wygląda lepiej i zajmuje więcej pliku, zostawiając mniej miejsca na dopełnienie." }, diff --git a/internal/gui/text/locale/said/pl.json b/internal/gui/text/locale/said/pl.json index 67adb0c7..0affa011 100644 --- a/internal/gui/text/locale/said/pl.json +++ b/internal/gui/text/locale/said/pl.json @@ -202,8 +202,8 @@ "engine.CrashPlanning": { "hash": "sha256-90f7d5da615f", "other": "planowania pliku" }, "engine.CrashWhile": { "hash": "sha256-071bf3d7823e", "other": "{{.While}}" }, "engine.CrashWriting": { "hash": "sha256-1e90995301d1", "other": "zapisu {{.File}}" }, - "engine.CutAfterContent": { "hash": "sha256-34a36096782c", "other": "Cięcie trafiło w wypełnienie za treścią, więc treść jest cała, a brakuje tylko końca pliku." }, - "engine.CutBeforeContent": { "hash": "sha256-ce3c81abf2d2", "other": "Cięcie trafiło w wypełnienie przed treścią, więc z treści nic nie zostało." }, + "engine.CutAfterContent": { "hash": "sha256-34a36096782c", "other": "Cięcie trafiło w dopełnienie za treścią, więc treść jest cała, a brakuje tylko końca pliku." }, + "engine.CutBeforeContent": { "hash": "sha256-ce3c81abf2d2", "other": "Cięcie trafiło w dopełnienie przed treścią, więc z treści nic nie zostało." }, "engine.DamagedAboveWhy": { "hash": "sha256-59fb1ae064c2", "other": "{{.Damage}} musi wyjść od pliku {{.Format}} o rozmiarze najwyżej {{.Maximum}}" }, "engine.DamagedBelowWhy": { "hash": "sha256-5fe7c6a1ed13", "other": "{{.Damage}} musi wyjść od pliku {{.Format}} o rozmiarze co najmniej {{.Minimum}}" }, "engine.DamagedFloorWhy": { "hash": "sha256-7829143089a2", "other": "{{.Damage}} potrzebuje co najmniej {{.Floor}}" }, diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index 75d3aad2..9924c060 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -63,6 +63,8 @@ def layout_png(data): raise NoMap(f"a chunk header runs past the end at {pos}") length = struct.unpack(">I", data[pos:pos + 4])[0] kind = data[pos + 4:pos + 8] + if pos + 12 + length > len(data): + raise NoMap(f"chunk {kind!r} at {pos} declares {length} B and runs past the end") out.append([PNG_ROLES.get(kind, STRUCTURE), pos, 12 + length]) pos += 12 + length return out