From 971b256069f23c8f399da2f9db21260d8d12f7d4 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 12 Nov 2026 00:48:18 +0100 Subject: [PATCH 1/5] A damage setting outside its range is refused, and a damaged file's label names its damage The command line and the window hand the engine a chain without a recipe, and the engine never asked whether its settings were allowed: truncate with missing=95 cut the default half, ended with code 0 and recorded 95 in the manifest. damage.Chain.Check is now asked first in prepareDamage and answers damage.SettingError, addressed targets[N].damage[M].key like the recipe reader's refusal and mapped to exit code 2. Descriptor.Check had no caller and is removed. The label of a damaged file stated the size the generator was asked to write - the whole file before a cut - and no damage. It now states the size on disk and every damage with its settings: "tfg - txt - 10485760 B - damage truncate:missing=50 - seed c2a547b2". core.Label is replaced by format.Request.LabelText and LabelSized, and the request carries Damage and OnDisk, so no format can leave the damage out. Undamaged files are byte for byte unchanged. A zero-head file with a label is not, which is the Breaking entry. The picture font gains = and ,. The office formats keep a label longer than their 64 bytes whole, and the damaged format checks the floor it names at the size on disk it leads to, because that label's length follows the digits of the size. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 14 +++ README.md | 10 +- internal/cli/errors.go | 25 ++++- internal/core/label.go | 17 --- internal/core/setting.go | 12 ++ internal/damage/apply.go | 66 ++++++++++- internal/damage/refusals.go | 38 ++++++- internal/engine/damage.go | 16 ++- internal/engine/damagedformat.go | 64 +++++++++-- internal/engine/drawsizes.go | 15 +-- internal/engine/plantarget.go | 33 ++++-- internal/format/avif/avif.go | 2 +- internal/format/bmp/bmp.go | 2 +- internal/format/docx/docx.go | 6 +- internal/format/eml/eml.go | 4 +- internal/format/format.go | 12 +- internal/format/gif/gif.go | 2 +- internal/format/htmlfile/html.go | 2 +- internal/format/ico/ico.go | 2 +- internal/format/imagelabel/font.go | 27 ++++- internal/format/jpg/jpg.go | 2 +- internal/format/jxl/jxl.go | 2 +- internal/format/label.go | 47 ++++++++ internal/format/logfile/log.go | 2 +- internal/format/md/md.go | 2 +- internal/format/opc/opc.go | 17 ++- internal/format/pdf/pdf.go | 2 +- internal/format/png/png.go | 2 +- internal/format/pptx/pptx.go | 6 +- internal/format/rtf/rtf.go | 2 +- internal/format/svgfile/svg.go | 2 +- internal/format/targz/targz.go | 4 +- internal/format/tiff/tiff.go | 2 +- internal/format/tomlfile/toml.go | 2 +- internal/format/txt/txt.go | 2 +- internal/format/video/plan.go | 2 +- internal/format/wav/wav.go | 2 +- internal/format/webp/webp.go | 2 +- internal/format/xlsx/xlsx.go | 6 +- internal/format/xmlfile/xml.go | 2 +- internal/format/yamlfile/yaml.go | 2 +- internal/format/zip/zip.go | 4 +- internal/guard/damagelabel_test.go | 140 ++++++++++++++++++++++++ internal/guard/damagelength_test.go | 13 ++- internal/guard/damagesettings_test.go | 137 +++++++++++++++++++++++ internal/guard/damagewindow_test.go | 43 ++++++++ internal/guard/damagewitness_test.go | 9 +- internal/guard/filmahead_test.go | 3 +- internal/guard/filmlayout_test.go | 3 +- internal/guard/filmtiles_test.go | 4 +- internal/guard/jxlladder_test.go | 3 +- internal/guard/labelclaim_test.go | 5 +- internal/guard/mutationcoverage_test.go | 10 ++ internal/guard/textencoding_test.go | 3 +- internal/recipe/compose.go | 2 +- 55 files changed, 731 insertions(+), 129 deletions(-) delete mode 100644 internal/core/label.go create mode 100644 internal/format/label.go create mode 100644 internal/guard/damagesettings_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index cf9cadd4..d16048b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,13 @@ because it turns other people's test suites red. was meant. Only pixels along those edges change, and many files do not change at all. A test that compares a file with one 0.5.0 generated from the same request can fail. +- **A damaged file names its damage in its label.** The label now reads + `tfg - txt - 10485760 B - damage truncate:missing=50 - seed c2a547b2`: the + size the file has on disk, then every damage with its settings, in the order + they were applied. A text cut by half and one cut by nine tenths used to begin + with the same line. A file damaged with `zero-head` and carrying a label is + therefore not byte for byte what 0.5.0 wrote. Files with no damage, and files + made with `--clean`, are unchanged. ### Added @@ -164,6 +171,13 @@ because it turns other people's test suites red. ### Fixed +- **A damage setting outside its range is refused on the command line and in + the window.** `--damage truncate:missing=95` cut the default half, wrote the + file, ended with code 0 and recorded `missing: 95` in the manifest, and + `--damage zero-head:bytes=99999999999` zeroed 8 bytes. A setting the damage + does not take, such as `truncate:foo=1`, went through as well. Now the run + ends with code 2 before anything is written and says what the setting takes, + and the window shows that under the field. A recipe was already refused. - **An AVIF picture wider than 4096 px, or larger than 4096 x 2304 px, opens now.** Every release since AVIF arrived in 0.3.0 wrote such a picture as a file no viewer reads: Pillow, ImageMagick and libaom all refused it as diff --git a/README.md b/README.md index e5ffc759..4c7795a6 100644 --- a/README.md +++ b/README.md @@ -514,9 +514,17 @@ tfg generate --format png --size 20kb --count 10 --damage zero-head:bytes=16 tfg generate --format zip --size 5mb --damage truncate:missing=30 ``` +A damaged file names its damage in its label, beside the size it has on disk: +`tfg - txt - 10485760 B - damage truncate:missing=50 - seed c2a547b2`. So two +texts cut by different shares can be told apart from their first line, although +the text after it is the same. The label is still written before the damage, so +the damage can overwrite it, and the manifest does not promise it is there. + The smallest file a damage can be given follows its settings, so the column is measured with the defaults. Ask for less and the run is refused before anything is -written, naming a size that would work. +written, naming a size that would work. A setting outside what the damage takes, +such as `truncate:missing=95`, is refused before anything is written as well, and +the run ends with code 2. Asking for `--expected accept` beside a damage that breaks the rules of the format is refused too, because nothing that follows them accepts the file. Where diff --git a/internal/cli/errors.go b/internal/cli/errors.go index 660226d4..1327d0ad 100644 --- a/internal/cli/errors.go +++ b/internal/cli/errors.go @@ -248,6 +248,21 @@ func classifyRequest(err error) (int, bool) { if errors.As(err, &unknownPreset) { return ExitUsage, true } + if code, ok := classifyDamage(err); ok { + return code, true + } + if code, ok := classifyFormat(err); ok { + return code, true + } + return 0, false +} + +// classifyDamage covers what was asked to be broken. +// +// Its own function rather than more of classifyRequest, which crossed the +// line the crowding guard draws at sixty lines when the setting refusal +// arrived. Asked before classifyFormat, for the reason the last branch gives. +func classifyDamage(err error) (int, bool) { // A damage this build does not know is a typo in the invocation, the same // class as an unknown preset - a recipe naming one is refused while the // recipe is read, so anything reaching here came off the command line. @@ -288,8 +303,14 @@ func classifyRequest(err error) (int, bool) { if errors.As(err, &unsized) { return ExitUsage, true } - if code, ok := classifyFormat(err); ok { - return code, true + // The same class once more: a setting a damage does not take, or a value + // it does not allow, is wrong for every format and size. Asked before + // classifyFormat on purpose, because the refusal it wraps is the one a + // format setting out of range ends with FORMAT for - and that one is a fact + // about the format, which this is not. + var setting *damage.SettingError + if errors.As(err, &setting) { + return ExitUsage, true } return 0, false } diff --git a/internal/core/label.go b/internal/core/label.go deleted file mode 100644 index 3f2ed6e3..00000000 --- a/internal/core/label.go +++ /dev/null @@ -1,17 +0,0 @@ -package core - -import "fmt" - -// The self describing label ends the folder full of test1.txt, test2.txt and -// test_final_v2.txt. On a screenshot attached to a bug report you can see at -// once which file it is and how to reproduce it. -// -// It stays plain ASCII on purpose. Text formats reach into encodings such as -// Windows-1252 and Shift-JIS where a typographic separator has no -// representation, and a label that fails to encode is worse than a plain one. - -// Label composes the self describing label for one file. Every format that -// carries a label uses this one, so the wording cannot drift between them. -func Label(formatID string, bytes int64, seed uint64) string { - return fmt.Sprintf("tfg - %s - %d B - seed %s", formatID, bytes, SeedLabel(seed)) -} diff --git a/internal/core/setting.go b/internal/core/setting.go index 8f6f6955..373e7756 100644 --- a/internal/core/setting.go +++ b/internal/core/setting.go @@ -116,6 +116,11 @@ const ( // that is where the box is, and a bare width would collide with a recipe // key of the same name the day one arrives. KeyProperties = "properties" + // KeyDamage is the list of damages a target applies. A refusal about one + // setting of one entry is addressed under it - targets[1].damage[2].bytes + // - by the recipe reader and the engine alike, so a report names the same + // box whichever of them refused it. + KeyDamage = "damage" ) // TargetAddress is where one setting of one target lives. @@ -152,6 +157,13 @@ func ContentAddress(target, entry int, setting string) string { return fmt.Sprintf("%s.%s[%d].%s", TargetPrefix(target), KeyContains, entry, setting) } +// DamageAddress is where one setting of one damage lives inside a target, +// without the target: the engine puts that in front when it knows which +// target it is (atTarget). The position counts from one, for the reason above. +func DamageAddress(entry int, setting string) string { + return fmt.Sprintf("%s[%d].%s", KeyDamage, entry, setting) +} + // AddressNamesATarget says whether an address carries the position of the // target it is about, rather than a setting name on its own. // diff --git a/internal/damage/apply.go b/internal/damage/apply.go index 5ee6f871..e8037f4a 100644 --- a/internal/damage/apply.go +++ b/internal/damage/apply.go @@ -4,6 +4,8 @@ import ( "fmt" "io" "sort" + "strconv" + "strings" "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" @@ -92,12 +94,30 @@ func (d Descriptor) checkOne(p format.Property, raw string) error { } } -// Check is the first problem with the values stated, for the one-target path -// from the command line flags where there is no form to lay four of them out -// on. -func (d Descriptor) Check(v Values) error { - if bad := d.CheckEach(v); len(bad) > 0 { - return bad[0] +// Check is the first problem with what this chain states: a damage this build +// does not know, or a setting its declaration does not take or does not allow. +// +// The engine asks it, for the command line and the window, which hand it a +// chain without reading a recipe. The recipe reader asks CheckEach of every +// entry and reports each problem with its address, so this is a second caller +// of the same rule rather than a copy of it - the shape O199 gave the +// expectation. The first problem only, as format.Descriptor.CheckProperties +// gives it beside this: a window draws one damage at a time, and the engine +// addresses one refusal per target. +// +// Asked before a byte is written, because a value past the declaration does +// not fail later. The settings are read with a fallback to the default +// (missingShare, junkBytes, headBytes), so without this the file would be cut +// by the default and the manifest would record the value as written. +func (c Chain) Check() error { + for i, s := range c { + d, err := Get(s.ID) + if err != nil { + return err + } + if bad := d.CheckEach(s.Values); len(bad) > 0 { + return &SettingError{Position: i + 1, Err: bad[0]} + } } return nil } @@ -347,6 +367,40 @@ func (c Chain) String() string { return fmt.Sprint(out) } +// Label is the chain as the label of a damaged file states it: every entry in +// order, its settings resolved the way the manifest records them, joined by +// " then " - "truncate:missing=50 then append-junk:bytes=1024". Empty for an +// empty chain, which is a file whose label does not change. +// +// Resolved rather than as written, for the reason Resolved gives: the label +// answers what was done to this file, and its reader should not need the +// defaults of the build that wrote it. ASCII whatever a setting holds, because +// the label goes into encodings where nothing else is sure to survive +// (format/label.go): a value with anything else in it, or with a space or one +// of the separators, is written quoted with escapes. +func (c Chain) Label() string { + out := make([]string, 0, len(c)) + for _, s := range c { + values := Values{} + for name, value := range s.Resolved() { + values[name] = plainOrQuoted(value) + } + out = append(out, Spec{ID: s.ID, Values: values}.String()) + } + return strings.Join(out, " then ") +} + +// plainOrQuoted is v as it is when every byte of it is printable ASCII other +// than the separators a label entry is written with, and quoted otherwise. +func plainOrQuoted(v string) string { + for i := 0; i < len(v); i++ { + if b := v[i]; b <= ' ' || b > '~' || b == ',' || b == '=' || b == ':' || b == '"' { + return strconv.QuoteToASCII(v) + } + } + return v +} + // Resolved is the settings this entry was applied with, defaults filled in. // // Filled in rather than left as written, because the manifest answers "what diff --git a/internal/damage/refusals.go b/internal/damage/refusals.go index 26148559..def30e23 100644 --- a/internal/damage/refusals.go +++ b/internal/damage/refusals.go @@ -1,6 +1,7 @@ package damage import ( + "errors" "strings" "github.com/donislawdev/TestingFilesGenerator/internal/core" @@ -63,6 +64,41 @@ func (e *UnknownError) Said() core.Said { core.A("What", e.what()), core.A("Known", strings.Join(e.Known, ", "))) } +// SettingError is one entry of a chain stating a setting its damage does not +// take, or a value the declaration of that setting does not allow. +// +// A wrapper around the declaration's own refusal rather than that refusal on +// its own, for two things the refusal underneath cannot know. Which entry of +// the list it is about, so a form marks the right box and the engine names the +// address the recipe reader gives for the same mistake. And its class: a value +// a damage does not allow is wrong whatever format and size are asked for, so +// the command line ends with the code a mistyped flag gets, not the code for a +// request no format can deliver - the call O199 made for an expectation. +// +// The words are the declaration's (format.Property.Allows and Instead), reached +// through Unwrap, so a damage setting and a format setting refuse alike. +type SettingError struct { + // Position is the entry's place in the chain, counted from one. + Position int + // Err is *format.PropertyValueError or *format.UnknownPropertyError. + Err error +} + +func (e *SettingError) Error() string { return e.Err.Error() } + +func (e *SettingError) Unwrap() error { return e.Err } + +// AboutSetting is the setting inside its entry - damage[1].missing. The +// engine puts the target in front, and a window showing one target keeps the +// last part, which is the name its box is registered under. +func (e *SettingError) AboutSetting() string { + var about interface{ AboutSetting() string } + if !errors.As(e.Err, &about) || about.AboutSetting() == "" { + return core.KeyDamage + } + return core.DamageAddress(e.Position, about.AboutSetting()) +} + // TooSmallError is a file smaller than the damage it was given. // // It is a refusal in the plan rather than a failure part way through, because @@ -234,7 +270,7 @@ type EatenError struct { // AboutSetting puts this on the damage, because the order of the list is the // thing to change. -func (e *EatenError) AboutSetting() string { return "damage" } +func (e *EatenError) AboutSetting() string { return core.KeyDamage } func (e *EatenError) What() string { return e.what().String() } diff --git a/internal/engine/damage.go b/internal/engine/damage.go index e6aceef6..153caba0 100644 --- a/internal/engine/damage.go +++ b/internal/engine/damage.go @@ -39,15 +39,21 @@ func checkDamage(t *Target, desc format.Descriptor) error { // prepareDamage is what a damaged target is refused for before a size is // drawn, and the format its files are planned through. // -// Before the draw, because a chain that cuts away what it added and a damage -// with no size to work from are wrong whatever size comes out of the seed. -// And the format it hands back is the one the draw has to ask: a damage that -// changes the length needs the generator asked for a different size than the -// one on disk (damagedformat.go). +// Before the draw, because a setting the damage does not allow, a chain that +// cuts away what it added and a damage with no size to work from are wrong +// whatever size comes out of the seed. And the format it hands back is the one +// the draw has to ask: a damage that changes the length needs the generator +// asked for a different size than the one on disk (damagedformat.go). +// +// The settings first, because every question after them reads the settings +// and would answer about the default where the value is out of range. func prepareDamage(t *Target, desc format.Descriptor) (format.Descriptor, error) { if len(t.Damage) == 0 { return desc, nil } + if err := t.Damage.Check(); err != nil { + return desc, err + } if eaten := t.Damage.Eaten(); eaten != nil { return desc, eaten } diff --git a/internal/engine/damagedformat.go b/internal/engine/damagedformat.go index c32d2eeb..36f8b70e 100644 --- a/internal/engine/damagedformat.go +++ b/internal/engine/damagedformat.go @@ -33,8 +33,10 @@ import ( // // For a caller that has to know before planning - a guard measuring each pair // at its smallest, or a list announcing the number. Asked of the same format a -// run plans through, so the number cannot be one the run refuses. +// run plans through, so the number cannot be one the run refuses - with the +// damage in the request, because the label names it and a run's does. func SmallestDamaged(d format.Descriptor, chain damage.Chain, r format.Request) int64 { + r.Damage = chain.Label() floor, _, err := chain.Floor() if err != nil { return d.SmallestAccepted(r) @@ -78,13 +80,9 @@ func (g *damagedGenerator) PlanContext(ctx context.Context, r format.Request) (f if err != nil { return format.Plan{}, err } - r.Bytes = want - p, err := g.plan(ctx, r) - if err != nil && g.chain.Stretches() { - p, err = g.stretch(ctx, r, err) - } + p, inner, err := g.whole(ctx, r, want, size) if err != nil { - return format.Plan{}, g.inSizeOnDisk(ctx, r, size, err) + return format.Plan{}, g.inSizeOnDisk(ctx, inner, size, err) } return format.Plan{ Bytes: size, @@ -98,6 +96,20 @@ func (g *damagedGenerator) PlanContext(ctx context.Context, r format.Request) (f }, nil } +// whole plans the file the damage starts from, want bytes long, for a file of +// size bytes on disk, and answers the request it asked with. +// +// The label inside it states the file on disk, which is size - not the size +// the generator writes, nor one stretch moves it to. +func (g *damagedGenerator) whole(ctx context.Context, r format.Request, want, size int64) (format.Plan, format.Request, error) { + r.Bytes, r.OnDisk = want, size + p, err := g.plan(ctx, r) + if err != nil && g.chain.Stretches() { + p, err = g.stretch(ctx, r, err) + } + return p, r, err +} + // plan asks the real generator, through the wrapper everything the engine asks // of a generator goes through. // @@ -215,11 +227,7 @@ func (g *damagedGenerator) firstWritable(ctx context.Context, r format.Request, func (g *damagedGenerator) inSizeOnDisk(ctx context.Context, r format.Request, size int64, err error) error { var below *format.BelowMinimumError if errors.As(err, &below) { - first := below.Minimum - if writable, werr := g.firstWritable(ctx, r, first); werr == nil { - first = writable - } - least := g.smallestReaching(first) + first, least := g.floorOnDisk(ctx, r, below.Minimum) return &format.BelowMinimumError{ Format: g.format, Requested: size, Minimum: least, Reason: core.Says("engine.DamagedBelowWhy", "%s has to start from a %s file of at least %s", @@ -238,6 +246,38 @@ func (g *damagedGenerator) inSizeOnDisk(ctx context.Context, r format.Request, s return err } +// floorOnDisk is the smallest file the damage can start from and the smallest +// size on disk that reaches it, from the size a refusal of r named. +// +// Walked to a size the generator writes first, for the reason inSizeOnDisk +// gives. Then asked again at the size on disk it comes to, because the label +// of a damaged file states its size on disk: where that label is longer than +// the room a format keeps for one, the floor moves with the number of digits. +// A refusal of 428 B measured a label one byte shorter than the one the 1284 B +// it named carries, and 1284 B was refused in turn (docx with append-junk, +// measured 2026-11-12). A few rounds settle it, because the floor only grows +// with the digits and the digits stop changing. +func (g *damagedGenerator) floorOnDisk(ctx context.Context, r format.Request, named int64) (first, least int64) { + first = named + for round := 0; round < 4; round++ { + if writable, err := g.firstWritable(ctx, r, first); err == nil { + first = writable + } + least = g.smallestReaching(first) + want, err := g.chain.Before(least) + if err != nil { + return first, least + } + _, inner, err := g.whole(ctx, r, want, least) + var below *format.BelowMinimumError + if err == nil || !errors.As(err, &below) || below.Minimum <= first { + return first, least + } + r, first = inner, below.Minimum + } + return first, least +} + // smallestReaching is the smallest size on disk whose damage needs at least // written bytes from the generator. What the damage needs grows with the size // asked for, so this is found by halving. diff --git a/internal/engine/drawsizes.go b/internal/engine/drawsizes.go index dee68a2d..75104294 100644 --- a/internal/engine/drawsizes.go +++ b/internal/engine/drawsizes.go @@ -109,12 +109,8 @@ func firstWritable(ctx context.Context, desc format.Descriptor, r format.Request // format.PlanWith asks the context before anything else, and a stop is not a // refusal about the size, so firstWritable hands it straight back. func drawSizes(ctx context.Context, t *Target, desc format.Descriptor, targetSeed uint64) error { - first := format.Request{ - Contains: t.Contains, - Seed: core.FileSeed(targetSeed, 0), - Label: t.Label, - Properties: t.Properties, - } + damaged := t.Damage.Label() + first := requestFor(t, core.FileSeed(targetSeed, 0), damaged) // The low end is judged against what this format can do AT ALL, and that // check is older than the snapping below it. The two answer different @@ -155,12 +151,7 @@ func drawSizes(ctx context.Context, t *Target, desc format.Descriptor, targetSee want = t.Range.Min + int64(r.Uint64N(span+1)) } - req := format.Request{ - Contains: t.Contains, - Seed: core.FileSeed(targetSeed, i), - Label: t.Label, - Properties: t.Properties, - } + req := requestFor(t, core.FileSeed(targetSeed, i), damaged) got, err := firstWritable(ctx, desc, req, want, t.Range.Max) if err != nil && want > t.Range.Min { diff --git a/internal/engine/plantarget.go b/internal/engine/plantarget.go index 0b5cd4d1..54458096 100644 --- a/internal/engine/plantarget.go +++ b/internal/engine/plantarget.go @@ -38,17 +38,13 @@ func (pl *planning) files(ctx context.Context, t *Target, desc format.Descriptor // One ending for every file of the target, because it follows the // target's settings and every file shares them. ext := desc.ExtensionFor(t.Properties) + damaged := t.Damage.Label() for idx, size := range t.Sizes { fileSeed := core.FileSeed(targetSeed, idx) - p, err := planWithoutCrashing(ctx, desc, format.Request{ - Bytes: size, - SizeFromContents: t.SizeFromContents, - Contains: t.Contains, - Seed: fileSeed, - Label: t.Label, - Properties: t.Properties, - }) + r := requestFor(t, fileSeed, damaged) + r.Bytes = size + p, err := planWithoutCrashing(ctx, desc, r) if err != nil { return atTarget(position, t, err) } @@ -176,3 +172,24 @@ func (pl *planning) startList(ctx context.Context) error { pl.out = make([]PlannedFile, 0, max(1, pl.reserve)) return nil } + +// requestFor is what one file of t asks its generator for, all but the size. +// +// One place for the three that ask - planning each file, and the floor and +// each draw of a size from a range - because a field one of them leaves out is +// a file drawn as one thing and planned as another. The label is where that +// shows first: it names the damage, and a label is part of the size a +// generator settles. +// +// damaged is t.Damage.Label(), asked once a target by the caller rather than +// once a file, because it resolves every entry against the registry. +func requestFor(t *Target, seed uint64, damaged string) format.Request { + return format.Request{ + SizeFromContents: t.SizeFromContents, + Contains: t.Contains, + Seed: seed, + Label: t.Label, + Damage: damaged, + Properties: t.Properties, + } +} diff --git a/internal/format/avif/avif.go b/internal/format/avif/avif.go index 1531d22b..d8a4906d 100644 --- a/internal/format/avif/avif.go +++ b/internal/format/avif/avif.go @@ -171,7 +171,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("avif", r.Bytes, r.Seed) + label = r.LabelText("avif") } q, err := quality(r.Properties) diff --git a/internal/format/bmp/bmp.go b/internal/format/bmp/bmp.go index ae27a640..485fae13 100644 --- a/internal/format/bmp/bmp.go +++ b/internal/format/bmp/bmp.go @@ -121,7 +121,7 @@ func pixelBytes(width, height int) int64 { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("bmp", r.Bytes, r.Seed) + label = r.LabelText("bmp") } w, h, err := chooseSize(r) diff --git a/internal/format/docx/docx.go b/internal/format/docx/docx.go index 81daa80a..cba77f24 100644 --- a/internal/format/docx/docx.go +++ b/internal/format/docx/docx.go @@ -88,7 +88,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("docx", r.Bytes, r.Seed) + label = r.LabelText("docx") } paragraphs, err := intProperty(r.Properties, "paragraphs", 1) @@ -197,7 +197,7 @@ func document(paragraphs int, seed uint64, label string) []byte { b.WriteString(``) b.WriteString(``) if label != "" { - b.WriteString(paragraph(opc.Text(opc.Fixed(label, opc.LabelWidth)))) + b.WriteString(paragraph(opc.Text(opc.FixedLabel(label)))) } for i := 0; i < paragraphs; i++ { b.WriteString(paragraph(opc.Line(rng, wordsPerParagraph))) @@ -233,7 +233,7 @@ func intProperty(props map[string]string, key string, fallback int) (int, error) // text of the first cell or slide, so there the labelled file is smaller. func minimumBytes() int64 { smallest := int64(1) << 62 - for _, label := range []string{"", core.Label("docx", 0, 0)} { + for _, label := range []string{"", format.Request{}.LabelText("docx")} { shape, err := opc.Measure(parts(1, 0, label)) if err != nil { continue diff --git a/internal/format/eml/eml.go b/internal/format/eml/eml.go index cd90b1ac..97b5592b 100644 --- a/internal/format/eml/eml.go +++ b/internal/format/eml/eml.go @@ -155,7 +155,7 @@ func fit(m *message, r format.Request) ([]format.Note, error) { if !r.Label { return nil, nil } - m.label = core.Label("eml", r.Bytes, r.Seed) + m.label = r.LabelText("eml") if m.size() <= r.Bytes { return nil, nil } @@ -177,7 +177,7 @@ func fit(m *message, r format.Request) ([]format.Note, error) { func settle(m *message, r format.Request) int64 { size := m.size() for round := 0; r.Label && round < 8; round++ { - m.label = core.Label("eml", size, r.Seed) + m.label = r.LabelSized("eml", size) next := m.size() if next == size { break diff --git a/internal/format/format.go b/internal/format/format.go index f746c6b6..d209eabe 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -668,8 +668,18 @@ type Request struct { // Seed determines the content. The same seed gives the same bytes. Seed uint64 // Label asks for the self describing label. On by default, turned off - // with --clean. + // with --clean. A generator composes it with LabelText (label.go). Label bool + // Damage is what the engine will break about this file once it is + // written, as the label states it - "truncate:missing=50" - or empty for + // a file nothing breaks. A generator does not apply it and reads it for + // nothing but the label, through LabelText. + Damage string + // OnDisk is the size the file has on disk once its damage has changed the + // length, where Bytes is the size of the whole file before it - a cut + // asks the generator for more than it keeps. Zero means Bytes is the size + // on disk. Read for the label alone, like Damage. + OnDisk int64 // Properties are format specific settings, straight from the recipe. Properties map[string]string } diff --git a/internal/format/gif/gif.go b/internal/format/gif/gif.go index 7d30b992..ca4f099d 100644 --- a/internal/format/gif/gif.go +++ b/internal/format/gif/gif.go @@ -189,7 +189,7 @@ func checkJointLimits(w, h int) error { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("gif", r.Bytes, r.Seed) + label = r.LabelText("gif") } m, err := chooseSize(r, label) diff --git a/internal/format/htmlfile/html.go b/internal/format/htmlfile/html.go index 5b95ae1f..a2cd0afe 100644 --- a/internal/format/htmlfile/html.go +++ b/internal/format/htmlfile/html.go @@ -181,7 +181,7 @@ func belowMinimum(s settings, requested, min int64) error { // is what a part does rather than how long it is: this answers "does the label // fit and what does it cost", and Plan answers "is this request askable at all". func labelledHead(s settings, r format.Request, min int64) (head, heading string, note *format.Note) { - label := core.Label("html", r.Bytes, r.Seed) + label := r.LabelText("html") m := s.spelling() heading = m[labelOpen] + label + m[labelClose] if s.indented() { diff --git a/internal/format/ico/ico.go b/internal/format/ico/ico.go index 8edf2c16..3fcca382 100644 --- a/internal/format/ico/ico.go +++ b/internal/format/ico/ico.go @@ -112,7 +112,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("ico", r.Bytes, r.Seed) + label = r.LabelText("ico") } embed, err := embedding(r.Properties) diff --git a/internal/format/imagelabel/font.go b/internal/format/imagelabel/font.go index 89e98c30..1fe818fa 100644 --- a/internal/format/imagelabel/font.go +++ b/internal/format/imagelabel/font.go @@ -64,18 +64,35 @@ var glyphs = map[rune]string{ '/': "..#" + "..#" + ".#." + "#.." + "#..", '+': "..." + ".#." + "###" + ".#." + "...", '#': "#.#" + "###" + "#.#" + "###" + "#.#", + // The label of a damaged file names its settings - missing=50, + // bytes=16,other=2 - and these two are how it writes them. + '=': "..." + "###" + "..." + "###" + "...", + ',': "..." + "..." + "..." + ".#." + "#..", } // unknown stands in for any character with no glyph, so a label never -// silently loses part of itself. +// silently loses part of itself. It is drawn the same as 0 and O, which is why +// HasGlyph asks the table rather than comparing with this. const unknown = "###" + "#.#" + "#.#" + "#.#" + "###" func glyph(r rune) string { - if r >= 'a' && r <= 'z' { - r -= 'a' - 'A' - } - if g, ok := glyphs[r]; ok { + if g, ok := glyphs[upper(r)]; ok { return g } return unknown } + +// HasGlyph says whether r is drawn as itself rather than as the stand-in, for +// the guard that asks it of every character a label can hold. +func HasGlyph(r rune) bool { + _, ok := glyphs[upper(r)] + return ok +} + +// upper is r in the case the table is drawn in - see the top of this file. +func upper(r rune) rune { + if r >= 'a' && r <= 'z' { + return r - ('a' - 'A') + } + return r +} diff --git a/internal/format/jpg/jpg.go b/internal/format/jpg/jpg.go index b103c7e5..c38e69f5 100644 --- a/internal/format/jpg/jpg.go +++ b/internal/format/jpg/jpg.go @@ -191,7 +191,7 @@ func segmentsFor(extra int64) (segments int, payload int64) { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("jpg", r.Bytes, r.Seed) + label = r.LabelText("jpg") } quality, err := qualityOf(r.Properties) diff --git a/internal/format/jxl/jxl.go b/internal/format/jxl/jxl.go index 0bd09a8f..b1966275 100644 --- a/internal/format/jxl/jxl.go +++ b/internal/format/jxl/jxl.go @@ -211,7 +211,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("jxl", r.Bytes, r.Seed) + label = r.LabelText("jxl") } q, err := quality(r.Properties) diff --git a/internal/format/label.go b/internal/format/label.go new file mode 100644 index 00000000..3283ea33 --- /dev/null +++ b/internal/format/label.go @@ -0,0 +1,47 @@ +package format + +import ( + "fmt" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" +) + +// The self describing label ends the folder full of test1.txt, test2.txt and +// test_final_v2.txt. On a screenshot attached to a bug report you can see at +// once which file it is and how to reproduce it. +// +// It stays plain ASCII on purpose. Text formats reach into encodings such as +// Windows-1252 and Shift-JIS where a typographic separator has no +// representation, and a label that fails to encode is worse than a plain one. +// The damage it names is ASCII for the same reason (damage.Chain.Label). +// +// A method of the request rather than a function a generator hands numbers +// to, because the request is what knows whether the file will be damaged and +// what size it keeps on disk. A generator asking anything else could only +// state the size it was asked to write, which for a cut file is the size of +// the whole file before the cut - the label of a truncated text then named a +// size nobody asked for and no damage at all (O333). + +// LabelText is the self describing label of the file r asks for. Every format +// that carries a label asks this, so the wording cannot drift between them. +func (r Request) LabelText(formatID string) string { + return r.LabelSized(formatID, r.Bytes) +} + +// LabelSized is the label of a file of size bytes, for a generator that works +// its size out from its contents and so states a size the request does not +// hold yet - zip, eml and targz settle the label and the size against each +// other. +// +// A damaged file states its size on disk wherever the request knows it, which +// is wherever a damage changes the length, and the damage before the seed. A +// file nothing damages carries the label it always carried, to the byte (D11). +func (r Request) LabelSized(formatID string, size int64) string { + if r.OnDisk > 0 { + size = r.OnDisk + } + if r.Damage == "" { + return fmt.Sprintf("tfg - %s - %d B - seed %s", formatID, size, core.SeedLabel(r.Seed)) + } + return fmt.Sprintf("tfg - %s - %d B - damage %s - seed %s", formatID, size, r.Damage, core.SeedLabel(r.Seed)) +} diff --git a/internal/format/logfile/log.go b/internal/format/logfile/log.go index 8d1eda8f..46de7069 100644 --- a/internal/format/logfile/log.go +++ b/internal/format/logfile/log.go @@ -185,7 +185,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { // label is a line of its own. It is the one line that is not an entry, // and it says so in words rather than pretending to be one - except in // JSON lines, where a comment is not a line any reader would take. - line := opt.shape.label(core.Label("log", r.Bytes, r.Seed), opt) + line := opt.shape.label(r.LabelText("log"), opt) // It has to leave room for at least one whole entry, or the file would // be a label and nothing else. if int64(len(line))+min <= r.Bytes { diff --git a/internal/format/md/md.go b/internal/format/md/md.go index 6d9f883c..2b81a394 100644 --- a/internal/format/md/md.go +++ b/internal/format/md/md.go @@ -119,7 +119,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { if r.Label { // A plain line followed by a blank one is a paragraph, which renders // visibly and cannot break the document wherever it sits. - line := core.Label("md", r.Bytes, r.Seed) + "\n\n" + line := r.LabelText("md") + "\n\n" if int64(len(line)) <= m.source { m.labelLine = line } else { diff --git a/internal/format/opc/opc.go b/internal/format/opc/opc.go index 8a9b7ae3..1b7b7ab0 100644 --- a/internal/format/opc/opc.go +++ b/internal/format/opc/opc.go @@ -453,11 +453,22 @@ func Line(rng *rand.Rand, _ int) string { return Fixed(b.String(), LineWidth) } -// LabelWidth is the room a label gets. Wider than any label core.Label can -// produce, so the text is never cut, and constant so the floor does not move -// with the number of digits in the size. +// LabelWidth is the room a label gets. Wider than the label of any file +// nothing damages, so that text is never cut, and constant so the floor does +// not move with the number of digits in the size. const LabelWidth = 64 +// FixedLabel is label padded to LabelWidth, or whole where it is longer. +// +// Longer is the label of a damaged file, which names its damage. It keeps its +// own length rather than being cut to the width, because a label missing the +// end of its seed points at a file nobody can make again - and the floor that +// then moves with the digits is the floor of a damaged file, whose generator +// is asked with the same label it writes, so it names that floor itself. +func FixedLabel(label string) string { + return Fixed(label, max(LabelWidth, len(label))) +} + // Fixed cuts or pads text to exactly n bytes. // // Padding is spaces, which XML keeps when the element asks it to and which diff --git a/internal/format/pdf/pdf.go b/internal/format/pdf/pdf.go index 864ed24b..c73dd125 100644 --- a/internal/format/pdf/pdf.go +++ b/internal/format/pdf/pdf.go @@ -62,7 +62,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("pdf", r.Bytes, r.Seed) + label = r.LabelText("pdf") } m := memo{opts: opts, seed: r.Seed, label: label} diff --git a/internal/format/png/png.go b/internal/format/png/png.go index 01daba80..09fdb501 100644 --- a/internal/format/png/png.go +++ b/internal/format/png/png.go @@ -159,7 +159,7 @@ func checkJointLimits(w, h int) error { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("png", r.Bytes, r.Seed) + label = r.LabelText("png") } m, err := chooseSize(r, label) diff --git a/internal/format/pptx/pptx.go b/internal/format/pptx/pptx.go index 7c7af6bf..0fa32e31 100644 --- a/internal/format/pptx/pptx.go +++ b/internal/format/pptx/pptx.go @@ -92,7 +92,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("pptx", r.Bytes, r.Seed) + label = r.LabelText("pptx") } slides, err := intProperty(r.Properties, "slides", 1) @@ -210,7 +210,7 @@ func parts(slides int, seed uint64, label string) []opc.Part { for i := 1; i <= slides; i++ { text := opc.Line(rng, 6) if i == 1 && label != "" { - text = opc.Text(opc.Fixed(label, opc.LabelWidth)) + text = opc.Text(opc.FixedLabel(label)) } out = append(out, opc.Part{ @@ -347,7 +347,7 @@ func intProperty(props map[string]string, key string, fallback int) (int, error) // document the label is an extra paragraph and it goes the other way. func minimumBytes() int64 { smallest := int64(1) << 62 - for _, label := range []string{"", core.Label("pptx", 0, 0)} { + for _, label := range []string{"", format.Request{}.LabelText("pptx")} { shape, err := opc.Measure(parts(1, 0, label)) if err != nil { continue diff --git a/internal/format/rtf/rtf.go b/internal/format/rtf/rtf.go index 3f2749ec..36254ea3 100644 --- a/internal/format/rtf/rtf.go +++ b/internal/format/rtf/rtf.go @@ -93,7 +93,7 @@ func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan describePictures(p.Properties, s.pictures, pictures) m := memo{settings: s, seed: r.Seed, pictures: pictures, pictureBytes: pictureBytes} if r.Label { - line := labelLine(core.Label("rtf", r.Bytes, r.Seed)) + line := labelLine(r.LabelText("rtf")) if min+int64(len(line)) <= r.Bytes { m.labelLine = line } else { diff --git a/internal/format/svgfile/svg.go b/internal/format/svgfile/svg.go index 8ef54923..91c2aadf 100644 --- a/internal/format/svgfile/svg.go +++ b/internal/format/svgfile/svg.go @@ -282,7 +282,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { m := memo{seed: r.Seed, width: w, height: h} if r.Label { - switch line := labelOpen(h) + core.Label("svg", r.Bytes, r.Seed) + textClose; { + switch line := labelOpen(h) + r.LabelText("svg") + textClose; { case !labelFits(h): // Room on the page rather than room in the byte count, so it is a // separate sentence from the one below. Both leave the file named diff --git a/internal/format/targz/targz.go b/internal/format/targz/targz.go index fd4715dd..ba751886 100644 --- a/internal/format/targz/targz.go +++ b/internal/format/targz/targz.go @@ -301,7 +301,7 @@ func planChildren(ctx context.Context, r format.Request, groups []format.Content // label. The two settle against each other the same way they do in ZIP. func settleSize(m *memo, r format.Request) (target int64, label string, err error) { if r.Label && !r.SizeFromContents { - label = core.Label("targz", r.Bytes, r.Seed) + label = r.LabelText("targz") } m.comment = label @@ -320,7 +320,7 @@ func settleSize(m *memo, r format.Request) (target int64, label string, err erro // the wrong size is worse than the work of finding out. size := bare for i := 0; i < 4; i++ { - m.comment = core.Label("targz", size, r.Seed) + m.comment = r.LabelSized("targz", size) measured := archiveSize(*m) if measured == size { return size, m.comment, nil diff --git a/internal/format/tiff/tiff.go b/internal/format/tiff/tiff.go index 0c0f0f94..b6a1eaf5 100644 --- a/internal/format/tiff/tiff.go +++ b/internal/format/tiff/tiff.go @@ -161,7 +161,7 @@ func pixelBytes(width, height int) int64 { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("tiff", r.Bytes, r.Seed) + label = r.LabelText("tiff") } w, h, err := chooseSize(r) diff --git a/internal/format/tomlfile/toml.go b/internal/format/tomlfile/toml.go index ff433935..cb081720 100644 --- a/internal/format/tomlfile/toml.go +++ b/internal/format/tomlfile/toml.go @@ -161,7 +161,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { m := memo{seed: r.Seed} if r.Label { - line := "# " + core.Label("toml", r.Bytes, r.Seed) + "\n" + line := "# " + r.LabelText("toml") + "\n" if int64(len(line))+min <= r.Bytes { m.comment = line } else { diff --git a/internal/format/txt/txt.go b/internal/format/txt/txt.go index b867b907..3dca0252 100644 --- a/internal/format/txt/txt.go +++ b/internal/format/txt/txt.go @@ -112,7 +112,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { m := memo{seed: r.Seed, codec: codec, source: codec.Source(r.Bytes)} if r.Label { - line := core.Label("txt", r.Bytes, r.Seed) + "\n" + line := r.LabelText("txt") + "\n" if int64(len(line)) <= m.source { m.labelLine = line } else { diff --git a/internal/format/video/plan.go b/internal/format/video/plan.go index 70ca5d52..7af9bed1 100644 --- a/internal/format/video/plan.go +++ b/internal/format/video/plan.go @@ -30,7 +30,7 @@ func (f Film) Stream() Stream { func Plan(formatID string, r format.Request, need func(Stream) int64, refuse func(requested, need int64, c Choice, s Settings) error) (format.Plan, error) { label := "" if r.Label { - label = core.Label(formatID, r.Bytes, r.Seed) + label = r.LabelText(formatID) } s, err := Read(formatID, r.Properties) if err != nil { diff --git a/internal/format/wav/wav.go b/internal/format/wav/wav.go index fc4c6f12..79f5dcb0 100644 --- a/internal/format/wav/wav.go +++ b/internal/format/wav/wav.go @@ -231,7 +231,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } if r.Label { - m.info = infoChunk(core.Label("wav", r.Bytes, r.Seed)) + m.info = infoChunk(r.LabelText("wav")) } fixed := int64(baseSize) + int64(len(m.info)) diff --git a/internal/format/webp/webp.go b/internal/format/webp/webp.go index 0ed89756..91259b14 100644 --- a/internal/format/webp/webp.go +++ b/internal/format/webp/webp.go @@ -137,7 +137,7 @@ func bareBytes(width, height int) int64 { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("webp", r.Bytes, r.Seed) + label = r.LabelText("webp") } w, h, err := chooseSize(r) diff --git a/internal/format/xlsx/xlsx.go b/internal/format/xlsx/xlsx.go index df7174f0..fa88e045 100644 --- a/internal/format/xlsx/xlsx.go +++ b/internal/format/xlsx/xlsx.go @@ -114,7 +114,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("xlsx", r.Bytes, r.Seed) + label = r.LabelText("xlsx") } rows, err := intProperty(r.Properties, "rows", 1, minRows, maxRows) @@ -252,7 +252,7 @@ func sheet(rows, columns int, seed uint64, label string) []byte { // The label goes in the first cell, which is where a person looks // and where a spreadsheet reader shows it without scrolling. if row == 1 && col == 1 && label != "" { - text = opc.Text(opc.Fixed(label, opc.LabelWidth)) + text = opc.Text(opc.FixedLabel(label)) } b.WriteString(``) b.WriteString(text) @@ -298,7 +298,7 @@ func intProperty(props map[string]string, key string, fallback, min, max int) (i // document the label is an extra paragraph and it goes the other way. func minimumBytes() int64 { smallest := int64(1) << 62 - for _, label := range []string{"", core.Label("xlsx", 0, 0)} { + for _, label := range []string{"", format.Request{}.LabelText("xlsx")} { shape, err := opc.Measure(parts(1, 1, 0, label)) if err != nil { continue diff --git a/internal/format/xmlfile/xml.go b/internal/format/xmlfile/xml.go index 72bcf384..c82bb8e3 100644 --- a/internal/format/xmlfile/xml.go +++ b/internal/format/xmlfile/xml.go @@ -173,7 +173,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { // A comment carries the label without touching the content, and it can // sit anywhere after the declaration. The label text uses spaced // hyphens and never a double one, which a comment may not contain. - line := "\n" + line := "\n" // It has to leave room for a whole document beside it, or the file would // be a comment and an empty root. The number is what the comment COSTS // in this encoding, not how long it is to read - in UTF-16 those differ diff --git a/internal/format/yamlfile/yaml.go b/internal/format/yamlfile/yaml.go index 19a8c75b..08432368 100644 --- a/internal/format/yamlfile/yaml.go +++ b/internal/format/yamlfile/yaml.go @@ -179,7 +179,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { // sits in front of the document where a person opening the file reads // it first. Nothing in the label needs escaping: it is plain ASCII and // a comment runs to the end of the line. - line := "# " + core.Label("yaml", r.Bytes, r.Seed) + "\n" + line := "# " + r.LabelText("yaml") + "\n" // It has to leave room for a whole document beside it, or the file // would be a comment and a root key with nothing under it. if int64(len(line))+min <= r.Bytes { diff --git a/internal/format/zip/zip.go b/internal/format/zip/zip.go index acf4dde1..7bfb4cf0 100644 --- a/internal/format/zip/zip.go +++ b/internal/format/zip/zip.go @@ -244,7 +244,7 @@ func withinZip32(total int64) error { // comment, then settle the label, then measure again. func settleSize(m *memo, r format.Request) (target, bare int64, label string, err error) { if r.Label && !r.SizeFromContents { - label = core.Label("zip", r.Bytes, r.Seed) + label = r.LabelText("zip") } m.comment = label @@ -270,7 +270,7 @@ func settleSize(m *memo, r format.Request) (target, bare int64, label string, er size := bare settled := false for i := 0; i < 4 && !settled; i++ { - m.comment = core.Label("zip", size, r.Seed) + m.comment = r.LabelSized("zip", size) measured, err := archiveSize(*m) if err != nil { return 0, 0, "", err diff --git a/internal/guard/damagelabel_test.go b/internal/guard/damagelabel_test.go index 53251924..b5cb8c54 100644 --- a/internal/guard/damagelabel_test.go +++ b/internal/guard/damagelabel_test.go @@ -1,13 +1,20 @@ package guard import ( + "bytes" + "fmt" + "os" "path/filepath" "slices" + "strconv" + "strings" "testing" + "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/damage" "github.com/donislawdev/TestingFilesGenerator/internal/engine" "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" "github.com/donislawdev/TestingFilesGenerator/internal/manifest" ) @@ -80,3 +87,136 @@ func TestADamagedFileNeverClaimsItsLabel(t *testing.T) { } t.Logf("the claim was taken away in %d format(s)", withdrawn) } + +// TestADamagedFileStatesItsSizeOnDiskAndItsDamage holds the label of a damaged +// file to what is on disk. +// +// Measured 2026-11-11 before this held (O333): a text of 10 MiB cut with +// missing=50 said "20971520 B" in its first line - the size of the whole file +// before the cut - and named no damage, so a file cut by half and one cut by +// nine tenths looked the same from the inside. append-junk stated the size +// without the junk. +// +// Every format whose healthy file carries its label as text is asked, with +// junk appended: that damage keeps every byte the generator wrote, so the +// label is where the generator put it and can be read back. Which formats +// those are is read from the healthy file, not listed here - a picture draws +// its label in pixels and is left to the guards on pictures. The expected text +// is written out here rather than asked of format.Request, so a wording that +// drops the damage cannot agree with itself. +func TestADamagedFileStatesItsSizeOnDiskAndItsDamage(t *testing.T) { + chain := damage.Chain{{ID: damage.AppendJunk}} + const size = witnessSize + before, err := chain.Before(size) + if err != nil { + t.Fatal(err) + } + brokenSeed := core.FileSeed(core.TargetSeed(lengthSeed, "broken"), 0) + // The canary: the label this guard was written against, which states the + // size the generator wrote and no damage, must not pass it. + planted := fmt.Sprintf("tfg - txt - %d B - seed %s", before, core.SeedLabel(brokenSeed)) + if statesItsDamage([]byte(planted), "txt", size, chain, brokenSeed) == "" { + t.Fatal("the check took a label naming the size before the damage, and no damage, for the label it asks for") + } + + dir := t.TempDir() + read := 0 + for _, f := range format.All() { + good := engine.Target{ID: "good", Format: f.ID, Sizes: engine.Uniform(1, size), Label: true} + broken := engine.Target{ID: "broken", Format: f.ID, Sizes: engine.Uniform(1, size), Damage: chain, Label: true} + _, paths := lengthRun(t, filepath.Join(dir, f.ID), []engine.Target{good, broken}) + healthy, err := os.ReadFile(paths["good"]) + if err != nil { + t.Fatal(err) + } + goodSeed := core.FileSeed(core.TargetSeed(lengthSeed, "good"), 0) + if !bytes.Contains(healthy, fmt.Appendf(nil, "tfg - %s - %d B - seed %s", f.ID, size, core.SeedLabel(goodSeed))) { + continue + } + got, err := os.ReadFile(paths["broken"]) + if err != nil { + t.Fatal(err) + } + if why := statesItsDamage(got, f.ID, size, chain, brokenSeed); why != "" { + t.Errorf("%s: %s", f.ID, why) + } + read++ + } + if read < 10 { + t.Fatalf("only %d format(s) carry their label as text the guard could find - it did not reach the formats it is about", read) + } + t.Logf("the label was read back from %d format(s)", read) +} + +// TestATruncatedTextSaysHowMuchWasCut is the case the owner met: two texts of +// the same size cut by different shares used to begin with the same line. +// Their labels are the first thing a person opening them sees, so the share +// has to be there. +func TestATruncatedTextSaysHowMuchWasCut(t *testing.T) { + const size = 64 << 10 + seed := core.FileSeed(core.TargetSeed(lengthSeed, "cut"), 0) + for _, missing := range []string{"50", "90"} { + chain := damage.Chain{{ID: damage.Truncate, Values: damage.Values{damage.SettingMissing: missing}}} + target := engine.Target{ID: "cut", Format: "txt", Sizes: engine.Uniform(1, size), Damage: chain, Label: true} + _, paths := lengthRun(t, filepath.Join(t.TempDir(), missing), []engine.Target{target}) + got, err := os.ReadFile(paths["cut"]) + if err != nil { + t.Fatal(err) + } + first, _, _ := bytes.Cut(got, []byte("\n")) + want := fmt.Sprintf("tfg - txt - %d B - damage truncate:missing=%s - seed %s", size, missing, core.SeedLabel(seed)) + if string(first) != want { + t.Errorf("a text cut with missing=%s begins %q, expected %q", missing, first, want) + } + } +} + +// statesItsDamage says what is wrong with the label in got, or nothing when +// it states size on disk and every entry of chain with its settings resolved. +func statesItsDamage(got []byte, id string, size int64, chain damage.Chain, seed uint64) string { + want := fmt.Sprintf("tfg - %s - %d B - damage %s - seed %s", id, size, chain.Label(), core.SeedLabel(seed)) + if !strings.Contains(chain.Label(), "=") { + return "the chain " + chain.Label() + " does not name its settings, so the label cannot say how much was broken" + } + if !bytes.Contains(got, []byte(want)) { + return fmt.Sprintf("the file does not carry the label %q", want) + } + return "" +} + +// TestEveryCharacterOfADamagedLabelCanBeDrawn holds the label a picture draws +// to the characters its font has. +// +// The font is 3 by 5 pixels and drawn by hand, and a character it lacks comes +// out as a box the shape of a 0. Before the label named a damage it never held +// an = or a comma, and the font had neither - a picture cut with missing=50 +// would have said MISSING050. Every damage from the registry, at each end of +// every setting it takes, so a damage added with a new character in its name +// or settings turns this red. +func TestEveryCharacterOfADamagedLabelCanBeDrawn(t *testing.T) { + if imagelabel.HasGlyph('~') { + t.Fatal("the font draws ~, which it never had - the question below cannot tell a glyph from the stand-in") + } + asked := 0 + for _, d := range damage.All() { + values := []damage.Values{{}} + for _, p := range d.Parameters { + values = append(values, + damage.Values{p.Name: strconv.FormatInt(p.Min, 10)}, + damage.Values{p.Name: strconv.FormatInt(p.Max, 10)}) + } + for _, v := range values { + chain := damage.Chain{{ID: d.ID, Values: v}, {ID: d.ID, Values: v}} + label := format.Request{Bytes: 1 << 40, Seed: 1, Damage: chain.Label()}.LabelText("png") + for _, r := range label { + if !imagelabel.HasGlyph(r) { + t.Errorf("%s: the picture font has no %q, so the label %q is drawn with a box in its place", d.ID, r, label) + } + } + asked++ + } + } + if asked == 0 { + t.Fatal("no damage was asked - the registry was not read") + } +} diff --git a/internal/guard/damagelength_test.go b/internal/guard/damagelength_test.go index b8fe7f77..13b6991e 100644 --- a/internal/guard/damagelength_test.go +++ b/internal/guard/damagelength_test.go @@ -214,8 +214,12 @@ func TestATruncatedFileIsTheStartOfTheWholeOne(t *testing.T) { } compared := 0 for _, f := range format.All() { - cut := engine.Target{ID: "same", Format: f.ID, Sizes: engine.Uniform(1, size), Damage: chain, Label: true} - full := engine.Target{ID: "same", Format: f.ID, Sizes: engine.Uniform(1, whole), Label: true} + // Without the label, because a cut file's label states the size it + // keeps and the damage, and the whole file's states neither - the + // two differ on purpose (TestADamagedFileStatesItsSizeOnDiskAndItsDamage), + // and what this guard holds is every byte under them. + cut := engine.Target{ID: "same", Format: f.ID, Sizes: engine.Uniform(1, size), Damage: chain} + full := engine.Target{ID: "same", Format: f.ID, Sizes: engine.Uniform(1, whole)} if _, err := engine.Plan([]engine.Target{full}, engine.Options{OutDir: t.TempDir(), Seed: lengthSeed}); err != nil { // A size this format cannot write exactly: the cut started from // the next one it can, which is a different whole file. @@ -260,8 +264,9 @@ func TestAppendedJunkIsTheWholeFileAndThenTheSameBytesEveryTime(t *testing.T) { } junk := size - before - broken := engine.Target{ID: "same", Format: "txt", Sizes: engine.Uniform(1, size), Damage: chain, Label: true} - whole := engine.Target{ID: "same", Format: "txt", Sizes: engine.Uniform(1, before), Label: true} + // Without the label, for the reason the guard above gives. + broken := engine.Target{ID: "same", Format: "txt", Sizes: engine.Uniform(1, size), Damage: chain} + whole := engine.Target{ID: "same", Format: "txt", Sizes: engine.Uniform(1, before)} _, brokenPath := lengthRun(t, filepath.Join(dir, "broken"), []engine.Target{broken}) _, wholePath := lengthRun(t, filepath.Join(dir, "whole"), []engine.Target{whole}) got, err := os.ReadFile(brokenPath["same"]) diff --git a/internal/guard/damagesettings_test.go b/internal/guard/damagesettings_test.go new file mode 100644 index 00000000..854d54b6 --- /dev/null +++ b/internal/guard/damagesettings_test.go @@ -0,0 +1,137 @@ +package guard + +import ( + "bytes" + "context" + "errors" + "os" + "strconv" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/cli" + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/damage" + "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// A damage setting its declaration does not allow is refused by the engine, +// for the command line and the window, which hand it a chain without a recipe. +// +// Measured 2026-11-11 before this held: truncate:missing=95, append-junk with +// bytes=65537 and zero-head with bytes=99999999999 each ended with code 0, the +// file was cut or zeroed by the default, and the manifest recorded the value +// as written (O332). The recipe reader refused all of them, so the rule +// existed - the paths without a recipe never reached it. +// +// Every damage and every setting comes from the registry, with values just +// outside the declared range, one that is not a value at all, and a setting the +// damage does not take. Each is asked of the engine, which must refuse it at +// the address the recipe reader gives (the damage is second in its chain, so +// a position read off the wrong entry shows), and of the command line, which +// must end with USAGE, name the setting and write nothing. +// +// The control is the default of every setting in the same chain: planned. A +// build refusing every damage setting whatever its value would pass the half +// above, and this half is what tells the two apart. +func TestEveryDamageSettingItsDeclarationForbidsIsRefusedWithoutARecipe(t *testing.T) { + // The canary: the check below on two answers it must not take - nothing + // refused, and the right refusal about the first entry of the chain. + planted := &damage.SettingError{Position: 1, Err: &format.PropertyValueError{Format: damage.ZeroHead, Key: damage.SettingBytes}} + if refusedAsTheSetting(nil, damage.SettingBytes) == "" || refusedAsTheSetting(planted, damage.SettingBytes) == "" { + t.Fatal("the check took a plan that went through, or a refusal about the wrong entry, for the refusal it asks for") + } + + asked := 0 + for _, d := range damage.All() { + type stated struct{ name, value string } + cases := []stated{{"no-such-setting", "1"}} + for _, p := range d.Parameters { + for _, bad := range valuesTheDeclarationForbids(p) { + cases = append(cases, stated{p.Name, bad}) + } + } + for _, c := range cases { + spec := damage.Spec{ID: d.ID, Values: damage.Values{c.name: c.value}} + if why := refusedAsTheSetting(planSecondInAChain(t, spec), c.name); why != "" { + t.Errorf("%s %s", spec, why) + } + code, said, files := damageFromTheCommandLine(t, spec.String()) + if code != cli.ExitUsage || files != 0 || !strings.Contains(said, c.name) { + t.Errorf("--damage %s ended with %d and wrote %d file(s), expected %d and none, with the setting named: %s", + spec, code, files, cli.ExitUsage, said) + } + asked++ + } + for _, p := range d.Parameters { + spec := damage.Spec{ID: d.ID, Values: damage.Values{p.Name: p.Default}} + if err := planSecondInAChain(t, spec); err != nil { + t.Errorf("%s states the declared default and was refused - the rule refuses what it should let through: %v", spec, err) + } + } + } + if asked < len(damage.All())*2 { + t.Fatalf("only %d value(s) were asked across %d damage(s) - the registry was not read", asked, len(damage.All())) + } +} + +// valuesTheDeclarationForbids is what a person can type that p does not take: +// one step past each end of a bounded number, and a word where a value is +// expected. Free text has none. +func valuesTheDeclarationForbids(p format.Property) []string { + switch p.Kind { + case format.PropertyInt: + out := []string{"abc"} + if p.Min != 0 || p.Max != 0 { + out = append(out, strconv.FormatInt(p.Min-1, 10), strconv.FormatInt(p.Max+1, 10)) + } + return out + case format.PropertyText: + return nil + } + return []string{"no-such-value"} +} + +// planSecondInAChain plans a txt file damaged by zero-head and then spec. +func planSecondInAChain(t *testing.T, spec damage.Spec) error { + t.Helper() + target := engine.Target{ID: "t", Format: "txt", Sizes: engine.Uniform(1, 64<<10), Label: true, + Damage: damage.Chain{{ID: damage.ZeroHead}, spec}} + _, err := engine.Plan([]engine.Target{target}, engine.Options{OutDir: t.TempDir(), Seed: 1}) + return err +} + +// refusedAsTheSetting says what is wrong with err as the refusal of setting in +// the second entry of a chain, or nothing when that is what it is. +func refusedAsTheSetting(err error, setting string) string { + if err == nil { + return "was planned" + } + var wrong *damage.SettingError + if !errors.As(err, &wrong) { + return "was refused, but not as a damage setting: " + err.Error() + } + var about interface{ AboutSetting() string } + want := core.TargetAddress(1, core.DamageAddress(2, setting)) + if !errors.As(err, &about) || about.AboutSetting() != want { + return "was refused somewhere other than " + want + ", where the recipe reader puts the same mistake: " + err.Error() + } + return "" +} + +// damageFromTheCommandLine runs generate with one --damage and says how it +// ended, what it said and how many files it left. +func damageFromTheCommandLine(t *testing.T, spec string) (int, string, int) { + t.Helper() + dir := t.TempDir() + var out, errOut bytes.Buffer + code := cli.Run(context.Background(), []string{ + "generate", "--format", "txt", "--size", "64kb", "--damage", spec, "--out", dir, + }, &out, &errOut) + written, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("reading the output directory: %v", err) + } + return code, errOut.String(), len(written) +} diff --git a/internal/guard/damagewindow_test.go b/internal/guard/damagewindow_test.go index ba7e2d30..9bac2adf 100644 --- a/internal/guard/damagewindow_test.go +++ b/internal/guard/damagewindow_test.go @@ -267,6 +267,49 @@ func TestARunFromTheWindowIsReallyDamaged(t *testing.T) { } } +// A damage setting outside its range, typed into the window, is refused under +// its own box and writes nothing. +// +// The window hands the engine a chain without a recipe, so until 2026-11-12 it +// sent bytes=99999 on, the engine zeroed the default eight and the manifest +// recorded 99999 (O332). Pressed rather than settled, for the reason the guard +// above gives, and asked of the box the value was typed into: a refusal at the +// foot of the form would stop the run and leave a person looking for the box. +func TestADamageSettingOutsideItsRangeIsRefusedUnderItsBox(t *testing.T) { + dir := t.TempDir() + + host := newFakeHost(t) + gen := window.NewGenerate(host) + content := gen.Object() + t.Cleanup(func() { join(host) }) + + fields := gen.Fields() + chooserIn(t, fields, "format").SetSelected("png") + setBox(t, fields, "size", "20kb") + + chooser, ok := controlUnder(content, text.FieldDamage()).(*parts.Chooser) + if !ok { + t.Fatalf("the damage field is not a list to choose from") + } + chooser.SetSelected(damage.ZeroHead) + setBox(t, fields, damage.SettingBytes, "99999") + if got := boxText(t, fields, damage.SettingBytes); got != "99999" { + t.Fatalf("the box holds %q, so there is no refusal to cause", got) + } + + entryUnder(t, content, text.FieldOutputDir()).SetText(dir) + press(t, content, text.ButtonGenerate()) + join(host) + + if made, _ := filepath.Glob(filepath.Join(dir, "*")); len(made) != 0 { + t.Errorf("a refused run left %v behind", made) + } + said := fields.Lookup(damage.SettingBytes).Saying() + if !strings.Contains(said, "99999") { + t.Errorf("the box that holds 99999 says %q - the refusal went somewhere else. Marked: %v", said, fields.Marked()) + } +} + // damageManifest reads a manifest as the manifest package spells it. // // Its own reader rather than the manifestShape the recipe guards share: that diff --git a/internal/guard/damagewitness_test.go b/internal/guard/damagewitness_test.go index eb9b8c5f..ea0f6236 100644 --- a/internal/guard/damagewitness_test.go +++ b/internal/guard/damagewitness_test.go @@ -66,8 +66,13 @@ var policyRefusals = map[[2]string][]string{ {damage.Truncate, "html"}: {"was ordered and the page is not", "the document does not end with a closing"}, {damage.Truncate, "md"}: {"so one block is never closed"}, {damage.Truncate, "eml"}: {"the message has no blank line after its header lines", "layout=newsletter and the plain text", - // A rule the cut breaks only when it falls between a CR and its LF. - "a carriage return or a line feed on its own"}, + // A multipart message cut inside its body has no closing boundary, + // which RFC 2046 asks for and readers skip: Python's email reads such + // a cut multipart/alternative with its text part whole and notes a + // CloseBoundaryNotFoundDefect (measured 2026-11-12). Until the label + // of a damaged file named its damage, the cut at this size fell + // between a CR and its LF, and that reason came first. + "does not open with its boundary or has no closing one"}, {damage.Truncate, "txt"}: {}, // Bytes past the format's own end, which its rules forbid and readers skip. {damage.AppendJunk, "png"}: {"declares"}, diff --git a/internal/guard/filmahead_test.go b/internal/guard/filmahead_test.go index af41a92f..813d6a83 100644 --- a/internal/guard/filmahead_test.go +++ b/internal/guard/filmahead_test.go @@ -8,7 +8,6 @@ import ( "slices" "testing" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/engine" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/video" @@ -40,7 +39,7 @@ func TestAFilmPaintsOnlyWhatChangesAndGetsTheWholePicture(t *testing.T) { } timelines[name] = tl } - labels := map[string]string{"labelled": core.Label("webm", 1<<20, 7), "unlabelled": ""} + labels := map[string]string{"labelled": format.Request{Bytes: 1 << 20, Seed: 7}.LabelText("webm"), "unlabelled": ""} compared, clocked, moved := 0, 0, 0 for _, size := range sizes { diff --git a/internal/guard/filmlayout_test.go b/internal/guard/filmlayout_test.go index c88f054b..d1d679b3 100644 --- a/internal/guard/filmlayout_test.go +++ b/internal/guard/filmlayout_test.go @@ -4,7 +4,6 @@ import ( "fmt" "testing" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/video" ) @@ -66,7 +65,7 @@ func TestEveryFilmLayoutIsOneTheAV1SpecificationAllows(t *testing.T) { t.Fatal(err) } } - labels := [2]string{core.Label("webm", 268435456, 7741), ""} + labels := [2]string{format.Request{Bytes: 268435456, Seed: 7741}.LabelText("webm"), ""} seen := layoutsSeen{} n := 0 for _, w := range sides { diff --git a/internal/guard/filmtiles_test.go b/internal/guard/filmtiles_test.go index 192ef12f..d6ed15a4 100644 --- a/internal/guard/filmtiles_test.go +++ b/internal/guard/filmtiles_test.go @@ -10,8 +10,8 @@ import ( "github.com/gen2brain/gav1d/av1" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/video" ) @@ -333,7 +333,7 @@ func tilesOfFilmHold(t *testing.T, props map[string]string, size int64, changes if err != nil { t.Fatal(err) } - label := core.Label("webm", size, seed) + label := format.Request{Bytes: size, Seed: seed}.LabelText("webm") var dec av1.Decoder if _, err := dec.DecodeOBUs(append([]byte{0x12, 0x00}, f.blocks[0].data...)); err != nil { t.Fatalf("%v: gav1d does not decode the film's first frame: %v", props, err) diff --git a/internal/guard/jxlladder_test.go b/internal/guard/jxlladder_test.go index b851fb01..d2b43f57 100644 --- a/internal/guard/jxlladder_test.go +++ b/internal/guard/jxlladder_test.go @@ -5,7 +5,6 @@ import ( "runtime" "testing" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" "github.com/donislawdev/TestingFilesGenerator/internal/format/jxl" @@ -123,7 +122,7 @@ func TestEveryJxlRungStillFitsInsideItsDeclaredCeiling(t *testing.T) { for _, withLabel := range []bool{true, false} { label := "" if withLabel { - label = core.Label("jxl", want, seed) + label = format.Request{Bytes: want, Seed: seed}.LabelText("jxl") } n, err := jxl.FileSizeFor(w, h, jxl.DefaultQuality, seed, label) if err != nil { diff --git a/internal/guard/labelclaim_test.go b/internal/guard/labelclaim_test.go index 5549de11..26bd4cb5 100644 --- a/internal/guard/labelclaim_test.go +++ b/internal/guard/labelclaim_test.go @@ -5,7 +5,6 @@ import ( "strconv" "testing" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" "github.com/donislawdev/TestingFilesGenerator/internal/format/video" @@ -77,7 +76,7 @@ func TestAPictureSaysItCarriesItsLabelOnlyWhenItsTextIsWhole(t *testing.T) { // drawn at. The guard above asks the lowest height and one with room, and an // answer one row out at the boundary passed it (found by review, 2026-10-27). func TestALabelIsReadableFromTheFirstHeightThatHoldsItsText(t *testing.T) { - label := core.Label("bmp", 2048, 1) + label := format.Request{Bytes: 2048, Seed: 1}.LabelText("bmp") across := len(label)*4 + 4 // a glyph and its gap is four columns at scale 1, with the padding either side for scale := 1; scale <= 4; scale++ { width := across * scale @@ -118,7 +117,7 @@ func TestALabelIsReadableFromTheFirstHeightThatHoldsItsText(t *testing.T) { // edges it can move at, and each series has to give both answers. func TestAFilmSaysItCarriesItsLabelOnlyWhenAPictureShowsItWhole(t *testing.T) { const seed, tall = 7, 256 - label := core.Label("webm", 1<<20, seed) + label := format.Request{Bytes: 1 << 20, Seed: seed}.LabelText("webm") across := len(label)*4 + 4 // the width that draws it at scale 1, as in the guard above timeline := func(changes int64) video.Timeline { tl, err := video.NewTimeline("webm", changes*1000, 60_000, 1000, 30) diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index b2970fbf..da34669f 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -136,6 +136,16 @@ var notProvenByMutation = map[string]bool{ // "proven another way" are different states and lumping them together would // send a later session to re-prove what is already proven. var provenByProbe = map[string]string{ + // Broken once each on 2026-11-12 while they were written, one change at a + // time and restored (docs/DAMAGE-SETTINGS-LABEL-2026-11-12.md). The first + // also carries a canary of its own check. + "TestEveryDamageSettingItsDeclarationForbidsIsRefusedWithoutARecipe": "its own canary - a plan that went through and a refusal about the first entry of the chain, both rejected by its check - and broken on 2026-11-12 " + + "by taking damage.Chain.Check out of engine.prepareDamage: red, every value went through.", + "TestADamageSettingOutsideItsRangeIsRefusedUnderItsBox": "broken on 2026-11-12 by taking damage.Chain.Check out of engine.prepareDamage: red, the window wrote the file and the box said nothing.", + "TestADamagedFileStatesItsSizeOnDiskAndItsDamage": "its own canary - the label it was written against, naming the size before the damage and no damage, rejected by its check - and broken on 2026-11-12 " + + "twice: Request.LabelSized ignoring OnDisk, and leaving the damage out. Red both times.", + "TestATruncatedTextSaysHowMuchWasCut": "broken on 2026-11-12 twice: Request.LabelSized ignoring OnDisk, and leaving the damage out. Red both times.", + "TestEveryCharacterOfADamagedLabelCanBeDrawn": "its own canary - a character the font never had must not count as drawn - and broken on 2026-11-12 by taking = out of the picture font: red.", // What breaks it lives in a module, not in a file of this repository, so // no mutation of this tree reaches it - the dependency it was written // against is the break. diff --git a/internal/guard/textencoding_test.go b/internal/guard/textencoding_test.go index 4bad04ae..b2786062 100644 --- a/internal/guard/textencoding_test.go +++ b/internal/guard/textencoding_test.go @@ -25,7 +25,6 @@ import ( "strings" "testing" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/textenc" "github.com/donislawdev/TestingFilesGenerator/internal/oracle" @@ -58,7 +57,7 @@ func (c encodingCase) refusedBy(id string) bool { // rather than silently measuring an empty wrapper. func labelLine(t *testing.T, id string, size int64, seed uint64) string { t.Helper() - body := core.Label(id, size, seed) + body := format.Request{Bytes: size, Seed: seed}.LabelText(id) switch id { case "txt": return body + "\n" diff --git a/internal/recipe/compose.go b/internal/recipe/compose.go index 838a98d0..b2efb27a 100644 --- a/internal/recipe/compose.go +++ b/internal/recipe/compose.go @@ -470,7 +470,7 @@ const ( KeySendAs = core.SettingSendAs KeyLabel = "label" KeyFill = "fill" - KeyDamage = "damage" + KeyDamage = core.KeyDamage KeyProperties = core.KeyProperties KeyExpected = "expected" KeyExpectedReason = "expected.reason" From d97d3980a610606d055fe35cb48e670e10855a9e Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 12 Nov 2026 01:25:59 +0100 Subject: [PATCH 2/5] A damage label has a length limit, and a mail writes its label in its own encoding A chain has no length limit, so the label naming it had none either, and a format whose label lives in a field with a limit of its own broke on a long one: a gzip comment Go's reader refuses from 512 bytes, a ZIP comment whose padding went negative and panicked. damage.LabelLimit bounds the damage text at 160 bytes in one place. A longer chain names the entries that fit and how many more there are, and the manifest still lists every one. eml wrote the label as it is into quoted-printable, where its = starts an escape, so append-junk:bytes=64 read back as bytesd. The label now goes through the encoder the frame uses, and the other encodings break a label longer than a line at its spaces. The JXL rung guard measures the longest damaged label too, and no rung passes its ceiling. Five mutation entries follow the code they named. CI asked for gofmt and an exhaustive switch. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d16048b7..78f4aa18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,10 +27,10 @@ because it turns other people's test suites red. - **A damaged file names its damage in its label.** The label now reads `tfg - txt - 10485760 B - damage truncate:missing=50 - seed c2a547b2`: the size the file has on disk, then every damage with its settings, in the order - they were applied. A text cut by half and one cut by nine tenths used to begin - with the same line. A file damaged with `zero-head` and carrying a label is - therefore not byte for byte what 0.5.0 wrote. Files with no damage, and files - made with `--clean`, are unchanged. + they were applied. A long list of damages names the first few and how many + more there are, and the manifest lists every one. A file damaged with + `zero-head` and carrying a label is therefore not byte for byte what 0.5.0 + wrote. Files with no damage, and files made with `--clean`, are unchanged. ### Added From 61d8d70fff1ac8fbfcf0b1b6129176761b42f1f0 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 12 Nov 2026 01:30:50 +0100 Subject: [PATCH 3/5] The code of the label limit and of the mail label, which the last commit left out The previous commit carries only its changelog entry. This is the change it describes: damage.LabelLimit holds the damage text of a label to 160 bytes, naming the entries that fit and how many more, so no format meets a label longer than its fields take - a gzip comment Go refuses from 512 bytes, a ZIP comment whose padding went negative. eml writes the label through the encoder its frame uses, so quoted-printable spells the = as =3D, and breaks a long label at its spaces elsewhere. Guards: every format with the longest label, the limit and its count of what it left out, a mail's label read back in every text encoding, the JXL rungs with the longest damaged label. Five mutation entries follow the code they named. CI asked for gofmt and an exhaustive switch. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +- internal/damage/apply.go | 48 ++++- internal/format/eml/message.go | 44 +++- internal/guard/damagelabel_test.go | 270 ++++++++++++++++++++++++ internal/guard/damagesettings_test.go | 14 +- internal/guard/damagewitness_test.go | 7 +- internal/guard/jxlladder_test.go | 11 +- internal/guard/mutation/entries.py | 16 +- internal/guard/mutationcoverage_test.go | 5 + 9 files changed, 399 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 4c7795a6..fce3f3b2 100644 --- a/README.md +++ b/README.md @@ -517,8 +517,10 @@ tfg generate --format zip --size 5mb --damage truncate:missing=30 A damaged file names its damage in its label, beside the size it has on disk: `tfg - txt - 10485760 B - damage truncate:missing=50 - seed c2a547b2`. So two texts cut by different shares can be told apart from their first line, although -the text after it is the same. The label is still written before the damage, so -the damage can overwrite it, and the manifest does not promise it is there. +the text after it is the same. A long list of damages names the first few and +how many more there are, and the manifest lists every one. The label is still +written before the damage, so the damage can overwrite it, and the manifest does +not promise it is there. The smallest file a damage can be given follows its settings, so the column is measured with the defaults. Ask for less and the run is refused before anything is diff --git a/internal/damage/apply.go b/internal/damage/apply.go index e8037f4a..03752ff7 100644 --- a/internal/damage/apply.go +++ b/internal/damage/apply.go @@ -367,6 +367,17 @@ func (c Chain) String() string { return fmt.Sprint(out) } +// LabelLimit is the most bytes the damage takes up in a label. +// +// The label lands in fields with limits of their own, and the tightest one +// measured is a gzip comment, which Go's reader refuses from 512 bytes +// (compress/gzip, a 512 byte buffer). A chain has no length limit of its own, +// so without this a long one would break a format before any damage was +// applied - an archive Go cannot open, a ZIP comment past its padding. Bounded +// here, once, so a format has one length to be ready for rather than every +// length a recipe can write (TestALongDamageLabelLeavesEveryFormatWhole). +const LabelLimit = 160 + // Label is the chain as the label of a damaged file states it: every entry in // order, its settings resolved the way the manifest records them, joined by // " then " - "truncate:missing=50 then append-junk:bytes=1024". Empty for an @@ -378,6 +389,10 @@ func (c Chain) String() string { // the label goes into encodings where nothing else is sure to survive // (format/label.go): a value with anything else in it, or with a space or one // of the separators, is written quoted with escapes. +// +// At most LabelLimit bytes. A chain longer than that names the entries that +// fit and how many more there are - "... and 12 more" - and the manifest lists +// every one. An entry too long to fit on its own is named by its damage alone. func (c Chain) Label() string { out := make([]string, 0, len(c)) for _, s := range c { @@ -387,7 +402,38 @@ func (c Chain) Label() string { } out = append(out, Spec{ID: s.ID, Values: values}.String()) } - return strings.Join(out, " then ") + if whole := strings.Join(out, " then "); len(whole) <= LabelLimit { + return whole + } + return shortened(c, out) +} + +// shortened is the entries of a chain that fit in LabelLimit with room left +// for saying how many did not. +func shortened(c Chain, entries []string) string { + kept := "" + for i, entry := range entries { + if i == 0 && len(entry) > LabelLimit/2 { + entry = c[0].ID + } + next := entry + if kept != "" { + next = kept + " then " + entry + } + if len(next)+len(moreThan(len(entries)-i-1)) > LabelLimit { + return kept + moreThan(len(entries)-i) + } + kept = next + } + return kept +} + +// moreThan is what a shortened label says about the entries it leaves out. +func moreThan(n int) string { + if n == 0 { + return "" + } + return " and " + strconv.Itoa(n) + " more" } // plainOrQuoted is v as it is when every byte of it is printable ASCII other diff --git a/internal/format/eml/message.go b/internal/format/eml/message.go index 8a662ad7..1a7f6d6c 100644 --- a/internal/format/eml/message.go +++ b/internal/format/eml/message.go @@ -3,6 +3,7 @@ package eml import ( "context" "io" + "strings" "github.com/donislawdev/TestingFilesGenerator/internal/core" ) @@ -177,7 +178,7 @@ func (m *message) textPart(o *out, contentType string, write func(*out), last bo func (m *message) plain(o *out) { o.put(m.frame.plainOpen) if m.label != "" { - o.put(m.label, o.eol) + o.put(m.labelText("", "", o.eol)) } o.put(m.plainLine) fill(o, m.text, m.seed) @@ -195,13 +196,52 @@ func (m *message) html(o *out) { } o.put(m.frame.htmlOpen) if m.label != "" { - o.put("

", m.label, "

", eol) + o.put(m.labelText("

", "

", eol)) } o.put(m.htmlLine, m.images, "

", eol) fill(o, m.text, m.seed) o.put(eol, "

", eol, m.frame.htmlClose) } +// labelText is the label as a part of the text writes it, between open and +// close and with its line endings: one line, as it always was where it fits. +// +// A damaged file's label names its damage, so it holds an = and can be longer +// than a line of the text. quoted-printable spells the = as =3D and breaks a +// long line softly (frameLine), so the text still reads as one line - written +// raw, append-junk:bytes=64 was read back as bytesd. The other encodings have +// no soft break, so there the label is broken at its spaces into lines of at +// most lineWidth, under the 78 characters RFC 5322 asks a line to keep to. +func (m *message) labelText(open, close, eol string) string { + line := open + m.label + close + if m.s.encoding == QuotedPrintable { + return frameLine(m.s, line) + eol + } + return strings.Join(atSpaces(line, lineWidth), eol) + eol +} + +// atSpaces breaks text at its spaces into lines of at most width, or as near +// as a word longer than width allows. Text that fits is one line. +func atSpaces(text string, width int) []string { + if len(text) <= width { + return []string{text} + } + var lines []string + line := "" + for _, word := range strings.Split(text, " ") { + switch { + case line == "": + line = word + case len(line)+1+len(word) > width: + lines = append(lines, line) + line = word + default: + line += " " + word + } + } + return append(lines, line) +} + // out is where a message goes - a file, or nowhere while it is being counted. // // The first error sticks and everything after it is counted and not written, diff --git a/internal/guard/damagelabel_test.go b/internal/guard/damagelabel_test.go index b5cb8c54..3e8163e8 100644 --- a/internal/guard/damagelabel_test.go +++ b/internal/guard/damagelabel_test.go @@ -2,13 +2,19 @@ package guard import ( "bytes" + "compress/gzip" + "encoding/base64" "fmt" + "io" + "mime/quotedprintable" + "net/mail" "os" "path/filepath" "slices" "strconv" "strings" "testing" + "unicode/utf8" "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/damage" @@ -16,6 +22,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" "github.com/donislawdev/TestingFilesGenerator/internal/manifest" + "github.com/donislawdev/TestingFilesGenerator/internal/oracle" ) // labelNotGuaranteed is the note a damaged file carries in place of a claim @@ -171,6 +178,269 @@ func TestATruncatedTextSaysHowMuchWasCut(t *testing.T) { } } +// TestALongDamageLabelLeavesEveryFormatWhole holds every format to the longest +// damage a label can name. +// +// Nothing limits how many damages a chain lists, so the label is input a +// person controls, and a format whose label lives in a field with a limit of +// its own can break on it before any damage is applied: a gzip comment Go's +// reader refuses from 512 bytes, a ZIP comment past its padding, a line of a +// mail. Found in review on 2026-11-12 (docs/REVIEW-234-2026-11-12.md). The +// answer is damage.LabelLimit, and this asks every format at it: each +// generator directly, with the label alone and no damage, so the file has to +// come out whole - planned, written, taken by the structural check, and a +// gzip stream opened by Go's own reader, which the structural check in Python +// does not hold to that limit. +func TestALongDamageLabelLeavesEveryFormatWhole(t *testing.T) { + long := longestDamageLabel(t) + + dir := t.TempDir() + var reqs []oracle.StrictRequest + for _, f := range format.All() { + path := filepath.Join(dir, f.ID) + if err := writeWithLabel(t, f, long, path); err != nil { + t.Errorf("%s with a label of %d B: %v", f.ID, len(long), err) + continue + } + reqs = append(reqs, oracle.StrictRequest{Format: f.ID, Path: path}) + } + got, available, err := oracle.StrictBatch(t.Context(), reqs) + if !available { + t.Skip("the structural checker needs python, and there is none on this machine") + } + if err != nil { + t.Fatal(err) + } + gzipped := 0 + for i, j := range got { + if j.Verdict != oracle.StrictOK { + t.Errorf("%s with a label of %d B is refused by the structural check: %s", reqs[i].Format, len(long), j.Reason) + } + if err := opensAsGzipWhereItIsOne(reqs[i].Path); err != nil { + t.Errorf("%s with a label of %d B: Go's gzip reader refuses it: %v", reqs[i].Format, len(long), err) + } else if isGzip(reqs[i].Path) { + gzipped++ + } + } + if len(got) < len(format.All())*3/4 || gzipped == 0 { + t.Fatalf("only %d of %d formats were judged, %d of them gzip streams", len(got), len(format.All()), gzipped) + } +} + +// longestDamageLabel is the damage text of a chain far longer than the label +// takes, after asserting that the label was shortened to what it takes and +// that the whole text would have passed the gzip limit - so the guard above is +// asking about the limit rather than about a label that never reached it. +func longestDamageLabel(t *testing.T) string { + t.Helper() + chain := damage.Chain{} + for range 40 { + chain = append(chain, damage.Spec{ID: damage.AppendJunk, Values: damage.Values{damage.SettingBytes: "65536"}}) + } + long := chain.Label() + whole := len(chain) * len(chain[0].String()) + if whole < 512 || len(long) > damage.LabelLimit || len(long) < damage.LabelLimit*3/4 { + t.Fatalf("forty damages name %d B whole and %d B in the label, with the label held to %d B - the guard does not reach the limit it is about", + whole, len(long), damage.LabelLimit) + } + return long +} + +// isGzip says whether the file at path starts as a gzip stream. +func isGzip(path string) bool { + head := make([]byte, 2) + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + n, _ := f.Read(head) + return n == 2 && head[0] == 0x1f && head[1] == 0x8b +} + +// opensAsGzipWhereItIsOne opens the header of a gzip stream with Go's reader, +// and says nothing about a file that is not one. +func opensAsGzipWhereItIsOne(path string) error { + if !isGzip(path) { + return nil + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + z, err := gzip.NewReader(f) + if err != nil { + return err + } + return z.Close() +} + +// TestADamageLabelIsBoundedAndSaysHowManyItLeftOut holds damage.Chain.Label +// to its limit for every length of chain, and to an honest count of what it +// did not name: the entries it names and the number it gives add up to the +// chain. The canary is the shortest chain over the limit, which has to be +// shortened, so a limit nothing reaches cannot pass. +func TestADamageLabelIsBoundedAndSaysHowManyItLeftOut(t *testing.T) { + entry := damage.Spec{ID: damage.AppendJunk, Values: damage.Values{damage.SettingBytes: "65536"}} + shortened := 0 + for n := 1; n <= 200; n++ { + chain := slices.Repeat(damage.Chain{entry}, n) + label := chain.Label() + if len(label) > damage.LabelLimit { + t.Fatalf("%d damages: the label is %d B, past the %d B it is held to", n, len(label), damage.LabelLimit) + } + named := strings.Count(label, damage.AppendJunk) + left := 0 + if before, more, cut := strings.Cut(label, " and "); cut { + left, _ = strconv.Atoi(strings.TrimSuffix(more, " more")) + named = strings.Count(before, damage.AppendJunk) + shortened++ + } + if named+left != n || named == 0 { + t.Errorf("%d damages: the label %q names %d and says %d more", n, label, named, left) + } + } + if shortened == 0 { + t.Fatal("no chain up to 200 damages was shortened, so the limit was never reached") + } +} + +// TestADamagedMailCarriesItsLabelWholeInEveryTextEncoding reads the label of +// a mail back the way a mail reader does, in every encoding of its text and +// both parts that carry it. +// +// Found 2026-11-12 while answering a review: the label was written as it is +// into quoted-printable, where its = starts an escape, so +// append-junk:bytes=64 read back as bytesd - and the structural check took it, +// because =64 is a valid escape. A label naming two damages is also longer +// than the 78 characters RFC 5322 asks a line to keep to. The encodings and +// parts are read from the declaration, and the label is decoded with Go's own +// readers rather than searched for in the bytes. +func TestADamagedMailCarriesItsLabelWholeInEveryTextEncoding(t *testing.T) { + d, err := format.Get("eml") + if err != nil { + t.Fatal(err) + } + encodings, eok := settingNamed(d, "text_encoding") + bodies, bok := settingNamed(d, "body") + if !eok || !bok || len(encodings.Choices) < 4 { + t.Fatalf("eml declares text_encoding %v and body %v - the guard does not reach the encodings it is about", encodings.Choices, bodies.Choices) + } + // One damage, whose label fits a line, so the = is asked on its own. And + // two, whose label does not, so the breaking is asked as well. + one := damage.Chain{{ID: damage.AppendJunk, Values: damage.Values{damage.SettingBytes: "1024"}}} + two := damage.Chain{{ID: damage.Truncate, Values: damage.Values{damage.SettingMissing: "10"}}, one[0]} + for _, chain := range []damage.Chain{one, two} { + for _, encoding := range encodings.Choices { + for _, body := range []string{"plain", "html"} { + mailCarriesItsLabel(t, d, chain, encoding, body) + } + } + } + if !fitsALine(d, one) || fitsALine(d, two) { + t.Fatal("the label of one damage has to fit a line and the label of two must not, or one half of this guard asks nothing") + } +} + +// fitsALine says whether the label of a 64 KiB mail damaged by chain fits the +// 78 characters a line keeps to. +func fitsALine(d format.Descriptor, chain damage.Chain) bool { + return len(format.Request{Bytes: 64 << 10, Seed: 3, Damage: chain.Label()}.LabelText(d.ID)) <= 78 +} + +// mailCarriesItsLabel writes one mail damaged by chain in one encoding and +// part, and reads its label back. +func mailCarriesItsLabel(t *testing.T, d format.Descriptor, chain damage.Chain, encoding, body string) { + t.Helper() + r := format.Request{Seed: 3, Label: true, Damage: chain.Label(), + Properties: map[string]string{"text_encoding": encoding, "body": body}} + r.Bytes = max(int64(64<<10), d.SmallestAccepted(r)) + want := fmt.Sprintf("tfg - eml - %d B - damage %s - seed %s", r.Bytes, chain.Label(), core.SeedLabel(r.Seed)) + text, err := mailText(t, d, r) + if err != nil { + t.Errorf("%s, text_encoding=%s body=%s: %v", chain.Label(), encoding, body, err) + return + } + if !strings.Contains(text, want) { + t.Errorf("%s, text_encoding=%s body=%s: the text read back does not carry %q", chain.Label(), encoding, body, want) + } +} + +// mailText writes the mail r asks for and reads its text back: the body +// decoded by its Content-Transfer-Encoding, every line break read as the space +// it was broken at. A line longer than 78 characters in the file is an error. +func mailText(t *testing.T, d format.Descriptor, r format.Request) (string, error) { + t.Helper() + p, err := d.Generator.Plan(r) + if err != nil { + return "", err + } + var file bytes.Buffer + if err := d.Generator.Write(t.Context(), &file, p); err != nil { + return "", err + } + for i, line := range strings.Split(file.String(), "\n") { + if n := utf8.RuneCountInString(strings.TrimSuffix(line, "\r")); n > 78 { + return "", fmt.Errorf("line %d of the file is %d characters, past the 78 RFC 5322 asks for", i+1, n) + } + } + msg, err := mail.ReadMessage(&file) + if err != nil { + return "", err + } + var body io.Reader = msg.Body + switch msg.Header.Get("Content-Transfer-Encoding") { + case "quoted-printable": + body = quotedprintable.NewReader(body) + case "base64": + body = base64.NewDecoder(base64.StdEncoding, newlineless{body}) + } + text, err := io.ReadAll(body) + if err != nil { + return "", err + } + return strings.NewReplacer("\r\n", " ", "\n", " ").Replace(string(text)), nil +} + +// newlineless is a reader with its line breaks taken out, which base64 in a +// mail carries every 76 characters and the decoder does not skip. +type newlineless struct{ r io.Reader } + +func (n newlineless) Read(p []byte) (int, error) { + k, err := n.r.Read(p) + out := p[:0] + for _, c := range p[:k] { + if c != '\r' && c != '\n' { + out = append(out, c) + } + } + return len(out), err +} + +// writeWithLabel plans and writes one file of f carrying the damage text long +// in its label, and turns a panic into the error it is. +func writeWithLabel(t *testing.T, f format.Descriptor, long, path string) (err error) { + t.Helper() + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("panicked: %v", p) + } + }() + r := format.Request{Seed: 1, Label: true, Damage: long} + r.Bytes = max(int64(witnessSize), f.SmallestAccepted(r)) + p, err := f.Generator.Plan(r) + if err != nil { + return err + } + out, err := os.Create(path) + if err != nil { + return err + } + defer out.Close() + return f.Generator.Write(t.Context(), out, p) +} + // statesItsDamage says what is wrong with the label in got, or nothing when // it states size on disk and every entry of chain with its settings resolved. func statesItsDamage(got []byte, id string, size int64, chain damage.Chain, seed uint64) string { diff --git a/internal/guard/damagesettings_test.go b/internal/guard/damagesettings_test.go index 854d54b6..f20aaa63 100644 --- a/internal/guard/damagesettings_test.go +++ b/internal/guard/damagesettings_test.go @@ -43,8 +43,12 @@ func TestEveryDamageSettingItsDeclarationForbidsIsRefusedWithoutARecipe(t *testi t.Fatal("the check took a plan that went through, or a refusal about the wrong entry, for the refusal it asks for") } + all := damage.All() + if len(all) == 0 { + t.Fatal("the damage registry is empty, so this guard would pass without asking anything") + } asked := 0 - for _, d := range damage.All() { + for _, d := range all { type stated struct{ name, value string } cases := []stated{{"no-such-setting", "1"}} for _, p := range d.Parameters { @@ -71,8 +75,8 @@ func TestEveryDamageSettingItsDeclarationForbidsIsRefusedWithoutARecipe(t *testi } } } - if asked < len(damage.All())*2 { - t.Fatalf("only %d value(s) were asked across %d damage(s) - the registry was not read", asked, len(damage.All())) + if asked < len(all)*2 { + t.Fatalf("only %d value(s) were asked across %d damage(s) - the registry was not read", asked, len(all)) } } @@ -87,10 +91,12 @@ func valuesTheDeclarationForbids(p format.Property) []string { out = append(out, strconv.FormatInt(p.Min-1, 10), strconv.FormatInt(p.Max+1, 10)) } return out + case format.PropertyChoice, format.PropertyBool, format.PropertySize, format.PropertyDuration: + return []string{"no-such-value"} case format.PropertyText: return nil } - return []string{"no-such-value"} + return nil } // planSecondInAChain plans a txt file damaged by zero-head and then spec. diff --git a/internal/guard/damagewitness_test.go b/internal/guard/damagewitness_test.go index ea0f6236..585ca67a 100644 --- a/internal/guard/damagewitness_test.go +++ b/internal/guard/damagewitness_test.go @@ -89,7 +89,12 @@ var policyRefusals = map[[2]string][]string{ "after its closing boundary", "is not a line of base64", // A rule the junk breaks only when its random bytes hold a CR or an // LF on its own - about four seeds in ten, so not every file. - "a carriage return or a line feed on its own"}, + "a carriage return or a line feed on its own", + // In quoted-printable the junk lengthens the last line past the 76 + // RFC 2045 allows. Python's email reads that text whole with no + // defect (measured 2026-11-12). It came first once the label's = + // was spelled =3D, which moved the text two bytes. + "a quoted-printable line is 79 characters or ends in white space"}, // UTF-16 in the next two: the junk breaks the encoding only when its // random bytes form a lone surrogate, which is a matter of the seed. {damage.AppendJunk, "txt"}: {"that are not printable ASCII", "does not decode as utf-16"}, diff --git a/internal/guard/jxlladder_test.go b/internal/guard/jxlladder_test.go index d2b43f57..8f6c64df 100644 --- a/internal/guard/jxlladder_test.go +++ b/internal/guard/jxlladder_test.go @@ -104,8 +104,11 @@ func TestEveryJxlRungStillFitsInsideItsDeclaredCeiling(t *testing.T) { } // The label carries the byte count, so its length moves with the size - // asked for, and a longer label is more ink in the picture. + // asked for, and a longer label is more ink in the picture. A damaged + // file's label names its damage as well, up to damage.LabelLimit, which is + // the most ink a label can put down (found in review, 2026-11-12). requested := []int64{int64(jxl.MinimumBytes), 1 << 14, 1 << 30} + damaged := longestDamageLabel(t) seeds := uint64(8) if raceEnabled { @@ -119,10 +122,10 @@ func TestEveryJxlRungStillFitsInsideItsDeclaredCeiling(t *testing.T) { var worstSeed uint64 for seed := uint64(0); seed < seeds; seed++ { for _, want := range requested { - for _, withLabel := range []bool{true, false} { + for _, damage := range []string{"", damaged, "none"} { label := "" - if withLabel { - label = format.Request{Bytes: want, Seed: seed}.LabelText("jxl") + if damage != "none" { + label = format.Request{Bytes: want, Seed: seed, Damage: damage}.LabelText("jxl") } n, err := jxl.FileSizeFor(w, h, jxl.DefaultQuality, seed, label) if err != nil { diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index dd9aa38a..fe41a711 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -395,7 +395,7 @@ ( "the smallest Office file is worked out for one label state instead of both", "internal/format/xlsx/xlsx.go", - 'for _, label := range []string{"", core.Label("xlsx", 0, 0)} {', + 'for _, label := range []string{"", format.Request{}.LabelText("xlsx")} {', 'for _, label := range []string{""} {', "TestSizeBelowTheMinimumIsRefused", ), @@ -1175,8 +1175,8 @@ # in, and the size stays exact because the filler absorbs it. "a toml document opens with a byte order mark, which no reader accepts", "internal/format/tomlfile/toml.go", - 'line := "# " + core.Label("toml", r.Bytes, r.Seed) + "\\n"', - 'line := "\\ufeff# " + core.Label("toml", r.Bytes, r.Seed) + "\\n"', + 'line := "# " + r.LabelText("toml") + "\\n"', + 'line := "\\ufeff# " + r.LabelText("toml") + "\\n"', "TestEveryFormatSurvivesItsReferenceTool", ), ( @@ -1255,7 +1255,7 @@ # bytes. The guard pins txt with the label and without it, and only # the first of the two may notice this. "the self describing label is reworded", - "internal/core/label.go", + "internal/format/label.go", '"tfg - %s - %d B - seed %s"', '"tfg %s %d B seed %s"', "TestOurOwnGeneratorsHaveNotDrifted", @@ -6274,8 +6274,8 @@ ( "the engine calls a generator directly again, so a crash while planning ends the process", "internal/engine/plantarget.go", - "planWithoutCrashing(ctx, desc, format.Request{", - "desc.Generator.Plan(format.Request{", + "p, err := planWithoutCrashing(ctx, desc, r)", + "p, err := desc.Generator.Plan(r)", "TestEveryCallIntoAGeneratorGoesThroughTheCrashWrapper", ), ( @@ -17321,8 +17321,8 @@ ( "a refusal of a damaged size names the size the generator needed", "internal/engine/damagedformat.go", - "\t\tleast := g.smallestReaching(first)", - "\t\tleast := first", + "\t\tleast = g.smallestReaching(first)", + "\t\tleast = first", "TestTheSmallestDamagedFileIsTakenAndOneByteLessIsNot", ), ( diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index da34669f..54f90578 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -145,6 +145,11 @@ var provenByProbe = map[string]string{ "TestADamagedFileStatesItsSizeOnDiskAndItsDamage": "its own canary - the label it was written against, naming the size before the damage and no damage, rejected by its check - and broken on 2026-11-12 " + "twice: Request.LabelSized ignoring OnDisk, and leaving the damage out. Red both times.", "TestATruncatedTextSaysHowMuchWasCut": "broken on 2026-11-12 twice: Request.LabelSized ignoring OnDisk, and leaving the damage out. Red both times.", + "TestALongDamageLabelLeavesEveryFormatWhole": "broken on 2026-11-12 by letting damage.Chain.Label return the whole chain past LabelLimit: red, " + + "its own reach check says forty damages no longer fit the limit (docs/REVIEW-234-2026-11-12.md).", + "TestADamageLabelIsBoundedAndSaysHowManyItLeftOut": "its own canary - no chain up to 200 damages shortened is a failure - and broken on 2026-11-12 by letting damage.Chain.Label return the whole chain: red.", + "TestADamagedMailCarriesItsLabelWholeInEveryTextEncoding": "broken on 2026-11-12 twice: eml writing the label raw in quoted-printable (red, the one damage label read back as bytesd), " + + "and eml leaving a long label on one line (red, a line of 92 characters).", "TestEveryCharacterOfADamagedLabelCanBeDrawn": "its own canary - a character the font never had must not count as drawn - and broken on 2026-11-12 by taking = out of the picture font: red.", // What breaks it lives in a module, not in a file of this repository, so // no mutation of this tree reaches it - the dependency it was written From 0ece8f2d8ae7bd808f32be379f33b787731caafa Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 12 Nov 2026 09:49:16 +0100 Subject: [PATCH 4/5] A damaged picture has a rung ceiling of its own, and AVIF ceilings keep a margin format.RequestKey carries Damage and OnDisk. TestEveryFieldOfARequestIsInItsKey sets every field of Request and stopped at the first string it could not set. A damaged file's label names its damage, and a longer label puts more ink into a picture until it is too long for its scale. The AVIF, JPEG XL and video ladders chose a picture by ceilings measured with the label of a file nothing damages, so an AVIF of 278533 B with four append-junk damages was refused by writing after planning had accepted it, for 36 of 40 seeds. Each rung now carries a second ceiling for a damaged file, a tenth above the heaviest picture measured with the longest damaged label each scale still draws (imagelabel.LongestAtEachScale). Files nothing damages keep their ceilings. The first AVIF ceilings had gone stale. A file asked for at the 640x480 ceiling, or 50 B above it, was not written in 45 of 512 tries on seeds near zero and 332 of 1024 far from it, and 0.5.0 failed the same way. They are now a tenth above what was measured over 768 seeds, so an AVIF between the old step and the new one gets the smaller picture. Guards: the AVIF and JPEG XL rung guards ask both ceilings, on the first seeds and on the heaviest seeds the probes found, without which their canaries stayed green. The PNG and GIF ceiling guard asks a damaged label at every scale. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 ++ internal/format/avif/avif.go | 59 ++++++++++-- internal/format/imagelabel/imagelabel.go | 18 ++++ internal/format/jxl/jxl.go | 31 ++++++- internal/format/jxl/measure.go | 12 +-- internal/format/registry.go | 3 + internal/format/video/choose.go | 43 ++++++--- internal/guard/avifladder_test.go | 101 +++++++++++++++++++++ internal/guard/damagelabel_test.go | 32 +++++++ internal/guard/jxlladder_test.go | 111 ++++++++++++++++------- internal/guard/ladderceiling_test.go | 38 +++++--- internal/guard/mutation/entries.py | 8 +- internal/guard/mutationcoverage_test.go | 3 + internal/guard/requestkey_test.go | 2 + 14 files changed, 390 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78f4aa18..1d30e7d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,11 @@ because it turns other people's test suites red. more there are, and the manifest lists every one. A file damaged with `zero-head` and carrying a label is therefore not byte for byte what 0.5.0 wrote. Files with no damage, and files made with `--clean`, are unchanged. +- **An AVIF file just above the size where its picture grows gets the smaller + picture.** The picture now steps up to 160x120 at 3865 B, to 256x256 at + 5493 B, to 320x240 at 7732 B and to 640x480 at 18722 B, about a tenth above + where it did. A file between the old step and the new one is not byte for + byte what 0.5.0 wrote. ### Added @@ -171,6 +176,11 @@ because it turns other people's test suites red. ### Fixed +- **An AVIF file asked for just above the size where its picture grows is + written.** In 0.5.0 about a quarter of the files asked for at 16389 or + 16439 B were not written at all, and the run ended with "1 file could not be + produced" although a preview had accepted them. The same happened, less + often, at the smaller steps. - **A damage setting outside its range is refused on the command line and in the window.** `--damage truncate:missing=95` cut the default half, wrote the file, ended with code 0 and recorded `missing: 95` in the manifest, and diff --git a/internal/format/avif/avif.go b/internal/format/avif/avif.go index d8a4906d..36ad4e8f 100644 --- a/internal/format/avif/avif.go +++ b/internal/format/avif/avif.go @@ -285,11 +285,21 @@ func checkFits(want int64, m memo) error { // who does want them says so - width and height are settings, and 40 megapixels // is the ceiling, which covers 4K and 8K. // -// Ceiling is the LARGEST file that rung has ever been seen to produce, the free -// box included, and it is what lets planning choose a picture without coding -// one. Measured on 2026-08-29 across all 256 seeds, four orders of magnitude of -// requested size - the label carries the byte count, so its length moves with -// it - and with the label on and off. +// Ceiling is a tenth above the LARGEST file that rung was seen to produce, the +// free box included, and it is what lets planning choose a picture without +// coding one. Measured on 2026-11-12 by tools/probes/avifladder over 256 seeds +// from zero and 512 far from it, because the label's text moves with the whole +// seed, with the label off and with the label of a file at the edge of the +// rung - a size with the digits of the ceiling, the lightest and the heaviest. +// 640x480 coded to 17020 B, 320x240 to 7029, 256x256 to 4993 and 160x120 to +// 3513. The rungs below draw no label and their picture moves only with the +// low byte of the seed, so their ceiling is the largest of all 256 and no more. +// +// The tenth is the margin the video ladder keeps, and it is not caution for its +// own sake. The ceilings measured on 2026-08-29 over 256 seeds kept none, and +// by 2026-11-12 a file asked for at the ceiling of 640x480, or 50 B above it, +// was not written in 45 of 512 tries on the 256 seeds near zero and in 332 of +// 1024 on 512 seeds far from it (O273). // // A ceiling rather than a floor, and the difference is the whole point. // Planning promises an exact size, so the rung it picks has to be one whose @@ -297,15 +307,46 @@ func checkFits(want int64, m memo) error { // costs a smaller picture than the file had room for and nothing else. Set too // low, it would let planning pick a picture that does not fit, so writing // checks and says so rather than trusting this table. +// +// The second number is the ceiling for a damaged file, whose label names its +// damage, by the same rule with the label at the longest length each scale +// still draws: 640x480 coded to 20024 B, 320x240 to 7556 and 256x256 to 6043, +// each above the first number. A rung too narrow to draw a damaged label keeps +// the first number. var sizeLadder = []rung{ - {640, 480, 16389}, {320, 240, 6792}, {256, 256, 4890}, {160, 120, 3392}, - {80, 60, 832}, {40, 30, 549}, {20, 15, 448}, {8, 6, 348}, - {4, 3, 344}, {2, 2, 327}, {1, 1, minimumBytes}, + {640, 480, 18722, 22027}, {320, 240, 7732, 8312}, {256, 256, 5493, 6648}, {160, 120, 3865, 3865}, + {80, 60, 832, 832}, {40, 30, 549, 549}, {20, 15, 448, 448}, {8, 6, 348, 348}, + {4, 3, 344, 344}, {2, 2, 327, 327}, {1, 1, minimumBytes, minimumBytes}, } type rung struct { width, height int ceiling int64 + // damaged is the ceiling for the label of a damaged file, which names its + // damage as well. A longer label can put more ink into the picture than + // any the ceiling was measured with, until it is too long to draw at all. + damaged int64 +} + +// bound is the ceiling that speaks for the label r carries. A file nothing +// damages keeps the ceiling it always had, so its picture and its bytes do not +// move (D11). +func (s rung) bound(r format.Request) int64 { + if r.Damage != "" { + return s.damaged + } + return s.ceiling +} + +// Rungs reports the ladder as the dimensions of each rung with its two +// ceilings - for a file nothing damages and for a damaged one - so a guard and +// a probe walk exactly the table planning walks rather than a second copy. +func Rungs() [][4]int64 { + out := make([][4]int64, 0, len(sizeLadder)) + for _, r := range sizeLadder { + out = append(out, [4]int64{int64(r.width), int64(r.height), r.ceiling, r.damaged}) + } + return out } // chooseSize settles the picture size and encodes it. @@ -330,7 +371,7 @@ func chooseSize(r format.Request, label string, q int) (memo, error) { // is for every other format. pick := sizeLadder[len(sizeLadder)-1] for _, step := range sizeLadder { - if r.Bytes >= step.ceiling { + if r.Bytes >= step.bound(r) { pick = step break } diff --git a/internal/format/imagelabel/imagelabel.go b/internal/format/imagelabel/imagelabel.go index 114238d0..3eeed87e 100644 --- a/internal/format/imagelabel/imagelabel.go +++ b/internal/format/imagelabel/imagelabel.go @@ -203,6 +203,24 @@ func (t laidText) ink(x, y int) bool { return off < t.Width && glyph(t.runes[k])[(y-pad)/t.scale*glyphWidth+off/t.scale] == '#' } +// LongestAtEachScale is the most characters a label can have and still be +// drawn at each scale a picture width wide draws one at, the largest scale +// first - empty for a picture too narrow to draw any. +// +// That, and not the longest label, is the most a label can put into a picture: +// a longer one is drawn smaller, and one too long for the smallest scale is not +// drawn at all. A size ladder whose ceilings stand for every label is measured +// at these lengths. +func LongestAtEachScale(width int) []int { + var out []int + for n := 1; scaleFor(width, n) > 0; n++ { + if scaleFor(width, n+1) != scaleFor(width, n) { + out = append(out, n) + } + } + return out +} + // BandHeight is how tall the band Draw paints for a label of chars characters // across width is, before it is cut to the picture - 0 when the label does not // fit. A film draws a second line under the first and asks this before it diff --git a/internal/format/jxl/jxl.go b/internal/format/jxl/jxl.go index b1966275..d2a7d919 100644 --- a/internal/format/jxl/jxl.go +++ b/internal/format/jxl/jxl.go @@ -337,15 +337,38 @@ func checkFits(want int64, m memo) error { // costs a smaller picture than the file had room for and nothing else. Set too // low, it would let planning pick a picture that does not fit, so writing // checks and says so rather than trusting this table. +// +// The second number is the ceiling for a damaged file, whose label names its +// damage. Measured on 2026-11-12 by tools/probes/jxlladder over 768 seeds, with +// the label at the longest length each scale still draws: 640x480 coded to +// 21424 B, 320x240 to 8049 and 256x256 to 6942, the last two above the first +// number. It is a tenth above what was measured and never below the first +// number, the margin the video ladder keeps, because the damage text is +// whatever the chain names. A rung too narrow to draw a damaged label keeps the +// first number. var sizeLadder = []rung{ - {640, 480, 22762}, {320, 240, 7949}, {256, 256, 6485}, {160, 120, 3542}, - {80, 60, 1267}, {40, 30, 731}, {20, 15, 452}, {8, 6, 222}, - {4, 3, 212}, {2, 2, 206}, {1, 1, minimumBytes}, + {640, 480, 22762, 23567}, {320, 240, 7949, 8854}, {256, 256, 6485, 7637}, {160, 120, 3542, 3542}, + {80, 60, 1267, 1267}, {40, 30, 731, 731}, {20, 15, 452, 452}, {8, 6, 222, 222}, + {4, 3, 212, 212}, {2, 2, 206, 206}, {1, 1, minimumBytes, minimumBytes}, } type rung struct { width, height int ceiling int64 + // damaged is the ceiling for the label of a damaged file, which names its + // damage as well. A longer label can put more ink into the picture than + // any the ceiling was measured with, until it is too long to draw at all. + damaged int64 +} + +// bound is the ceiling that speaks for the label r carries. A file nothing +// damages keeps the ceiling it always had, so its picture and its bytes do not +// move (D11). +func (s rung) bound(r format.Request) int64 { + if r.Damage != "" { + return s.damaged + } + return s.ceiling } // chooseSize settles the picture size, and codes it only when it has to. @@ -370,7 +393,7 @@ func chooseSize(r format.Request, label string, q int) (memo, error) { // is for every other format. pick := sizeLadder[len(sizeLadder)-1] for _, step := range sizeLadder { - if r.Bytes >= step.ceiling { + if r.Bytes >= step.bound(r) { pick = step break } diff --git a/internal/format/jxl/measure.go b/internal/format/jxl/measure.go index 85dc2a6c..581553a7 100644 --- a/internal/format/jxl/measure.go +++ b/internal/format/jxl/measure.go @@ -28,13 +28,13 @@ func FileSizeFor(width, height, quality int, seed uint64, label string) (int, er return len(coded) + containerOverhead + boxHeader, nil } -// Rungs reports the ladder as pairs of dimensions with the ceiling each one -// carries, so a probe and a guard can walk exactly the table planning walks -// rather than a second copy of it. -func Rungs() [][3]int64 { - out := make([][3]int64, 0, len(sizeLadder)) +// Rungs reports the ladder as the dimensions of each rung with its two +// ceilings - for a file nothing damages and for a damaged one - so a probe and +// a guard walk exactly the table planning walks rather than a second copy. +func Rungs() [][4]int64 { + out := make([][4]int64, 0, len(sizeLadder)) for _, r := range sizeLadder { - out = append(out, [3]int64{int64(r.width), int64(r.height), r.ceiling}) + out = append(out, [4]int64{int64(r.width), int64(r.height), r.ceiling, r.damaged}) } return out } diff --git a/internal/format/registry.go b/internal/format/registry.go index f8048773..62679884 100644 --- a/internal/format/registry.go +++ b/internal/format/registry.go @@ -148,6 +148,9 @@ func RequestKey(id string, r Request) (string, bool) { sort.Strings(names) var b strings.Builder fmt.Fprintf(&b, "%q %d %t %t %d", id, r.Bytes, r.SizeFromContents, r.Label, r.Seed) + // The damage and the size on disk are words of the label, so they change + // its length and with it the smallest file that carries it. + fmt.Fprintf(&b, " %d %q", r.OnDisk, r.Damage) for _, name := range names { fmt.Fprintf(&b, " %q=%q", name, r.Properties[name]) } diff --git a/internal/format/video/choose.go b/internal/format/video/choose.go index 5f2ffda9..c129178f 100644 --- a/internal/format/video/choose.go +++ b/internal/format/video/choose.go @@ -14,6 +14,19 @@ import ( type Rung struct { Width, Height int Ceiling int + // Damaged is the ceiling for a film whose label names its damage. A + // longer label can put more ink into a picture than any the first ceiling + // was measured with, until it is too long to draw at all. + Damaged int +} + +// bound is the ceiling that speaks for the label r carries. A film nothing +// damages keeps the ceiling it always had, so its bytes do not move (D11). +func (g Rung) bound(r format.Request) int { + if r.Damage != "" { + return g.Damaged + } + return g.Ceiling } // Ladder is tried from the largest down when the request names no picture @@ -47,10 +60,18 @@ type Rung struct { // room for and nothing else. One too low would let planning promise a size // writing cannot keep, so writing checks every picture and says so rather // than trusting this table. +// +// The second number is the ceiling for a damaged film, whose label names its +// damage, by the same rule: a tenth above the largest picture measured with +// the label at the longest length each scale still draws, never below the +// first number. Measured on 2026-11-12 by tools/probes/videoladder, the same +// three passes: on 640x360 a damaged label coded to 16946 B, which left less +// than a fiftieth of the first number above it, and on every other rung it +// coded smaller than the label of a film nothing damages. var Ladder = []Rung{ - {640, 360, 17277}, {426, 240, 11156}, {320, 180, 8322}, {256, 144, 6754}, - {160, 90, 3951}, {80, 45, 1454}, {40, 23, 825}, {16, 9, 149}, - {4, 3, 46}, {2, 2, 31}, {1, 1, 6}, + {640, 360, 17277, 18641}, {426, 240, 11156, 11156}, {320, 180, 8322, 8322}, {256, 144, 6754, 6754}, + {160, 90, 3951, 3951}, {80, 45, 1454, 1454}, {40, 23, 825, 825}, {16, 9, 149, 149}, + {4, 3, 46, 46}, {2, 2, 31, 31}, {1, 1, 6, 6}, } // Choice is the picture a film shows. @@ -216,7 +237,7 @@ func Choose(formatID string, r format.Request, s Settings, label string, fits fu var last Choice var lastStream Stream for _, rung := range Ladder { - c, st, err := onRung(base, rung, s) + c, st, err := onRung(base, rung, rung.bound(r), s) if err != nil { return Choice{}, Stream{}, err } @@ -229,15 +250,15 @@ func Choose(formatID string, r format.Request, s Settings, label string, fits fu } // onRung is the picture of one rung and the stream planning sizes it by: the -// rung's ceiling for every picture, or - when the request named a quality the -// ceilings were not measured at - a reserve from a sample of the film's -// pictures, coded (sampleReserve). That is the slow road AVIF takes for the -// same reason, and only runs that asked for it pay. -func onRung(base Choice, rung Rung, s Settings) (Choice, Stream, error) { +// rung's ceiling for the label the film carries, for every picture, or - when +// the request named a quality the ceilings were not measured at - a reserve +// from a sample of the film's pictures, coded (sampleReserve). That is the slow +// road AVIF takes for the same reason, and only runs that asked for it pay. +func onRung(base Choice, rung Rung, ceiling int, s Settings) (Choice, Stream, error) { c := base - c.Width, c.Height, c.Ceiling = rung.Width, rung.Height, rung.Ceiling + c.Width, c.Height, c.Ceiling = rung.Width, rung.Height, ceiling if !s.QualityNamed { - return c, NewStream(s.Timeline, rung.Ceiling, rung.Width, rung.Height, c.Label), nil + return c, NewStream(s.Timeline, ceiling, rung.Width, rung.Height, c.Label), nil } tiles, reserve, err := sampleReserve(c.Width, c.Height, c.Seed, c.Label, s.Timeline, c.QIndex) if err != nil { diff --git a/internal/guard/avifladder_test.go b/internal/guard/avifladder_test.go index c855c8e2..9abb9716 100644 --- a/internal/guard/avifladder_test.go +++ b/internal/guard/avifladder_test.go @@ -2,11 +2,15 @@ package guard import ( "context" + "errors" "runtime" + "slices" + "strconv" "testing" "github.com/donislawdev/TestingFilesGenerator/internal/format" _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" + "github.com/donislawdev/TestingFilesGenerator/internal/format/avif" ) // AVIF is the one format here whose file size is decided by a compressor rather @@ -77,6 +81,103 @@ func TestAvifDrawsAPictureThatGrowsWithTheFileAndAlwaysFits(t *testing.T) { t.Logf("largest picture reached over the sweep: %d pixels", biggest) } +// heaviestAvifSeeds are the seeds the thorough sweep (tools/probes/avifladder, +// 768 seeds) found heaviest on 2026-11-12, asked beside the first few from +// zero. The first few alone stayed green with both ceilings set back to the +// numbers that were too low, so they could not see what the sweep found. A +// codec raised underneath moves every rung at once, so the heaviest seeds of +// today are a fair sample of tomorrow's. +var heaviestAvifSeeds = []uint64{ + 0x65a47abe4364edd2, 0xfb0ac60c4be71cd6, 0x949ac3a5a267fad4, 0xf3fc5ddbb41fd92, + 0xd2, 0xea, 0xf4, 0xfb, 0xfe, 0xff, +} + +// Every rung's two ceilings still cover what that rung codes to: the first +// with the label of a file at the edge of the rung, the second with the label +// of a damaged file. +// +// A file at the edge of a rung - asked for at its ceiling, where the picture +// leaves the least room - states a size with the digits of the ceiling, so the +// first ceiling is asked with those. A damaged file's label names its damage, +// and a longer label is more ink until it is too long for its scale, so the +// second is asked with the longest damaged label every scale still draws. +// Asking one byte of a named picture size makes the format state the file +// that picture needs, which is what a ceiling is a bound on (2026-11-12, when +// both were found too low - O273 and the review of #234). +// +// A handful of seeds from zero, as the JPEG XL guard takes, and the heaviest +// the thorough sweep found (heaviestAvifSeeds). +func TestEveryAvifRungStillFitsInsideItsDeclaredCeilings(t *testing.T) { + d, err := format.Get("avif") + if err != nil { + t.Fatal(err) + } + rungs := avif.Rungs() + if len(rungs) == 0 { + t.Fatal("the ladder is empty, so this guard measured nothing") + } + first := uint64(8) + if raceEnabled { + first = 2 + } + seeds := slices.Clone(heaviestAvifSeeds) + for seed := range first { + seeds = append(seeds, seed) + } + + type ask struct { + r format.Request + ceiling int64 + } + drawn := 0 + for _, rung := range rungs { + w, h := int(rung[0]), int(rung[1]) + named := map[string]string{"width": strconv.Itoa(w), "height": strconv.Itoa(h)} + over, needs, worst := int64(0), int64(0), ask{} + for _, seed := range seeds { + asks := []ask{{format.Request{Bytes: 1, Seed: seed, Properties: named}, rung[2]}} + // The size on disk is what the label states: the ceiling itself, + // a tenth above it, and the lightest and the heaviest number with + // as many digits. + eights, lightest := int64(0), int64(1) + for n := rung[2]; n > 0; n /= 10 { + eights, lightest = eights*10+8, lightest*10 + } + lightest /= 10 + for _, size := range []int64{rung[2], rung[2] + rung[2]/10, lightest, eights} { + asks = append(asks, ask{format.Request{Bytes: 1, Seed: seed, Label: true, OnDisk: size, Properties: named}, rung[2]}) + } + // A damaged file states a size of any length the damage leaves it. + plain := format.Request{Bytes: 1, Seed: seed, Label: true, OnDisk: 1e12, Properties: named} + for _, damage := range damagesDrawnAtEveryScale(t, plain, "avif", w) { + r := plain + r.Damage = damage + asks = append(asks, ask{r, rung[3]}) + drawn++ + } + for _, a := range asks { + _, err := d.Generator.Plan(a.r) + var below *format.BelowMinimumError + if !errors.As(err, &below) { + t.Fatalf("a %dx%d picture refused one byte with %T rather than a BelowMinimumError: %v", w, h, err, err) + } + if below.Minimum-a.ceiling > over { + over, needs, worst = below.Minimum-a.ceiling, below.Minimum, a + } + } + } + if over > 0 { + t.Errorf("the %dx%d rung declares a ceiling of %d B and its picture needs %d B (seed %d, label %t, size %d, damage %q).\n"+ + "A ceiling below what its rung produces lets planning pick a picture the file cannot hold, and writing then refuses a size planning accepted.\n"+ + "What to do: run tools/probes/avifladder and record what it measures.", + w, h, worst.ceiling, needs, worst.r.Seed, worst.r.Label, worst.r.OnDisk, worst.r.Damage) + } + } + if drawn == 0 { + t.Fatal("no rung is wide enough to draw a damaged label, so this guard asked nothing about one") + } +} + // Planning must not code the picture. // // It used to, and the price was measured on 2026-08-29: a preview of a thousand diff --git a/internal/guard/damagelabel_test.go b/internal/guard/damagelabel_test.go index 3e8163e8..d254b0f6 100644 --- a/internal/guard/damagelabel_test.go +++ b/internal/guard/damagelabel_test.go @@ -246,6 +246,38 @@ func longestDamageLabel(t *testing.T) string { return long } +// damagesDrawnAtEveryScale is, for a picture width wide, the damage that makes +// the label of r the longest one still drawn at each scale the picture can draw +// a damaged label at, the largest scale first. +// +// The longest label is not the heaviest. A label too long for the picture is +// not drawn at all, and a shorter one is drawn larger, so the most a damaged +// label can put into a picture is the longest that still fits at some scale. +// The longest damage label puts no ink at all into a picture 640 wide (found in +// review, 2026-11-12). The lengths come from imagelabel, so the scales live in +// one place. +// +// Empty for a picture too narrow to draw any damaged label. +func damagesDrawnAtEveryScale(t *testing.T, r format.Request, id string, width int) []string { + t.Helper() + long := longestDamageLabel(t) + r.Damage = "x" + fixed := len(r.LabelText(id)) - len(r.Damage) + var out []string + for _, n := range imagelabel.LongestAtEachScale(width) { + if n <= fixed { + continue // no damaged label is this short + } + r.Damage = long[:min(n-fixed, len(long))] + if got, want := imagelabel.BandHeight(width, len(r.LabelText(id))), imagelabel.BandHeight(width, n); got != want || got == 0 { + t.Fatalf("a %s label of %d characters across %d pixels draws a band %d high where %d was worked out - the guard does not reach the scale it measures", + id, len(r.LabelText(id)), width, got, want) + } + out = append(out, r.Damage) + } + return out +} + // isGzip says whether the file at path starts as a gzip stream. func isGzip(path string) bool { head := make([]byte, 2) diff --git a/internal/guard/jxlladder_test.go b/internal/guard/jxlladder_test.go index 8f6c64df..01e20c4f 100644 --- a/internal/guard/jxlladder_test.go +++ b/internal/guard/jxlladder_test.go @@ -3,6 +3,7 @@ package guard import ( "context" "runtime" + "slices" "testing" "github.com/donislawdev/TestingFilesGenerator/internal/format" @@ -78,16 +79,38 @@ func TestJxlDrawsAPictureThatGrowsWithTheFileAndAlwaysFits(t *testing.T) { t.Logf("largest picture reached over the sweep: %d pixels", biggest) } -// Every rung's declared ceiling still covers what that rung produces. +// heaviestJxlSeeds are, for each rung that draws a label, the seeds the +// thorough sweep (tools/probes/jxlladder, 768 seeds) found heaviest on +// 2026-11-12 - for the label of a file at the edge of the rung and for the +// label of a damaged one. The first few seeds alone stayed green with the +// ceiling for a damaged file set back to the number that was too low, so they +// could not see what the sweep found. +var heaviestJxlSeeds = map[int][]uint64{ + 640: {0xc76d0104c41a71bd, 0x8f27bfdd1384288d}, + 320: {0x29b351201f9ae4c5, 0xe162c2724da869d3}, + 256: {0x3d, 0x80e38da90dd80935}, + 160: {0xb9f4567924888695}, +} + +// Every rung's two ceilings still cover what that rung produces: the first +// with the label of a file at the edge of the rung, the second with the label +// of a damaged file. // // The guard above walks the ladder the way a run does, so it only ever sees // the rung each size happens to select, at one seed. This one asks each rung // directly, across seeds, which is the question the ceiling is an answer to. // -// A handful of seeds rather than all 256: the thorough sweep is -// tools/probes/jxlladder, which takes minutes. This is the version that runs -// on every push and would still catch a codec raised underneath us, because a -// new encoder moves every rung at once rather than one seed of one rung. +// A file at the edge of a rung - asked for at its ceiling, where the picture +// leaves the least room - states a size with the digits of the ceiling, so the +// first ceiling is asked with those. A label with more digits is a file with +// far more room than its picture needs. A damaged file's label names its +// damage, and a longer label is more ink until it is too long for its scale, so +// the second is asked with the longest damaged label every scale still draws. +// +// A handful of seeds from zero and the heaviest the thorough sweep found +// (heaviestJxlSeeds): this is the version that runs on every push and would +// still catch a codec raised underneath us, because a new encoder moves every +// rung at once rather than one seed of one rung. // // Fewer seeds again under the race detector, and that needs saying rather than // hiding. This encoder is slow and the detector makes it slower: measured on @@ -103,47 +126,65 @@ func TestEveryJxlRungStillFitsInsideItsDeclaredCeiling(t *testing.T) { t.Fatal("the ladder is empty, so this guard measured nothing") } - // The label carries the byte count, so its length moves with the size - // asked for, and a longer label is more ink in the picture. A damaged - // file's label names its damage as well, up to damage.LabelLimit, which is - // the most ink a label can put down (found in review, 2026-11-12). - requested := []int64{int64(jxl.MinimumBytes), 1 << 14, 1 << 30} - damaged := longestDamageLabel(t) - - seeds := uint64(8) + first := uint64(8) if raceEnabled { - seeds = 2 - t.Logf("the race detector is on, so this sweeps %d seeds rather than 8 - it costs minutes here and finds no races either way", seeds) + first = 2 + t.Logf("the race detector is on, so this sweeps %d seeds from zero rather than 8 - it costs minutes here and finds no races either way", first) } + type ask struct { + label string + ceiling int64 + } + drawn := 0 for _, r := range rungs { - w, h, ceiling := int(r[0]), int(r[1]), r[2] - worst := int64(0) - var worstSeed uint64 - for seed := uint64(0); seed < seeds; seed++ { - for _, want := range requested { - for _, damage := range []string{"", damaged, "none"} { - label := "" - if damage != "none" { - label = format.Request{Bytes: want, Seed: seed, Damage: damage}.LabelText("jxl") - } - n, err := jxl.FileSizeFor(w, h, jxl.DefaultQuality, seed, label) - if err != nil { - t.Fatalf("%dx%d seed %d: %v", w, h, seed, err) - } - if int64(n) > worst { - worst, worstSeed = int64(n), seed - } + w, h := int(r[0]), int(r[1]) + seeds := slices.Clone(heaviestJxlSeeds[w]) + for seed := range first { + seeds = append(seeds, seed) + } + // The size the label states: the ceiling itself, a tenth above it, + // and the lightest and the heaviest number with as many digits. + eights, lightest := int64(0), int64(1) + for n := r[2]; n > 0; n /= 10 { + eights, lightest = eights*10+8, lightest*10 + } + lightest /= 10 + var worst ask + over, size, worstSeed := int64(0), int64(0), uint64(0) + for _, seed := range seeds { + asks := []ask{{"", r[2]}} + for _, want := range []int64{r[2], r[2] + r[2]/10, lightest, eights} { + asks = append(asks, ask{format.Request{Bytes: want, Seed: seed}.LabelText("jxl"), r[2]}) + } + // A damaged file states a size of any length the damage leaves it. + plain := format.Request{Bytes: 1e12, Seed: seed} + for _, damage := range damagesDrawnAtEveryScale(t, plain, "jxl", w) { + damaged := plain + damaged.Damage = damage + asks = append(asks, ask{damaged.LabelText("jxl"), r[3]}) + drawn++ + } + for _, a := range asks { + n, err := jxl.FileSizeFor(w, h, jxl.DefaultQuality, seed, a.label) + if err != nil { + t.Fatalf("%dx%d seed %d: %v", w, h, seed, err) + } + if int64(n)-a.ceiling > over { + over, size, worst, worstSeed = int64(n)-a.ceiling, int64(n), a, seed } } } - if worst > ceiling { - t.Errorf("the %dx%d rung declares a ceiling of %d B and produced %d B at seed %d.\n"+ + if over > 0 { + t.Errorf("the %dx%d rung declares a ceiling of %d B and produced %d B at seed %#x, with the label %q.\n"+ "A ceiling below what its rung produces lets planning pick a picture the file cannot hold.\n"+ "What to do: run tools/probes/jxlladder and record what it measures.", - w, h, ceiling, worst, worstSeed) + w, h, worst.ceiling, size, worstSeed, worst.label) } } + if drawn == 0 { + t.Fatal("no rung is wide enough to draw a damaged label, so this guard asked nothing about one") + } } // Planning must not code the picture. diff --git a/internal/guard/ladderceiling_test.go b/internal/guard/ladderceiling_test.go index b983ef42..07a0cdf3 100644 --- a/internal/guard/ladderceiling_test.go +++ b/internal/guard/ladderceiling_test.go @@ -71,24 +71,35 @@ func TestTheLadderCeilingIsAboveEveryPictureTheTopRungMakes(t *testing.T) { worst := int64(0) var worstAt string + drawn := 0 for _, extra := range c.extra { for _, seed := range seeds { - for _, label := range []bool{true, false} { - props := map[string]string{ - "width": strconv.Itoa(c.topWidth), - "height": strconv.Itoa(c.topHigh), - } - for k, v := range extra { - props[k] = v - } + props := map[string]string{ + "width": strconv.Itoa(c.topWidth), + "height": strconv.Itoa(c.topHigh), + } + for k, v := range extra { + props[k] = v + } + // Without a label, with one, and with the label of a damaged + // file at every scale the picture can draw it at - a longer + // label is more ink until it no longer fits its scale. + plain := format.Request{Bytes: 1, Seed: seed, Label: true, Properties: props} + asks := []format.Request{plain, {Bytes: 1, Seed: seed, Properties: props}} + for _, damage := range damagesDrawnAtEveryScale(t, plain, c.id, c.topWidth) { + damaged := plain + damaged.Damage = damage + asks = append(asks, damaged) + drawn++ + } + + for _, r := range asks { // Asking for one byte at the top rung makes the format // state its own floor, and that floor is the encoded // picture plus whatever it always carries. Nothing else // reports the encoded size from outside the package. - _, err := d.Generator.Plan(format.Request{ - Bytes: 1, Seed: seed, Label: label, Properties: props, - }) + _, err := d.Generator.Plan(r) var below *format.BelowMinimumError if !errors.As(err, &below) { t.Fatalf("%s at %dx%d refused one byte with %T rather than a BelowMinimumError: %v", @@ -96,11 +107,14 @@ func TestTheLadderCeilingIsAboveEveryPictureTheTopRungMakes(t *testing.T) { } if below.Minimum > worst { worst = below.Minimum - worstAt = fmt.Sprintf("seed %d, label %v, %v", seed, label, extra) + worstAt = fmt.Sprintf("seed %d, label %v, damage %q, %v", seed, r.Label, r.Damage, extra) } } } } + if drawn == 0 { + t.Fatalf("%s at %d wide draws no damaged label, so this guard asked nothing about one", c.id, c.topWidth) + } if worst > ceiling { t.Errorf("%s: the top rung reaches %d B (%s) but ladderCeiling is %d.\n"+ diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index fe41a711..cef2b58c 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -7413,8 +7413,8 @@ ( "the AVIF ladder claims a picture fits a file far too small to hold it, so the size it promised cannot be met", "internal/format/avif/avif.go", - "\t{640, 480, 16389}, {320, 240, 6792}, {256, 256, 4890}, {160, 120, 3392},", - "\t{640, 480, 311}, {320, 240, 6792}, {256, 256, 4890}, {160, 120, 3392},", + "\t{640, 480, 18722, 22027}, {320, 240, 7732, 8312}, {256, 256, 5493, 6648}, {160, 120, 3865, 3865},", + "\t{640, 480, 311, 22027}, {320, 240, 7732, 8312}, {256, 256, 5493, 6648}, {160, 120, 3865, 3865},", "TestAvifDrawsAPictureThatGrowsWithTheFileAndAlwaysFits", ), ( @@ -7511,8 +7511,8 @@ ( "the JPEG XL ladder claims a picture fits a file far too small to hold it, so the size it promised cannot be met", "internal/format/jxl/jxl.go", - " {640, 480, 22762}, {320, 240, 7949}, {256, 256, 6485}, {160, 120, 3542},", - " {640, 480, 147}, {320, 240, 7949}, {256, 256, 6485}, {160, 120, 3542},", + " {640, 480, 22762, 23567}, {320, 240, 7949, 8854}, {256, 256, 6485, 7637}, {160, 120, 3542, 3542},", + " {640, 480, 147, 23567}, {320, 240, 7949, 8854}, {256, 256, 6485, 7637}, {160, 120, 3542, 3542},", "TestJxlDrawsAPictureThatGrowsWithTheFileAndAlwaysFits", ), ( diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index 54f90578..feca6ddf 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -147,6 +147,9 @@ var provenByProbe = map[string]string{ "TestATruncatedTextSaysHowMuchWasCut": "broken on 2026-11-12 twice: Request.LabelSized ignoring OnDisk, and leaving the damage out. Red both times.", "TestALongDamageLabelLeavesEveryFormatWhole": "broken on 2026-11-12 by letting damage.Chain.Label return the whole chain past LabelLimit: red, " + "its own reach check says forty damages no longer fit the limit (docs/REVIEW-234-2026-11-12.md).", + "TestEveryAvifRungStillFitsInsideItsDeclaredCeilings": "its own reach check - no damaged label drawn is a failure - and broken on 2026-11-12 twice, with " + + "the heaviest seeds: the first ceiling of 640x480 set back to 16389 (red, 17314 B) and the second to 18722 (red, 20024 B). " + + "Without those seeds both stayed green (docs/REVIEW-234-2026-11-12.md).", "TestADamageLabelIsBoundedAndSaysHowManyItLeftOut": "its own canary - no chain up to 200 damages shortened is a failure - and broken on 2026-11-12 by letting damage.Chain.Label return the whole chain: red.", "TestADamagedMailCarriesItsLabelWholeInEveryTextEncoding": "broken on 2026-11-12 twice: eml writing the label raw in quoted-printable (red, the one damage label read back as bytesd), " + "and eml leaving a long label on one line (red, a line of 92 characters).", diff --git a/internal/guard/requestkey_test.go b/internal/guard/requestkey_test.go index 10adb046..73793c72 100644 --- a/internal/guard/requestkey_test.go +++ b/internal/guard/requestkey_test.go @@ -36,6 +36,8 @@ func TestEveryFieldOfARequestIsInItsKey(t *testing.T) { v.SetUint(1) case reflect.Bool: v.SetBool(true) + case reflect.String: + v.SetString("truncate:missing=90") case reflect.Map: v.Set(reflect.ValueOf(map[string]string{"delimiter": ";"})) case reflect.Slice: From c742dec25b6729df845b250617f65e065ca30532 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Thu, 12 Nov 2026 10:26:28 +0100 Subject: [PATCH 5/5] The AVIF rung guard is named in the race weights TestEveryAvifRungStillFitsInsideItsDeclaredCeilings took 73.6 s under the race detector, past the 60 s an unlisted test may take before it fails its part. Co-Authored-By: Claude Opus 5.5 --- .github/race-weights | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/race-weights b/.github/race-weights index 0ba8b1a2..55d4494f 100644 --- a/.github/race-weights +++ b/.github/race-weights @@ -23,6 +23,7 @@ unlisted 0.58 117.9 TestFreeBoxesCarryPaddingOfAnyLength 99.9 TestEveryBase64TextSizeIsExact 93.8 TestAFilmPictureLargerThanAnyAV1LevelIsRefusedNotBroken +73.6 TestEveryAvifRungStillFitsInsideItsDeclaredCeilings 67.9 TestADeclaredDefaultIsTheOneTheFormatUses 64.3 TestNoTrackedFileCarriesPrivateContent 63.8 TestNoGeneratorHoldsTheWholeFileInMemory