diff --git a/.github/race-weights b/.github/race-weights index 0ba8b1a2..55d4494f 100644 --- a/.github/race-weights +++ b/.github/race-weights @@ -23,6 +23,7 @@ unlisted 0.58 117.9 TestFreeBoxesCarryPaddingOfAnyLength 99.9 TestEveryBase64TextSizeIsExact 93.8 TestAFilmPictureLargerThanAnyAV1LevelIsRefusedNotBroken +73.6 TestEveryAvifRungStillFitsInsideItsDeclaredCeilings 67.9 TestADeclaredDefaultIsTheOneTheFormatUses 64.3 TestNoTrackedFileCarriesPrivateContent 63.8 TestNoGeneratorHoldsTheWholeFileInMemory diff --git a/CHANGELOG.md b/CHANGELOG.md index cf9cadd4..1d30e7d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,18 @@ because it turns other people's test suites red. was meant. Only pixels along those edges change, and many files do not change at all. A test that compares a file with one 0.5.0 generated from the same request can fail. +- **A damaged file names its damage in its label.** The label now reads + `tfg - txt - 10485760 B - damage truncate:missing=50 - seed c2a547b2`: the + size the file has on disk, then every damage with its settings, in the order + they were applied. A long list of damages names the first few and how many + more there are, and the manifest lists every one. A file damaged with + `zero-head` and carrying a label is therefore not byte for byte what 0.5.0 + wrote. Files with no damage, and files made with `--clean`, are unchanged. +- **An AVIF file just above the size where its picture grows gets the smaller + picture.** The picture now steps up to 160x120 at 3865 B, to 256x256 at + 5493 B, to 320x240 at 7732 B and to 640x480 at 18722 B, about a tenth above + where it did. A file between the old step and the new one is not byte for + byte what 0.5.0 wrote. ### Added @@ -164,6 +176,18 @@ because it turns other people's test suites red. ### Fixed +- **An AVIF file asked for just above the size where its picture grows is + written.** In 0.5.0 about a quarter of the files asked for at 16389 or + 16439 B were not written at all, and the run ended with "1 file could not be + produced" although a preview had accepted them. The same happened, less + often, at the smaller steps. +- **A damage setting outside its range is refused on the command line and in + the window.** `--damage truncate:missing=95` cut the default half, wrote the + file, ended with code 0 and recorded `missing: 95` in the manifest, and + `--damage zero-head:bytes=99999999999` zeroed 8 bytes. A setting the damage + does not take, such as `truncate:foo=1`, went through as well. Now the run + ends with code 2 before anything is written and says what the setting takes, + and the window shows that under the field. A recipe was already refused. - **An AVIF picture wider than 4096 px, or larger than 4096 x 2304 px, opens now.** Every release since AVIF arrived in 0.3.0 wrote such a picture as a file no viewer reads: Pillow, ImageMagick and libaom all refused it as diff --git a/README.md b/README.md index e5ffc759..fce3f3b2 100644 --- a/README.md +++ b/README.md @@ -514,9 +514,19 @@ tfg generate --format png --size 20kb --count 10 --damage zero-head:bytes=16 tfg generate --format zip --size 5mb --damage truncate:missing=30 ``` +A damaged file names its damage in its label, beside the size it has on disk: +`tfg - txt - 10485760 B - damage truncate:missing=50 - seed c2a547b2`. So two +texts cut by different shares can be told apart from their first line, although +the text after it is the same. A long list of damages names the first few and +how many more there are, and the manifest lists every one. The label is still +written before the damage, so the damage can overwrite it, and the manifest does +not promise it is there. + The smallest file a damage can be given follows its settings, so the column is measured with the defaults. Ask for less and the run is refused before anything is -written, naming a size that would work. +written, naming a size that would work. A setting outside what the damage takes, +such as `truncate:missing=95`, is refused before anything is written as well, and +the run ends with code 2. Asking for `--expected accept` beside a damage that breaks the rules of the format is refused too, because nothing that follows them accepts the file. Where diff --git a/internal/cli/errors.go b/internal/cli/errors.go index 660226d4..1327d0ad 100644 --- a/internal/cli/errors.go +++ b/internal/cli/errors.go @@ -248,6 +248,21 @@ func classifyRequest(err error) (int, bool) { if errors.As(err, &unknownPreset) { return ExitUsage, true } + if code, ok := classifyDamage(err); ok { + return code, true + } + if code, ok := classifyFormat(err); ok { + return code, true + } + return 0, false +} + +// classifyDamage covers what was asked to be broken. +// +// Its own function rather than more of classifyRequest, which crossed the +// line the crowding guard draws at sixty lines when the setting refusal +// arrived. Asked before classifyFormat, for the reason the last branch gives. +func classifyDamage(err error) (int, bool) { // A damage this build does not know is a typo in the invocation, the same // class as an unknown preset - a recipe naming one is refused while the // recipe is read, so anything reaching here came off the command line. @@ -288,8 +303,14 @@ func classifyRequest(err error) (int, bool) { if errors.As(err, &unsized) { return ExitUsage, true } - if code, ok := classifyFormat(err); ok { - return code, true + // The same class once more: a setting a damage does not take, or a value + // it does not allow, is wrong for every format and size. Asked before + // classifyFormat on purpose, because the refusal it wraps is the one a + // format setting out of range ends with FORMAT for - and that one is a fact + // about the format, which this is not. + var setting *damage.SettingError + if errors.As(err, &setting) { + return ExitUsage, true } return 0, false } diff --git a/internal/core/label.go b/internal/core/label.go deleted file mode 100644 index 3f2ed6e3..00000000 --- a/internal/core/label.go +++ /dev/null @@ -1,17 +0,0 @@ -package core - -import "fmt" - -// The self describing label ends the folder full of test1.txt, test2.txt and -// test_final_v2.txt. On a screenshot attached to a bug report you can see at -// once which file it is and how to reproduce it. -// -// It stays plain ASCII on purpose. Text formats reach into encodings such as -// Windows-1252 and Shift-JIS where a typographic separator has no -// representation, and a label that fails to encode is worse than a plain one. - -// Label composes the self describing label for one file. Every format that -// carries a label uses this one, so the wording cannot drift between them. -func Label(formatID string, bytes int64, seed uint64) string { - return fmt.Sprintf("tfg - %s - %d B - seed %s", formatID, bytes, SeedLabel(seed)) -} diff --git a/internal/core/setting.go b/internal/core/setting.go index 8f6f6955..373e7756 100644 --- a/internal/core/setting.go +++ b/internal/core/setting.go @@ -116,6 +116,11 @@ const ( // that is where the box is, and a bare width would collide with a recipe // key of the same name the day one arrives. KeyProperties = "properties" + // KeyDamage is the list of damages a target applies. A refusal about one + // setting of one entry is addressed under it - targets[1].damage[2].bytes + // - by the recipe reader and the engine alike, so a report names the same + // box whichever of them refused it. + KeyDamage = "damage" ) // TargetAddress is where one setting of one target lives. @@ -152,6 +157,13 @@ func ContentAddress(target, entry int, setting string) string { return fmt.Sprintf("%s.%s[%d].%s", TargetPrefix(target), KeyContains, entry, setting) } +// DamageAddress is where one setting of one damage lives inside a target, +// without the target: the engine puts that in front when it knows which +// target it is (atTarget). The position counts from one, for the reason above. +func DamageAddress(entry int, setting string) string { + return fmt.Sprintf("%s[%d].%s", KeyDamage, entry, setting) +} + // AddressNamesATarget says whether an address carries the position of the // target it is about, rather than a setting name on its own. // diff --git a/internal/damage/apply.go b/internal/damage/apply.go index 5ee6f871..03752ff7 100644 --- a/internal/damage/apply.go +++ b/internal/damage/apply.go @@ -4,6 +4,8 @@ import ( "fmt" "io" "sort" + "strconv" + "strings" "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" @@ -92,12 +94,30 @@ func (d Descriptor) checkOne(p format.Property, raw string) error { } } -// Check is the first problem with the values stated, for the one-target path -// from the command line flags where there is no form to lay four of them out -// on. -func (d Descriptor) Check(v Values) error { - if bad := d.CheckEach(v); len(bad) > 0 { - return bad[0] +// Check is the first problem with what this chain states: a damage this build +// does not know, or a setting its declaration does not take or does not allow. +// +// The engine asks it, for the command line and the window, which hand it a +// chain without reading a recipe. The recipe reader asks CheckEach of every +// entry and reports each problem with its address, so this is a second caller +// of the same rule rather than a copy of it - the shape O199 gave the +// expectation. The first problem only, as format.Descriptor.CheckProperties +// gives it beside this: a window draws one damage at a time, and the engine +// addresses one refusal per target. +// +// Asked before a byte is written, because a value past the declaration does +// not fail later. The settings are read with a fallback to the default +// (missingShare, junkBytes, headBytes), so without this the file would be cut +// by the default and the manifest would record the value as written. +func (c Chain) Check() error { + for i, s := range c { + d, err := Get(s.ID) + if err != nil { + return err + } + if bad := d.CheckEach(s.Values); len(bad) > 0 { + return &SettingError{Position: i + 1, Err: bad[0]} + } } return nil } @@ -347,6 +367,86 @@ func (c Chain) String() string { return fmt.Sprint(out) } +// LabelLimit is the most bytes the damage takes up in a label. +// +// The label lands in fields with limits of their own, and the tightest one +// measured is a gzip comment, which Go's reader refuses from 512 bytes +// (compress/gzip, a 512 byte buffer). A chain has no length limit of its own, +// so without this a long one would break a format before any damage was +// applied - an archive Go cannot open, a ZIP comment past its padding. Bounded +// here, once, so a format has one length to be ready for rather than every +// length a recipe can write (TestALongDamageLabelLeavesEveryFormatWhole). +const LabelLimit = 160 + +// Label is the chain as the label of a damaged file states it: every entry in +// order, its settings resolved the way the manifest records them, joined by +// " then " - "truncate:missing=50 then append-junk:bytes=1024". Empty for an +// empty chain, which is a file whose label does not change. +// +// Resolved rather than as written, for the reason Resolved gives: the label +// answers what was done to this file, and its reader should not need the +// defaults of the build that wrote it. ASCII whatever a setting holds, because +// the label goes into encodings where nothing else is sure to survive +// (format/label.go): a value with anything else in it, or with a space or one +// of the separators, is written quoted with escapes. +// +// At most LabelLimit bytes. A chain longer than that names the entries that +// fit and how many more there are - "... and 12 more" - and the manifest lists +// every one. An entry too long to fit on its own is named by its damage alone. +func (c Chain) Label() string { + out := make([]string, 0, len(c)) + for _, s := range c { + values := Values{} + for name, value := range s.Resolved() { + values[name] = plainOrQuoted(value) + } + out = append(out, Spec{ID: s.ID, Values: values}.String()) + } + if whole := strings.Join(out, " then "); len(whole) <= LabelLimit { + return whole + } + return shortened(c, out) +} + +// shortened is the entries of a chain that fit in LabelLimit with room left +// for saying how many did not. +func shortened(c Chain, entries []string) string { + kept := "" + for i, entry := range entries { + if i == 0 && len(entry) > LabelLimit/2 { + entry = c[0].ID + } + next := entry + if kept != "" { + next = kept + " then " + entry + } + if len(next)+len(moreThan(len(entries)-i-1)) > LabelLimit { + return kept + moreThan(len(entries)-i) + } + kept = next + } + return kept +} + +// moreThan is what a shortened label says about the entries it leaves out. +func moreThan(n int) string { + if n == 0 { + return "" + } + return " and " + strconv.Itoa(n) + " more" +} + +// plainOrQuoted is v as it is when every byte of it is printable ASCII other +// than the separators a label entry is written with, and quoted otherwise. +func plainOrQuoted(v string) string { + for i := 0; i < len(v); i++ { + if b := v[i]; b <= ' ' || b > '~' || b == ',' || b == '=' || b == ':' || b == '"' { + return strconv.QuoteToASCII(v) + } + } + return v +} + // Resolved is the settings this entry was applied with, defaults filled in. // // Filled in rather than left as written, because the manifest answers "what diff --git a/internal/damage/refusals.go b/internal/damage/refusals.go index 26148559..def30e23 100644 --- a/internal/damage/refusals.go +++ b/internal/damage/refusals.go @@ -1,6 +1,7 @@ package damage import ( + "errors" "strings" "github.com/donislawdev/TestingFilesGenerator/internal/core" @@ -63,6 +64,41 @@ func (e *UnknownError) Said() core.Said { core.A("What", e.what()), core.A("Known", strings.Join(e.Known, ", "))) } +// SettingError is one entry of a chain stating a setting its damage does not +// take, or a value the declaration of that setting does not allow. +// +// A wrapper around the declaration's own refusal rather than that refusal on +// its own, for two things the refusal underneath cannot know. Which entry of +// the list it is about, so a form marks the right box and the engine names the +// address the recipe reader gives for the same mistake. And its class: a value +// a damage does not allow is wrong whatever format and size are asked for, so +// the command line ends with the code a mistyped flag gets, not the code for a +// request no format can deliver - the call O199 made for an expectation. +// +// The words are the declaration's (format.Property.Allows and Instead), reached +// through Unwrap, so a damage setting and a format setting refuse alike. +type SettingError struct { + // Position is the entry's place in the chain, counted from one. + Position int + // Err is *format.PropertyValueError or *format.UnknownPropertyError. + Err error +} + +func (e *SettingError) Error() string { return e.Err.Error() } + +func (e *SettingError) Unwrap() error { return e.Err } + +// AboutSetting is the setting inside its entry - damage[1].missing. The +// engine puts the target in front, and a window showing one target keeps the +// last part, which is the name its box is registered under. +func (e *SettingError) AboutSetting() string { + var about interface{ AboutSetting() string } + if !errors.As(e.Err, &about) || about.AboutSetting() == "" { + return core.KeyDamage + } + return core.DamageAddress(e.Position, about.AboutSetting()) +} + // TooSmallError is a file smaller than the damage it was given. // // It is a refusal in the plan rather than a failure part way through, because @@ -234,7 +270,7 @@ type EatenError struct { // AboutSetting puts this on the damage, because the order of the list is the // thing to change. -func (e *EatenError) AboutSetting() string { return "damage" } +func (e *EatenError) AboutSetting() string { return core.KeyDamage } func (e *EatenError) What() string { return e.what().String() } diff --git a/internal/engine/damage.go b/internal/engine/damage.go index e6aceef6..153caba0 100644 --- a/internal/engine/damage.go +++ b/internal/engine/damage.go @@ -39,15 +39,21 @@ func checkDamage(t *Target, desc format.Descriptor) error { // prepareDamage is what a damaged target is refused for before a size is // drawn, and the format its files are planned through. // -// Before the draw, because a chain that cuts away what it added and a damage -// with no size to work from are wrong whatever size comes out of the seed. -// And the format it hands back is the one the draw has to ask: a damage that -// changes the length needs the generator asked for a different size than the -// one on disk (damagedformat.go). +// Before the draw, because a setting the damage does not allow, a chain that +// cuts away what it added and a damage with no size to work from are wrong +// whatever size comes out of the seed. And the format it hands back is the one +// the draw has to ask: a damage that changes the length needs the generator +// asked for a different size than the one on disk (damagedformat.go). +// +// The settings first, because every question after them reads the settings +// and would answer about the default where the value is out of range. func prepareDamage(t *Target, desc format.Descriptor) (format.Descriptor, error) { if len(t.Damage) == 0 { return desc, nil } + if err := t.Damage.Check(); err != nil { + return desc, err + } if eaten := t.Damage.Eaten(); eaten != nil { return desc, eaten } diff --git a/internal/engine/damagedformat.go b/internal/engine/damagedformat.go index c32d2eeb..36f8b70e 100644 --- a/internal/engine/damagedformat.go +++ b/internal/engine/damagedformat.go @@ -33,8 +33,10 @@ import ( // // For a caller that has to know before planning - a guard measuring each pair // at its smallest, or a list announcing the number. Asked of the same format a -// run plans through, so the number cannot be one the run refuses. +// run plans through, so the number cannot be one the run refuses - with the +// damage in the request, because the label names it and a run's does. func SmallestDamaged(d format.Descriptor, chain damage.Chain, r format.Request) int64 { + r.Damage = chain.Label() floor, _, err := chain.Floor() if err != nil { return d.SmallestAccepted(r) @@ -78,13 +80,9 @@ func (g *damagedGenerator) PlanContext(ctx context.Context, r format.Request) (f if err != nil { return format.Plan{}, err } - r.Bytes = want - p, err := g.plan(ctx, r) - if err != nil && g.chain.Stretches() { - p, err = g.stretch(ctx, r, err) - } + p, inner, err := g.whole(ctx, r, want, size) if err != nil { - return format.Plan{}, g.inSizeOnDisk(ctx, r, size, err) + return format.Plan{}, g.inSizeOnDisk(ctx, inner, size, err) } return format.Plan{ Bytes: size, @@ -98,6 +96,20 @@ func (g *damagedGenerator) PlanContext(ctx context.Context, r format.Request) (f }, nil } +// whole plans the file the damage starts from, want bytes long, for a file of +// size bytes on disk, and answers the request it asked with. +// +// The label inside it states the file on disk, which is size - not the size +// the generator writes, nor one stretch moves it to. +func (g *damagedGenerator) whole(ctx context.Context, r format.Request, want, size int64) (format.Plan, format.Request, error) { + r.Bytes, r.OnDisk = want, size + p, err := g.plan(ctx, r) + if err != nil && g.chain.Stretches() { + p, err = g.stretch(ctx, r, err) + } + return p, r, err +} + // plan asks the real generator, through the wrapper everything the engine asks // of a generator goes through. // @@ -215,11 +227,7 @@ func (g *damagedGenerator) firstWritable(ctx context.Context, r format.Request, func (g *damagedGenerator) inSizeOnDisk(ctx context.Context, r format.Request, size int64, err error) error { var below *format.BelowMinimumError if errors.As(err, &below) { - first := below.Minimum - if writable, werr := g.firstWritable(ctx, r, first); werr == nil { - first = writable - } - least := g.smallestReaching(first) + first, least := g.floorOnDisk(ctx, r, below.Minimum) return &format.BelowMinimumError{ Format: g.format, Requested: size, Minimum: least, Reason: core.Says("engine.DamagedBelowWhy", "%s has to start from a %s file of at least %s", @@ -238,6 +246,38 @@ func (g *damagedGenerator) inSizeOnDisk(ctx context.Context, r format.Request, s return err } +// floorOnDisk is the smallest file the damage can start from and the smallest +// size on disk that reaches it, from the size a refusal of r named. +// +// Walked to a size the generator writes first, for the reason inSizeOnDisk +// gives. Then asked again at the size on disk it comes to, because the label +// of a damaged file states its size on disk: where that label is longer than +// the room a format keeps for one, the floor moves with the number of digits. +// A refusal of 428 B measured a label one byte shorter than the one the 1284 B +// it named carries, and 1284 B was refused in turn (docx with append-junk, +// measured 2026-11-12). A few rounds settle it, because the floor only grows +// with the digits and the digits stop changing. +func (g *damagedGenerator) floorOnDisk(ctx context.Context, r format.Request, named int64) (first, least int64) { + first = named + for round := 0; round < 4; round++ { + if writable, err := g.firstWritable(ctx, r, first); err == nil { + first = writable + } + least = g.smallestReaching(first) + want, err := g.chain.Before(least) + if err != nil { + return first, least + } + _, inner, err := g.whole(ctx, r, want, least) + var below *format.BelowMinimumError + if err == nil || !errors.As(err, &below) || below.Minimum <= first { + return first, least + } + r, first = inner, below.Minimum + } + return first, least +} + // smallestReaching is the smallest size on disk whose damage needs at least // written bytes from the generator. What the damage needs grows with the size // asked for, so this is found by halving. diff --git a/internal/engine/drawsizes.go b/internal/engine/drawsizes.go index dee68a2d..75104294 100644 --- a/internal/engine/drawsizes.go +++ b/internal/engine/drawsizes.go @@ -109,12 +109,8 @@ func firstWritable(ctx context.Context, desc format.Descriptor, r format.Request // format.PlanWith asks the context before anything else, and a stop is not a // refusal about the size, so firstWritable hands it straight back. func drawSizes(ctx context.Context, t *Target, desc format.Descriptor, targetSeed uint64) error { - first := format.Request{ - Contains: t.Contains, - Seed: core.FileSeed(targetSeed, 0), - Label: t.Label, - Properties: t.Properties, - } + damaged := t.Damage.Label() + first := requestFor(t, core.FileSeed(targetSeed, 0), damaged) // The low end is judged against what this format can do AT ALL, and that // check is older than the snapping below it. The two answer different @@ -155,12 +151,7 @@ func drawSizes(ctx context.Context, t *Target, desc format.Descriptor, targetSee want = t.Range.Min + int64(r.Uint64N(span+1)) } - req := format.Request{ - Contains: t.Contains, - Seed: core.FileSeed(targetSeed, i), - Label: t.Label, - Properties: t.Properties, - } + req := requestFor(t, core.FileSeed(targetSeed, i), damaged) got, err := firstWritable(ctx, desc, req, want, t.Range.Max) if err != nil && want > t.Range.Min { diff --git a/internal/engine/plantarget.go b/internal/engine/plantarget.go index 0b5cd4d1..54458096 100644 --- a/internal/engine/plantarget.go +++ b/internal/engine/plantarget.go @@ -38,17 +38,13 @@ func (pl *planning) files(ctx context.Context, t *Target, desc format.Descriptor // One ending for every file of the target, because it follows the // target's settings and every file shares them. ext := desc.ExtensionFor(t.Properties) + damaged := t.Damage.Label() for idx, size := range t.Sizes { fileSeed := core.FileSeed(targetSeed, idx) - p, err := planWithoutCrashing(ctx, desc, format.Request{ - Bytes: size, - SizeFromContents: t.SizeFromContents, - Contains: t.Contains, - Seed: fileSeed, - Label: t.Label, - Properties: t.Properties, - }) + r := requestFor(t, fileSeed, damaged) + r.Bytes = size + p, err := planWithoutCrashing(ctx, desc, r) if err != nil { return atTarget(position, t, err) } @@ -176,3 +172,24 @@ func (pl *planning) startList(ctx context.Context) error { pl.out = make([]PlannedFile, 0, max(1, pl.reserve)) return nil } + +// requestFor is what one file of t asks its generator for, all but the size. +// +// One place for the three that ask - planning each file, and the floor and +// each draw of a size from a range - because a field one of them leaves out is +// a file drawn as one thing and planned as another. The label is where that +// shows first: it names the damage, and a label is part of the size a +// generator settles. +// +// damaged is t.Damage.Label(), asked once a target by the caller rather than +// once a file, because it resolves every entry against the registry. +func requestFor(t *Target, seed uint64, damaged string) format.Request { + return format.Request{ + SizeFromContents: t.SizeFromContents, + Contains: t.Contains, + Seed: seed, + Label: t.Label, + Damage: damaged, + Properties: t.Properties, + } +} diff --git a/internal/format/avif/avif.go b/internal/format/avif/avif.go index 1531d22b..36ad4e8f 100644 --- a/internal/format/avif/avif.go +++ b/internal/format/avif/avif.go @@ -171,7 +171,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("avif", r.Bytes, r.Seed) + label = r.LabelText("avif") } q, err := quality(r.Properties) @@ -285,11 +285,21 @@ func checkFits(want int64, m memo) error { // who does want them says so - width and height are settings, and 40 megapixels // is the ceiling, which covers 4K and 8K. // -// Ceiling is the LARGEST file that rung has ever been seen to produce, the free -// box included, and it is what lets planning choose a picture without coding -// one. Measured on 2026-08-29 across all 256 seeds, four orders of magnitude of -// requested size - the label carries the byte count, so its length moves with -// it - and with the label on and off. +// Ceiling is a tenth above the LARGEST file that rung was seen to produce, the +// free box included, and it is what lets planning choose a picture without +// coding one. Measured on 2026-11-12 by tools/probes/avifladder over 256 seeds +// from zero and 512 far from it, because the label's text moves with the whole +// seed, with the label off and with the label of a file at the edge of the +// rung - a size with the digits of the ceiling, the lightest and the heaviest. +// 640x480 coded to 17020 B, 320x240 to 7029, 256x256 to 4993 and 160x120 to +// 3513. The rungs below draw no label and their picture moves only with the +// low byte of the seed, so their ceiling is the largest of all 256 and no more. +// +// The tenth is the margin the video ladder keeps, and it is not caution for its +// own sake. The ceilings measured on 2026-08-29 over 256 seeds kept none, and +// by 2026-11-12 a file asked for at the ceiling of 640x480, or 50 B above it, +// was not written in 45 of 512 tries on the 256 seeds near zero and in 332 of +// 1024 on 512 seeds far from it (O273). // // A ceiling rather than a floor, and the difference is the whole point. // Planning promises an exact size, so the rung it picks has to be one whose @@ -297,15 +307,46 @@ func checkFits(want int64, m memo) error { // costs a smaller picture than the file had room for and nothing else. Set too // low, it would let planning pick a picture that does not fit, so writing // checks and says so rather than trusting this table. +// +// The second number is the ceiling for a damaged file, whose label names its +// damage, by the same rule with the label at the longest length each scale +// still draws: 640x480 coded to 20024 B, 320x240 to 7556 and 256x256 to 6043, +// each above the first number. A rung too narrow to draw a damaged label keeps +// the first number. var sizeLadder = []rung{ - {640, 480, 16389}, {320, 240, 6792}, {256, 256, 4890}, {160, 120, 3392}, - {80, 60, 832}, {40, 30, 549}, {20, 15, 448}, {8, 6, 348}, - {4, 3, 344}, {2, 2, 327}, {1, 1, minimumBytes}, + {640, 480, 18722, 22027}, {320, 240, 7732, 8312}, {256, 256, 5493, 6648}, {160, 120, 3865, 3865}, + {80, 60, 832, 832}, {40, 30, 549, 549}, {20, 15, 448, 448}, {8, 6, 348, 348}, + {4, 3, 344, 344}, {2, 2, 327, 327}, {1, 1, minimumBytes, minimumBytes}, } type rung struct { width, height int ceiling int64 + // damaged is the ceiling for the label of a damaged file, which names its + // damage as well. A longer label can put more ink into the picture than + // any the ceiling was measured with, until it is too long to draw at all. + damaged int64 +} + +// bound is the ceiling that speaks for the label r carries. A file nothing +// damages keeps the ceiling it always had, so its picture and its bytes do not +// move (D11). +func (s rung) bound(r format.Request) int64 { + if r.Damage != "" { + return s.damaged + } + return s.ceiling +} + +// Rungs reports the ladder as the dimensions of each rung with its two +// ceilings - for a file nothing damages and for a damaged one - so a guard and +// a probe walk exactly the table planning walks rather than a second copy. +func Rungs() [][4]int64 { + out := make([][4]int64, 0, len(sizeLadder)) + for _, r := range sizeLadder { + out = append(out, [4]int64{int64(r.width), int64(r.height), r.ceiling, r.damaged}) + } + return out } // chooseSize settles the picture size and encodes it. @@ -330,7 +371,7 @@ func chooseSize(r format.Request, label string, q int) (memo, error) { // is for every other format. pick := sizeLadder[len(sizeLadder)-1] for _, step := range sizeLadder { - if r.Bytes >= step.ceiling { + if r.Bytes >= step.bound(r) { pick = step break } diff --git a/internal/format/bmp/bmp.go b/internal/format/bmp/bmp.go index ae27a640..485fae13 100644 --- a/internal/format/bmp/bmp.go +++ b/internal/format/bmp/bmp.go @@ -121,7 +121,7 @@ func pixelBytes(width, height int) int64 { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("bmp", r.Bytes, r.Seed) + label = r.LabelText("bmp") } w, h, err := chooseSize(r) diff --git a/internal/format/docx/docx.go b/internal/format/docx/docx.go index 81daa80a..cba77f24 100644 --- a/internal/format/docx/docx.go +++ b/internal/format/docx/docx.go @@ -88,7 +88,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("docx", r.Bytes, r.Seed) + label = r.LabelText("docx") } paragraphs, err := intProperty(r.Properties, "paragraphs", 1) @@ -197,7 +197,7 @@ func document(paragraphs int, seed uint64, label string) []byte { b.WriteString(``) b.WriteString(``) if label != "" { - b.WriteString(paragraph(opc.Text(opc.Fixed(label, opc.LabelWidth)))) + b.WriteString(paragraph(opc.Text(opc.FixedLabel(label)))) } for i := 0; i < paragraphs; i++ { b.WriteString(paragraph(opc.Line(rng, wordsPerParagraph))) @@ -233,7 +233,7 @@ func intProperty(props map[string]string, key string, fallback int) (int, error) // text of the first cell or slide, so there the labelled file is smaller. func minimumBytes() int64 { smallest := int64(1) << 62 - for _, label := range []string{"", core.Label("docx", 0, 0)} { + for _, label := range []string{"", format.Request{}.LabelText("docx")} { shape, err := opc.Measure(parts(1, 0, label)) if err != nil { continue diff --git a/internal/format/eml/eml.go b/internal/format/eml/eml.go index cd90b1ac..97b5592b 100644 --- a/internal/format/eml/eml.go +++ b/internal/format/eml/eml.go @@ -155,7 +155,7 @@ func fit(m *message, r format.Request) ([]format.Note, error) { if !r.Label { return nil, nil } - m.label = core.Label("eml", r.Bytes, r.Seed) + m.label = r.LabelText("eml") if m.size() <= r.Bytes { return nil, nil } @@ -177,7 +177,7 @@ func fit(m *message, r format.Request) ([]format.Note, error) { func settle(m *message, r format.Request) int64 { size := m.size() for round := 0; r.Label && round < 8; round++ { - m.label = core.Label("eml", size, r.Seed) + m.label = r.LabelSized("eml", size) next := m.size() if next == size { break diff --git a/internal/format/eml/message.go b/internal/format/eml/message.go index 8a662ad7..1a7f6d6c 100644 --- a/internal/format/eml/message.go +++ b/internal/format/eml/message.go @@ -3,6 +3,7 @@ package eml import ( "context" "io" + "strings" "github.com/donislawdev/TestingFilesGenerator/internal/core" ) @@ -177,7 +178,7 @@ func (m *message) textPart(o *out, contentType string, write func(*out), last bo func (m *message) plain(o *out) { o.put(m.frame.plainOpen) if m.label != "" { - o.put(m.label, o.eol) + o.put(m.labelText("", "", o.eol)) } o.put(m.plainLine) fill(o, m.text, m.seed) @@ -195,13 +196,52 @@ func (m *message) html(o *out) { } o.put(m.frame.htmlOpen) if m.label != "" { - o.put("

", m.label, "

", eol) + o.put(m.labelText("

", "

", eol)) } o.put(m.htmlLine, m.images, "

", eol) fill(o, m.text, m.seed) o.put(eol, "

", eol, m.frame.htmlClose) } +// labelText is the label as a part of the text writes it, between open and +// close and with its line endings: one line, as it always was where it fits. +// +// A damaged file's label names its damage, so it holds an = and can be longer +// than a line of the text. quoted-printable spells the = as =3D and breaks a +// long line softly (frameLine), so the text still reads as one line - written +// raw, append-junk:bytes=64 was read back as bytesd. The other encodings have +// no soft break, so there the label is broken at its spaces into lines of at +// most lineWidth, under the 78 characters RFC 5322 asks a line to keep to. +func (m *message) labelText(open, close, eol string) string { + line := open + m.label + close + if m.s.encoding == QuotedPrintable { + return frameLine(m.s, line) + eol + } + return strings.Join(atSpaces(line, lineWidth), eol) + eol +} + +// atSpaces breaks text at its spaces into lines of at most width, or as near +// as a word longer than width allows. Text that fits is one line. +func atSpaces(text string, width int) []string { + if len(text) <= width { + return []string{text} + } + var lines []string + line := "" + for _, word := range strings.Split(text, " ") { + switch { + case line == "": + line = word + case len(line)+1+len(word) > width: + lines = append(lines, line) + line = word + default: + line += " " + word + } + } + return append(lines, line) +} + // out is where a message goes - a file, or nowhere while it is being counted. // // The first error sticks and everything after it is counted and not written, diff --git a/internal/format/format.go b/internal/format/format.go index f746c6b6..d209eabe 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -668,8 +668,18 @@ type Request struct { // Seed determines the content. The same seed gives the same bytes. Seed uint64 // Label asks for the self describing label. On by default, turned off - // with --clean. + // with --clean. A generator composes it with LabelText (label.go). Label bool + // Damage is what the engine will break about this file once it is + // written, as the label states it - "truncate:missing=50" - or empty for + // a file nothing breaks. A generator does not apply it and reads it for + // nothing but the label, through LabelText. + Damage string + // OnDisk is the size the file has on disk once its damage has changed the + // length, where Bytes is the size of the whole file before it - a cut + // asks the generator for more than it keeps. Zero means Bytes is the size + // on disk. Read for the label alone, like Damage. + OnDisk int64 // Properties are format specific settings, straight from the recipe. Properties map[string]string } diff --git a/internal/format/gif/gif.go b/internal/format/gif/gif.go index 7d30b992..ca4f099d 100644 --- a/internal/format/gif/gif.go +++ b/internal/format/gif/gif.go @@ -189,7 +189,7 @@ func checkJointLimits(w, h int) error { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("gif", r.Bytes, r.Seed) + label = r.LabelText("gif") } m, err := chooseSize(r, label) diff --git a/internal/format/htmlfile/html.go b/internal/format/htmlfile/html.go index 5b95ae1f..a2cd0afe 100644 --- a/internal/format/htmlfile/html.go +++ b/internal/format/htmlfile/html.go @@ -181,7 +181,7 @@ func belowMinimum(s settings, requested, min int64) error { // is what a part does rather than how long it is: this answers "does the label // fit and what does it cost", and Plan answers "is this request askable at all". func labelledHead(s settings, r format.Request, min int64) (head, heading string, note *format.Note) { - label := core.Label("html", r.Bytes, r.Seed) + label := r.LabelText("html") m := s.spelling() heading = m[labelOpen] + label + m[labelClose] if s.indented() { diff --git a/internal/format/ico/ico.go b/internal/format/ico/ico.go index 8edf2c16..3fcca382 100644 --- a/internal/format/ico/ico.go +++ b/internal/format/ico/ico.go @@ -112,7 +112,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("ico", r.Bytes, r.Seed) + label = r.LabelText("ico") } embed, err := embedding(r.Properties) diff --git a/internal/format/imagelabel/font.go b/internal/format/imagelabel/font.go index 89e98c30..1fe818fa 100644 --- a/internal/format/imagelabel/font.go +++ b/internal/format/imagelabel/font.go @@ -64,18 +64,35 @@ var glyphs = map[rune]string{ '/': "..#" + "..#" + ".#." + "#.." + "#..", '+': "..." + ".#." + "###" + ".#." + "...", '#': "#.#" + "###" + "#.#" + "###" + "#.#", + // The label of a damaged file names its settings - missing=50, + // bytes=16,other=2 - and these two are how it writes them. + '=': "..." + "###" + "..." + "###" + "...", + ',': "..." + "..." + "..." + ".#." + "#..", } // unknown stands in for any character with no glyph, so a label never -// silently loses part of itself. +// silently loses part of itself. It is drawn the same as 0 and O, which is why +// HasGlyph asks the table rather than comparing with this. const unknown = "###" + "#.#" + "#.#" + "#.#" + "###" func glyph(r rune) string { - if r >= 'a' && r <= 'z' { - r -= 'a' - 'A' - } - if g, ok := glyphs[r]; ok { + if g, ok := glyphs[upper(r)]; ok { return g } return unknown } + +// HasGlyph says whether r is drawn as itself rather than as the stand-in, for +// the guard that asks it of every character a label can hold. +func HasGlyph(r rune) bool { + _, ok := glyphs[upper(r)] + return ok +} + +// upper is r in the case the table is drawn in - see the top of this file. +func upper(r rune) rune { + if r >= 'a' && r <= 'z' { + return r - ('a' - 'A') + } + return r +} diff --git a/internal/format/imagelabel/imagelabel.go b/internal/format/imagelabel/imagelabel.go index 114238d0..3eeed87e 100644 --- a/internal/format/imagelabel/imagelabel.go +++ b/internal/format/imagelabel/imagelabel.go @@ -203,6 +203,24 @@ func (t laidText) ink(x, y int) bool { return off < t.Width && glyph(t.runes[k])[(y-pad)/t.scale*glyphWidth+off/t.scale] == '#' } +// LongestAtEachScale is the most characters a label can have and still be +// drawn at each scale a picture width wide draws one at, the largest scale +// first - empty for a picture too narrow to draw any. +// +// That, and not the longest label, is the most a label can put into a picture: +// a longer one is drawn smaller, and one too long for the smallest scale is not +// drawn at all. A size ladder whose ceilings stand for every label is measured +// at these lengths. +func LongestAtEachScale(width int) []int { + var out []int + for n := 1; scaleFor(width, n) > 0; n++ { + if scaleFor(width, n+1) != scaleFor(width, n) { + out = append(out, n) + } + } + return out +} + // BandHeight is how tall the band Draw paints for a label of chars characters // across width is, before it is cut to the picture - 0 when the label does not // fit. A film draws a second line under the first and asks this before it diff --git a/internal/format/jpg/jpg.go b/internal/format/jpg/jpg.go index b103c7e5..c38e69f5 100644 --- a/internal/format/jpg/jpg.go +++ b/internal/format/jpg/jpg.go @@ -191,7 +191,7 @@ func segmentsFor(extra int64) (segments int, payload int64) { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("jpg", r.Bytes, r.Seed) + label = r.LabelText("jpg") } quality, err := qualityOf(r.Properties) diff --git a/internal/format/jxl/jxl.go b/internal/format/jxl/jxl.go index 0bd09a8f..d2a7d919 100644 --- a/internal/format/jxl/jxl.go +++ b/internal/format/jxl/jxl.go @@ -211,7 +211,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("jxl", r.Bytes, r.Seed) + label = r.LabelText("jxl") } q, err := quality(r.Properties) @@ -337,15 +337,38 @@ func checkFits(want int64, m memo) error { // costs a smaller picture than the file had room for and nothing else. Set too // low, it would let planning pick a picture that does not fit, so writing // checks and says so rather than trusting this table. +// +// The second number is the ceiling for a damaged file, whose label names its +// damage. Measured on 2026-11-12 by tools/probes/jxlladder over 768 seeds, with +// the label at the longest length each scale still draws: 640x480 coded to +// 21424 B, 320x240 to 8049 and 256x256 to 6942, the last two above the first +// number. It is a tenth above what was measured and never below the first +// number, the margin the video ladder keeps, because the damage text is +// whatever the chain names. A rung too narrow to draw a damaged label keeps the +// first number. var sizeLadder = []rung{ - {640, 480, 22762}, {320, 240, 7949}, {256, 256, 6485}, {160, 120, 3542}, - {80, 60, 1267}, {40, 30, 731}, {20, 15, 452}, {8, 6, 222}, - {4, 3, 212}, {2, 2, 206}, {1, 1, minimumBytes}, + {640, 480, 22762, 23567}, {320, 240, 7949, 8854}, {256, 256, 6485, 7637}, {160, 120, 3542, 3542}, + {80, 60, 1267, 1267}, {40, 30, 731, 731}, {20, 15, 452, 452}, {8, 6, 222, 222}, + {4, 3, 212, 212}, {2, 2, 206, 206}, {1, 1, minimumBytes, minimumBytes}, } type rung struct { width, height int ceiling int64 + // damaged is the ceiling for the label of a damaged file, which names its + // damage as well. A longer label can put more ink into the picture than + // any the ceiling was measured with, until it is too long to draw at all. + damaged int64 +} + +// bound is the ceiling that speaks for the label r carries. A file nothing +// damages keeps the ceiling it always had, so its picture and its bytes do not +// move (D11). +func (s rung) bound(r format.Request) int64 { + if r.Damage != "" { + return s.damaged + } + return s.ceiling } // chooseSize settles the picture size, and codes it only when it has to. @@ -370,7 +393,7 @@ func chooseSize(r format.Request, label string, q int) (memo, error) { // is for every other format. pick := sizeLadder[len(sizeLadder)-1] for _, step := range sizeLadder { - if r.Bytes >= step.ceiling { + if r.Bytes >= step.bound(r) { pick = step break } diff --git a/internal/format/jxl/measure.go b/internal/format/jxl/measure.go index 85dc2a6c..581553a7 100644 --- a/internal/format/jxl/measure.go +++ b/internal/format/jxl/measure.go @@ -28,13 +28,13 @@ func FileSizeFor(width, height, quality int, seed uint64, label string) (int, er return len(coded) + containerOverhead + boxHeader, nil } -// Rungs reports the ladder as pairs of dimensions with the ceiling each one -// carries, so a probe and a guard can walk exactly the table planning walks -// rather than a second copy of it. -func Rungs() [][3]int64 { - out := make([][3]int64, 0, len(sizeLadder)) +// Rungs reports the ladder as the dimensions of each rung with its two +// ceilings - for a file nothing damages and for a damaged one - so a probe and +// a guard walk exactly the table planning walks rather than a second copy. +func Rungs() [][4]int64 { + out := make([][4]int64, 0, len(sizeLadder)) for _, r := range sizeLadder { - out = append(out, [3]int64{int64(r.width), int64(r.height), r.ceiling}) + out = append(out, [4]int64{int64(r.width), int64(r.height), r.ceiling, r.damaged}) } return out } diff --git a/internal/format/label.go b/internal/format/label.go new file mode 100644 index 00000000..3283ea33 --- /dev/null +++ b/internal/format/label.go @@ -0,0 +1,47 @@ +package format + +import ( + "fmt" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" +) + +// The self describing label ends the folder full of test1.txt, test2.txt and +// test_final_v2.txt. On a screenshot attached to a bug report you can see at +// once which file it is and how to reproduce it. +// +// It stays plain ASCII on purpose. Text formats reach into encodings such as +// Windows-1252 and Shift-JIS where a typographic separator has no +// representation, and a label that fails to encode is worse than a plain one. +// The damage it names is ASCII for the same reason (damage.Chain.Label). +// +// A method of the request rather than a function a generator hands numbers +// to, because the request is what knows whether the file will be damaged and +// what size it keeps on disk. A generator asking anything else could only +// state the size it was asked to write, which for a cut file is the size of +// the whole file before the cut - the label of a truncated text then named a +// size nobody asked for and no damage at all (O333). + +// LabelText is the self describing label of the file r asks for. Every format +// that carries a label asks this, so the wording cannot drift between them. +func (r Request) LabelText(formatID string) string { + return r.LabelSized(formatID, r.Bytes) +} + +// LabelSized is the label of a file of size bytes, for a generator that works +// its size out from its contents and so states a size the request does not +// hold yet - zip, eml and targz settle the label and the size against each +// other. +// +// A damaged file states its size on disk wherever the request knows it, which +// is wherever a damage changes the length, and the damage before the seed. A +// file nothing damages carries the label it always carried, to the byte (D11). +func (r Request) LabelSized(formatID string, size int64) string { + if r.OnDisk > 0 { + size = r.OnDisk + } + if r.Damage == "" { + return fmt.Sprintf("tfg - %s - %d B - seed %s", formatID, size, core.SeedLabel(r.Seed)) + } + return fmt.Sprintf("tfg - %s - %d B - damage %s - seed %s", formatID, size, r.Damage, core.SeedLabel(r.Seed)) +} diff --git a/internal/format/logfile/log.go b/internal/format/logfile/log.go index 8d1eda8f..46de7069 100644 --- a/internal/format/logfile/log.go +++ b/internal/format/logfile/log.go @@ -185,7 +185,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { // label is a line of its own. It is the one line that is not an entry, // and it says so in words rather than pretending to be one - except in // JSON lines, where a comment is not a line any reader would take. - line := opt.shape.label(core.Label("log", r.Bytes, r.Seed), opt) + line := opt.shape.label(r.LabelText("log"), opt) // It has to leave room for at least one whole entry, or the file would // be a label and nothing else. if int64(len(line))+min <= r.Bytes { diff --git a/internal/format/md/md.go b/internal/format/md/md.go index 6d9f883c..2b81a394 100644 --- a/internal/format/md/md.go +++ b/internal/format/md/md.go @@ -119,7 +119,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { if r.Label { // A plain line followed by a blank one is a paragraph, which renders // visibly and cannot break the document wherever it sits. - line := core.Label("md", r.Bytes, r.Seed) + "\n\n" + line := r.LabelText("md") + "\n\n" if int64(len(line)) <= m.source { m.labelLine = line } else { diff --git a/internal/format/opc/opc.go b/internal/format/opc/opc.go index 8a9b7ae3..1b7b7ab0 100644 --- a/internal/format/opc/opc.go +++ b/internal/format/opc/opc.go @@ -453,11 +453,22 @@ func Line(rng *rand.Rand, _ int) string { return Fixed(b.String(), LineWidth) } -// LabelWidth is the room a label gets. Wider than any label core.Label can -// produce, so the text is never cut, and constant so the floor does not move -// with the number of digits in the size. +// LabelWidth is the room a label gets. Wider than the label of any file +// nothing damages, so that text is never cut, and constant so the floor does +// not move with the number of digits in the size. const LabelWidth = 64 +// FixedLabel is label padded to LabelWidth, or whole where it is longer. +// +// Longer is the label of a damaged file, which names its damage. It keeps its +// own length rather than being cut to the width, because a label missing the +// end of its seed points at a file nobody can make again - and the floor that +// then moves with the digits is the floor of a damaged file, whose generator +// is asked with the same label it writes, so it names that floor itself. +func FixedLabel(label string) string { + return Fixed(label, max(LabelWidth, len(label))) +} + // Fixed cuts or pads text to exactly n bytes. // // Padding is spaces, which XML keeps when the element asks it to and which diff --git a/internal/format/pdf/pdf.go b/internal/format/pdf/pdf.go index 864ed24b..c73dd125 100644 --- a/internal/format/pdf/pdf.go +++ b/internal/format/pdf/pdf.go @@ -62,7 +62,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("pdf", r.Bytes, r.Seed) + label = r.LabelText("pdf") } m := memo{opts: opts, seed: r.Seed, label: label} diff --git a/internal/format/png/png.go b/internal/format/png/png.go index 01daba80..09fdb501 100644 --- a/internal/format/png/png.go +++ b/internal/format/png/png.go @@ -159,7 +159,7 @@ func checkJointLimits(w, h int) error { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("png", r.Bytes, r.Seed) + label = r.LabelText("png") } m, err := chooseSize(r, label) diff --git a/internal/format/pptx/pptx.go b/internal/format/pptx/pptx.go index 7c7af6bf..0fa32e31 100644 --- a/internal/format/pptx/pptx.go +++ b/internal/format/pptx/pptx.go @@ -92,7 +92,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("pptx", r.Bytes, r.Seed) + label = r.LabelText("pptx") } slides, err := intProperty(r.Properties, "slides", 1) @@ -210,7 +210,7 @@ func parts(slides int, seed uint64, label string) []opc.Part { for i := 1; i <= slides; i++ { text := opc.Line(rng, 6) if i == 1 && label != "" { - text = opc.Text(opc.Fixed(label, opc.LabelWidth)) + text = opc.Text(opc.FixedLabel(label)) } out = append(out, opc.Part{ @@ -347,7 +347,7 @@ func intProperty(props map[string]string, key string, fallback int) (int, error) // document the label is an extra paragraph and it goes the other way. func minimumBytes() int64 { smallest := int64(1) << 62 - for _, label := range []string{"", core.Label("pptx", 0, 0)} { + for _, label := range []string{"", format.Request{}.LabelText("pptx")} { shape, err := opc.Measure(parts(1, 0, label)) if err != nil { continue diff --git a/internal/format/registry.go b/internal/format/registry.go index f8048773..62679884 100644 --- a/internal/format/registry.go +++ b/internal/format/registry.go @@ -148,6 +148,9 @@ func RequestKey(id string, r Request) (string, bool) { sort.Strings(names) var b strings.Builder fmt.Fprintf(&b, "%q %d %t %t %d", id, r.Bytes, r.SizeFromContents, r.Label, r.Seed) + // The damage and the size on disk are words of the label, so they change + // its length and with it the smallest file that carries it. + fmt.Fprintf(&b, " %d %q", r.OnDisk, r.Damage) for _, name := range names { fmt.Fprintf(&b, " %q=%q", name, r.Properties[name]) } diff --git a/internal/format/rtf/rtf.go b/internal/format/rtf/rtf.go index 3f2749ec..36254ea3 100644 --- a/internal/format/rtf/rtf.go +++ b/internal/format/rtf/rtf.go @@ -93,7 +93,7 @@ func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan describePictures(p.Properties, s.pictures, pictures) m := memo{settings: s, seed: r.Seed, pictures: pictures, pictureBytes: pictureBytes} if r.Label { - line := labelLine(core.Label("rtf", r.Bytes, r.Seed)) + line := labelLine(r.LabelText("rtf")) if min+int64(len(line)) <= r.Bytes { m.labelLine = line } else { diff --git a/internal/format/svgfile/svg.go b/internal/format/svgfile/svg.go index 8ef54923..91c2aadf 100644 --- a/internal/format/svgfile/svg.go +++ b/internal/format/svgfile/svg.go @@ -282,7 +282,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { m := memo{seed: r.Seed, width: w, height: h} if r.Label { - switch line := labelOpen(h) + core.Label("svg", r.Bytes, r.Seed) + textClose; { + switch line := labelOpen(h) + r.LabelText("svg") + textClose; { case !labelFits(h): // Room on the page rather than room in the byte count, so it is a // separate sentence from the one below. Both leave the file named diff --git a/internal/format/targz/targz.go b/internal/format/targz/targz.go index fd4715dd..ba751886 100644 --- a/internal/format/targz/targz.go +++ b/internal/format/targz/targz.go @@ -301,7 +301,7 @@ func planChildren(ctx context.Context, r format.Request, groups []format.Content // label. The two settle against each other the same way they do in ZIP. func settleSize(m *memo, r format.Request) (target int64, label string, err error) { if r.Label && !r.SizeFromContents { - label = core.Label("targz", r.Bytes, r.Seed) + label = r.LabelText("targz") } m.comment = label @@ -320,7 +320,7 @@ func settleSize(m *memo, r format.Request) (target int64, label string, err erro // the wrong size is worse than the work of finding out. size := bare for i := 0; i < 4; i++ { - m.comment = core.Label("targz", size, r.Seed) + m.comment = r.LabelSized("targz", size) measured := archiveSize(*m) if measured == size { return size, m.comment, nil diff --git a/internal/format/tiff/tiff.go b/internal/format/tiff/tiff.go index 0c0f0f94..b6a1eaf5 100644 --- a/internal/format/tiff/tiff.go +++ b/internal/format/tiff/tiff.go @@ -161,7 +161,7 @@ func pixelBytes(width, height int) int64 { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("tiff", r.Bytes, r.Seed) + label = r.LabelText("tiff") } w, h, err := chooseSize(r) diff --git a/internal/format/tomlfile/toml.go b/internal/format/tomlfile/toml.go index ff433935..cb081720 100644 --- a/internal/format/tomlfile/toml.go +++ b/internal/format/tomlfile/toml.go @@ -161,7 +161,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { m := memo{seed: r.Seed} if r.Label { - line := "# " + core.Label("toml", r.Bytes, r.Seed) + "\n" + line := "# " + r.LabelText("toml") + "\n" if int64(len(line))+min <= r.Bytes { m.comment = line } else { diff --git a/internal/format/txt/txt.go b/internal/format/txt/txt.go index b867b907..3dca0252 100644 --- a/internal/format/txt/txt.go +++ b/internal/format/txt/txt.go @@ -112,7 +112,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { m := memo{seed: r.Seed, codec: codec, source: codec.Source(r.Bytes)} if r.Label { - line := core.Label("txt", r.Bytes, r.Seed) + "\n" + line := r.LabelText("txt") + "\n" if int64(len(line)) <= m.source { m.labelLine = line } else { diff --git a/internal/format/video/choose.go b/internal/format/video/choose.go index 5f2ffda9..c129178f 100644 --- a/internal/format/video/choose.go +++ b/internal/format/video/choose.go @@ -14,6 +14,19 @@ import ( type Rung struct { Width, Height int Ceiling int + // Damaged is the ceiling for a film whose label names its damage. A + // longer label can put more ink into a picture than any the first ceiling + // was measured with, until it is too long to draw at all. + Damaged int +} + +// bound is the ceiling that speaks for the label r carries. A film nothing +// damages keeps the ceiling it always had, so its bytes do not move (D11). +func (g Rung) bound(r format.Request) int { + if r.Damage != "" { + return g.Damaged + } + return g.Ceiling } // Ladder is tried from the largest down when the request names no picture @@ -47,10 +60,18 @@ type Rung struct { // room for and nothing else. One too low would let planning promise a size // writing cannot keep, so writing checks every picture and says so rather // than trusting this table. +// +// The second number is the ceiling for a damaged film, whose label names its +// damage, by the same rule: a tenth above the largest picture measured with +// the label at the longest length each scale still draws, never below the +// first number. Measured on 2026-11-12 by tools/probes/videoladder, the same +// three passes: on 640x360 a damaged label coded to 16946 B, which left less +// than a fiftieth of the first number above it, and on every other rung it +// coded smaller than the label of a film nothing damages. var Ladder = []Rung{ - {640, 360, 17277}, {426, 240, 11156}, {320, 180, 8322}, {256, 144, 6754}, - {160, 90, 3951}, {80, 45, 1454}, {40, 23, 825}, {16, 9, 149}, - {4, 3, 46}, {2, 2, 31}, {1, 1, 6}, + {640, 360, 17277, 18641}, {426, 240, 11156, 11156}, {320, 180, 8322, 8322}, {256, 144, 6754, 6754}, + {160, 90, 3951, 3951}, {80, 45, 1454, 1454}, {40, 23, 825, 825}, {16, 9, 149, 149}, + {4, 3, 46, 46}, {2, 2, 31, 31}, {1, 1, 6, 6}, } // Choice is the picture a film shows. @@ -216,7 +237,7 @@ func Choose(formatID string, r format.Request, s Settings, label string, fits fu var last Choice var lastStream Stream for _, rung := range Ladder { - c, st, err := onRung(base, rung, s) + c, st, err := onRung(base, rung, rung.bound(r), s) if err != nil { return Choice{}, Stream{}, err } @@ -229,15 +250,15 @@ func Choose(formatID string, r format.Request, s Settings, label string, fits fu } // onRung is the picture of one rung and the stream planning sizes it by: the -// rung's ceiling for every picture, or - when the request named a quality the -// ceilings were not measured at - a reserve from a sample of the film's -// pictures, coded (sampleReserve). That is the slow road AVIF takes for the -// same reason, and only runs that asked for it pay. -func onRung(base Choice, rung Rung, s Settings) (Choice, Stream, error) { +// rung's ceiling for the label the film carries, for every picture, or - when +// the request named a quality the ceilings were not measured at - a reserve +// from a sample of the film's pictures, coded (sampleReserve). That is the slow +// road AVIF takes for the same reason, and only runs that asked for it pay. +func onRung(base Choice, rung Rung, ceiling int, s Settings) (Choice, Stream, error) { c := base - c.Width, c.Height, c.Ceiling = rung.Width, rung.Height, rung.Ceiling + c.Width, c.Height, c.Ceiling = rung.Width, rung.Height, ceiling if !s.QualityNamed { - return c, NewStream(s.Timeline, rung.Ceiling, rung.Width, rung.Height, c.Label), nil + return c, NewStream(s.Timeline, ceiling, rung.Width, rung.Height, c.Label), nil } tiles, reserve, err := sampleReserve(c.Width, c.Height, c.Seed, c.Label, s.Timeline, c.QIndex) if err != nil { diff --git a/internal/format/video/plan.go b/internal/format/video/plan.go index 70ca5d52..7af9bed1 100644 --- a/internal/format/video/plan.go +++ b/internal/format/video/plan.go @@ -30,7 +30,7 @@ func (f Film) Stream() Stream { func Plan(formatID string, r format.Request, need func(Stream) int64, refuse func(requested, need int64, c Choice, s Settings) error) (format.Plan, error) { label := "" if r.Label { - label = core.Label(formatID, r.Bytes, r.Seed) + label = r.LabelText(formatID) } s, err := Read(formatID, r.Properties) if err != nil { diff --git a/internal/format/wav/wav.go b/internal/format/wav/wav.go index fc4c6f12..79f5dcb0 100644 --- a/internal/format/wav/wav.go +++ b/internal/format/wav/wav.go @@ -231,7 +231,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } if r.Label { - m.info = infoChunk(core.Label("wav", r.Bytes, r.Seed)) + m.info = infoChunk(r.LabelText("wav")) } fixed := int64(baseSize) + int64(len(m.info)) diff --git a/internal/format/webp/webp.go b/internal/format/webp/webp.go index 0ed89756..91259b14 100644 --- a/internal/format/webp/webp.go +++ b/internal/format/webp/webp.go @@ -137,7 +137,7 @@ func bareBytes(width, height int) int64 { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("webp", r.Bytes, r.Seed) + label = r.LabelText("webp") } w, h, err := chooseSize(r) diff --git a/internal/format/xlsx/xlsx.go b/internal/format/xlsx/xlsx.go index df7174f0..fa88e045 100644 --- a/internal/format/xlsx/xlsx.go +++ b/internal/format/xlsx/xlsx.go @@ -114,7 +114,7 @@ type memo struct { func (generator) Plan(r format.Request) (format.Plan, error) { label := "" if r.Label { - label = core.Label("xlsx", r.Bytes, r.Seed) + label = r.LabelText("xlsx") } rows, err := intProperty(r.Properties, "rows", 1, minRows, maxRows) @@ -252,7 +252,7 @@ func sheet(rows, columns int, seed uint64, label string) []byte { // The label goes in the first cell, which is where a person looks // and where a spreadsheet reader shows it without scrolling. if row == 1 && col == 1 && label != "" { - text = opc.Text(opc.Fixed(label, opc.LabelWidth)) + text = opc.Text(opc.FixedLabel(label)) } b.WriteString(``) b.WriteString(text) @@ -298,7 +298,7 @@ func intProperty(props map[string]string, key string, fallback, min, max int) (i // document the label is an extra paragraph and it goes the other way. func minimumBytes() int64 { smallest := int64(1) << 62 - for _, label := range []string{"", core.Label("xlsx", 0, 0)} { + for _, label := range []string{"", format.Request{}.LabelText("xlsx")} { shape, err := opc.Measure(parts(1, 1, 0, label)) if err != nil { continue diff --git a/internal/format/xmlfile/xml.go b/internal/format/xmlfile/xml.go index 72bcf384..c82bb8e3 100644 --- a/internal/format/xmlfile/xml.go +++ b/internal/format/xmlfile/xml.go @@ -173,7 +173,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { // A comment carries the label without touching the content, and it can // sit anywhere after the declaration. The label text uses spaced // hyphens and never a double one, which a comment may not contain. - line := "\n" + line := "\n" // It has to leave room for a whole document beside it, or the file would // be a comment and an empty root. The number is what the comment COSTS // in this encoding, not how long it is to read - in UTF-16 those differ diff --git a/internal/format/yamlfile/yaml.go b/internal/format/yamlfile/yaml.go index 19a8c75b..08432368 100644 --- a/internal/format/yamlfile/yaml.go +++ b/internal/format/yamlfile/yaml.go @@ -179,7 +179,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { // sits in front of the document where a person opening the file reads // it first. Nothing in the label needs escaping: it is plain ASCII and // a comment runs to the end of the line. - line := "# " + core.Label("yaml", r.Bytes, r.Seed) + "\n" + line := "# " + r.LabelText("yaml") + "\n" // It has to leave room for a whole document beside it, or the file // would be a comment and a root key with nothing under it. if int64(len(line))+min <= r.Bytes { diff --git a/internal/format/zip/zip.go b/internal/format/zip/zip.go index acf4dde1..7bfb4cf0 100644 --- a/internal/format/zip/zip.go +++ b/internal/format/zip/zip.go @@ -244,7 +244,7 @@ func withinZip32(total int64) error { // comment, then settle the label, then measure again. func settleSize(m *memo, r format.Request) (target, bare int64, label string, err error) { if r.Label && !r.SizeFromContents { - label = core.Label("zip", r.Bytes, r.Seed) + label = r.LabelText("zip") } m.comment = label @@ -270,7 +270,7 @@ func settleSize(m *memo, r format.Request) (target, bare int64, label string, er size := bare settled := false for i := 0; i < 4 && !settled; i++ { - m.comment = core.Label("zip", size, r.Seed) + m.comment = r.LabelSized("zip", size) measured, err := archiveSize(*m) if err != nil { return 0, 0, "", err diff --git a/internal/guard/avifladder_test.go b/internal/guard/avifladder_test.go index c855c8e2..9abb9716 100644 --- a/internal/guard/avifladder_test.go +++ b/internal/guard/avifladder_test.go @@ -2,11 +2,15 @@ package guard import ( "context" + "errors" "runtime" + "slices" + "strconv" "testing" "github.com/donislawdev/TestingFilesGenerator/internal/format" _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" + "github.com/donislawdev/TestingFilesGenerator/internal/format/avif" ) // AVIF is the one format here whose file size is decided by a compressor rather @@ -77,6 +81,103 @@ func TestAvifDrawsAPictureThatGrowsWithTheFileAndAlwaysFits(t *testing.T) { t.Logf("largest picture reached over the sweep: %d pixels", biggest) } +// heaviestAvifSeeds are the seeds the thorough sweep (tools/probes/avifladder, +// 768 seeds) found heaviest on 2026-11-12, asked beside the first few from +// zero. The first few alone stayed green with both ceilings set back to the +// numbers that were too low, so they could not see what the sweep found. A +// codec raised underneath moves every rung at once, so the heaviest seeds of +// today are a fair sample of tomorrow's. +var heaviestAvifSeeds = []uint64{ + 0x65a47abe4364edd2, 0xfb0ac60c4be71cd6, 0x949ac3a5a267fad4, 0xf3fc5ddbb41fd92, + 0xd2, 0xea, 0xf4, 0xfb, 0xfe, 0xff, +} + +// Every rung's two ceilings still cover what that rung codes to: the first +// with the label of a file at the edge of the rung, the second with the label +// of a damaged file. +// +// A file at the edge of a rung - asked for at its ceiling, where the picture +// leaves the least room - states a size with the digits of the ceiling, so the +// first ceiling is asked with those. A damaged file's label names its damage, +// and a longer label is more ink until it is too long for its scale, so the +// second is asked with the longest damaged label every scale still draws. +// Asking one byte of a named picture size makes the format state the file +// that picture needs, which is what a ceiling is a bound on (2026-11-12, when +// both were found too low - O273 and the review of #234). +// +// A handful of seeds from zero, as the JPEG XL guard takes, and the heaviest +// the thorough sweep found (heaviestAvifSeeds). +func TestEveryAvifRungStillFitsInsideItsDeclaredCeilings(t *testing.T) { + d, err := format.Get("avif") + if err != nil { + t.Fatal(err) + } + rungs := avif.Rungs() + if len(rungs) == 0 { + t.Fatal("the ladder is empty, so this guard measured nothing") + } + first := uint64(8) + if raceEnabled { + first = 2 + } + seeds := slices.Clone(heaviestAvifSeeds) + for seed := range first { + seeds = append(seeds, seed) + } + + type ask struct { + r format.Request + ceiling int64 + } + drawn := 0 + for _, rung := range rungs { + w, h := int(rung[0]), int(rung[1]) + named := map[string]string{"width": strconv.Itoa(w), "height": strconv.Itoa(h)} + over, needs, worst := int64(0), int64(0), ask{} + for _, seed := range seeds { + asks := []ask{{format.Request{Bytes: 1, Seed: seed, Properties: named}, rung[2]}} + // The size on disk is what the label states: the ceiling itself, + // a tenth above it, and the lightest and the heaviest number with + // as many digits. + eights, lightest := int64(0), int64(1) + for n := rung[2]; n > 0; n /= 10 { + eights, lightest = eights*10+8, lightest*10 + } + lightest /= 10 + for _, size := range []int64{rung[2], rung[2] + rung[2]/10, lightest, eights} { + asks = append(asks, ask{format.Request{Bytes: 1, Seed: seed, Label: true, OnDisk: size, Properties: named}, rung[2]}) + } + // A damaged file states a size of any length the damage leaves it. + plain := format.Request{Bytes: 1, Seed: seed, Label: true, OnDisk: 1e12, Properties: named} + for _, damage := range damagesDrawnAtEveryScale(t, plain, "avif", w) { + r := plain + r.Damage = damage + asks = append(asks, ask{r, rung[3]}) + drawn++ + } + for _, a := range asks { + _, err := d.Generator.Plan(a.r) + var below *format.BelowMinimumError + if !errors.As(err, &below) { + t.Fatalf("a %dx%d picture refused one byte with %T rather than a BelowMinimumError: %v", w, h, err, err) + } + if below.Minimum-a.ceiling > over { + over, needs, worst = below.Minimum-a.ceiling, below.Minimum, a + } + } + } + if over > 0 { + t.Errorf("the %dx%d rung declares a ceiling of %d B and its picture needs %d B (seed %d, label %t, size %d, damage %q).\n"+ + "A ceiling below what its rung produces lets planning pick a picture the file cannot hold, and writing then refuses a size planning accepted.\n"+ + "What to do: run tools/probes/avifladder and record what it measures.", + w, h, worst.ceiling, needs, worst.r.Seed, worst.r.Label, worst.r.OnDisk, worst.r.Damage) + } + } + if drawn == 0 { + t.Fatal("no rung is wide enough to draw a damaged label, so this guard asked nothing about one") + } +} + // Planning must not code the picture. // // It used to, and the price was measured on 2026-08-29: a preview of a thousand diff --git a/internal/guard/damagelabel_test.go b/internal/guard/damagelabel_test.go index 53251924..d254b0f6 100644 --- a/internal/guard/damagelabel_test.go +++ b/internal/guard/damagelabel_test.go @@ -1,14 +1,28 @@ package guard import ( + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "io" + "mime/quotedprintable" + "net/mail" + "os" "path/filepath" "slices" + "strconv" + "strings" "testing" + "unicode/utf8" + "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/damage" "github.com/donislawdev/TestingFilesGenerator/internal/engine" "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" "github.com/donislawdev/TestingFilesGenerator/internal/manifest" + "github.com/donislawdev/TestingFilesGenerator/internal/oracle" ) // labelNotGuaranteed is the note a damaged file carries in place of a claim @@ -80,3 +94,431 @@ func TestADamagedFileNeverClaimsItsLabel(t *testing.T) { } t.Logf("the claim was taken away in %d format(s)", withdrawn) } + +// TestADamagedFileStatesItsSizeOnDiskAndItsDamage holds the label of a damaged +// file to what is on disk. +// +// Measured 2026-11-11 before this held (O333): a text of 10 MiB cut with +// missing=50 said "20971520 B" in its first line - the size of the whole file +// before the cut - and named no damage, so a file cut by half and one cut by +// nine tenths looked the same from the inside. append-junk stated the size +// without the junk. +// +// Every format whose healthy file carries its label as text is asked, with +// junk appended: that damage keeps every byte the generator wrote, so the +// label is where the generator put it and can be read back. Which formats +// those are is read from the healthy file, not listed here - a picture draws +// its label in pixels and is left to the guards on pictures. The expected text +// is written out here rather than asked of format.Request, so a wording that +// drops the damage cannot agree with itself. +func TestADamagedFileStatesItsSizeOnDiskAndItsDamage(t *testing.T) { + chain := damage.Chain{{ID: damage.AppendJunk}} + const size = witnessSize + before, err := chain.Before(size) + if err != nil { + t.Fatal(err) + } + brokenSeed := core.FileSeed(core.TargetSeed(lengthSeed, "broken"), 0) + // The canary: the label this guard was written against, which states the + // size the generator wrote and no damage, must not pass it. + planted := fmt.Sprintf("tfg - txt - %d B - seed %s", before, core.SeedLabel(brokenSeed)) + if statesItsDamage([]byte(planted), "txt", size, chain, brokenSeed) == "" { + t.Fatal("the check took a label naming the size before the damage, and no damage, for the label it asks for") + } + + dir := t.TempDir() + read := 0 + for _, f := range format.All() { + good := engine.Target{ID: "good", Format: f.ID, Sizes: engine.Uniform(1, size), Label: true} + broken := engine.Target{ID: "broken", Format: f.ID, Sizes: engine.Uniform(1, size), Damage: chain, Label: true} + _, paths := lengthRun(t, filepath.Join(dir, f.ID), []engine.Target{good, broken}) + healthy, err := os.ReadFile(paths["good"]) + if err != nil { + t.Fatal(err) + } + goodSeed := core.FileSeed(core.TargetSeed(lengthSeed, "good"), 0) + if !bytes.Contains(healthy, fmt.Appendf(nil, "tfg - %s - %d B - seed %s", f.ID, size, core.SeedLabel(goodSeed))) { + continue + } + got, err := os.ReadFile(paths["broken"]) + if err != nil { + t.Fatal(err) + } + if why := statesItsDamage(got, f.ID, size, chain, brokenSeed); why != "" { + t.Errorf("%s: %s", f.ID, why) + } + read++ + } + if read < 10 { + t.Fatalf("only %d format(s) carry their label as text the guard could find - it did not reach the formats it is about", read) + } + t.Logf("the label was read back from %d format(s)", read) +} + +// TestATruncatedTextSaysHowMuchWasCut is the case the owner met: two texts of +// the same size cut by different shares used to begin with the same line. +// Their labels are the first thing a person opening them sees, so the share +// has to be there. +func TestATruncatedTextSaysHowMuchWasCut(t *testing.T) { + const size = 64 << 10 + seed := core.FileSeed(core.TargetSeed(lengthSeed, "cut"), 0) + for _, missing := range []string{"50", "90"} { + chain := damage.Chain{{ID: damage.Truncate, Values: damage.Values{damage.SettingMissing: missing}}} + target := engine.Target{ID: "cut", Format: "txt", Sizes: engine.Uniform(1, size), Damage: chain, Label: true} + _, paths := lengthRun(t, filepath.Join(t.TempDir(), missing), []engine.Target{target}) + got, err := os.ReadFile(paths["cut"]) + if err != nil { + t.Fatal(err) + } + first, _, _ := bytes.Cut(got, []byte("\n")) + want := fmt.Sprintf("tfg - txt - %d B - damage truncate:missing=%s - seed %s", size, missing, core.SeedLabel(seed)) + if string(first) != want { + t.Errorf("a text cut with missing=%s begins %q, expected %q", missing, first, want) + } + } +} + +// TestALongDamageLabelLeavesEveryFormatWhole holds every format to the longest +// damage a label can name. +// +// Nothing limits how many damages a chain lists, so the label is input a +// person controls, and a format whose label lives in a field with a limit of +// its own can break on it before any damage is applied: a gzip comment Go's +// reader refuses from 512 bytes, a ZIP comment past its padding, a line of a +// mail. Found in review on 2026-11-12 (docs/REVIEW-234-2026-11-12.md). The +// answer is damage.LabelLimit, and this asks every format at it: each +// generator directly, with the label alone and no damage, so the file has to +// come out whole - planned, written, taken by the structural check, and a +// gzip stream opened by Go's own reader, which the structural check in Python +// does not hold to that limit. +func TestALongDamageLabelLeavesEveryFormatWhole(t *testing.T) { + long := longestDamageLabel(t) + + dir := t.TempDir() + var reqs []oracle.StrictRequest + for _, f := range format.All() { + path := filepath.Join(dir, f.ID) + if err := writeWithLabel(t, f, long, path); err != nil { + t.Errorf("%s with a label of %d B: %v", f.ID, len(long), err) + continue + } + reqs = append(reqs, oracle.StrictRequest{Format: f.ID, Path: path}) + } + got, available, err := oracle.StrictBatch(t.Context(), reqs) + if !available { + t.Skip("the structural checker needs python, and there is none on this machine") + } + if err != nil { + t.Fatal(err) + } + gzipped := 0 + for i, j := range got { + if j.Verdict != oracle.StrictOK { + t.Errorf("%s with a label of %d B is refused by the structural check: %s", reqs[i].Format, len(long), j.Reason) + } + if err := opensAsGzipWhereItIsOne(reqs[i].Path); err != nil { + t.Errorf("%s with a label of %d B: Go's gzip reader refuses it: %v", reqs[i].Format, len(long), err) + } else if isGzip(reqs[i].Path) { + gzipped++ + } + } + if len(got) < len(format.All())*3/4 || gzipped == 0 { + t.Fatalf("only %d of %d formats were judged, %d of them gzip streams", len(got), len(format.All()), gzipped) + } +} + +// longestDamageLabel is the damage text of a chain far longer than the label +// takes, after asserting that the label was shortened to what it takes and +// that the whole text would have passed the gzip limit - so the guard above is +// asking about the limit rather than about a label that never reached it. +func longestDamageLabel(t *testing.T) string { + t.Helper() + chain := damage.Chain{} + for range 40 { + chain = append(chain, damage.Spec{ID: damage.AppendJunk, Values: damage.Values{damage.SettingBytes: "65536"}}) + } + long := chain.Label() + whole := len(chain) * len(chain[0].String()) + if whole < 512 || len(long) > damage.LabelLimit || len(long) < damage.LabelLimit*3/4 { + t.Fatalf("forty damages name %d B whole and %d B in the label, with the label held to %d B - the guard does not reach the limit it is about", + whole, len(long), damage.LabelLimit) + } + return long +} + +// damagesDrawnAtEveryScale is, for a picture width wide, the damage that makes +// the label of r the longest one still drawn at each scale the picture can draw +// a damaged label at, the largest scale first. +// +// The longest label is not the heaviest. A label too long for the picture is +// not drawn at all, and a shorter one is drawn larger, so the most a damaged +// label can put into a picture is the longest that still fits at some scale. +// The longest damage label puts no ink at all into a picture 640 wide (found in +// review, 2026-11-12). The lengths come from imagelabel, so the scales live in +// one place. +// +// Empty for a picture too narrow to draw any damaged label. +func damagesDrawnAtEveryScale(t *testing.T, r format.Request, id string, width int) []string { + t.Helper() + long := longestDamageLabel(t) + r.Damage = "x" + fixed := len(r.LabelText(id)) - len(r.Damage) + var out []string + for _, n := range imagelabel.LongestAtEachScale(width) { + if n <= fixed { + continue // no damaged label is this short + } + r.Damage = long[:min(n-fixed, len(long))] + if got, want := imagelabel.BandHeight(width, len(r.LabelText(id))), imagelabel.BandHeight(width, n); got != want || got == 0 { + t.Fatalf("a %s label of %d characters across %d pixels draws a band %d high where %d was worked out - the guard does not reach the scale it measures", + id, len(r.LabelText(id)), width, got, want) + } + out = append(out, r.Damage) + } + return out +} + +// isGzip says whether the file at path starts as a gzip stream. +func isGzip(path string) bool { + head := make([]byte, 2) + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + n, _ := f.Read(head) + return n == 2 && head[0] == 0x1f && head[1] == 0x8b +} + +// opensAsGzipWhereItIsOne opens the header of a gzip stream with Go's reader, +// and says nothing about a file that is not one. +func opensAsGzipWhereItIsOne(path string) error { + if !isGzip(path) { + return nil + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + z, err := gzip.NewReader(f) + if err != nil { + return err + } + return z.Close() +} + +// TestADamageLabelIsBoundedAndSaysHowManyItLeftOut holds damage.Chain.Label +// to its limit for every length of chain, and to an honest count of what it +// did not name: the entries it names and the number it gives add up to the +// chain. The canary is the shortest chain over the limit, which has to be +// shortened, so a limit nothing reaches cannot pass. +func TestADamageLabelIsBoundedAndSaysHowManyItLeftOut(t *testing.T) { + entry := damage.Spec{ID: damage.AppendJunk, Values: damage.Values{damage.SettingBytes: "65536"}} + shortened := 0 + for n := 1; n <= 200; n++ { + chain := slices.Repeat(damage.Chain{entry}, n) + label := chain.Label() + if len(label) > damage.LabelLimit { + t.Fatalf("%d damages: the label is %d B, past the %d B it is held to", n, len(label), damage.LabelLimit) + } + named := strings.Count(label, damage.AppendJunk) + left := 0 + if before, more, cut := strings.Cut(label, " and "); cut { + left, _ = strconv.Atoi(strings.TrimSuffix(more, " more")) + named = strings.Count(before, damage.AppendJunk) + shortened++ + } + if named+left != n || named == 0 { + t.Errorf("%d damages: the label %q names %d and says %d more", n, label, named, left) + } + } + if shortened == 0 { + t.Fatal("no chain up to 200 damages was shortened, so the limit was never reached") + } +} + +// TestADamagedMailCarriesItsLabelWholeInEveryTextEncoding reads the label of +// a mail back the way a mail reader does, in every encoding of its text and +// both parts that carry it. +// +// Found 2026-11-12 while answering a review: the label was written as it is +// into quoted-printable, where its = starts an escape, so +// append-junk:bytes=64 read back as bytesd - and the structural check took it, +// because =64 is a valid escape. A label naming two damages is also longer +// than the 78 characters RFC 5322 asks a line to keep to. The encodings and +// parts are read from the declaration, and the label is decoded with Go's own +// readers rather than searched for in the bytes. +func TestADamagedMailCarriesItsLabelWholeInEveryTextEncoding(t *testing.T) { + d, err := format.Get("eml") + if err != nil { + t.Fatal(err) + } + encodings, eok := settingNamed(d, "text_encoding") + bodies, bok := settingNamed(d, "body") + if !eok || !bok || len(encodings.Choices) < 4 { + t.Fatalf("eml declares text_encoding %v and body %v - the guard does not reach the encodings it is about", encodings.Choices, bodies.Choices) + } + // One damage, whose label fits a line, so the = is asked on its own. And + // two, whose label does not, so the breaking is asked as well. + one := damage.Chain{{ID: damage.AppendJunk, Values: damage.Values{damage.SettingBytes: "1024"}}} + two := damage.Chain{{ID: damage.Truncate, Values: damage.Values{damage.SettingMissing: "10"}}, one[0]} + for _, chain := range []damage.Chain{one, two} { + for _, encoding := range encodings.Choices { + for _, body := range []string{"plain", "html"} { + mailCarriesItsLabel(t, d, chain, encoding, body) + } + } + } + if !fitsALine(d, one) || fitsALine(d, two) { + t.Fatal("the label of one damage has to fit a line and the label of two must not, or one half of this guard asks nothing") + } +} + +// fitsALine says whether the label of a 64 KiB mail damaged by chain fits the +// 78 characters a line keeps to. +func fitsALine(d format.Descriptor, chain damage.Chain) bool { + return len(format.Request{Bytes: 64 << 10, Seed: 3, Damage: chain.Label()}.LabelText(d.ID)) <= 78 +} + +// mailCarriesItsLabel writes one mail damaged by chain in one encoding and +// part, and reads its label back. +func mailCarriesItsLabel(t *testing.T, d format.Descriptor, chain damage.Chain, encoding, body string) { + t.Helper() + r := format.Request{Seed: 3, Label: true, Damage: chain.Label(), + Properties: map[string]string{"text_encoding": encoding, "body": body}} + r.Bytes = max(int64(64<<10), d.SmallestAccepted(r)) + want := fmt.Sprintf("tfg - eml - %d B - damage %s - seed %s", r.Bytes, chain.Label(), core.SeedLabel(r.Seed)) + text, err := mailText(t, d, r) + if err != nil { + t.Errorf("%s, text_encoding=%s body=%s: %v", chain.Label(), encoding, body, err) + return + } + if !strings.Contains(text, want) { + t.Errorf("%s, text_encoding=%s body=%s: the text read back does not carry %q", chain.Label(), encoding, body, want) + } +} + +// mailText writes the mail r asks for and reads its text back: the body +// decoded by its Content-Transfer-Encoding, every line break read as the space +// it was broken at. A line longer than 78 characters in the file is an error. +func mailText(t *testing.T, d format.Descriptor, r format.Request) (string, error) { + t.Helper() + p, err := d.Generator.Plan(r) + if err != nil { + return "", err + } + var file bytes.Buffer + if err := d.Generator.Write(t.Context(), &file, p); err != nil { + return "", err + } + for i, line := range strings.Split(file.String(), "\n") { + if n := utf8.RuneCountInString(strings.TrimSuffix(line, "\r")); n > 78 { + return "", fmt.Errorf("line %d of the file is %d characters, past the 78 RFC 5322 asks for", i+1, n) + } + } + msg, err := mail.ReadMessage(&file) + if err != nil { + return "", err + } + var body io.Reader = msg.Body + switch msg.Header.Get("Content-Transfer-Encoding") { + case "quoted-printable": + body = quotedprintable.NewReader(body) + case "base64": + body = base64.NewDecoder(base64.StdEncoding, newlineless{body}) + } + text, err := io.ReadAll(body) + if err != nil { + return "", err + } + return strings.NewReplacer("\r\n", " ", "\n", " ").Replace(string(text)), nil +} + +// newlineless is a reader with its line breaks taken out, which base64 in a +// mail carries every 76 characters and the decoder does not skip. +type newlineless struct{ r io.Reader } + +func (n newlineless) Read(p []byte) (int, error) { + k, err := n.r.Read(p) + out := p[:0] + for _, c := range p[:k] { + if c != '\r' && c != '\n' { + out = append(out, c) + } + } + return len(out), err +} + +// writeWithLabel plans and writes one file of f carrying the damage text long +// in its label, and turns a panic into the error it is. +func writeWithLabel(t *testing.T, f format.Descriptor, long, path string) (err error) { + t.Helper() + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("panicked: %v", p) + } + }() + r := format.Request{Seed: 1, Label: true, Damage: long} + r.Bytes = max(int64(witnessSize), f.SmallestAccepted(r)) + p, err := f.Generator.Plan(r) + if err != nil { + return err + } + out, err := os.Create(path) + if err != nil { + return err + } + defer out.Close() + return f.Generator.Write(t.Context(), out, p) +} + +// statesItsDamage says what is wrong with the label in got, or nothing when +// it states size on disk and every entry of chain with its settings resolved. +func statesItsDamage(got []byte, id string, size int64, chain damage.Chain, seed uint64) string { + want := fmt.Sprintf("tfg - %s - %d B - damage %s - seed %s", id, size, chain.Label(), core.SeedLabel(seed)) + if !strings.Contains(chain.Label(), "=") { + return "the chain " + chain.Label() + " does not name its settings, so the label cannot say how much was broken" + } + if !bytes.Contains(got, []byte(want)) { + return fmt.Sprintf("the file does not carry the label %q", want) + } + return "" +} + +// TestEveryCharacterOfADamagedLabelCanBeDrawn holds the label a picture draws +// to the characters its font has. +// +// The font is 3 by 5 pixels and drawn by hand, and a character it lacks comes +// out as a box the shape of a 0. Before the label named a damage it never held +// an = or a comma, and the font had neither - a picture cut with missing=50 +// would have said MISSING050. Every damage from the registry, at each end of +// every setting it takes, so a damage added with a new character in its name +// or settings turns this red. +func TestEveryCharacterOfADamagedLabelCanBeDrawn(t *testing.T) { + if imagelabel.HasGlyph('~') { + t.Fatal("the font draws ~, which it never had - the question below cannot tell a glyph from the stand-in") + } + asked := 0 + for _, d := range damage.All() { + values := []damage.Values{{}} + for _, p := range d.Parameters { + values = append(values, + damage.Values{p.Name: strconv.FormatInt(p.Min, 10)}, + damage.Values{p.Name: strconv.FormatInt(p.Max, 10)}) + } + for _, v := range values { + chain := damage.Chain{{ID: d.ID, Values: v}, {ID: d.ID, Values: v}} + label := format.Request{Bytes: 1 << 40, Seed: 1, Damage: chain.Label()}.LabelText("png") + for _, r := range label { + if !imagelabel.HasGlyph(r) { + t.Errorf("%s: the picture font has no %q, so the label %q is drawn with a box in its place", d.ID, r, label) + } + } + asked++ + } + } + if asked == 0 { + t.Fatal("no damage was asked - the registry was not read") + } +} diff --git a/internal/guard/damagelength_test.go b/internal/guard/damagelength_test.go index b8fe7f77..13b6991e 100644 --- a/internal/guard/damagelength_test.go +++ b/internal/guard/damagelength_test.go @@ -214,8 +214,12 @@ func TestATruncatedFileIsTheStartOfTheWholeOne(t *testing.T) { } compared := 0 for _, f := range format.All() { - cut := engine.Target{ID: "same", Format: f.ID, Sizes: engine.Uniform(1, size), Damage: chain, Label: true} - full := engine.Target{ID: "same", Format: f.ID, Sizes: engine.Uniform(1, whole), Label: true} + // Without the label, because a cut file's label states the size it + // keeps and the damage, and the whole file's states neither - the + // two differ on purpose (TestADamagedFileStatesItsSizeOnDiskAndItsDamage), + // and what this guard holds is every byte under them. + cut := engine.Target{ID: "same", Format: f.ID, Sizes: engine.Uniform(1, size), Damage: chain} + full := engine.Target{ID: "same", Format: f.ID, Sizes: engine.Uniform(1, whole)} if _, err := engine.Plan([]engine.Target{full}, engine.Options{OutDir: t.TempDir(), Seed: lengthSeed}); err != nil { // A size this format cannot write exactly: the cut started from // the next one it can, which is a different whole file. @@ -260,8 +264,9 @@ func TestAppendedJunkIsTheWholeFileAndThenTheSameBytesEveryTime(t *testing.T) { } junk := size - before - broken := engine.Target{ID: "same", Format: "txt", Sizes: engine.Uniform(1, size), Damage: chain, Label: true} - whole := engine.Target{ID: "same", Format: "txt", Sizes: engine.Uniform(1, before), Label: true} + // Without the label, for the reason the guard above gives. + broken := engine.Target{ID: "same", Format: "txt", Sizes: engine.Uniform(1, size), Damage: chain} + whole := engine.Target{ID: "same", Format: "txt", Sizes: engine.Uniform(1, before)} _, brokenPath := lengthRun(t, filepath.Join(dir, "broken"), []engine.Target{broken}) _, wholePath := lengthRun(t, filepath.Join(dir, "whole"), []engine.Target{whole}) got, err := os.ReadFile(brokenPath["same"]) diff --git a/internal/guard/damagesettings_test.go b/internal/guard/damagesettings_test.go new file mode 100644 index 00000000..f20aaa63 --- /dev/null +++ b/internal/guard/damagesettings_test.go @@ -0,0 +1,143 @@ +package guard + +import ( + "bytes" + "context" + "errors" + "os" + "strconv" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/cli" + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/damage" + "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// A damage setting its declaration does not allow is refused by the engine, +// for the command line and the window, which hand it a chain without a recipe. +// +// Measured 2026-11-11 before this held: truncate:missing=95, append-junk with +// bytes=65537 and zero-head with bytes=99999999999 each ended with code 0, the +// file was cut or zeroed by the default, and the manifest recorded the value +// as written (O332). The recipe reader refused all of them, so the rule +// existed - the paths without a recipe never reached it. +// +// Every damage and every setting comes from the registry, with values just +// outside the declared range, one that is not a value at all, and a setting the +// damage does not take. Each is asked of the engine, which must refuse it at +// the address the recipe reader gives (the damage is second in its chain, so +// a position read off the wrong entry shows), and of the command line, which +// must end with USAGE, name the setting and write nothing. +// +// The control is the default of every setting in the same chain: planned. A +// build refusing every damage setting whatever its value would pass the half +// above, and this half is what tells the two apart. +func TestEveryDamageSettingItsDeclarationForbidsIsRefusedWithoutARecipe(t *testing.T) { + // The canary: the check below on two answers it must not take - nothing + // refused, and the right refusal about the first entry of the chain. + planted := &damage.SettingError{Position: 1, Err: &format.PropertyValueError{Format: damage.ZeroHead, Key: damage.SettingBytes}} + if refusedAsTheSetting(nil, damage.SettingBytes) == "" || refusedAsTheSetting(planted, damage.SettingBytes) == "" { + t.Fatal("the check took a plan that went through, or a refusal about the wrong entry, for the refusal it asks for") + } + + all := damage.All() + if len(all) == 0 { + t.Fatal("the damage registry is empty, so this guard would pass without asking anything") + } + asked := 0 + for _, d := range all { + type stated struct{ name, value string } + cases := []stated{{"no-such-setting", "1"}} + for _, p := range d.Parameters { + for _, bad := range valuesTheDeclarationForbids(p) { + cases = append(cases, stated{p.Name, bad}) + } + } + for _, c := range cases { + spec := damage.Spec{ID: d.ID, Values: damage.Values{c.name: c.value}} + if why := refusedAsTheSetting(planSecondInAChain(t, spec), c.name); why != "" { + t.Errorf("%s %s", spec, why) + } + code, said, files := damageFromTheCommandLine(t, spec.String()) + if code != cli.ExitUsage || files != 0 || !strings.Contains(said, c.name) { + t.Errorf("--damage %s ended with %d and wrote %d file(s), expected %d and none, with the setting named: %s", + spec, code, files, cli.ExitUsage, said) + } + asked++ + } + for _, p := range d.Parameters { + spec := damage.Spec{ID: d.ID, Values: damage.Values{p.Name: p.Default}} + if err := planSecondInAChain(t, spec); err != nil { + t.Errorf("%s states the declared default and was refused - the rule refuses what it should let through: %v", spec, err) + } + } + } + if asked < len(all)*2 { + t.Fatalf("only %d value(s) were asked across %d damage(s) - the registry was not read", asked, len(all)) + } +} + +// valuesTheDeclarationForbids is what a person can type that p does not take: +// one step past each end of a bounded number, and a word where a value is +// expected. Free text has none. +func valuesTheDeclarationForbids(p format.Property) []string { + switch p.Kind { + case format.PropertyInt: + out := []string{"abc"} + if p.Min != 0 || p.Max != 0 { + out = append(out, strconv.FormatInt(p.Min-1, 10), strconv.FormatInt(p.Max+1, 10)) + } + return out + case format.PropertyChoice, format.PropertyBool, format.PropertySize, format.PropertyDuration: + return []string{"no-such-value"} + case format.PropertyText: + return nil + } + return nil +} + +// planSecondInAChain plans a txt file damaged by zero-head and then spec. +func planSecondInAChain(t *testing.T, spec damage.Spec) error { + t.Helper() + target := engine.Target{ID: "t", Format: "txt", Sizes: engine.Uniform(1, 64<<10), Label: true, + Damage: damage.Chain{{ID: damage.ZeroHead}, spec}} + _, err := engine.Plan([]engine.Target{target}, engine.Options{OutDir: t.TempDir(), Seed: 1}) + return err +} + +// refusedAsTheSetting says what is wrong with err as the refusal of setting in +// the second entry of a chain, or nothing when that is what it is. +func refusedAsTheSetting(err error, setting string) string { + if err == nil { + return "was planned" + } + var wrong *damage.SettingError + if !errors.As(err, &wrong) { + return "was refused, but not as a damage setting: " + err.Error() + } + var about interface{ AboutSetting() string } + want := core.TargetAddress(1, core.DamageAddress(2, setting)) + if !errors.As(err, &about) || about.AboutSetting() != want { + return "was refused somewhere other than " + want + ", where the recipe reader puts the same mistake: " + err.Error() + } + return "" +} + +// damageFromTheCommandLine runs generate with one --damage and says how it +// ended, what it said and how many files it left. +func damageFromTheCommandLine(t *testing.T, spec string) (int, string, int) { + t.Helper() + dir := t.TempDir() + var out, errOut bytes.Buffer + code := cli.Run(context.Background(), []string{ + "generate", "--format", "txt", "--size", "64kb", "--damage", spec, "--out", dir, + }, &out, &errOut) + written, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("reading the output directory: %v", err) + } + return code, errOut.String(), len(written) +} diff --git a/internal/guard/damagewindow_test.go b/internal/guard/damagewindow_test.go index ba7e2d30..9bac2adf 100644 --- a/internal/guard/damagewindow_test.go +++ b/internal/guard/damagewindow_test.go @@ -267,6 +267,49 @@ func TestARunFromTheWindowIsReallyDamaged(t *testing.T) { } } +// A damage setting outside its range, typed into the window, is refused under +// its own box and writes nothing. +// +// The window hands the engine a chain without a recipe, so until 2026-11-12 it +// sent bytes=99999 on, the engine zeroed the default eight and the manifest +// recorded 99999 (O332). Pressed rather than settled, for the reason the guard +// above gives, and asked of the box the value was typed into: a refusal at the +// foot of the form would stop the run and leave a person looking for the box. +func TestADamageSettingOutsideItsRangeIsRefusedUnderItsBox(t *testing.T) { + dir := t.TempDir() + + host := newFakeHost(t) + gen := window.NewGenerate(host) + content := gen.Object() + t.Cleanup(func() { join(host) }) + + fields := gen.Fields() + chooserIn(t, fields, "format").SetSelected("png") + setBox(t, fields, "size", "20kb") + + chooser, ok := controlUnder(content, text.FieldDamage()).(*parts.Chooser) + if !ok { + t.Fatalf("the damage field is not a list to choose from") + } + chooser.SetSelected(damage.ZeroHead) + setBox(t, fields, damage.SettingBytes, "99999") + if got := boxText(t, fields, damage.SettingBytes); got != "99999" { + t.Fatalf("the box holds %q, so there is no refusal to cause", got) + } + + entryUnder(t, content, text.FieldOutputDir()).SetText(dir) + press(t, content, text.ButtonGenerate()) + join(host) + + if made, _ := filepath.Glob(filepath.Join(dir, "*")); len(made) != 0 { + t.Errorf("a refused run left %v behind", made) + } + said := fields.Lookup(damage.SettingBytes).Saying() + if !strings.Contains(said, "99999") { + t.Errorf("the box that holds 99999 says %q - the refusal went somewhere else. Marked: %v", said, fields.Marked()) + } +} + // damageManifest reads a manifest as the manifest package spells it. // // Its own reader rather than the manifestShape the recipe guards share: that diff --git a/internal/guard/damagewitness_test.go b/internal/guard/damagewitness_test.go index eb9b8c5f..585ca67a 100644 --- a/internal/guard/damagewitness_test.go +++ b/internal/guard/damagewitness_test.go @@ -66,8 +66,13 @@ var policyRefusals = map[[2]string][]string{ {damage.Truncate, "html"}: {"was ordered and the page is not", "the document does not end with a closing"}, {damage.Truncate, "md"}: {"so one block is never closed"}, {damage.Truncate, "eml"}: {"the message has no blank line after its header lines", "layout=newsletter and the plain text", - // A rule the cut breaks only when it falls between a CR and its LF. - "a carriage return or a line feed on its own"}, + // A multipart message cut inside its body has no closing boundary, + // which RFC 2046 asks for and readers skip: Python's email reads such + // a cut multipart/alternative with its text part whole and notes a + // CloseBoundaryNotFoundDefect (measured 2026-11-12). Until the label + // of a damaged file named its damage, the cut at this size fell + // between a CR and its LF, and that reason came first. + "does not open with its boundary or has no closing one"}, {damage.Truncate, "txt"}: {}, // Bytes past the format's own end, which its rules forbid and readers skip. {damage.AppendJunk, "png"}: {"declares"}, @@ -84,7 +89,12 @@ var policyRefusals = map[[2]string][]string{ "after its closing boundary", "is not a line of base64", // A rule the junk breaks only when its random bytes hold a CR or an // LF on its own - about four seeds in ten, so not every file. - "a carriage return or a line feed on its own"}, + "a carriage return or a line feed on its own", + // In quoted-printable the junk lengthens the last line past the 76 + // RFC 2045 allows. Python's email reads that text whole with no + // defect (measured 2026-11-12). It came first once the label's = + // was spelled =3D, which moved the text two bytes. + "a quoted-printable line is 79 characters or ends in white space"}, // UTF-16 in the next two: the junk breaks the encoding only when its // random bytes form a lone surrogate, which is a matter of the seed. {damage.AppendJunk, "txt"}: {"that are not printable ASCII", "does not decode as utf-16"}, diff --git a/internal/guard/filmahead_test.go b/internal/guard/filmahead_test.go index af41a92f..813d6a83 100644 --- a/internal/guard/filmahead_test.go +++ b/internal/guard/filmahead_test.go @@ -8,7 +8,6 @@ import ( "slices" "testing" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/engine" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/video" @@ -40,7 +39,7 @@ func TestAFilmPaintsOnlyWhatChangesAndGetsTheWholePicture(t *testing.T) { } timelines[name] = tl } - labels := map[string]string{"labelled": core.Label("webm", 1<<20, 7), "unlabelled": ""} + labels := map[string]string{"labelled": format.Request{Bytes: 1 << 20, Seed: 7}.LabelText("webm"), "unlabelled": ""} compared, clocked, moved := 0, 0, 0 for _, size := range sizes { diff --git a/internal/guard/filmlayout_test.go b/internal/guard/filmlayout_test.go index c88f054b..d1d679b3 100644 --- a/internal/guard/filmlayout_test.go +++ b/internal/guard/filmlayout_test.go @@ -4,7 +4,6 @@ import ( "fmt" "testing" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/video" ) @@ -66,7 +65,7 @@ func TestEveryFilmLayoutIsOneTheAV1SpecificationAllows(t *testing.T) { t.Fatal(err) } } - labels := [2]string{core.Label("webm", 268435456, 7741), ""} + labels := [2]string{format.Request{Bytes: 268435456, Seed: 7741}.LabelText("webm"), ""} seen := layoutsSeen{} n := 0 for _, w := range sides { diff --git a/internal/guard/filmtiles_test.go b/internal/guard/filmtiles_test.go index 192ef12f..d6ed15a4 100644 --- a/internal/guard/filmtiles_test.go +++ b/internal/guard/filmtiles_test.go @@ -10,8 +10,8 @@ import ( "github.com/gen2brain/gav1d/av1" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/video" ) @@ -333,7 +333,7 @@ func tilesOfFilmHold(t *testing.T, props map[string]string, size int64, changes if err != nil { t.Fatal(err) } - label := core.Label("webm", size, seed) + label := format.Request{Bytes: size, Seed: seed}.LabelText("webm") var dec av1.Decoder if _, err := dec.DecodeOBUs(append([]byte{0x12, 0x00}, f.blocks[0].data...)); err != nil { t.Fatalf("%v: gav1d does not decode the film's first frame: %v", props, err) diff --git a/internal/guard/jxlladder_test.go b/internal/guard/jxlladder_test.go index b851fb01..01e20c4f 100644 --- a/internal/guard/jxlladder_test.go +++ b/internal/guard/jxlladder_test.go @@ -3,9 +3,9 @@ package guard import ( "context" "runtime" + "slices" "testing" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" "github.com/donislawdev/TestingFilesGenerator/internal/format/jxl" @@ -79,16 +79,38 @@ func TestJxlDrawsAPictureThatGrowsWithTheFileAndAlwaysFits(t *testing.T) { t.Logf("largest picture reached over the sweep: %d pixels", biggest) } -// Every rung's declared ceiling still covers what that rung produces. +// heaviestJxlSeeds are, for each rung that draws a label, the seeds the +// thorough sweep (tools/probes/jxlladder, 768 seeds) found heaviest on +// 2026-11-12 - for the label of a file at the edge of the rung and for the +// label of a damaged one. The first few seeds alone stayed green with the +// ceiling for a damaged file set back to the number that was too low, so they +// could not see what the sweep found. +var heaviestJxlSeeds = map[int][]uint64{ + 640: {0xc76d0104c41a71bd, 0x8f27bfdd1384288d}, + 320: {0x29b351201f9ae4c5, 0xe162c2724da869d3}, + 256: {0x3d, 0x80e38da90dd80935}, + 160: {0xb9f4567924888695}, +} + +// Every rung's two ceilings still cover what that rung produces: the first +// with the label of a file at the edge of the rung, the second with the label +// of a damaged file. // // The guard above walks the ladder the way a run does, so it only ever sees // the rung each size happens to select, at one seed. This one asks each rung // directly, across seeds, which is the question the ceiling is an answer to. // -// A handful of seeds rather than all 256: the thorough sweep is -// tools/probes/jxlladder, which takes minutes. This is the version that runs -// on every push and would still catch a codec raised underneath us, because a -// new encoder moves every rung at once rather than one seed of one rung. +// A file at the edge of a rung - asked for at its ceiling, where the picture +// leaves the least room - states a size with the digits of the ceiling, so the +// first ceiling is asked with those. A label with more digits is a file with +// far more room than its picture needs. A damaged file's label names its +// damage, and a longer label is more ink until it is too long for its scale, so +// the second is asked with the longest damaged label every scale still draws. +// +// A handful of seeds from zero and the heaviest the thorough sweep found +// (heaviestJxlSeeds): this is the version that runs on every push and would +// still catch a codec raised underneath us, because a new encoder moves every +// rung at once rather than one seed of one rung. // // Fewer seeds again under the race detector, and that needs saying rather than // hiding. This encoder is slow and the detector makes it slower: measured on @@ -104,44 +126,65 @@ func TestEveryJxlRungStillFitsInsideItsDeclaredCeiling(t *testing.T) { t.Fatal("the ladder is empty, so this guard measured nothing") } - // The label carries the byte count, so its length moves with the size - // asked for, and a longer label is more ink in the picture. - requested := []int64{int64(jxl.MinimumBytes), 1 << 14, 1 << 30} - - seeds := uint64(8) + first := uint64(8) if raceEnabled { - seeds = 2 - t.Logf("the race detector is on, so this sweeps %d seeds rather than 8 - it costs minutes here and finds no races either way", seeds) + first = 2 + t.Logf("the race detector is on, so this sweeps %d seeds from zero rather than 8 - it costs minutes here and finds no races either way", first) } + type ask struct { + label string + ceiling int64 + } + drawn := 0 for _, r := range rungs { - w, h, ceiling := int(r[0]), int(r[1]), r[2] - worst := int64(0) - var worstSeed uint64 - for seed := uint64(0); seed < seeds; seed++ { - for _, want := range requested { - for _, withLabel := range []bool{true, false} { - label := "" - if withLabel { - label = core.Label("jxl", want, seed) - } - n, err := jxl.FileSizeFor(w, h, jxl.DefaultQuality, seed, label) - if err != nil { - t.Fatalf("%dx%d seed %d: %v", w, h, seed, err) - } - if int64(n) > worst { - worst, worstSeed = int64(n), seed - } + w, h := int(r[0]), int(r[1]) + seeds := slices.Clone(heaviestJxlSeeds[w]) + for seed := range first { + seeds = append(seeds, seed) + } + // The size the label states: the ceiling itself, a tenth above it, + // and the lightest and the heaviest number with as many digits. + eights, lightest := int64(0), int64(1) + for n := r[2]; n > 0; n /= 10 { + eights, lightest = eights*10+8, lightest*10 + } + lightest /= 10 + var worst ask + over, size, worstSeed := int64(0), int64(0), uint64(0) + for _, seed := range seeds { + asks := []ask{{"", r[2]}} + for _, want := range []int64{r[2], r[2] + r[2]/10, lightest, eights} { + asks = append(asks, ask{format.Request{Bytes: want, Seed: seed}.LabelText("jxl"), r[2]}) + } + // A damaged file states a size of any length the damage leaves it. + plain := format.Request{Bytes: 1e12, Seed: seed} + for _, damage := range damagesDrawnAtEveryScale(t, plain, "jxl", w) { + damaged := plain + damaged.Damage = damage + asks = append(asks, ask{damaged.LabelText("jxl"), r[3]}) + drawn++ + } + for _, a := range asks { + n, err := jxl.FileSizeFor(w, h, jxl.DefaultQuality, seed, a.label) + if err != nil { + t.Fatalf("%dx%d seed %d: %v", w, h, seed, err) + } + if int64(n)-a.ceiling > over { + over, size, worst, worstSeed = int64(n)-a.ceiling, int64(n), a, seed } } } - if worst > ceiling { - t.Errorf("the %dx%d rung declares a ceiling of %d B and produced %d B at seed %d.\n"+ + if over > 0 { + t.Errorf("the %dx%d rung declares a ceiling of %d B and produced %d B at seed %#x, with the label %q.\n"+ "A ceiling below what its rung produces lets planning pick a picture the file cannot hold.\n"+ "What to do: run tools/probes/jxlladder and record what it measures.", - w, h, ceiling, worst, worstSeed) + w, h, worst.ceiling, size, worstSeed, worst.label) } } + if drawn == 0 { + t.Fatal("no rung is wide enough to draw a damaged label, so this guard asked nothing about one") + } } // Planning must not code the picture. diff --git a/internal/guard/labelclaim_test.go b/internal/guard/labelclaim_test.go index 5549de11..26bd4cb5 100644 --- a/internal/guard/labelclaim_test.go +++ b/internal/guard/labelclaim_test.go @@ -5,7 +5,6 @@ import ( "strconv" "testing" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" "github.com/donislawdev/TestingFilesGenerator/internal/format/video" @@ -77,7 +76,7 @@ func TestAPictureSaysItCarriesItsLabelOnlyWhenItsTextIsWhole(t *testing.T) { // drawn at. The guard above asks the lowest height and one with room, and an // answer one row out at the boundary passed it (found by review, 2026-10-27). func TestALabelIsReadableFromTheFirstHeightThatHoldsItsText(t *testing.T) { - label := core.Label("bmp", 2048, 1) + label := format.Request{Bytes: 2048, Seed: 1}.LabelText("bmp") across := len(label)*4 + 4 // a glyph and its gap is four columns at scale 1, with the padding either side for scale := 1; scale <= 4; scale++ { width := across * scale @@ -118,7 +117,7 @@ func TestALabelIsReadableFromTheFirstHeightThatHoldsItsText(t *testing.T) { // edges it can move at, and each series has to give both answers. func TestAFilmSaysItCarriesItsLabelOnlyWhenAPictureShowsItWhole(t *testing.T) { const seed, tall = 7, 256 - label := core.Label("webm", 1<<20, seed) + label := format.Request{Bytes: 1 << 20, Seed: seed}.LabelText("webm") across := len(label)*4 + 4 // the width that draws it at scale 1, as in the guard above timeline := func(changes int64) video.Timeline { tl, err := video.NewTimeline("webm", changes*1000, 60_000, 1000, 30) diff --git a/internal/guard/ladderceiling_test.go b/internal/guard/ladderceiling_test.go index b983ef42..07a0cdf3 100644 --- a/internal/guard/ladderceiling_test.go +++ b/internal/guard/ladderceiling_test.go @@ -71,24 +71,35 @@ func TestTheLadderCeilingIsAboveEveryPictureTheTopRungMakes(t *testing.T) { worst := int64(0) var worstAt string + drawn := 0 for _, extra := range c.extra { for _, seed := range seeds { - for _, label := range []bool{true, false} { - props := map[string]string{ - "width": strconv.Itoa(c.topWidth), - "height": strconv.Itoa(c.topHigh), - } - for k, v := range extra { - props[k] = v - } + props := map[string]string{ + "width": strconv.Itoa(c.topWidth), + "height": strconv.Itoa(c.topHigh), + } + for k, v := range extra { + props[k] = v + } + // Without a label, with one, and with the label of a damaged + // file at every scale the picture can draw it at - a longer + // label is more ink until it no longer fits its scale. + plain := format.Request{Bytes: 1, Seed: seed, Label: true, Properties: props} + asks := []format.Request{plain, {Bytes: 1, Seed: seed, Properties: props}} + for _, damage := range damagesDrawnAtEveryScale(t, plain, c.id, c.topWidth) { + damaged := plain + damaged.Damage = damage + asks = append(asks, damaged) + drawn++ + } + + for _, r := range asks { // Asking for one byte at the top rung makes the format // state its own floor, and that floor is the encoded // picture plus whatever it always carries. Nothing else // reports the encoded size from outside the package. - _, err := d.Generator.Plan(format.Request{ - Bytes: 1, Seed: seed, Label: label, Properties: props, - }) + _, err := d.Generator.Plan(r) var below *format.BelowMinimumError if !errors.As(err, &below) { t.Fatalf("%s at %dx%d refused one byte with %T rather than a BelowMinimumError: %v", @@ -96,11 +107,14 @@ func TestTheLadderCeilingIsAboveEveryPictureTheTopRungMakes(t *testing.T) { } if below.Minimum > worst { worst = below.Minimum - worstAt = fmt.Sprintf("seed %d, label %v, %v", seed, label, extra) + worstAt = fmt.Sprintf("seed %d, label %v, damage %q, %v", seed, r.Label, r.Damage, extra) } } } } + if drawn == 0 { + t.Fatalf("%s at %d wide draws no damaged label, so this guard asked nothing about one", c.id, c.topWidth) + } if worst > ceiling { t.Errorf("%s: the top rung reaches %d B (%s) but ladderCeiling is %d.\n"+ diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index dd9aa38a..cef2b58c 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -395,7 +395,7 @@ ( "the smallest Office file is worked out for one label state instead of both", "internal/format/xlsx/xlsx.go", - 'for _, label := range []string{"", core.Label("xlsx", 0, 0)} {', + 'for _, label := range []string{"", format.Request{}.LabelText("xlsx")} {', 'for _, label := range []string{""} {', "TestSizeBelowTheMinimumIsRefused", ), @@ -1175,8 +1175,8 @@ # in, and the size stays exact because the filler absorbs it. "a toml document opens with a byte order mark, which no reader accepts", "internal/format/tomlfile/toml.go", - 'line := "# " + core.Label("toml", r.Bytes, r.Seed) + "\\n"', - 'line := "\\ufeff# " + core.Label("toml", r.Bytes, r.Seed) + "\\n"', + 'line := "# " + r.LabelText("toml") + "\\n"', + 'line := "\\ufeff# " + r.LabelText("toml") + "\\n"', "TestEveryFormatSurvivesItsReferenceTool", ), ( @@ -1255,7 +1255,7 @@ # bytes. The guard pins txt with the label and without it, and only # the first of the two may notice this. "the self describing label is reworded", - "internal/core/label.go", + "internal/format/label.go", '"tfg - %s - %d B - seed %s"', '"tfg %s %d B seed %s"', "TestOurOwnGeneratorsHaveNotDrifted", @@ -6274,8 +6274,8 @@ ( "the engine calls a generator directly again, so a crash while planning ends the process", "internal/engine/plantarget.go", - "planWithoutCrashing(ctx, desc, format.Request{", - "desc.Generator.Plan(format.Request{", + "p, err := planWithoutCrashing(ctx, desc, r)", + "p, err := desc.Generator.Plan(r)", "TestEveryCallIntoAGeneratorGoesThroughTheCrashWrapper", ), ( @@ -7413,8 +7413,8 @@ ( "the AVIF ladder claims a picture fits a file far too small to hold it, so the size it promised cannot be met", "internal/format/avif/avif.go", - "\t{640, 480, 16389}, {320, 240, 6792}, {256, 256, 4890}, {160, 120, 3392},", - "\t{640, 480, 311}, {320, 240, 6792}, {256, 256, 4890}, {160, 120, 3392},", + "\t{640, 480, 18722, 22027}, {320, 240, 7732, 8312}, {256, 256, 5493, 6648}, {160, 120, 3865, 3865},", + "\t{640, 480, 311, 22027}, {320, 240, 7732, 8312}, {256, 256, 5493, 6648}, {160, 120, 3865, 3865},", "TestAvifDrawsAPictureThatGrowsWithTheFileAndAlwaysFits", ), ( @@ -7511,8 +7511,8 @@ ( "the JPEG XL ladder claims a picture fits a file far too small to hold it, so the size it promised cannot be met", "internal/format/jxl/jxl.go", - " {640, 480, 22762}, {320, 240, 7949}, {256, 256, 6485}, {160, 120, 3542},", - " {640, 480, 147}, {320, 240, 7949}, {256, 256, 6485}, {160, 120, 3542},", + " {640, 480, 22762, 23567}, {320, 240, 7949, 8854}, {256, 256, 6485, 7637}, {160, 120, 3542, 3542},", + " {640, 480, 147, 23567}, {320, 240, 7949, 8854}, {256, 256, 6485, 7637}, {160, 120, 3542, 3542},", "TestJxlDrawsAPictureThatGrowsWithTheFileAndAlwaysFits", ), ( @@ -17321,8 +17321,8 @@ ( "a refusal of a damaged size names the size the generator needed", "internal/engine/damagedformat.go", - "\t\tleast := g.smallestReaching(first)", - "\t\tleast := first", + "\t\tleast = g.smallestReaching(first)", + "\t\tleast = first", "TestTheSmallestDamagedFileIsTakenAndOneByteLessIsNot", ), ( diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index b2970fbf..feca6ddf 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -136,6 +136,24 @@ var notProvenByMutation = map[string]bool{ // "proven another way" are different states and lumping them together would // send a later session to re-prove what is already proven. var provenByProbe = map[string]string{ + // Broken once each on 2026-11-12 while they were written, one change at a + // time and restored (docs/DAMAGE-SETTINGS-LABEL-2026-11-12.md). The first + // also carries a canary of its own check. + "TestEveryDamageSettingItsDeclarationForbidsIsRefusedWithoutARecipe": "its own canary - a plan that went through and a refusal about the first entry of the chain, both rejected by its check - and broken on 2026-11-12 " + + "by taking damage.Chain.Check out of engine.prepareDamage: red, every value went through.", + "TestADamageSettingOutsideItsRangeIsRefusedUnderItsBox": "broken on 2026-11-12 by taking damage.Chain.Check out of engine.prepareDamage: red, the window wrote the file and the box said nothing.", + "TestADamagedFileStatesItsSizeOnDiskAndItsDamage": "its own canary - the label it was written against, naming the size before the damage and no damage, rejected by its check - and broken on 2026-11-12 " + + "twice: Request.LabelSized ignoring OnDisk, and leaving the damage out. Red both times.", + "TestATruncatedTextSaysHowMuchWasCut": "broken on 2026-11-12 twice: Request.LabelSized ignoring OnDisk, and leaving the damage out. Red both times.", + "TestALongDamageLabelLeavesEveryFormatWhole": "broken on 2026-11-12 by letting damage.Chain.Label return the whole chain past LabelLimit: red, " + + "its own reach check says forty damages no longer fit the limit (docs/REVIEW-234-2026-11-12.md).", + "TestEveryAvifRungStillFitsInsideItsDeclaredCeilings": "its own reach check - no damaged label drawn is a failure - and broken on 2026-11-12 twice, with " + + "the heaviest seeds: the first ceiling of 640x480 set back to 16389 (red, 17314 B) and the second to 18722 (red, 20024 B). " + + "Without those seeds both stayed green (docs/REVIEW-234-2026-11-12.md).", + "TestADamageLabelIsBoundedAndSaysHowManyItLeftOut": "its own canary - no chain up to 200 damages shortened is a failure - and broken on 2026-11-12 by letting damage.Chain.Label return the whole chain: red.", + "TestADamagedMailCarriesItsLabelWholeInEveryTextEncoding": "broken on 2026-11-12 twice: eml writing the label raw in quoted-printable (red, the one damage label read back as bytesd), " + + "and eml leaving a long label on one line (red, a line of 92 characters).", + "TestEveryCharacterOfADamagedLabelCanBeDrawn": "its own canary - a character the font never had must not count as drawn - and broken on 2026-11-12 by taking = out of the picture font: red.", // What breaks it lives in a module, not in a file of this repository, so // no mutation of this tree reaches it - the dependency it was written // against is the break. diff --git a/internal/guard/requestkey_test.go b/internal/guard/requestkey_test.go index 10adb046..73793c72 100644 --- a/internal/guard/requestkey_test.go +++ b/internal/guard/requestkey_test.go @@ -36,6 +36,8 @@ func TestEveryFieldOfARequestIsInItsKey(t *testing.T) { v.SetUint(1) case reflect.Bool: v.SetBool(true) + case reflect.String: + v.SetString("truncate:missing=90") case reflect.Map: v.Set(reflect.ValueOf(map[string]string{"delimiter": ";"})) case reflect.Slice: diff --git a/internal/guard/textencoding_test.go b/internal/guard/textencoding_test.go index 4bad04ae..b2786062 100644 --- a/internal/guard/textencoding_test.go +++ b/internal/guard/textencoding_test.go @@ -25,7 +25,6 @@ import ( "strings" "testing" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/textenc" "github.com/donislawdev/TestingFilesGenerator/internal/oracle" @@ -58,7 +57,7 @@ func (c encodingCase) refusedBy(id string) bool { // rather than silently measuring an empty wrapper. func labelLine(t *testing.T, id string, size int64, seed uint64) string { t.Helper() - body := core.Label(id, size, seed) + body := format.Request{Bytes: size, Seed: seed}.LabelText(id) switch id { case "txt": return body + "\n" diff --git a/internal/recipe/compose.go b/internal/recipe/compose.go index 838a98d0..b2efb27a 100644 --- a/internal/recipe/compose.go +++ b/internal/recipe/compose.go @@ -470,7 +470,7 @@ const ( KeySendAs = core.SettingSendAs KeyLabel = "label" KeyFill = "fill" - KeyDamage = "damage" + KeyDamage = core.KeyDamage KeyProperties = core.KeyProperties KeyExpected = "expected" KeyExpectedReason = "expected.reason"