diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f6f3771..ac8f4b0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,17 @@ because it turns other people's test suites red. ## [Unreleased] +### Breaking + +- **Some AVIF pictures and WebM and MP4 films are not byte for byte what + 0.5.0 wrote.** The AV1 encoder underneath, gav1d, is now 0.2.6. Where the + right or bottom edge of a picture cuts through a block, the encoder now + predicts that block's colour the way every decoder does. Before, a viewer + showed those pixels along the edge up to 7 of 255 off from the picture that + was meant. Only pixels along those edges change, and many files do not + change at all. A test that compares a file with one 0.5.0 generated from the + same request can fail. + ### Added - **A new format: `otlp`, OpenTelemetry traces and logs as OTLP JSON.** @@ -145,6 +156,13 @@ because it turns other people's test suites red. ### Fixed +- **An AVIF picture wider than 4096 px, or larger than 4096 x 2304 px, opens + now.** Every release since AVIF arrived in 0.3.0 wrote such a picture as a + file no viewer reads: Pillow, ImageMagick and libaom all refused it as + corrupt, and tfg said nothing. The picture is now cut into the tiles AV1 + requires at that size, and all three read it. A picture of 8000 x 5000 px is + also written about three times faster on a machine with several cores. + Smaller pictures and films were never affected. - **A damaged file no longer claims to carry its label.** Its manifest entry said `label_embedded: true`, although the damage comes after the label is written and can overwrite it. On a text file, `zero-head` zeroes the start of diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index f970e202..d72b3a89 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -121,7 +121,7 @@ Source: ## github.com/gen2brain/gav1d -Version 0.2.5. Codes and reads AV1, which is what the picture inside an +Version 0.2.6. Codes and reads AV1, which is what the picture inside an AVIF is. It is written in Go and assembly with no C in it and no module of its own behind it, so it brings nothing else along. diff --git a/go.mod b/go.mod index 2adb8582..ae306763 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ go 1.27.4 require ( fyne.io/fyne/v2 v2.8.2 - github.com/gen2brain/gav1d v0.2.5 + github.com/gen2brain/gav1d v0.2.6 github.com/gen2brain/jxl v0.2.0 github.com/goccy/go-yaml v1.19.2 github.com/nicksnyder/go-i18n/v2 v2.7.0 diff --git a/go.sum b/go.sum index e211979a..061edca5 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/fyne-io/image v0.1.1 h1:WH0z4H7qfvNUw5l4p3bC1q70sa5+YWVt6HCj7y4VNyA= github.com/fyne-io/image v0.1.1/go.mod h1:xrfYBh6yspc+KjkgdZU/ifUC9sPA5Iv7WYUBzQKK7JM= github.com/fyne-io/oksvg v0.2.1-0.20260918172519-a9af55fa95d9 h1:owsdpq6a2KhaAA/hoHGQmFM7kOgKo3srXZVGnbEUzr0= github.com/fyne-io/oksvg v0.2.1-0.20260918172519-a9af55fa95d9/go.mod h1:dJ9oEkPiWhnTFNCmRgEze+YNprJF7YRbpjgpWS4kzoI= -github.com/gen2brain/gav1d v0.2.5 h1:Zg5/DE1JBdf8kxZqIvNo2ZEizF9DVF9jdPTAKJQZbOY= -github.com/gen2brain/gav1d v0.2.5/go.mod h1:ReFKjHq7pvRhEk+pgRPAo9gOlcrf51sdzVcAbbX7e0E= +github.com/gen2brain/gav1d v0.2.6 h1:7TSkrWTdE22k5Ul7vYZML7Nu80gNW6OovV7FSCITSE0= +github.com/gen2brain/gav1d v0.2.6/go.mod h1:ReFKjHq7pvRhEk+pgRPAo9gOlcrf51sdzVcAbbX7e0E= github.com/gen2brain/jxl v0.2.0 h1:cXSQnfbcC0JGFzaI344VzPMbe7mh6rYc0tjPcBUsE4U= github.com/gen2brain/jxl v0.2.0/go.mod h1:VjJHRai/8I8Es01mULqyiOihRBOjJr1Zo+NoN/MyIwo= github.com/go-gl/glfw/v3.4/glfw v0.1.0-pre.2 h1:ez76TQInhkDIMobWbOFcPJw6FeB7coXRAVZpKwF8NBg= diff --git a/internal/format/avif/avif.go b/internal/format/avif/avif.go index a2973bc1..1531d22b 100644 --- a/internal/format/avif/avif.go +++ b/internal/format/avif/avif.go @@ -76,6 +76,12 @@ const ( // Measured on 2026-08-29 with the encoder that is actually here: 1080p // costs 16 MB and 266 ms, 4K costs 63 MB and 1.06 s, 8K costs 253 MB and // 4.7 s. All of them encode, none of them fail. + // + // What nobody did then was decode them. Up to gav1d v0.2.5 a picture wider + // than 4096 px or past 4096x2304, 8K among them, came out as a file no + // decoder reads, and v0.2.6 codes it in tiles + // (TestAnAVIFLargerThanOneAV1TileSurvivesItsReferenceTool, + // docs/GAV1D-026-2026-11-11.md). maxPixels = 40_000_000 // minimumBytes is the smallest AVIF this generator produces, and it is one diff --git a/internal/format/mp4/mp4.go b/internal/format/mp4/mp4.go index 2567160f..2862b580 100644 --- a/internal/format/mp4/mp4.go +++ b/internal/format/mp4/mp4.go @@ -96,7 +96,12 @@ type generator struct{} // 904), the sample lengths and the second pass's crew the difference - // and 1241 with one object allocated per frame, the defect the ceiling // exists for. 1024 sits between the two. -func (generator) AllocCeiling() int64 { return 1024 } +// +// Raised to 1536 on 2026-11-11 with WebM's, by the owner's decision and for +// WebM's reason, gav1d v0.2.6 allocating more a call. The default film is +// 1398 to 1400 objects at one, four and sixteen threads, and one object a +// frame takes it to 1698 to 1701 (docs/GAV1D-026-2026-11-11.md). +func (generator) AllocCeiling() int64 { return 1536 } func (generator) Plan(r format.Request) (format.Plan, error) { need := func(st video.Stream) int64 { return newLayout(st).need() } diff --git a/internal/format/rtf/settings.go b/internal/format/rtf/settings.go index 92d52ce9..5395748a 100644 --- a/internal/format/rtf/settings.go +++ b/internal/format/rtf/settings.go @@ -101,7 +101,7 @@ func ownProperties() []format.Property { "codepage writes only the code page bytes, and the emoji still as numbers, since no code page holds them. " + "literal writes the question mark as a plain character. " + "LibreOffice, Tika and the RichEdit control of Windows read all four. " + - "pandoc shows codepage text in the wrong letters and drops the character after every letter written literal, and pandoc 3.1 and 3.7 drop the emoji in all four. " + + "pandoc shows codepage text in the wrong letters. pandoc before 3.13 also drops the character after every letter written literal, and pandoc 3.1 and 3.7 drop the emoji in all four. " + "striprtf returns an emoji as two halves, which Python refuses to write as UTF-8.", }, { Name: Elements, Kind: format.PropertyChoice, diff --git a/internal/format/video/settings.go b/internal/format/video/settings.go index 37a441a5..526f9852 100644 --- a/internal/format/video/settings.go +++ b/internal/format/video/settings.go @@ -41,12 +41,14 @@ const ( maxQuality = 100 defaultQuality = 60 - // The picture's bounds are AV1's, not the encoder's. gav1d codes one tile - // correctly and no more - a larger frame comes out of it with a header - // that announces several tiles over the data of one (measured 2026-10-06, - // docs/REVIEW-165-2026-10-06.md), and until 2026-10-07 that held the - // picture to 4096x2304. A picture is cut into tiles here now (grid.go) and - // every tile is within one of gav1d's, so the bound left is the format's: + // The picture's bounds are AV1's, not the encoder's. gav1d up to v0.2.5 + // coded one tile correctly and no more - a larger frame came out of it + // with a header that announced several tiles over the data of one + // (measured 2026-10-06, docs/REVIEW-165-2026-10-06.md), and until + // 2026-10-07 that held the picture to 4096x2304. A picture is cut into + // tiles here now (grid.go) and every tile is within one of gav1d's - still + // so with v0.2.6, which codes several, because a tile no picture changes + // is then coded once a film - so the bound left is the format's: // maxPixels is the largest picture an AV1 level describes, 8192 by 4352 // (MaxPicSize of the 6.x levels, annex.a.levels.md lines 89-92). A larger // one could declare no level but the maximum parameters one, which a diff --git a/internal/format/video/still.go b/internal/format/video/still.go index 2b92dd89..eff00056 100644 --- a/internal/format/video/still.go +++ b/internal/format/video/still.go @@ -33,9 +33,10 @@ type tileCoded struct { // which every tile of one frame has to be: gav1d writes them from the // quantizer alone (gav1d v0.2.5, av1/encode_hdr.go, writeFrameHdr - the loop // filter off, no delta, no tx_mode_select), measured on 2026-10-06 as one -// suffix per quantizer across 27 pictures of 9 sizes. A later gav1d that chose -// them from the picture would break that, and the frame is refused rather -// than written with one tile's header over another's data. +// suffix per quantizer across 27 pictures of 9 sizes. v0.2.6 changed only +// tile_info in that function (docs/GAV1D-026-2026-11-11.md). A later gav1d +// that chose them from the picture would break that, and the frame is refused +// rather than written with one tile's header over another's data. func (t tileCoded) sameHeader(o tileCoded) bool { if t.to-t.from != o.to-o.from { return false diff --git a/internal/format/webm/webm.go b/internal/format/webm/webm.go index 6f51def1..0d3f42f2 100644 --- a/internal/format/webm/webm.go +++ b/internal/format/webm/webm.go @@ -102,7 +102,14 @@ type generator struct{} // ceiling exists for, takes it to 1177. 1024 sits between the two, as // 512 sat between 474 and 698 (docs/WEBM-WYDAJNOSC-2026-10-06.md // section 10). -func (generator) AllocCeiling() int64 { return 1024 } +// +// Raised to 1536 on 2026-11-11 by the owner's decision, with gav1d v0.2.6, +// which no longer reserves its entropy coder's buffer up front and grows it +// instead: 36 to 57 objects a call where v0.2.5 made 21 to 22 +// (tools/probes/videotiles/allocs, docs/GAV1D-026-2026-11-11.md). The +// default film is 1341 to 1350 objects at one, four and sixteen threads, and +// one object a frame takes it to 1634 to 1644. 1536 sits between the two. +func (generator) AllocCeiling() int64 { return 1536 } func (generator) Plan(r format.Request) (format.Plan, error) { need := func(st video.Stream) int64 { return newLayout(st).boundBytes() + minVoid } diff --git a/internal/guard/avifwide_test.go b/internal/guard/avifwide_test.go new file mode 100644 index 00000000..5aa011e1 --- /dev/null +++ b/internal/guard/avifwide_test.go @@ -0,0 +1,109 @@ +package guard + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/gen2brain/gav1d/avif" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/oracle" +) + +// An AVIF picture larger than one AV1 tile is coded as tiles, and it is read +// back whole - by the specification's tile_info, by gav1d's own decoder and +// by the format's reference tool. +// +// AV1 holds a tile to 4096 px across and to 4096x2304 px in area +// (06.bitstream.syntax.md lines 1176-1192), so a picture past either has to be +// coded as several. gav1d up to v0.2.5 wrote such a picture's tile_info as +// several tiles over the data of one (docs/REVIEW-165-2026-10-06.md), and +// this format takes pictures to 16384 px and 40 megapixels - so every AVIF +// wider than 4096 px or past that area was a file no decoder reads. Pillow, +// libaom and ImageMagick refused all four such sizes measured on 2026-11-11, +// and read them once gav1d v0.2.6 coded them in tiles +// (docs/GAV1D-026-2026-11-11.md). Films were never hit: +// they cut a picture into tiles themselves (internal/format/video, grid.go). +// +// The two pictures are the two ways tile_info cuts one: a width a few pixels +// past one column, and an area one superblock row past the limit at the full +// width of a tile, which is cut into rows. Each is held to its tiles read +// from the file first, so this guard knows it asked about a picture of tiles +// rather than assuming it. A missing reference tool skips that part loudly. +func TestAnAVIFLargerThanOneAV1TileSurvivesItsReferenceTool(t *testing.T) { + d, err := format.Get("avif") + if err != nil { + t.Fatal(err) + } + checker, known := oracle.For(d.Oracle) + if !known { + t.Fatalf("avif declares the oracle %q and nothing implements it", d.Oracle) + } + for _, c := range []struct { + width, height int + cols, rows int + }{ + {width: 4100, height: 64, cols: 2, rows: 1}, + {width: 4096, height: 2305, cols: 1, rows: 2}, + } { + name := strconv.Itoa(c.width) + "x" + strconv.Itoa(c.height) + plan, err := d.Generator.Plan(format.Request{Bytes: 4 << 20, Seed: goldenSeed, Label: true, + Properties: map[string]string{"width": strconv.Itoa(c.width), "height": strconv.Itoa(c.height)}}) + if err != nil { + t.Fatalf("%s: planning failed: %v", name, err) + } + var buf bytes.Buffer + if err := d.Generator.Write(context.Background(), &buf, plan); err != nil { + t.Fatalf("%s: writing failed: %v", name, err) + } + b := buf.Bytes() + + mdat, err := mp4Child(b, "mdat") + if err != nil { + t.Fatalf("%s: %v", name, err) + } + frame, _, err := frameOBU(mdat) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + // A still picture's frame header has three bits ahead of tile_info: + // disable_cdf_update, allow_screen_content_tools and + // render_and_frame_size_different (internal/format/video, still.go). + tiles, err := readFilmTiles(frame, 3, c.width, c.height) + if err != nil { + t.Errorf("%s: the picture's tiles do not read back from the file: %v", name, err) + continue + } + if len(tiles.rects) != c.cols*c.rows { + t.Fatalf("%s: %d tiles read back where tile_info derives %d columns and %d rows - the case no longer asks about a picture of tiles", + name, len(tiles.rects), c.cols, c.rows) + } + + m, err := avif.Decode(bytes.NewReader(b)) + if err != nil { + t.Errorf("%s: gav1d's decoder refused it: %v", name, err) + continue + } + if got := m.Bounds(); got.Dx() != c.width || got.Dy() != c.height { + t.Errorf("%s: gav1d's decoder read a %dx%d picture", name, got.Dx(), got.Dy()) + } + + path := filepath.Join(t.TempDir(), name+".avif") + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatal(err) + } + res := checker.Check(path) + switch { + case !res.Available: + t.Logf("SKIPPED %s: %s is not installed, so only the tiles and gav1d's decoder were asked", name, checker.Name) + case res.Err != nil: + t.Errorf("%s: %v", name, res.Err) + default: + t.Logf("%s: %d tiles, %s accepted it - %s", name, len(tiles.rects), checker.Name, firstLine(res.Output)) + } + } +} diff --git a/internal/guard/film_test.go b/internal/guard/film_test.go index 15b42420..3be68e48 100644 --- a/internal/guard/film_test.go +++ b/internal/guard/film_test.go @@ -490,7 +490,7 @@ func TestEveryFrameOfAFilmSurvivesItsReferenceTool(t *testing.T) { // before coding, and the pictures that used to be refused for not fitting one // tile are not. // -// Until 2026-10-07 the bound was one AV1 tile, 4096x2304, because gav1d codes +// Until 2026-10-07 the bound was one AV1 tile, 4096x2304, because gav1d coded // one tile correctly and no more (docs/REVIEW-165-2026-10-06.md). A picture // is cut into tiles of at most that now, and the bound is AV1's own: 8192 by // 4352, the 6.x levels' MaxPicSize, with either side up to 16384 (the owner's diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index a0c413b5..dd9aa38a 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -13695,7 +13695,8 @@ ), ( # One object per frame, the defect the film's own allocation ceiling - # sits below: measured at 698 objects against a ceiling of 512. + # sits below: measured at 698 objects against a ceiling of 512, and + # with gav1d v0.2.6 at 1634 to 1644 against 1536. "a film allocates a block header for every frame it writes", "internal/format/webm/webm.go", " out.write(block[:])", @@ -14022,7 +14023,8 @@ ), ( # One object per frame, the defect the film's own allocation ceiling - # sits below: measured at 1241 objects against a ceiling of 1024. + # sits below: measured at 1241 objects against a ceiling of 1024, and + # with gav1d v0.2.6 at 1698 to 1701 against 1536. "an MP4 allocates a copy of every frame it writes", "internal/format/mp4/mp4.go", " o.write(frame[t.SampleAt(i)])", @@ -17705,4 +17707,18 @@ "fmt.Fprintf(out, \" extension %s\\n\", d.Extension)", "TestTheFormatsCommandSaysWhichSettingGivesWhichEnding", ), + ( + "pandoc's version line is read only without the .exe Windows prints, so pandoc 3.13 there is a version nobody knows", + 'internal/oracle/rtfscripts.go', + '`(?m)^pandoc(?:\\.exe)? (\\d+)\\.(\\d+)`', + '`(?m)^pandoc (\\d+)\\.(\\d+)`', + 'TestThePandocVersionDecidesWhatALiteralDocumentReadsAs', + ), + ( + "pandoc's minor version is read from the major's digits, so 3.13 reads as 3.3 and is held to the old reading of literal", + 'internal/oracle/rtfscripts.go', + 'minor, errMinor := strconv.Atoi(m[2])', + 'minor, errMinor := strconv.Atoi(m[1])', + 'TestThePandocVersionDecidesWhatALiteralDocumentReadsAs', + ), ] diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index 3b9f42df..7013c4c1 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -136,6 +136,11 @@ var notProvenByMutation = map[string]bool{ // "proven another way" are different states and lumping them together would // send a later session to re-prove what is already proven. var provenByProbe = map[string]string{ + // What breaks it lives in a module, not in a file of this repository, so + // no mutation of this tree reaches it - the dependency it was written + // against is the break. + "TestAnAVIFLargerThanOneAV1TileSurvivesItsReferenceTool": "broken on 2026-11-11 by gav1d v0.2.5, go.mod as it stood before v0.2.6: both pictures went red saying tile_start_and_end_present_flag is set in an OBU_FRAME, " + + "the tile_info of several tiles read over the data of one. With v0.2.6 both read back as 2 tiles, gav1d's decoder read them whole and Pillow accepted them (docs/GAV1D-026-2026-11-11.md).", // Runs only in the CI job that has the Collector, so a mutation job without // one would meet a skip. What the guard adds to the structural check is the // oracle itself, and the oracle was handed three files it has to refuse. diff --git a/internal/guard/rtf_test.go b/internal/guard/rtf_test.go index 2c932cfc..a2cffee0 100644 --- a/internal/guard/rtf_test.go +++ b/internal/guard/rtf_test.go @@ -3,6 +3,7 @@ package guard import ( "bytes" "context" + "fmt" "os" "path/filepath" "sort" @@ -56,13 +57,17 @@ var ( // The structural checker is told the settings and has to find the document made // the way they say, and pandoc has to read it without a word of complaint. -// pandoc's own readings of good files are held as they were measured on three -// versions of it, 3.1.3, 3.7.0.2 and 3.12.1: the four scripts read right in -// unicode and both, the Polish line wrong in codepage and literal, the -// footnote kept and the hidden text left out. Those readings are what the -// sentences of text_encoding and elements tell a user, so this goes red the day -// pandoc reads differently, and the sentence is what has to change. Emoji and -// the text of a link differ between the three versions and are not asked. +// pandoc's own readings of good files are held as they were measured on four +// versions of it, 3.1.3, 3.7.0.2, 3.12.1 and 3.13: the four scripts read right +// in unicode and both, the Polish line wrong in codepage, the footnote kept +// and the hidden text left out. literal is the Polish line wrong before 3.13 +// and the four scripts right from it, the version that stopped dropping the +// character after a letter written literal (jgm/pandoc#11981, measured +// 2026-11-11) - so which one is asked is the version pandoc says it is. Those +// readings are what the sentences of text_encoding and elements tell a user, +// so this goes red the day pandoc reads differently, and the sentence is what +// has to change. Emoji and the text of a link differ between the versions and +// are not asked. // // Every size from the smallest the settings allow, where the label does not fit // and the document has no paragraph, through the next two, to a document with @@ -75,6 +80,11 @@ func TestEveryRTFSettingSurvivesItsReferenceTool(t *testing.T) { t.Fatal(err) } dir := t.TempDir() + major, minor, ok := oracle.PandocVersion() + pv := pandocVersionOf(major, minor, ok) + if ok { + t.Logf("pandoc %d.%d answers", major, minor) + } var strict, byPandoc atomic.Int64 t.Run("cases", func(t *testing.T) { for i, props := range rtfCases { @@ -90,7 +100,7 @@ func TestEveryRTFSettingSurvivesItsReferenceTool(t *testing.T) { if err := os.WriteFile(path, b, 0o600); err != nil { t.Fatal(err) } - s, pd := rtfSurvivesReaders(t, path, b, props, p, c.size) + s, pd := rtfSurvivesReaders(t, path, b, props, p, c.size, pv) strict.Add(int64(s)) byPandoc.Add(int64(pd)) } @@ -103,9 +113,61 @@ func TestEveryRTFSettingSurvivesItsReferenceTool(t *testing.T) { } } +// pandocVersion is what the guard knows of the pandoc that answers: whether +// --version said, and whether it is 3.13 or later, which reads a letter +// written literal right where the versions before it dropped the character +// after it (jgm/pandoc#11981). +type pandocVersion struct { + known, readsLiteral bool +} + +// pandocVersionOf is what oracle.PandocVersion's answer means to this guard. +func pandocVersionOf(major, minor int, ok bool) pandocVersion { + if !ok { + return pandocVersion{} + } + return pandocVersion{known: true, readsLiteral: major > 3 || major == 3 && minor >= 13} +} + +// The version pandoc says it is decides what a document written literal has +// to read as, and that is held here on words written out rather than on the +// pandoc installed: CI has 3.1.3 and this machine 3.13, so each of them only +// ever asks one side of the boundary (jgm/pandoc#11981). Each version is held +// to its reading and has to refuse the other side's. +func TestThePandocVersionDecidesWhatALiteralDocumentReadsAs(t *testing.T) { + right := strings.Join([]string{rtfPolish, rtfRussian, rtfJapanese, rtfArabic}, "\n") + lost := "Za g ja." // the Polish line with the letter after each one written literal dropped + literal := map[string]string{rtf.Characters: rtf.International, rtf.TextEncoding: rtf.Literal} + for _, c := range []struct { + out string + known, reads bool + }{ + {"pandoc 3.1.3\nFeatures: +server +lua\n", true, false}, + {"pandoc 3.12.1\n", true, false}, + {"pandoc.exe 3.13\n", true, true}, + {"pandoc 3.13.1\n", true, true}, + {"pandoc 4.0\n", true, true}, + {"pandoc: unknown flag --version\n", false, false}, + {"", false, false}, + } { + pv := pandocVersionOf(oracle.ParsePandocVersion(c.out)) + if pv.known != c.known || pv.readsLiteral != c.reads { + t.Errorf("%q: known %v and reads literal %v, where %v and %v were meant", c.out, pv.known, pv.readsLiteral, c.known, c.reads) + continue + } + if !pv.known { + continue + } + onRight, onLost := rtfScriptsProblem(right, literal, pv.readsLiteral), rtfScriptsProblem(lost, literal, pv.readsLiteral) + if c.reads && (onRight != "" || onLost == "") || !c.reads && (onRight == "" || onLost != "") { + t.Errorf("%q: the four scripts read right say %q and the Polish line lost says %q", c.out, onRight, onLost) + } + } +} + // rtfSurvivesReaders asks the checker and pandoc about one document and says // which of them read it. -func rtfSurvivesReaders(t *testing.T, path string, body []byte, props map[string]string, p format.Plan, size int64) (strict, byPandoc int) { +func rtfSurvivesReaders(t *testing.T, path string, body []byte, props map[string]string, p format.Plan, size int64, pv pandocVersion) (strict, byPandoc int) { t.Helper() labelled, _ := p.Properties[format.PropertyLabelEmbedded].(bool) s := oracle.Strict("rtf", path, rtfSettings(props, labelled)...) @@ -131,7 +193,10 @@ func rtfSurvivesReaders(t *testing.T, path string, body []byte, props map[string if got := strings.Contains(text, "tfg - rtf - "); got != labelled { t.Fatalf("%v at %d B: the manifest says the label is there %v, and pandoc read it %v", props, size, labelled, got) } - rtfPandocReadings(t, norm.NFC.String(text), props, size) + if !pv.known { + t.Fatalf("%v at %d B: pandoc read the document and its --version says nothing this guard reads, so what it should make of literal is unknown", props, size) + } + rtfPandocReadings(t, norm.NFC.String(text), props, size, pv.readsLiteral) rtfPandocKeepsHexPictures(t, path, body, props, size) return 1, 1 } @@ -182,20 +247,12 @@ func rtfPandocBreaksBinaryPictures(t *testing.T, path string, body []byte, props } // rtfPandocReadings is what pandoc was measured to make of the paragraphs in -// other scripts and of the catalogue of every part. -func rtfPandocReadings(t *testing.T, text string, props map[string]string, size int64) { +// other scripts and of the catalogue of every part. readsLiteral is pandoc +// 3.13 or later, which reads literal as it reads unicode. +func rtfPandocReadings(t *testing.T, text string, props map[string]string, size int64, readsLiteral bool) { t.Helper() - if props[rtf.Characters] == rtf.International { - enc := props[rtf.TextEncoding] - if enc == "" || enc == rtf.Unicode || enc == rtf.Both { - for _, line := range []string{rtfPolish, rtfRussian, rtfJapanese, rtfArabic} { - if !strings.Contains(text, line) { - t.Fatalf("%v at %d B: pandoc did not read %q", props, size, line) - } - } - } else if strings.Contains(text, rtfPolish) { - t.Fatalf("%v at %d B: pandoc read the Polish line right, and text_encoding says it shows %s wrong - the sentence has to change", props, size, enc) - } + if problem := rtfScriptsProblem(text, props, readsLiteral); problem != "" { + t.Fatalf("%v at %d B: %s", props, size, problem) } if props[rtf.Elements] == rtf.Every { if !strings.Contains(text, "This sentence is in a footnote.") || strings.Contains(text, "This sentence is hidden text.") { @@ -204,6 +261,28 @@ func rtfPandocReadings(t *testing.T, text string, props map[string]string, size } } +// rtfScriptsProblem is what is wrong with pandoc's reading of the paragraphs +// in other scripts, or nothing: read right in unicode and both, and in literal +// from pandoc 3.13, and the Polish line read wrong otherwise. +func rtfScriptsProblem(text string, props map[string]string, readsLiteral bool) string { + if props[rtf.Characters] != rtf.International { + return "" + } + enc := props[rtf.TextEncoding] + if enc == "" || enc == rtf.Unicode || enc == rtf.Both || enc == rtf.Literal && readsLiteral { + for _, line := range []string{rtfPolish, rtfRussian, rtfJapanese, rtfArabic} { + if !strings.Contains(text, line) { + return fmt.Sprintf("pandoc did not read %q", line) + } + } + return "" + } + if strings.Contains(text, rtfPolish) { + return fmt.Sprintf("pandoc read the Polish line right, and text_encoding says it shows %s wrong - the sentence has to change", enc) + } + return "" +} + // rtfSettings is a case as the words the structural checker is told, with the // label the manifest says the file carries. func rtfSettings(props map[string]string, labelled bool) []string { diff --git a/internal/guard/testdata/generator-golden.json b/internal/guard/testdata/generator-golden.json index 4c581b6c..359acb5b 100644 --- a/internal/guard/testdata/generator-golden.json +++ b/internal/guard/testdata/generator-golden.json @@ -408,8 +408,8 @@ }, "webm_wide": { "bytes": 524288, - "sha256": "270aab596f2a59abe47cf9ab35236964b88dcd1be2ea24247cbce8c72c3c1cd0", - "measured_on": "2026-10-07" + "sha256": "a7b10a755815dfc0082c70318ca72fc890b6b04c3212e67bfab1d691cc36f932", + "measured_on": "2026-11-11" }, "mp4_64kib": { "bytes": 65536, @@ -433,8 +433,8 @@ }, "mp4_wide": { "bytes": 524288, - "sha256": "ae5d267ff067caf04aa42e72916f67a89a19584d2ea30445a955202571fc45c3", - "measured_on": "2026-10-07" + "sha256": "f2a09c9a2dacccdb778a8e828b3afa2fdf0b98e4d91f79f41b6488c4a67112bb", + "measured_on": "2026-11-11" }, "geojson_8kib": { "bytes": 8192, @@ -823,6 +823,14 @@ "wav_32bit_past_one_cycle", "wav_past_one_cycle" ] + }, + { + "on": "2026-11-11", + "why": "gav1d v0.2.6 under every film and AVIF picture. Its encoder now pads the luma it predicts chroma from (CfL) the way the decoder does, by the luma transform rather than the chroma one, so a block cut by the right or bottom edge of a picture is coded against the prediction a decoder makes - before, the picture decoded differed from the one coded in those blocks by up to 7 of 255. The two wide films have such blocks in their bottom row and change there, every frame, by up to 5 of 255. The CI of the Dependabot pull request measured both on windows, linux and macos alike. A breaking change, taken with the bump, docs/GAV1D-026-2026-11-11.md.", + "files": [ + "mp4_wide", + "webm_wide" + ] } ] } diff --git a/internal/gui/text/locale/registry/en.json b/internal/gui/text/locale/registry/en.json index f1a839be..60737cf8 100644 --- a/internal/gui/text/locale/registry/en.json +++ b/internal/gui/text/locale/registry/en.json @@ -1751,8 +1751,8 @@ }, "Detail.format/rtf.text_encoding": { "description": "The sentence under the text_encoding setting of the rtf format, after what it takes.", - "hash": "sha256-e6a5e55837f3", - "other": "How the letters outside ASCII are written, which changes nothing while characters is ascii. unicode writes each one as a number with a question mark after it for a reader that does not know numbers. both puts the letter in the code page of its font after the number, two bytes for Japanese, the way Word writes it. codepage writes only the code page bytes, and the emoji still as numbers, since no code page holds them. literal writes the question mark as a plain character. LibreOffice, Tika and the RichEdit control of Windows read all four. pandoc shows codepage text in the wrong letters and drops the character after every letter written literal, and pandoc 3.1 and 3.7 drop the emoji in all four. striprtf returns an emoji as two halves, which Python refuses to write as UTF-8." + "hash": "sha256-3766bb620616", + "other": "How the letters outside ASCII are written, which changes nothing while characters is ascii. unicode writes each one as a number with a question mark after it for a reader that does not know numbers. both puts the letter in the code page of its font after the number, two bytes for Japanese, the way Word writes it. codepage writes only the code page bytes, and the emoji still as numbers, since no code page holds them. literal writes the question mark as a plain character. LibreOffice, Tika and the RichEdit control of Windows read all four. pandoc shows codepage text in the wrong letters. pandoc before 3.13 also drops the character after every letter written literal, and pandoc 3.1 and 3.7 drop the emoji in all four. striprtf returns an emoji as two halves, which Python refuses to write as UTF-8." }, "Detail.format/svg.height": { "description": "The sentence under the height setting of the svg format, after what it takes.", diff --git a/internal/gui/text/locale/registry/pl.json b/internal/gui/text/locale/registry/pl.json index fe734e1b..db4b1025 100644 --- a/internal/gui/text/locale/registry/pl.json +++ b/internal/gui/text/locale/registry/pl.json @@ -349,7 +349,7 @@ "Detail.format/rtf.links": { "hash": "sha256-9e3ca1ff09e6", "other": "Ile odnośników niesie dokument, każdy jako pole HYPERLINK w osobnym akapicie, nad obrazkami i blokami. Przechodzą po kolei przez 17 rodzajów i potem się powtarzają - do zakładki w dokumencie, względny, do katalogu wyżej, od korzenia, bezwzględny z zapytaniem i fragmentem, bezwzględny z położeniem, http, bez schematu, z kodowaniem procentowym, domena międzynarodowa, port, mailto:, tel:, ftp:, nowe okno, podpowiedź i data: - a tekst każdego mówi, którym jest. Każdy adres leży w domenie zarezerwowanej na przykłady. LibreOffice zamienia odnośnik bez schematu na adres file:// i przepisuje względne od miejsca, w którym zapisuje dokument. pandoc 3.12 zostawia odnośnik do zakładki bez adresu, pandoc 3.1 i 3.7 gubią każdy odnośnik razem z tekstem, a RichEdit oddaje adres każdego odnośnika przed jego tekstem. LibreOffice czytał milion odnośników ponad 11 minut. Katalog wszystkich części ma jeden własny odnośnik, którego to ustawienie nie liczy." }, "Detail.format/rtf.nesting": { "hash": "sha256-e1d640adb186", "other": "W ilu grupach, jedna w drugiej, siedzi treść. 0 umieszcza ją wprost w dokumencie, a etykieta zostaje na zewnątrz, więc plik dalej mówi, czym jest. Żaden zmierzony czytnik nie zgubił treści przy 10 000 000 poziomów, ale LibreOffice zużył na nie 18,5 GB pamięci, a pandoc 14,5 GB, wobec około 2 GB każdy przy 1 000 000." }, "Detail.format/rtf.picture_data": { "hash": "sha256-f91b5cd2e506", "other": "Jak zapisane są bajty każdego obrazka, co niczego nie zmienia, dopóki inline wynosi 0. hex zapisuje je jako tekst, dwa razy większy, tak jak zapisuje je LibreOffice. binarnie zapisuje je takie, jakie są, więc dokument przestaje być tekstem, a program, który liczy klamry albo czyta go jako tekst, gubi się. pandoc zmienia końce linii wewnątrz obrazka zapisanego binarnie, więc żaden png nie wychodzi cały, a pandoc może zgubić tekst za obrazkiem albo odrzucić cały dokument. LibreOffice, Tika i RichEdit czytają oba zapisy." }, - "Detail.format/rtf.text_encoding": { "hash": "sha256-e6a5e55837f3", "other": "Jak zapisane są litery spoza ASCII. Gdy Znaki to ascii, nic to nie zmienia. unicode zapisuje każdą jako liczbę ze znakiem zapytania po niej dla czytnika, który nie zna liczb. oba dopisuje po liczbie literę w stronie kodowej jej fontu, dla japońskiego dwa bajty, tak jak zapisuje ją Word. strona kodowa zapisuje tylko bajty strony kodowej, a emoji dalej jako liczby, bo żadna strona kodowa ich nie ma. dosłowny zapisuje znak zapytania jako zwykły znak. LibreOffice, Tika i kontrolka RichEdit z Windows czytają wszystkie cztery. pandoc pokazuje tekst ze strony kodowej złymi literami i gubi znak po każdej literze zapisanej dosłownie, a pandoc 3.1 i 3.7 gubią emoji we wszystkich czterech. striprtf oddaje emoji jako dwie połówki, których Python nie zapisze w UTF-8." }, + "Detail.format/rtf.text_encoding": { "hash": "sha256-3766bb620616", "other": "Jak zapisane są litery spoza ASCII. Gdy Znaki to ascii, nic to nie zmienia. unicode zapisuje każdą jako liczbę ze znakiem zapytania po niej dla czytnika, który nie zna liczb. oba dopisuje po liczbie literę w stronie kodowej jej fontu, dla japońskiego dwa bajty, tak jak zapisuje ją Word. strona kodowa zapisuje tylko bajty strony kodowej, a emoji dalej jako liczby, bo żadna strona kodowa ich nie ma. dosłowny zapisuje znak zapytania jako zwykły znak. LibreOffice, Tika i kontrolka RichEdit z Windows czytają wszystkie cztery. pandoc pokazuje tekst ze strony kodowej złymi literami. pandoc przed 3.13 gubi też znak po każdej literze zapisanej dosłownie, a pandoc 3.1 i 3.7 gubią emoji we wszystkich czterech. striprtf oddaje emoji jako dwie połówki, których Python nie zapisze w UTF-8." }, "Detail.format/svg.height": { "hash": "sha256-093ec883d633", "other": "Jaką wysokość deklaruje rysunek. Etykieta biegnie wzdłuż dolnej krawędzi, więc rysunek niższy od tego pasa nie ma widocznej etykiety." }, "Detail.format/svg.width": { "hash": "sha256-5dc327958f8d", "other": "Jaką szerokość deklaruje rysunek. Nic nie jest tu rysowane w pikselach, więc duża liczba kosztuje kilka bajtów w pliku i bardzo dużo pamięci w tym, co go otworzy." }, "Detail.format/targz.compression": { "hash": "sha256-82bb36e51fe2", "other": "Jak mocno ściskane są pliki w środku. Archiwum i tak ma zamówiony rozmiar - zmienia się to, ile z niego jest prawdziwą treścią, a ile dopełnieniem." }, diff --git a/internal/oracle/rtfscripts.go b/internal/oracle/rtfscripts.go index f955bd63..b85d7245 100644 --- a/internal/oracle/rtfscripts.go +++ b/internal/oracle/rtfscripts.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "regexp" + "strconv" "strings" ) @@ -14,11 +15,12 @@ import ( // our language. // // It is also the strictest reader in a way that is not a virtue - it shows -// codepage text in the wrong letters and drops a character after every letter -// written literal. Those are its readings of good files, which the guard of the -// format holds as such, so this checker asks only whether it read the file and -// said nothing about it. A warning on standard error counts as a complaint, -// the way it does for ffprobe. +// codepage text in the wrong letters, and before 3.13 it dropped a character +// after every letter written literal (jgm/pandoc#11981). Those are its +// readings of good files, which the guard of the format holds as such, by +// PandocVersion where they differ, so this checker asks only whether it read +// the file and said nothing about it. A warning on standard error counts as a +// complaint, the way it does for ffprobe. var pandocRTF = Checker{ Name: "pandoc", find: inPath("pandoc"), @@ -51,6 +53,44 @@ func PandocRTF(path string) (text, warnings string, res Result) { return out, strings.TrimSpace(errOut), res } +// pandocVersionLine is the line pandoc --version opens with, "pandoc 3.13", +// with the major and minor version in it. +var pandocVersionLine = regexp.MustCompile(`(?m)^pandoc(?:\.exe)? (\d+)\.(\d+)`) + +// PandocVersion is the major and minor version of the pandoc PandocRTF runs, +// for a guard whose reading of a good file changed between versions. ok is +// false when pandoc is missing, refuses --version or opens with something +// else. +func PandocVersion() (major, minor int, ok bool) { + c := pandocRTF + var out string + c.args = func(string) []string { return []string{"--version"} } + c.accept = func(stdout, _ string, code int) error { + out = stdout + if code != 0 { + return fmt.Errorf("pandoc --version ended with %d", code) + } + return nil + } + if res := c.Check(""); !res.Available || res.Err != nil { + return 0, 0, false + } + return ParsePandocVersion(out) +} + +// ParsePandocVersion reads the major and minor version out of what pandoc +// --version printed, "pandoc 3.13" or "pandoc.exe 3.1.3" on its first line. +// ok is false when the output opens with anything else. +func ParsePandocVersion(out string) (major, minor int, ok bool) { + m := pandocVersionLine.FindStringSubmatch(out) + if m == nil { + return 0, 0, false + } + major, errMajor := strconv.Atoi(m[1]) + minor, errMinor := strconv.Atoi(m[2]) + return major, minor, errMajor == nil && errMinor == nil +} + // pandocPicture is how pandoc's markdown points at a picture it wrote out. var pandocPicture = regexp.MustCompile(`!\[[^\]]*\]\(([^)]+)\)`)