From 7757618714889c81742de59a4794f5806e0e026c Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 10:43:57 +0000 Subject: [PATCH 01/14] fix(cli): default to native parsers The CLI used pandoc whenever a writer was on PATH, and that writer rewrote GFM tables and list markers. Native parsers are the default again. --use-pandoc or use_pandoc = true opts in, and markdown on that path is read and written as gfm. A render mismatch leaves the input unchanged and warns on stderr. --- README.md | 9 +-- docs/orgmode/howto/pandoc-backend.org | 34 ++++++----- docs/orgmode/reference/cli.org | 9 ++- docs/orgmode/reference/config.org | 18 ++++++ docs/orgmode/tutorials/quickstart.org | 9 +-- docs/source/cli-reference.md | 4 +- readme_src.org | 9 +-- src/cli.rs | 61 +++++++++++-------- src/config.rs | 13 ++++ src/init.rs | 4 ++ src/lib.rs | 51 ++++++++++++---- src/main.rs | 5 +- src/parser/pandoc/mod.rs | 3 +- src/watch.rs | 10 +++- tests/cli_integration.rs | 77 ++++++++++++++++++++++++ tests/pandoc_ast_backend.rs | 86 +++++++++++++++++---------- 16 files changed, 296 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index 0ca6ae20..6594ab2f 100644 --- a/README.md +++ b/README.md @@ -122,10 +122,11 @@ Pipe through stdin (for editor integration): cat draft.org | snapper --native --format org -The CLI uses pandoc (auto FFI, then CLI) when an FFI writer or `pandoc` on `PATH` is available. -Otherwise it keeps the native line parsers (no error). -Pass `--native` to force today's parsers. -Pass `--use-pandoc` to require pandoc (error if missing). +The CLI uses the native line parsers unless `--use-pandoc` is set or `.snapperrc.toml` sets `use_pandoc = true`. +`--native` is accepted and selects those same parsers. +`--use-pandoc` requires pandoc (error if missing). +Markdown on that path is read and written as `gfm`. +A render mismatch leaves the input unchanged and warns on stderr. Editors, wasm, and LSP stay native. Check formatting without modifying (for CI): diff --git a/docs/orgmode/howto/pandoc-backend.org b/docs/orgmode/howto/pandoc-backend.org index 62e45863..9b437508 100644 --- a/docs/orgmode/howto/pandoc-backend.org +++ b/docs/orgmode/howto/pandoc-backend.org @@ -33,10 +33,12 @@ Writer still needs =pandoc= on =PATH= when the library has no writer. **CLI backend.** Pandoc 2.x or 3.x on =PATH=. -The CLI default uses pandoc when an FFI writer or =pandoc= on =PATH= is available. -Otherwise it keeps the native line parsers (no error, no silent all-prose). +The CLI default is the native line parsers. +=--use-pandoc=, or =use_pandoc = true= in =.snapperrc.toml=, opts into pandoc. With =--use-pandoc=, a missing or failing pandoc is an explicit error (no silent all-prose). -Pass =--native= to force today's line parsers. +=--native= is accepted and selects the native parsers. +Markdown on the pandoc path is read and written as =gfm=. +A render mismatch leaves the input unchanged and warns on stderr. **Writer.** The library =libsnapper_pandoc= is reader-only. @@ -66,21 +68,21 @@ See =native/snapper-pandoc/README.md=. :END: #+begin_src bash -# Default: pandoc when FFI or pandoc on PATH is available +# Default: native line parsers snapper paper.md -snapper --pandoc-backend auto paper.md -snapper --pandoc-backend cli guide.adoc -snapper --pandoc-backend ffi paper.md -# Require pandoc (error if missing) +# Require pandoc (error if missing). Markdown is read and written as gfm. +snapper --use-pandoc paper.md +snapper --use-pandoc --pandoc-backend auto paper.md +snapper --use-pandoc --pandoc-backend cli guide.adoc +snapper --use-pandoc --pandoc-backend ffi paper.md snapper --use-pandoc paper.typ -# Force today's line parsers +# Accepted no-op: the default is already native snapper --native paper.md #+end_src -The CLI default uses pandoc (auto FFI, then CLI) when an in-process writer or =pandoc= on =PATH= is available. -Otherwise it keeps the native line parsers (no error, no silent all-prose). -Pass =--native= to force today's line parsers. -Pass =--use-pandoc= to require the pandoc path (explicit error if FFI and =pandoc= are missing). +The CLI default is the native line parsers. +=--use-pandoc=, or =use_pandoc = true= in =.snapperrc.toml=, requires the pandoc path (explicit error if FFI and =pandoc= are missing). +=--native= is accepted and selects those same native parsers. The writer still needs =pandoc= on =PATH= unless the library exports a writer. **Speed.** @@ -95,8 +97,8 @@ Disable with =SNAPPER_PANDOC_CACHE=0=; override dir with =SNAPPER_PANDOC_CACHE_D :CUSTOM_ID: when-to-use :END: -- **Pandoc** (CLI default when available): structure from pandoc’s model and coverage of formats without a built-in parser. -- **Built-in** (=--native=): fast, source-oriented parsers for Org / LaTeX / Markdown / RST (e.g. full ATX lines as structure). +- **Built-in** (CLI default): fast, source-oriented parsers for Org / LaTeX / Markdown / RST (e.g. full ATX lines as structure). +- **Pandoc** (=--use-pandoc= or =use_pandoc = true=): structure from pandoc’s model and coverage of formats without a built-in parser. The two paths are different pipelines, not two ways to fake the same source lines. @@ -110,4 +112,4 @@ The two paths are different pipelines, not two ways to fake the same source line - Other HTML / Org / LaTeX comments that are not snapper pragmas may still be dropped. - Round-trip is through the AST (not a perfect source-preserving rewrite of every markup character) - FFI library is optional; wasm/editor bundles do not embed GHC/pandoc -- CLI pandoc versus editor native is the contract: the CLI default may use pandoc; VS Code, Obsidian, Word, wasm, and LSP stay on native parsers. +- CLI pandoc versus editor native is the contract: the CLI default is native; =--use-pandoc= opts into pandoc; VS Code, Obsidian, Word, wasm, and LSP stay on native parsers. diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 56167744..5000f75b 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -186,16 +186,19 @@ Parse may use in-process FFI when =libsnapper_pandoc= loads; the writer still ne Without FFI, both parse and write use the =pandoc= CLI. A missing binary is an explicit error (never silent all-prose). Enables support for typst, asciidoc, docx, html, and all other pandoc-supported formats. -The default (omit this flag) already uses pandoc when an FFI writer or =pandoc= on =PATH= is available. +The default is the native line parsers. +This flag, or =use_pandoc = true= in =.snapperrc.toml=, opts into pandoc. +Markdown is read and written as =gfm=. +A render mismatch leaves the input unchanged and warns on stderr. *** =--native= :PROPERTIES: :CUSTOM_ID: opt-native :END: -Force today's native line parsers (markdown / org / rst / latex / plaintext). +Accepted for compatibility. +The default is already the native line parsers (markdown / org / rst / latex / plaintext). Conflicts with =--use-pandoc=. -Use this when you want source-oriented splice instead of the pandoc writer. Editors, wasm, and LSP stay native even without this flag. *** =--pandoc-backend = diff --git a/docs/orgmode/reference/config.org b/docs/orgmode/reference/config.org index fe5f1654..026578fc 100644 --- a/docs/orgmode/reference/config.org +++ b/docs/orgmode/reference/config.org @@ -37,6 +37,10 @@ max_width = 0 # that is already followed by whitespace. # clause_breaks = false +# Opt into the pandoc writer. The default is the native parsers. +# Markdown on this path is read and written as gfm. +# use_pandoc = false + # Per-format overrides [org] extra_abbreviations = ["PROPERTIES", "DEADLINE"] @@ -135,6 +139,20 @@ When true, snapper inserts a soft break after independent-clause punctuation (co Tokens such as =1,000=, =10:30=, URLs, =--flags=, and unspaced dashes never split. =--check= uses the same mode when this flag or config is on. +** use_pandoc +:PROPERTIES: +:CUSTOM_ID: field-use-pandoc +:END: + +Boolean. +Default: =false=. +Also selectable via =--use-pandoc= (and forced off by =--native=). + +When true, the CLI takes the pandoc path for every format. +Markdown is read and written as =gfm=. +A render mismatch leaves the file unchanged and warns on stderr. +Editors, wasm, and LSP stay on the native parsers. + ** long_threshold :PROPERTIES: :CUSTOM_ID: field-long-threshold diff --git a/docs/orgmode/tutorials/quickstart.org b/docs/orgmode/tutorials/quickstart.org index 875a56e9..3fbac9cd 100644 --- a/docs/orgmode/tutorials/quickstart.org +++ b/docs/orgmode/tutorials/quickstart.org @@ -79,10 +79,11 @@ Pipe through stdin (for editor integration): cat draft.org | snapper --format org #+end_src -The CLI uses pandoc (auto FFI, then CLI) when an FFI writer or =pandoc= on =PATH= is available. -Otherwise it keeps the native line parsers (no error). -Pass =--native= to force today's parsers. -Pass =--use-pandoc= to require pandoc (error if missing). +The CLI uses the native line parsers unless =--use-pandoc= is set or =.snapperrc.toml= sets =use_pandoc = true=. +=--native= is accepted and selects those same parsers. +=--use-pandoc= requires pandoc (error if missing). +Markdown on that path is read and written as =gfm=. +A render mismatch leaves the input unchanged and warns on stderr. Editors, wasm, and LSP stay native. ** What it does diff --git a/docs/source/cli-reference.md b/docs/source/cli-reference.md index ea73778d..9284c7ec 100644 --- a/docs/source/cli-reference.md +++ b/docs/source/cli-reference.md @@ -49,8 +49,8 @@ Semantic line break formatter * `--neural` — Use neural sentence detection (nnsplit LSTM model) * `--lang ` — Language for neural sentence detection (default: en). Available: en, de, fr, no, sv, zh, tr, ru, uk * `--model-path ` — Path to custom ONNX model file for neural detection -* `--use-pandoc` — Force the pandoc path (auto FFI then CLI). Errors if FFI and `pandoc` on PATH are both missing (no silent all-prose). The default (omit this flag) already uses pandoc when an FFI writer or `pandoc` on PATH is available. RST `..` comments and `snapper:off` / `snapper:on` are written through so the output still contains that text. Parse may use in-process FFI; the writer still needs `pandoc` on PATH (`libsnapper_pandoc` is reader-only) unless that library exports a writer -* `--native` — Force today's native line parsers (markdown/org/rst/latex/plaintext). The default (omit this flag) uses pandoc when an FFI writer or `pandoc` on PATH is available; otherwise it keeps these parsers (no error, no silent all-prose). Editors, wasm, and LSP stay native +* `--use-pandoc` — Force the pandoc path (auto FFI then CLI). Errors if FFI and `pandoc` on PATH are both missing (no silent all-prose). The default is the native line parsers. This flag, or `use_pandoc = true` in `.snapperrc.toml`, opts into pandoc. Markdown is read and written as `gfm`. A render mismatch leaves the input unchanged and warns on stderr. RST `..` comments and `snapper:off` / `snapper:on` are written through so the output still contains that text. Parse may use in-process FFI; the writer still needs `pandoc` on PATH (`libsnapper_pandoc` is reader-only) unless that library exports a writer +* `--native` — Accepted for compatibility. The default is already the native line parsers (markdown/org/rst/latex/plaintext). Editors, wasm, and LSP stay native. Conflicts with `--use-pandoc` * `--pandoc-backend ` — Pandoc AST source when the pandoc path is used: `auto` (prefer in-process FFI, else CLI), `ffi` (`libsnapper_pandoc`), or `cli` (`pandoc` subprocess). Default: `auto`. `ffi` fails explicitly if the library is missing. The writer still needs `pandoc` on PATH (no silent spawn surprise) Default value: `auto` diff --git a/readme_src.org b/readme_src.org index 51ac3603..ced9872b 100644 --- a/readme_src.org +++ b/readme_src.org @@ -87,10 +87,11 @@ Pipe through stdin (for editor integration): #+begin_src bash cat draft.org | snapper --native --format org #+end_src -The CLI uses pandoc (auto FFI, then CLI) when an FFI writer or =pandoc= on =PATH= is available. -Otherwise it keeps the native line parsers (no error). -Pass =--native= to force today's parsers. -Pass =--use-pandoc= to require pandoc (error if missing). +The CLI uses the native line parsers unless =--use-pandoc= is set or =.snapperrc.toml= sets =use_pandoc = true=. +=--native= is accepted and selects those same parsers. +=--use-pandoc= requires pandoc (error if missing). +Markdown on that path is read and written as =gfm=. +A render mismatch leaves the input unchanged and warns on stderr. Editors, wasm, and LSP stay native. Check formatting without modifying (for CI): #+begin_src bash diff --git a/src/cli.rs b/src/cli.rs index ffe94586..e61d42c6 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -95,8 +95,10 @@ pub struct Cli { /// Force the pandoc path (auto FFI then CLI). /// Errors if FFI and `pandoc` on PATH are both missing (no silent - /// all-prose). The default (omit this flag) already uses pandoc when - /// an FFI writer or `pandoc` on PATH is available. + /// all-prose). The default is the native line parsers. + /// This flag, or `use_pandoc = true` in `.snapperrc.toml`, opts into pandoc. + /// Markdown is read and written as `gfm`. + /// A render mismatch leaves the input unchanged and warns on stderr. /// RST `..` comments and `snapper:off` / `snapper:on` are written /// through so the output still contains that text. /// Parse may use in-process FFI; the writer still needs `pandoc` on PATH @@ -105,10 +107,11 @@ pub struct Cli { #[arg(long, conflicts_with = "native")] pub use_pandoc: bool, - /// Force today's native line parsers (markdown/org/rst/latex/plaintext). - /// The default (omit this flag) uses pandoc when an FFI writer or - /// `pandoc` on PATH is available; otherwise it keeps these parsers - /// (no error, no silent all-prose). Editors, wasm, and LSP stay native. + /// Accepted for compatibility. + /// The default is already the native line parsers + /// (markdown/org/rst/latex/plaintext). + /// Editors, wasm, and LSP stay native. + /// Conflicts with `--use-pandoc`. #[arg(long, conflicts_with = "use_pandoc")] pub native: bool, @@ -217,32 +220,32 @@ pub enum Commands { } impl Cli { - /// Whether this invocation should take the pandoc write path. + /// Whether the flags alone select the pandoc write path. /// - /// `--native` forces today's line parsers. `--use-pandoc` requires - /// pandoc (error if missing). With neither flag, use pandoc when an - /// FFI writer or `pandoc` on PATH is available, else native. + /// `--use-pandoc` is true. `--native` is accepted and stays false, + /// the same as omitting both flags. `.snapperrc.toml` `use_pandoc` + /// is applied by the caller when neither flag is set. pub fn resolve_use_pandoc(&self) -> bool { - if self.native { - false - } else if self.use_pandoc { - true - } else { - Self::default_use_pandoc() - } + self.use_pandoc } - /// CLI default (no `--native` / `--use-pandoc`): pandoc when the - /// write path can complete, else native. Library/wasm/LSP stay + /// CLI default (no `--native` / `--use-pandoc` and no config opt-in): + /// native parsers for every format. Library/wasm/LSP stay /// `FormatConfig::default().use_pandoc == false`. pub fn default_use_pandoc() -> bool { - #[cfg(feature = "pandoc")] - { - crate::parser::pandoc::pandoc_default_available() - } - #[cfg(not(feature = "pandoc"))] - { + false + } + + /// `--native` forces native parsers. `--use-pandoc` forces pandoc. + /// With neither flag, `configured` (`use_pandoc` in `.snapperrc.toml`) + /// opts in. Missing config stays native. + pub fn resolve_use_pandoc_config(&self, configured: Option) -> bool { + if self.native { false + } else if self.use_pandoc { + true + } else { + configured.unwrap_or(false) } } } @@ -316,7 +319,13 @@ mod tests { let cli = Cli::parse_from(["snapper"]); assert!(!cli.native); assert!(!cli.use_pandoc); - assert_eq!(cli.resolve_use_pandoc(), Cli::default_use_pandoc()); + assert!(!Cli::default_use_pandoc()); + assert!(!cli.resolve_use_pandoc()); + assert!(!cli.resolve_use_pandoc_config(None)); + assert!(!cli.resolve_use_pandoc_config(Some(false))); + assert!(cli.resolve_use_pandoc_config(Some(true))); + let native = Cli::parse_from(["snapper", "--native"]); + assert!(!native.resolve_use_pandoc_config(Some(true))); } #[test] diff --git a/src/config.rs b/src/config.rs index b47baacf..61418d25 100644 --- a/src/config.rs +++ b/src/config.rs @@ -119,6 +119,9 @@ pub struct ProjectConfig { /// `max_width = 0` this still inserts a newline after each mark that /// is already followed by whitespace. pub clause_breaks: Option, + /// When true, the CLI takes the pandoc path without `--use-pandoc`. + /// `--native` still forces the native parsers. Default false. + pub use_pandoc: Option, /// Default language for abbreviation sets. pub lang: Option, @@ -286,6 +289,16 @@ lang = "de" assert_eq!(config.long_threshold, Some(100)); } + #[test] + fn parse_use_pandoc() { + let on = ProjectConfig::parse("use_pandoc = true\n").unwrap(); + assert_eq!(on.use_pandoc, Some(true)); + let off = ProjectConfig::parse("use_pandoc = false\n").unwrap(); + assert_eq!(off.use_pandoc, Some(false)); + let missing = ProjectConfig::parse("").unwrap(); + assert_eq!(missing.use_pandoc, None); + } + #[test] fn parse_comments_and_blanks() { let toml = "# comment\n\nextra_abbreviations = [\"Fig\"]\n"; diff --git a/src/init.rs b/src/init.rs index b9af60d4..2defc483 100644 --- a/src/init.rs +++ b/src/init.rs @@ -68,6 +68,10 @@ max_width = 0 # that is already followed by whitespace. # clause_breaks = false +# Opt into the pandoc writer. The default is the native parsers. +# Markdown on this path is read and written as gfm. +# use_pandoc = false + # Extra LaTeX environments / commands (added to the built-in lists). # [latex] # verbatim_envs = ["Verbatim"] diff --git a/src/lib.rs b/src/lib.rs index aed7acd4..cf13afde 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -303,8 +303,26 @@ pub fn format_text_with_splitter( // The pandoc writer is not a native splice: it normalizes markup, so // the native region-tree oracle and byte fixpoint would veto a correct // write-through and return the original (no sentence breaks). + // Markdown opted in via `--use-pandoc` is the exception: it is read and + // written as gfm, and a render mismatch returns the input unchanged. let candidate = if config.use_pandoc { - format_once(work_input, config, splitter, config.format_code)? + let written = format_once(work_input, config, splitter, config.format_code)?; + if pandoc_markdown_oracle(config) + && config.render_backstop + && written != work_input + && !oracle::matches_ex( + config.format, + work_input, + &written, + config.format_code, + Some(config), + ) + { + eprintln!("snapper: pandoc markdown render check failed; left input unchanged"); + work_input.to_string() + } else { + written + } } else { let once = format_once(work_input, config, splitter, config.format_code)?; // Later passes prove prose stability. External code formatters @@ -353,6 +371,26 @@ pub fn format_text_with_splitter( Ok(output) } +/// Pandoc reader/writer name for this config. +/// Markdown defaults to `gfm` so `--use-pandoc` does not emit pandoc tables. +fn pandoc_input_format(config: &FormatConfig) -> &str { + if let Some(fmt) = config.pandoc_format.as_deref() { + return fmt; + } + match config.format { + Format::Org => "org", + Format::Latex => "latex", + Format::Markdown => "gfm", + Format::Rst => "rst", + Format::Plaintext => "markdown", + } +} + +/// True when a pandoc markdown write should be checked with the render oracle. +fn pandoc_markdown_oracle(config: &FormatConfig) -> bool { + config.format == Format::Markdown && pandoc_input_format(config) == "gfm" +} + /// One parse+reflow pass. Native parsers splice into original bytes; /// pandoc reflows the AST and writes through pandoc's writer. fn format_once( @@ -378,16 +416,7 @@ fn format_once( if config.use_pandoc { #[cfg(feature = "pandoc")] { - let pandoc_fmt = config - .pandoc_format - .as_deref() - .unwrap_or(match config.format { - Format::Org => "org", - Format::Latex => "latex", - Format::Markdown => "markdown", - Format::Rst => "rst", - Format::Plaintext => "markdown", - }); + let pandoc_fmt = pandoc_input_format(config); return parser::pandoc::format_via_pandoc( work_input, pandoc_fmt, diff --git a/src/main.rs b/src/main.rs index a70b5f64..8bdf1b35 100644 --- a/src/main.rs +++ b/src/main.rs @@ -108,7 +108,8 @@ fn run() -> Result<()> { patterns, fmt, cli.config.as_deref(), - cli.resolve_use_pandoc(), + cli.native, + cli.use_pandoc, ); } } @@ -409,7 +410,7 @@ fn build_format_config( latex_verbatim_envs: project_config.latex_verbatim_envs(), latex_structure_envs: project_config.latex_structure_envs(), latex_verbatim_commands: project_config.latex_verbatim_commands(), - use_pandoc: cli.resolve_use_pandoc(), + use_pandoc: cli.resolve_use_pandoc_config(project_config.use_pandoc), #[cfg(feature = "pandoc")] pandoc_backend: cli .pandoc_backend diff --git a/src/parser/pandoc/mod.rs b/src/parser/pandoc/mod.rs index ff3817b0..37f825fa 100644 --- a/src/parser/pandoc/mod.rs +++ b/src/parser/pandoc/mod.rs @@ -112,10 +112,11 @@ pub enum PandocError { DropsComments(String), } -/// Whether the CLI default can run the pandoc path without erroring. +/// Whether a pandoc write can finish without erroring. /// /// True when an in-process FFI writer is loaded or `pandoc` is on PATH. /// Reader-only FFI without PATH is not enough (write would fail). +/// The CLI default does not use this; `--use-pandoc` or `use_pandoc` does. pub fn pandoc_default_available() -> bool { ffi_write_available() || pandoc_available() } diff --git a/src/watch.rs b/src/watch.rs index b40147cb..ece7368a 100644 --- a/src/watch.rs +++ b/src/watch.rs @@ -14,9 +14,17 @@ pub fn run_watch( patterns: &[String], format_override: Option, config_path: Option<&std::path::Path>, - use_pandoc: bool, + native: bool, + use_pandoc_flag: bool, ) -> Result<()> { let project_config = ProjectConfig::resolve(config_path).unwrap_or_default(); + let use_pandoc = if native { + false + } else if use_pandoc_flag { + true + } else { + project_config.use_pandoc.unwrap_or(false) + }; // Resolve patterns to actual file paths let files = resolve_patterns(patterns)?; diff --git a/tests/cli_integration.rs b/tests/cli_integration.rs index 6e2faa91..47f8f407 100644 --- a/tests/cli_integration.rs +++ b/tests/cli_integration.rs @@ -473,6 +473,83 @@ fn idempotent_edge_cases_plaintext() { ); } +#[test] +fn default_cli_keeps_gfm_table_list_and_splits_prose() { + let dir = tempfile::tempdir().unwrap(); + let input = "Intro. Second sentence.\n\n| Col | Value |\n|---|---|\n| a | 1. Next |\n\n- item one\n- item two\n"; + let expected = "Intro.\nSecond sentence.\n\n| Col | Value |\n|---|---|\n| a | 1. Next |\n\n- item one\n- item two\n"; + let mut child = snapper_binary(); + child + .args(["--format", "markdown"]) + .current_dir(dir.path()) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + let mut spawned = child.spawn().expect("spawn snapper"); + { + use std::io::Write; + spawned + .stdin + .take() + .unwrap() + .write_all(input.as_bytes()) + .unwrap(); + } + let out = spawned.wait_with_output().expect("wait"); + assert!( + out.status.success(), + "default must stay native; stderr={}", + String::from_utf8_lossy(&out.stderr) + ); + let stdout = String::from_utf8(out.stdout).unwrap(); + assert_eq!(stdout, expected); + assert!( + String::from_utf8_lossy(&out.stderr).is_empty(), + "native default must not warn" + ); +} + +#[test] +fn snapperrc_use_pandoc_matches_flag() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join(".snapperrc.toml"), "use_pandoc = true\n").unwrap(); + let input = "Hello world. Second sentence.\n"; + let run = |args: &[&str]| { + let mut child = snapper_binary(); + child + .args(args) + .current_dir(dir.path()) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + let mut spawned = child.spawn().expect("spawn snapper"); + { + use std::io::Write; + spawned + .stdin + .take() + .unwrap() + .write_all(input.as_bytes()) + .unwrap(); + } + spawned.wait_with_output().expect("wait") + }; + let from_config = run(&["--format", "markdown"]); + let from_flag = run(&["--use-pandoc", "--format", "markdown"]); + assert_eq!(from_config.status.success(), from_flag.status.success()); + if from_flag.status.success() { + assert_eq!( + String::from_utf8_lossy(&from_config.stdout), + String::from_utf8_lossy(&from_flag.stdout) + ); + let stdout = String::from_utf8_lossy(&from_config.stdout); + assert!( + stdout.contains("Hello world.\n") && stdout.contains("Second sentence"), + "config use_pandoc must split prose:\n{stdout}" + ); + } +} + // ---- FEATURE 7: snapper init ---- #[test] diff --git a/tests/pandoc_ast_backend.rs b/tests/pandoc_ast_backend.rs index 98efe667..ddc6419f 100644 --- a/tests/pandoc_ast_backend.rs +++ b/tests/pandoc_ast_backend.rs @@ -539,33 +539,62 @@ fn pandoc_org_verbatim_inner_equals_stays_one_span() { ); } -/// `snapper --use-pandoc FILE` must exit 0 and emit sentence breaks. +/// `snapper --use-pandoc` exits 0. +/// Prose that round-trips through gfm is sentence-split. +/// A file whose gfm write changes structure (tables, fences) is left unchanged +/// and warned, because that write fails the render oracle. #[test] fn snapper_cli_use_pandoc_exits_0() { if !require_writer() { return; } let bin = env!("CARGO_BIN_EXE_snapper"); - let input = fixture("mixed.md"); - let out = std::process::Command::new(bin) - .args(["--use-pandoc", "--format", "markdown"]) - .arg(&input) + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join("prose.md"), "Hello world. Second sentence.\n").unwrap(); + let prose = std::process::Command::new(bin) + .args(["--use-pandoc", "--format", "markdown", "prose.md"]) + .current_dir(dir.path()) .output() .expect("spawn snapper"); assert!( - out.status.success(), + prose.status.success(), "snapper --use-pandoc must exit 0; stderr={} stdout={}", - String::from_utf8_lossy(&out.stderr), - String::from_utf8_lossy(&out.stdout) + String::from_utf8_lossy(&prose.stderr), + String::from_utf8_lossy(&prose.stdout) ); - let stdout = String::from_utf8_lossy(&out.stdout); + let stdout = String::from_utf8_lossy(&prose.stdout); assert!( stdout.contains("Hello world.\n") && stdout.contains("Second sentence"), "CLI write must break sentences:\n{stdout}" ); + + let mixed = std::process::Command::new(bin) + .args(["--use-pandoc", "--format", "markdown"]) + .arg(fixture("mixed.md")) + .output() + .expect("spawn snapper mixed"); assert!( - stdout.contains("```") || stdout.contains("print(1)"), - "CLI write keeps CodeBlock:\n{stdout}" + mixed.status.success(), + "mixed.md --use-pandoc must exit 0; stderr={}", + String::from_utf8_lossy(&mixed.stderr) + ); + let mixed_out = String::from_utf8_lossy(&mixed.stdout); + let mixed_in = read_fixture("mixed.md"); + let err = String::from_utf8_lossy(&mixed.stderr); + if mixed_out == mixed_in { + assert!( + err.contains("render check failed"), + "unchanged gfm write must warn: {err}" + ); + } else { + assert!( + mixed_out.contains("Hello world.\n") && mixed_out.contains("Second sentence"), + "accepted gfm write must break sentences:\n{mixed_out}" + ); + } + assert!( + mixed_out.contains("```") || mixed_out.contains("print(1)"), + "CLI write keeps CodeBlock:\n{mixed_out}" ); } @@ -913,7 +942,7 @@ fn snapper_ffi_without_pandoc_on_path_is_inprocess_or_explicit() { ); } -/// CLI default (no flag) uses pandoc when a writer runtime exists. +/// CLI default (no flag) stays on the native parsers even when pandoc can write. #[test] fn snapper_cli_default_uses_pandoc_when_available() { if !pandoc_default_available() { @@ -927,26 +956,26 @@ fn snapper_cli_default_uses_pandoc_when_available() { .arg(&input) .output() .expect("spawn default"); - let forced = std::process::Command::new(bin) - .args(["--use-pandoc", "--format", "markdown"]) + let native = std::process::Command::new(bin) + .args(["--native", "--format", "markdown"]) .arg(&input) .output() - .expect("spawn --use-pandoc"); + .expect("spawn --native"); assert!( default_out.status.success(), - "default must use pandoc without error; stderr={} stdout={}", + "default must stay native without error; stderr={} stdout={}", String::from_utf8_lossy(&default_out.stderr), String::from_utf8_lossy(&default_out.stdout) ); assert!( - forced.status.success(), - "--use-pandoc must still work; stderr={}", - String::from_utf8_lossy(&forced.stderr) + native.status.success(), + "--native must still work; stderr={}", + String::from_utf8_lossy(&native.stderr) ); assert_eq!( String::from_utf8_lossy(&default_out.stdout), - String::from_utf8_lossy(&forced.stdout), - "default CLI output must match --use-pandoc when runtime exists" + String::from_utf8_lossy(&native.stdout), + "default CLI output must match --native even when pandoc can write" ); let stdout = String::from_utf8_lossy(&default_out.stdout); assert_has_sentence_break(&stdout, "Hello world.", "Second sentence"); @@ -990,14 +1019,6 @@ fn snapper_cli_default_is_native_when_pandoc_missing() { .env_remove("SNAPPER_PANDOC_LIB_DIR") .output() .expect("spawn default without pandoc"); - if ffi_write_available() { - assert!( - out.status.success(), - "FFI writer default must still succeed; stderr={}", - String::from_utf8_lossy(&out.stderr) - ); - return; - } assert!( out.status.success(), "default without pandoc must stay native, not error; stderr={} stdout={}", @@ -1007,7 +1028,7 @@ fn snapper_cli_default_is_native_when_pandoc_missing() { let stdout = String::from_utf8_lossy(&out.stdout); assert!( stdout.contains("### 1. `cargo binstall` (preferred binary install)"), - "missing pandoc must keep native ATX line, not all-prose:\n{stdout}" + "default must keep native ATX line, not all-prose:\n{stdout}" ); } @@ -1063,8 +1084,9 @@ fn snapper_help_names_native_and_default_pandoc() { "help must describe the native flag:\n{text}" ); assert!( - text.contains("when") && (text.contains("available") || text.contains("PATH")), - "help must say the default uses pandoc when available:\n{text}" + text.contains("The default is the native line parsers") + || text.contains("The default is already the native line parsers"), + "help must say the default is the native parsers:\n{text}" ); } From 87fa6a2f5242e41a4108768fb44810c3478c267c Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 10:46:24 +0000 Subject: [PATCH 02/14] feat(config): expose lowercase sentence starts UnicodeSentenceSplitter already opened a sentence at a lowercase word, but nothing turned that on. FormatConfig, the CLI, .snapperrc.toml, LSP, and MCP can now set lowercase_starts. Abbreviations such as e.g. and vs. still join. --- README.md | 4 ++ docs/orgmode/reference/cli.org | 10 +++++ docs/orgmode/reference/config.org | 19 ++++++++++ docs/source/cli-reference.md | 1 + readme_src.org | 5 +++ src/cli.rs | 8 ++++ src/config.rs | 38 +++++++++++++++++++ src/init.rs | 4 ++ src/lib.rs | 62 ++++++++++++++++++++++++++++++- src/lsp.rs | 1 + src/main.rs | 5 +++ src/mcp.rs | 30 ++++++++++++++- src/sentence/neural.rs | 6 +++ src/watch.rs | 1 + tests/cli_integration.rs | 27 ++++++++++++++ tests/pandoc_ast_backend.rs | 6 ++- 16 files changed, 223 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6594ab2f..c4b89d05 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,10 @@ With the default unlimited width this inserts a newline after every such mark th With `--max-width` set, overflowing sentences prefer those marks. A one-clause sentence stays one line. +Split before a lowercase word after a terminator (`pass. pytest` becomes two lines). +Abbreviations such as `e.g.` and `vs.` still join: + + snapper --lowercase-starts paper.md Preview changes as a unified diff before committing: snapper --diff paper.org diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 5000f75b..a9e83e40 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -89,6 +89,16 @@ With =--max-width= set, overflowing sentences prefer those marks; a sentence tha Tokens such as =1,000=, =10:30=, URLs, =--flags=, and unspaced dashes never split. =--check= uses the same mode when the flag or =clause_breaks= config is on. +*** =--lowercase-starts= +:PROPERTIES: +:CUSTOM_ID: opt-lowercase-starts +:END: + +Open a sentence at a lowercase word after a terminator and a space. +Abbreviations such as =e.g.= and =vs.= still join. +Off by default (UAX #29 SB8). +Also set with =lowercase_starts= in =.snapperrc.toml= (top level, or per format). + *** =--format-code= :PROPERTIES: :CUSTOM_ID: opt-format-code diff --git a/docs/orgmode/reference/config.org b/docs/orgmode/reference/config.org index 026578fc..791d33ff 100644 --- a/docs/orgmode/reference/config.org +++ b/docs/orgmode/reference/config.org @@ -41,6 +41,10 @@ max_width = 0 # Markdown on this path is read and written as gfm. # use_pandoc = false +# Open a sentence at a lowercase word after a terminator (default false). +# Abbreviations such as e.g. and vs. still join. +# lowercase_starts = false + # Per-format overrides [org] extra_abbreviations = ["PROPERTIES", "DEADLINE"] @@ -139,6 +143,20 @@ When true, snapper inserts a soft break after independent-clause punctuation (co Tokens such as =1,000=, =10:30=, URLs, =--flags=, and unspaced dashes never split. =--check= uses the same mode when this flag or config is on. +** lowercase_starts +:PROPERTIES: +:CUSTOM_ID: field-lowercase-starts +:END: + +Boolean. +Default: =false=. +Also selectable via =--lowercase-starts=. + +When true, a lowercase word after a terminator and a space opens a sentence. +Abbreviations such as =e.g.= and =vs.= still join. +A per-format =lowercase_starts= overrides this top-level value. +Editors and the LSP read the same key. + ** use_pandoc :PROPERTIES: :CUSTOM_ID: field-use-pandoc @@ -172,6 +190,7 @@ Add =[org]=, =[latex]=, =[markdown]=, =[rst]=, or =[plaintext]= sections to over Per-format =extra_abbreviations= merge with the top-level list. Per-format =max_width= overrides the top-level value. +Per-format =lowercase_starts= overrides the top-level value. The following keys are meaningful under =[latex]= only. Missing keys (or empty arrays) keep the built-in lists; present entries are added to those lists. diff --git a/docs/source/cli-reference.md b/docs/source/cli-reference.md index 9284c7ec..933bb39f 100644 --- a/docs/source/cli-reference.md +++ b/docs/source/cli-reference.md @@ -44,6 +44,7 @@ Semantic line break formatter Default value: `0` * `--clause-breaks` — Prefer soft breaks after independent-clause punctuation (comma, semicolon, colon, em dash). With `--max-width 0` (the default), insert a newline after every such mark that is already followed by whitespace. With `--max-width` set, prefer those marks when wrapping an overflowing sentence +* `--lowercase-starts` — Open a sentence at a lowercase word after a terminator and a space. Abbreviations such as `e.g.` and `vs.` still join. Off by default (UAX #29 SB8). Also set with `lowercase_starts` in `.snapperrc.toml` (top level, or per format) Default value: `false` * `--neural` — Use neural sentence detection (nnsplit LSTM model) diff --git a/readme_src.org b/readme_src.org index ced9872b..e39d873b 100644 --- a/readme_src.org +++ b/readme_src.org @@ -108,6 +108,11 @@ snapper --clause-breaks paper.org #+end_src With =--max-width= set, overflowing sentences prefer those marks. A one-clause sentence stays one line. +Split before a lowercase word after a terminator (=pass. pytest= becomes two lines). +Abbreviations such as =e.g.= and =vs.= still join. +#+begin_src bash +snapper --lowercase-starts paper.md +#+end_src Preview changes as a unified diff before committing: #+begin_src bash snapper --diff paper.org diff --git a/src/cli.rs b/src/cli.rs index e61d42c6..29fa76d4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -80,6 +80,14 @@ pub struct Cli { #[arg(long, default_value_t = false)] pub clause_breaks: bool, + /// Open a sentence at a lowercase word after a terminator and a space. + /// Abbreviations such as `e.g.` and `vs.` still join. + /// Off by default (UAX #29 SB8). + /// Also set with `lowercase_starts` in `.snapperrc.toml` + /// (top level, or per format). + #[arg(long, default_value_t = false)] + pub lowercase_starts: bool, + /// Use neural sentence detection (nnsplit LSTM model). #[arg(long)] pub neural: bool, diff --git a/src/config.rs b/src/config.rs index 61418d25..8e3fec5a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -53,6 +53,9 @@ pub struct FormatOverrides { /// Meaningful under `[latex]` only. Missing or empty keeps /// verb/lstinline/lstinputlisting/spverb/mintinline/inputminted/mint/Verb/SaveVerb/VerbatimInput/piton/verbatiminput/PitonInputFile/PitonInputFileT/PitonInputFileF/PitonInputFileTF/tcbinputlisting/tcboxverb/inputpy/inputpycon/inputpylab/inputpylabcon/inputsympy/inputsympycon/py/pyc/pys/pyb/pyv/pycon and twins/sympy/pylab and twins/usefamily ruby/rb/julia/jl/matlab/octave/bash/sage/rust/rs/R/perl/pl/perlsix/psix/javascript/js and twins/pyq/pycq/pyfile/pyfileq/pyfilerepl/inputpygments/pygment/pythontexcustomc/inputpython/inputpythonfile/CatchFileBetweenTags/CatchFileBetweenDelims/ExecuteMetaData/CatchFileDef/CatchFileEdef/listinginput/verbatimtabinput/verbatimwrite/listingcont/sageinput/sageplot/sagestr/pyth/Scontents/typestored/getstored/mergesc/meaningsc/foreachsc/setupsc/countsc/cleanseqsc/UseVerb/UseVerbatim/LUseVerbatim/BUseVerbatim/DefineShortVerb/UndefineShortVerb/EscVerb/inputsc/VerbatimInsertBuffer/VerbatimClearBuffer/InsertBuffer/IterateBuffer/ClearBuffer/WriteBuffer/lstset/lstdefinestyle/lstMakeShortInline/lstDeleteShortInline/lstnewenvironment/verbinput/verbwrite. pub verbatim_commands: Vec, + /// Open a sentence at a lowercase word after a terminator. + /// Overrides the top-level `lowercase_starts` for this format. + pub lowercase_starts: Option, } /// Per-language entry under the `[code]` table. @@ -119,6 +122,9 @@ pub struct ProjectConfig { /// `max_width = 0` this still inserts a newline after each mark that /// is already followed by whitespace. pub clause_breaks: Option, + /// Open a sentence at a lowercase word after a terminator and a space. + /// Per-format sections override this. Default false. + pub lowercase_starts: Option, /// When true, the CLI takes the pandoc path without `--use-pandoc`. /// `--native` still forces the native parsers. Default false. pub use_pandoc: Option, @@ -208,6 +214,24 @@ impl ProjectConfig { overrides.and_then(|ov| ov.max_width).or(self.max_width) } + /// Whether a lowercase word after a terminator opens a sentence. + /// Per-format `lowercase_starts` overrides the top-level key. + /// Missing keys stay false. + pub fn lowercase_starts_for_format(&self, format: &str) -> bool { + let overrides = match format { + "org" => self.org.as_ref(), + "latex" => self.latex.as_ref(), + "markdown" => self.markdown.as_ref(), + "rst" => self.rst.as_ref(), + "plaintext" => self.plaintext.as_ref(), + _ => None, + }; + overrides + .and_then(|ov| ov.lowercase_starts) + .or(self.lowercase_starts) + .unwrap_or(false) + } + /// Extra `[latex].verbatim_envs` names. Empty when the key is missing. pub fn latex_verbatim_envs(&self) -> Vec { self.latex @@ -289,6 +313,20 @@ lang = "de" assert_eq!(config.long_threshold, Some(100)); } + #[test] + fn parse_lowercase_starts() { + let top = ProjectConfig::parse("lowercase_starts = true\n").unwrap(); + assert!(top.lowercase_starts_for_format("markdown")); + assert!(top.lowercase_starts_for_format("org")); + let over = + ProjectConfig::parse("lowercase_starts = true\n[markdown]\nlowercase_starts = false\n") + .unwrap(); + assert!(!over.lowercase_starts_for_format("markdown")); + assert!(over.lowercase_starts_for_format("org")); + let missing = ProjectConfig::parse("").unwrap(); + assert!(!missing.lowercase_starts_for_format("plaintext")); + } + #[test] fn parse_use_pandoc() { let on = ProjectConfig::parse("use_pandoc = true\n").unwrap(); diff --git a/src/init.rs b/src/init.rs index 2defc483..b353c7a9 100644 --- a/src/init.rs +++ b/src/init.rs @@ -72,6 +72,10 @@ max_width = 0 # Markdown on this path is read and written as gfm. # use_pandoc = false +# Open a sentence at a lowercase word after a terminator (default false). +# Abbreviations such as e.g. and vs. still join. +# lowercase_starts = false + # Extra LaTeX environments / commands (added to the built-in lists). # [latex] # verbatim_envs = ["Verbatim"] diff --git a/src/lib.rs b/src/lib.rs index cf13afde..a9c3830d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -106,6 +106,9 @@ pub struct FormatConfig { /// Default `false` keeps one sentence per line (greedy wrap only /// under `max_width`). pub clause_breaks: bool, + /// Open a sentence at a lowercase word after a terminator and a space. + /// Abbreviations such as `e.g.` still join. Default `false` keeps UAX #29 SB8. + pub lowercase_starts: bool, /// Run `format_text` to a byte fixpoint (cap 4). Production default /// `true`; tests set `false` so a planner that needs the backstop fails. pub fixpoint_backstop: bool, @@ -169,6 +172,7 @@ impl Default for FormatConfig { code: HashMap::new(), format_code: false, clause_breaks: false, + lowercase_starts: false, fixpoint_backstop: true, render_backstop: true, latex_verbatim_envs: vec![], @@ -253,7 +257,8 @@ pub fn build_splitter(config: &FormatConfig) -> Result Ok(Box::new( neural .map_err(|e| anyhow::anyhow!("{e}"))? - .with_verbatim_commands(config.latex_verbatim_commands.clone()), + .with_verbatim_commands(config.latex_verbatim_commands.clone()) + .with_lowercase_starts(config.lowercase_starts), )) } #[cfg(not(feature = "neural"))] @@ -265,7 +270,8 @@ pub fn build_splitter(config: &FormatConfig) -> Result } else { Ok(Box::new( UnicodeSentenceSplitter::for_lang(&config.neural_lang, &config.extra_abbreviations) - .with_verbatim_commands(config.latex_verbatim_commands.clone()), + .with_verbatim_commands(config.latex_verbatim_commands.clone()) + .with_lowercase_starts(config.lowercase_starts), )) } } @@ -499,3 +505,55 @@ pub fn format_range( Ok(result) } + +#[cfg(test)] +mod lowercase_starts_tests { + use super::*; + use crate::format::Format; + + fn cfg(on: bool) -> FormatConfig { + FormatConfig { + format: Format::Markdown, + lowercase_starts: on, + ..Default::default() + } + } + + fn lines(on: bool, input: &str) -> Vec { + format_text(input, &cfg(on)) + .unwrap() + .lines() + .map(str::to_string) + .collect() + } + + #[test] + fn lowercase_starts_split_the_documented_cases() { + let tool = "There are two. setup-python moved to PyPy 8.0.0."; + let meson = "`meson test`: 58 of 58 pass. pytest: 661 pass and 31 skip."; + let abbrev = "Use e.g. this, i.e. that, vs. the other, etc. and more. Dr. Smith agreed."; + assert_eq!(lines(false, tool), vec![tool]); + assert_eq!(lines(false, meson), vec![meson]); + assert_eq!( + lines(true, tool), + vec![ + "There are two.".to_string(), + "setup-python moved to PyPy 8.0.0.".to_string() + ] + ); + assert_eq!( + lines(true, meson), + vec![ + "`meson test`: 58 of 58 pass.".to_string(), + "pytest: 661 pass and 31 skip.".to_string() + ] + ); + assert_eq!( + lines(true, abbrev), + vec![ + "Use e.g. this, i.e. that, vs. the other, etc. and more.".to_string(), + "Dr. Smith agreed.".to_string() + ] + ); + } +} diff --git a/src/lsp.rs b/src/lsp.rs index d4245435..8803098c 100644 --- a/src/lsp.rs +++ b/src/lsp.rs @@ -55,6 +55,7 @@ impl SnapperLsp { latex_structure_envs: project.latex_structure_envs(), latex_verbatim_commands: project.latex_verbatim_commands(), clause_breaks: project.clause_breaks.unwrap_or(false), + lowercase_starts: project.lowercase_starts_for_format(format_str), // VS Code / Helix LSP stay native even if the CLI default flips. use_pandoc: false, ..Default::default() diff --git a/src/main.rs b/src/main.rs index 8bdf1b35..e612ebb8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -334,6 +334,7 @@ struct SplitterKey { neural_lang: String, neural_model: Option, extras: Vec, + lowercase_starts: bool, } impl SplitterKey { @@ -344,6 +345,7 @@ impl SplitterKey { neural_lang: config.neural_lang.clone(), neural_model: config.neural_model_path.clone(), extras: config.extra_abbreviations.clone(), + lowercase_starts: config.lowercase_starts, } } } @@ -399,6 +401,8 @@ fn build_format_config( // CLI --clause-breaks wins when set; otherwise project config (default false). let clause_breaks = cli.clause_breaks || project_config.clause_breaks.unwrap_or(false); + let lowercase_starts = + cli.lowercase_starts || project_config.lowercase_starts_for_format(format_key); FormatConfig { format, @@ -419,6 +423,7 @@ fn build_format_config( code: project_config.code.clone(), format_code: cli.format_code, clause_breaks, + lowercase_starts, ..Default::default() } } diff --git a/src/mcp.rs b/src/mcp.rs index f4556b4d..2eb0997f 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -41,6 +41,10 @@ pub struct FormatTextParams { /// `max_width` greater than 0 is wrap-prefer. #[serde(default)] pub clause_breaks: bool, + /// Open a sentence at a lowercase word after a terminator + /// (same as CLI `--lowercase-starts`). + #[serde(default)] + pub lowercase_starts: bool, /// Optional 1-indexed inclusive line range. Same meaning as CLI `--range`. #[serde(default)] pub range: Option, @@ -68,6 +72,10 @@ pub struct CheckFormattingParams { /// `max_width` greater than 0 is wrap-prefer. #[serde(default)] pub clause_breaks: bool, + /// Open a sentence at a lowercase word after a terminator + /// (same as CLI `--lowercase-starts`; default false). + #[serde(default)] + pub lowercase_starts: bool, } #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -89,12 +97,14 @@ fn make_config( max_width: usize, extra_abbreviations: Vec, clause_breaks: bool, + lowercase_starts: bool, ) -> FormatConfig { FormatConfig { format, max_width, extra_abbreviations, clause_breaks, + lowercase_starts, ..Default::default() } } @@ -177,6 +187,7 @@ impl SnapperMcpServer { params.max_width, params.extra_abbreviations, params.clause_breaks, + params.lowercase_starts, ); let result = if let Some(range) = params.range { crate::format_range(¶ms.text, &config, range.start, range.end) @@ -215,7 +226,13 @@ impl SnapperMcpServer { Parameters(params): Parameters, ) -> Json { let format = parse_format(¶ms.format); - let config = make_config(format, params.max_width, vec![], params.clause_breaks); + let config = make_config( + format, + params.max_width, + vec![], + params.clause_breaks, + params.lowercase_starts, + ); let splitter = crate::build_splitter(&config).unwrap(); let would = would_reformat(¶ms.text, &config).unwrap_or(true); let threshold = resolve_long_threshold(params.max_width, None); @@ -350,6 +367,7 @@ mod tests { max_width: 0, extra_abbreviations: vec![], clause_breaks: false, + lowercase_starts: false, range: None, } } @@ -366,6 +384,7 @@ mod tests { format: "plaintext".to_string(), max_width: 0, clause_breaks, + lowercase_starts: false, })) .0 } @@ -459,6 +478,15 @@ mod tests { assert_eq!(params.max_width, 80); } + #[test] + fn format_text_lowercase_starts_splits_a_tool_name() { + let mut params = plaintext("There are two. setup-python moved to PyPy 8.0.0."); + params.format = "markdown".to_string(); + params.lowercase_starts = true; + let out = format(params); + assert_eq!(out, "There are two.\nsetup-python moved to PyPy 8.0.0."); + } + #[test] fn format_text_clause_breaks_wraps_after_commas() { let sentence = "It contains rules which govern how the Objectives are orchestrated, along with rules which can automatically activate the Objectives in the plan, without additional human intervention."; diff --git a/src/sentence/neural.rs b/src/sentence/neural.rs index 638dfa11..14db73df 100644 --- a/src/sentence/neural.rs +++ b/src/sentence/neural.rs @@ -52,6 +52,12 @@ impl NeuralSentenceSplitter { self } + /// Open a sentence at a lowercase word after a terminator and a space. + pub fn with_lowercase_starts(mut self, on: bool) -> Self { + self.post = self.post.with_lowercase_starts(on); + self + } + /// Load from a custom ONNX model file path. pub fn from_path(path: &std::path::Path) -> Result> { Self::from_path_with_extras(path, "en", &[]) diff --git a/src/watch.rs b/src/watch.rs index ece7368a..7905ce40 100644 --- a/src/watch.rs +++ b/src/watch.rs @@ -142,6 +142,7 @@ fn format_file_in_place( latex_structure_envs: project_config.latex_structure_envs(), latex_verbatim_commands: project_config.latex_verbatim_commands(), clause_breaks: project_config.clause_breaks.unwrap_or(false), + lowercase_starts: project_config.lowercase_starts_for_format(format_key), use_pandoc, ..Default::default() }; diff --git a/tests/cli_integration.rs b/tests/cli_integration.rs index 47f8f407..ed1d8ba2 100644 --- a/tests/cli_integration.rs +++ b/tests/cli_integration.rs @@ -473,6 +473,33 @@ fn idempotent_edge_cases_plaintext() { ); } +#[test] +fn lowercase_starts_flag_and_config_split_tool_names() { + let input = "There are two. setup-python moved to PyPy 8.0.0.\n"; + let off = pipe_stdin(input, &["--format", "markdown"]); + assert!(off.status.success(), "{:?}", off.stderr); + assert_eq!(String::from_utf8(off.stdout).unwrap(), input); + let on = pipe_stdin(input, &["--format", "markdown", "--lowercase-starts"]); + assert!(on.status.success(), "{:?}", on.stderr); + assert_eq!( + String::from_utf8(on.stdout).unwrap(), + "There are two.\nsetup-python moved to PyPy 8.0.0.\n" + ); + + let dir = tempfile::tempdir().unwrap(); + fs::write( + dir.path().join(".snapperrc.toml"), + "lowercase_starts = false\n[markdown]\nlowercase_starts = true\n", + ) + .unwrap(); + let configured = pipe_stdin_in_dir(input, &["--format", "markdown"], dir.path()); + assert!(configured.status.success(), "{:?}", configured.stderr); + assert_eq!( + String::from_utf8(configured.stdout).unwrap(), + "There are two.\nsetup-python moved to PyPy 8.0.0.\n" + ); +} + #[test] fn default_cli_keeps_gfm_table_list_and_splits_prose() { let dir = tempfile::tempdir().unwrap(); diff --git a/tests/pandoc_ast_backend.rs b/tests/pandoc_ast_backend.rs index ddc6419f..c50c711b 100644 --- a/tests/pandoc_ast_backend.rs +++ b/tests/pandoc_ast_backend.rs @@ -550,7 +550,11 @@ fn snapper_cli_use_pandoc_exits_0() { } let bin = env!("CARGO_BIN_EXE_snapper"); let dir = tempfile::tempdir().expect("tempdir"); - std::fs::write(dir.path().join("prose.md"), "Hello world. Second sentence.\n").unwrap(); + std::fs::write( + dir.path().join("prose.md"), + "Hello world. Second sentence.\n", + ) + .unwrap(); let prose = std::process::Command::new(bin) .args(["--use-pandoc", "--format", "markdown", "prose.md"]) .current_dir(dir.path()) From c1fcd61923026478b7bb2865e99596265c53a2ed Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 10:47:23 +0000 Subject: [PATCH 03/14] fix(sentence): split after an emphasis closer before a code span or lowercase start A closer run only opened the next sentence at a capital or a quote. An opening backtick now splits always, and a lowercase word splits when lowercase starts are on. Abbreviations before the closer, and an open delimiter span, still stay joined. --- src/sentence/neural.rs | 8 +-- src/sentence/unicode.rs | 145 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/src/sentence/neural.rs b/src/sentence/neural.rs index 14db73df..e5b67c86 100644 --- a/src/sentence/neural.rs +++ b/src/sentence/neural.rs @@ -3,10 +3,7 @@ use std::sync::Mutex; use nnsplit::NNSplit; use super::SentenceSplitter; -use super::unicode::{ - UnicodeSentenceSplitter, protect_inline_tokens_with, restore_inline_tokens, - split_after_markup_sentence_end, -}; +use super::unicode::{UnicodeSentenceSplitter, protect_inline_tokens_with, restore_inline_tokens}; /// nnsplit downloads models to `~/.cache/nnsplit/` on first use; concurrent /// loads race the download and can observe a partially written model @@ -104,7 +101,8 @@ impl SentenceSplitter for NeuralSentenceSplitter { }; let restored = restore_inline_tokens(raw, &placeholders); - split_after_markup_sentence_end(self.post.refine_segments(restored)) + self.post + .split_markup_sentences(self.post.refine_segments(restored)) } } diff --git a/src/sentence/unicode.rs b/src/sentence/unicode.rs index 50b231ca..03e337cf 100644 --- a/src/sentence/unicode.rs +++ b/src/sentence/unicode.rs @@ -185,6 +185,18 @@ impl UnicodeSentenceSplitter { pub(crate) fn verbatim_commands(&self) -> &[String] { &self.extra_verbatim_commands } + + /// Split after a markup closer when the next sentence is a capital, a + /// code span, or (when lowercase starts are on) a lowercase word. + pub(crate) fn split_markup_sentences(&self, segments: Vec) -> Vec { + split_after_markup_sentence_end( + segments, + self.lowercase_starts, + &self.lang_abbrev_pattern, + &self.lang_multi_pattern, + self.extra_pattern.as_ref(), + ) + } } impl Default for UnicodeSentenceSplitter { @@ -3153,7 +3165,7 @@ impl SentenceSplitter for UnicodeSentenceSplitter { let refs: Vec<&str> = expanded.iter().map(String::as_str).collect(); let merged = self.refine_segments_from_strs(&refs); let restored = restore_inline_tokens(merged, &placeholders); - split_after_markup_sentence_end(restored) + self.split_markup_sentences(restored) } } @@ -3388,46 +3400,79 @@ fn piece_starts_sentence_after_ticks(piece: &str) -> bool { /// sees one sentence. This post-pass runs after restore. Quotes are not /// paired spans, so they already split. Mid-span periods (`**the end. Still bold**`) /// have no closers after the period and stay one sentence. -pub(crate) fn split_after_markup_sentence_end(segments: Vec) -> Vec { +fn split_after_markup_sentence_end( + segments: Vec, + lowercase: bool, + abbrev_re: &Regex, + multi_re: &Regex, + extra: Option<&Regex>, +) -> Vec { let mut out = Vec::new(); for seg in segments { - push_markup_sentence_splits(&mut out, seg.trim()); + push_markup_sentence_splits(&mut out, seg.trim(), lowercase, abbrev_re, multi_re, extra); } out.into_iter().filter(|s| !s.is_empty()).collect() } -fn push_markup_sentence_splits(out: &mut Vec, seg: &str) { - if let Some((head, rest)) = take_markup_terminal_sentence(seg) { +fn push_markup_sentence_splits( + out: &mut Vec, + seg: &str, + lowercase: bool, + abbrev_re: &Regex, + multi_re: &Regex, + extra: Option<&Regex>, +) { + if let Some((head, rest)) = + take_markup_terminal_sentence(seg, lowercase, abbrev_re, multi_re, extra) + { out.push(head); - push_markup_sentence_splits(out, &rest); + push_markup_sentence_splits(out, &rest, lowercase, abbrev_re, multi_re, extra); } else if !seg.is_empty() { out.push(seg.to_string()); } } -fn take_markup_terminal_sentence(seg: &str) -> Option<(String, String)> { +fn take_markup_terminal_sentence( + seg: &str, + lowercase: bool, + abbrev_re: &Regex, + multi_re: &Regex, + extra: Option<&Regex>, +) -> Option<(String, String)> { // Terminal `.!?` immediately before `**` / `*` / `_` / backticks / // `~~` / Org `/` `=` `+` / `](url)`, or an RST interpreted-text - // closer then `.!?`, then whitespace, then a new sentence - // (uppercase or opening quote). + // closer then `.!?`, then whitespace, then a new sentence. + // A new sentence is an uppercase word, an opening quote plus an + // uppercase word, or an opening backtick (a code span). + // When lowercase starts are on, a lowercase word counts too. // Prefix closer is `:role:`text`. / `:domain:role:`text`. Suffix // closer is `text`:role:. / `text`:domain:role:. (the closer is // `:role:`, not the backtick). Opening ```` is not a closer. Role // charset has no `.` so `email:user.name:`host`. does not - // false-split. A lowercase continuation (`[Example Inc.](url) now.`) - // stays one sentence. + // false-split. An abbreviation before the closer + // (`**e.g.** foo`, `[Example Inc.](url) now`) stays one sentence. static CAP: LazyLock = LazyLock::new(|| { Regex::new( - r#"(?s)^(.*?(?:[.!?](?:\*{1,3}|_{1,3}|`+|~{1,2}|[/=+]|\]\([^)]*\))+|:[A-Za-z][A-Za-z0-9_-]*(?::[A-Za-z][A-Za-z0-9_-]*)*:`[^`\n]+`[.!?]|`[^`\n]+`:[A-Za-z][A-Za-z0-9_-]*(?::[A-Za-z][A-Za-z0-9_-]*)*:[.!?]))\s+([A-Z][\s\S]*|["'][A-Z][\s\S]*)$"#, + r#"(?s)^(.*?(?:[.!?](?:\*{1,3}|_{1,3}|`+|~{1,2}|[/=+]|\]\([^)]*\))+|:[A-Za-z][A-Za-z0-9_-]*(?::[A-Za-z][A-Za-z0-9_-]*)*:`[^`\n]+`[.!?]|`[^`\n]+`:[A-Za-z][A-Za-z0-9_-]*(?::[A-Za-z][A-Za-z0-9_-]*)*:[.!?]))\s+([A-Z][\s\S]*|["'][A-Z][\s\S]*|`[\s\S]*)$"#, ) .expect("valid markup-terminal sentence regex") }); - let c = CAP.captures(seg)?; + static CAP_LOWER: LazyLock = LazyLock::new(|| { + Regex::new( + r#"(?s)^(.*?(?:[.!?](?:\*{1,3}|_{1,3}|`+|~{1,2}|[/=+]|\]\([^)]*\))+|:[A-Za-z][A-Za-z0-9_-]*(?::[A-Za-z][A-Za-z0-9_-]*)*:`[^`\n]+`[.!?]|`[^`\n]+`:[A-Za-z][A-Za-z0-9_-]*(?::[A-Za-z][A-Za-z0-9_-]*)*:[.!?]))\s+([A-Za-z][\s\S]*|["'][A-Za-z][\s\S]*|`[\s\S]*)$"#, + ) + .expect("valid markup-terminal lowercase sentence regex") + }); + let re = if lowercase { &*CAP_LOWER } else { &*CAP }; + let c = re.captures(seg)?; let head = c.get(1)?.as_str().trim(); let rest = c.get(2)?.as_str().trim(); if head.is_empty() || rest.is_empty() { return None; } + if markup_head_is_abbreviation(head, abbrev_re, multi_re, extra) { + return None; + } // GitHub #374: `.` plus a capital looks like a markup sentence end, // but the backticks may sit inside a still-open quote/paren span. // Same protect + DelimState feed as newlines_respect_delimiter_spans. @@ -3437,6 +3482,47 @@ fn take_markup_terminal_sentence(seg: &str) -> Option<(String, String)> { Some((head.to_string(), rest.to_string())) } +/// True when the word before the closer's period is an abbreviation +/// (`**e.g.**`, `[Example Inc.](url)`). +fn markup_head_is_abbreviation( + head: &str, + abbrev_re: &Regex, + multi_re: &Regex, + extra: Option<&Regex>, +) -> bool { + let core = strip_trailing_markup_for_abbrev(head); + if !core.ends_with('.') { + return false; + } + let last = core.split_whitespace().last().unwrap_or(core); + let probe = last.trim_start_matches(['*', '_', '`', '~', '/', '=', '+', '[', '(']); + if probe.is_empty() { + return false; + } + is_abbreviation_ending(probe, abbrev_re, multi_re, extra) +} + +fn strip_trailing_markup_for_abbrev(s: &str) -> &str { + let mut core = s.trim_end(); + loop { + let next = strip_md_link_suffix(core) + .trim_end_matches(['*', '_', '`', '~', '/', '=', '+', ']', ')', '"', '\'']); + if next.len() == core.len() { + break; + } + core = next.trim_end(); + } + core +} + +fn strip_md_link_suffix(s: &str) -> &str { + let Some(open) = s.rfind("](") else { + return s; + }; + let dest = &s[open + 2..]; + if dest.ends_with(')') { &s[..open] } else { s } +} + /// True when `head` still has an open DelimState span after inline tokens /// are protected. Used so markup closer-split cannot invent a newline /// inside a balanced quote (GitHub #374). @@ -3918,6 +4004,39 @@ mod tests { ); } + #[test] + fn markup_closer_splits_before_code_span_and_lowercase() { + assert_eq!( + split("**Note.** `foo` runs now. Fine."), + vec!["**Note.**", "`foo` runs now.", "Fine."] + ); + let collection = "**Collection error on 0004c3d (ubuntu and macos).** setup-python now installs PyPy 8.0.0. numpy 2.4.6 only ships pp73 wheels."; + assert_eq!( + split(collection), + vec![collection.to_string()], + "lowercase after a closer stays joined until asked" + ); + let on = UnicodeSentenceSplitter::new().with_lowercase_starts(true); + assert_eq!( + on.split(collection), + vec![ + "**Collection error on 0004c3d (ubuntu and macos).**", + "setup-python now installs PyPy 8.0.0.", + "numpy 2.4.6 only ships pp73 wheels.", + ] + ); + assert_eq!( + split("[Example Inc.](url) now works."), + vec!["[Example Inc.](url) now works."] + ); + assert_eq!(split("**e.g.** foo bar."), vec!["**e.g.** foo bar."]); + assert_eq!( + on.split("[Example Inc.](url) now works."), + vec!["[Example Inc.](url) now works."] + ); + assert_eq!(on.split("**e.g.** foo bar."), vec!["**e.g.** foo bar."]); + } + #[test] fn a_lowercase_start_opens_a_sentence_only_when_asked() { let text = "Restart it. packsetd reloads the pack."; From ff2f33fac662d87e43aa19ebf89ac10f0b5636e8 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 10:48:38 +0000 Subject: [PATCH 04/14] fix(format): fall back per region, not per file A render mismatch used to return the whole input, so one bad paragraph blocked every other split. Each region is retried on its own, and only a mismatch keeps that region's original bytes. Stderr names the line ranges that were left unchanged. --- src/lib.rs | 99 +++++++++++++++++++++++++++++++++++++++- src/reflow.rs | 37 +++++++++++++-- tests/cli_integration.rs | 21 +++++++++ 3 files changed, 151 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a9c3830d..eff95bfd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -352,7 +352,7 @@ pub fn format_text_with_splitter( Some(config), ) { - work_input.to_string() + salvage_regions(work_input, config, splitter)? } else { candidate } @@ -377,6 +377,75 @@ pub fn format_text_with_splitter( Ok(output) } +/// When the whole-document render check fails, keep each region whose +/// own rewrite still matches and leave the rest as original bytes. +fn salvage_regions( + work_input: &str, + config: &FormatConfig, + splitter: &dyn SentenceSplitter, +) -> Result { + use crate::reflow::{ReflowConfig, apply_rewrites, region_rewrites}; + + let spanned = + parser::parser_for_format_config(config.format, Some(config)).parse_full(work_input); + let reflow_config = ReflowConfig { + max_width: config.max_width, + code: Some(&config.code), + format_code: config.format_code, + clause_breaks: config.clause_breaks, + format: config.format, + }; + let rewrites = match region_rewrites(work_input, &spanned, splitter, &reflow_config) { + Ok(rewrites) => rewrites, + Err(_) => { + let (start, end) = byte_line_span(work_input, 0, work_input.len()); + eprintln!("snapper: render check failed; left lines {start}-{end} unchanged"); + return Ok(work_input.to_string()); + } + }; + let mut kept = Vec::new(); + let mut rejected = Vec::new(); + for (start, end, repl) in rewrites { + let mut candidate = String::with_capacity(work_input.len() - (end - start) + repl.len()); + candidate.push_str(&work_input[..start]); + candidate.push_str(&repl); + candidate.push_str(&work_input[end..]); + if oracle::matches_ex( + config.format, + work_input, + &candidate, + config.format_code, + Some(config), + ) { + kept.push((start, end, repl)); + } else { + rejected.push(byte_line_span(work_input, start, end)); + } + } + if !rejected.is_empty() { + let ranges = rejected + .iter() + .map(|(start, end)| format!("{start}-{end}")) + .collect::>() + .join(", "); + eprintln!("snapper: render check failed; left lines {ranges} unchanged"); + } + apply_rewrites(work_input, &kept).map_err(|e| anyhow::anyhow!("{e}")) +} + +/// 1-indexed inclusive line numbers covering `[start, end)`. +fn byte_line_span(source: &str, start: usize, end: usize) -> (usize, usize) { + let start_line = byte_to_line(source, start); + let end_at = end.saturating_sub(1).max(start); + let end_line = byte_to_line(source, end_at); + (start_line, end_line.max(start_line)) +} + +fn byte_to_line(source: &str, byte: usize) -> usize { + let byte = byte.min(source.len()); + source[..byte].bytes().filter(|&b| b == b'\n').count() + 1 +} + /// Pandoc reader/writer name for this config. /// Markdown defaults to `gfm` so `--use-pandoc` does not emit pandoc tables. fn pandoc_input_format(config: &FormatConfig) -> &str { @@ -506,6 +575,34 @@ pub fn format_range( Ok(result) } +#[cfg(test)] +mod region_fallback_tests { + use super::*; + use crate::format::Format; + + #[test] + fn rejected_paragraph_does_not_revert_the_file() { + let input = + "**Note.** Next sentence here. Third one.\n\nThis is done.** Next starts here.\n"; + let out = format_text( + input, + &FormatConfig { + format: Format::Markdown, + ..Default::default() + }, + ) + .unwrap(); + assert!( + out.starts_with("**Note.**\nNext sentence here.\nThird one.\n\n"), + "first paragraph must split, got:\n{out}" + ); + assert!( + out.ends_with("This is done.** Next starts here.\n"), + "second paragraph must stay unchanged, got:\n{out}" + ); + } +} + #[cfg(test)] mod lowercase_starts_tests { use super::*; diff --git a/src/reflow.rs b/src/reflow.rs index 9e91338d..31fa15aa 100644 --- a/src/reflow.rs +++ b/src/reflow.rs @@ -91,6 +91,20 @@ fn splice( splitter: &dyn SentenceSplitter, config: &ReflowConfig, ) -> Result { + let rewrites = region_rewrites(source, spanned, splitter, config)?; + apply_rewrites(source, &rewrites) +} + +/// Prose and rewritten code-body replacements, in source order. +/// +/// A span whose text did not change is omitted. Callers that salvage a +/// rejected document try each entry on its own. +pub(crate) fn region_rewrites( + source: &str, + spanned: &[SpannedRegion], + splitter: &dyn SentenceSplitter, + config: &ReflowConfig, +) -> Result, SpliceError> { let regions: Vec = spanned.iter().map(|s| s.region.clone()).collect(); let mut rewrites: Vec<(usize, usize, String)> = Vec::new(); for (idx, sr) in spanned.iter().enumerate() { @@ -104,7 +118,9 @@ fn splice( continue; } let replacement = reflow_prose(text, idx, ®ions, splitter, config); - rewrites.push((span.start, span.end, replacement)); + if source.get(span.start..span.end) != Some(replacement.as_str()) { + rewrites.push((span.start, span.end, replacement)); + } } ( Region::Code { lang, body, .. }, @@ -132,18 +148,29 @@ fn splice( } } rewrites.sort_by_key(|(start, _, _)| *start); + Ok(rewrites) +} + +pub(crate) fn apply_rewrites( + source: &str, + rewrites: &[(usize, usize, String)], +) -> Result { let mut out = String::with_capacity(source.len()); let mut cursor = 0usize; for (start, end, repl) in rewrites { - if start < cursor || end > source.len() || start > end || source.get(start..end).is_none() { + if *start < cursor + || *end > source.len() + || *start > *end + || source.get(*start..*end).is_none() + { return Err(SpliceError(format!( "invalid splice span {start}..{end} (cursor={cursor}, len={})", source.len() ))); } - out.push_str(&source[cursor..start]); - out.push_str(&repl); - cursor = end; + out.push_str(&source[cursor..*start]); + out.push_str(repl); + cursor = *end; } out.push_str(&source[cursor..]); Ok(out) diff --git a/tests/cli_integration.rs b/tests/cli_integration.rs index ed1d8ba2..079d716f 100644 --- a/tests/cli_integration.rs +++ b/tests/cli_integration.rs @@ -473,6 +473,27 @@ fn idempotent_edge_cases_plaintext() { ); } +#[test] +fn render_mismatch_warns_and_keeps_the_good_paragraph() { + let input = "**Note.** Next sentence here. Third one.\n\nThis is done.** Next starts here.\n"; + let out = pipe_stdin(input, &["--format", "markdown"]); + assert!(out.status.success(), "{:?}", out.stderr); + let stdout = String::from_utf8(out.stdout).unwrap(); + assert!( + stdout.starts_with("**Note.**\nNext sentence here.\nThird one.\n\n"), + "{stdout}" + ); + assert!( + stdout.ends_with("This is done.** Next starts here.\n"), + "{stdout}" + ); + let err = String::from_utf8(out.stderr).unwrap(); + assert!( + err.contains("render check failed") && err.contains("left lines") && err.contains("3-3"), + "stderr must name the unchanged line range, got: {err}" + ); +} + #[test] fn lowercase_starts_flag_and_config_split_tool_names() { let input = "There are two. setup-python moved to PyPy 8.0.0.\n"; From bcafa3282d7a0872347a30aace48dcf1b6c3e068 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 10:57:09 +0000 Subject: [PATCH 05/14] fix(sentence): keep a closing emphasis run on the sentence it closes A segment that begins with a closer run and whitespace belongs to the previous sentence. The run is moved back onto that sentence before the lines are emitted. --- src/lib.rs | 25 ++++++++++++++++---- src/sentence/neural.rs | 6 +++-- src/sentence/unicode.rs | 49 +++++++++++++++++++++++++++++++++++++++- tests/cli_integration.rs | 7 ++---- 4 files changed, 75 insertions(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index eff95bfd..6328b7cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -582,8 +582,7 @@ mod region_fallback_tests { #[test] fn rejected_paragraph_does_not_revert_the_file() { - let input = - "**Note.** Next sentence here. Third one.\n\nThis is done.** Next starts here.\n"; + let input = "**Note.** Next sentence here. Third one.\n\nA. a0.A.\n"; let out = format_text( input, &FormatConfig { @@ -597,8 +596,26 @@ mod region_fallback_tests { "first paragraph must split, got:\n{out}" ); assert!( - out.ends_with("This is done.** Next starts here.\n"), - "second paragraph must stay unchanged, got:\n{out}" + out.ends_with("A. a0.A.\n"), + "rejected paragraph must stay unchanged, got:\n{out}" + ); + } + + #[test] + fn emphasis_closer_paragraph_splits_without_reverting() { + let input = + "**Note.** Next sentence here. Third one.\n\nThis is done.** Next starts here.\n"; + let out = format_text( + input, + &FormatConfig { + format: Format::Markdown, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + out, + "**Note.**\nNext sentence here.\nThird one.\n\nThis is done.**\nNext starts here.\n" ); } } diff --git a/src/sentence/neural.rs b/src/sentence/neural.rs index e5b67c86..767fb8a4 100644 --- a/src/sentence/neural.rs +++ b/src/sentence/neural.rs @@ -101,8 +101,10 @@ impl SentenceSplitter for NeuralSentenceSplitter { }; let restored = restore_inline_tokens(raw, &placeholders); - self.post - .split_markup_sentences(self.post.refine_segments(restored)) + crate::sentence::unicode::reattach_leading_closer_runs( + self.post + .split_markup_sentences(self.post.refine_segments(restored)), + ) } } diff --git a/src/sentence/unicode.rs b/src/sentence/unicode.rs index 03e337cf..4f9da6a2 100644 --- a/src/sentence/unicode.rs +++ b/src/sentence/unicode.rs @@ -3165,7 +3165,7 @@ impl SentenceSplitter for UnicodeSentenceSplitter { let refs: Vec<&str> = expanded.iter().map(String::as_str).collect(); let merged = self.refine_segments_from_strs(&refs); let restored = restore_inline_tokens(merged, &placeholders); - self.split_markup_sentences(restored) + reattach_leading_closer_runs(self.split_markup_sentences(restored)) } } @@ -3387,6 +3387,45 @@ fn piece_starts_sentence_after_ticks(piece: &str) -> bool { rest.len() < piece.len() && rest.starts_with(|c: char| c.is_uppercase()) } +/// A segment that begins with `*`, `_`, `~`, or backticks and then +/// whitespace is a closer that belongs on the previous sentence. +pub(crate) fn reattach_leading_closer_runs(segments: Vec) -> Vec { + let mut out: Vec = Vec::with_capacity(segments.len()); + for seg in segments { + let trimmed = seg.trim(); + if trimmed.is_empty() { + continue; + } + if let Some((run, rest)) = leading_closer_run(trimmed) { + if let Some(prev) = out.last_mut() { + prev.push_str(run); + if !rest.is_empty() { + out.push(rest.to_string()); + } + continue; + } + } + out.push(trimmed.to_string()); + } + out +} + +fn leading_closer_run(seg: &str) -> Option<(&str, &str)> { + let bytes = seg.as_bytes(); + let mut i = 0; + while i < bytes.len() && matches!(bytes[i], b'*' | b'_' | b'~' | b'`') { + i += 1; + } + if i == 0 || i >= bytes.len() || !bytes[i].is_ascii_whitespace() { + return None; + } + let mut j = i; + while j < bytes.len() && bytes[j].is_ascii_whitespace() { + j += 1; + } + Some((&seg[..i], seg[j..].trim())) +} + /// Merge false splits caused by sentence punctuation inside quotes or parens. /// E.g., `He said "wow!"` + `and left.` should stay as one sentence when /// the next segment starts with a lowercase letter. @@ -4004,6 +4043,14 @@ mod tests { ); } + #[test] + fn closing_emphasis_stays_on_the_sentence_it_closes() { + assert_eq!( + split("It works.** Next one."), + vec!["It works.**", "Next one."] + ); + } + #[test] fn markup_closer_splits_before_code_span_and_lowercase() { assert_eq!( diff --git a/tests/cli_integration.rs b/tests/cli_integration.rs index 079d716f..66d719c4 100644 --- a/tests/cli_integration.rs +++ b/tests/cli_integration.rs @@ -475,7 +475,7 @@ fn idempotent_edge_cases_plaintext() { #[test] fn render_mismatch_warns_and_keeps_the_good_paragraph() { - let input = "**Note.** Next sentence here. Third one.\n\nThis is done.** Next starts here.\n"; + let input = "**Note.** Next sentence here. Third one.\n\nA. a0.A.\n"; let out = pipe_stdin(input, &["--format", "markdown"]); assert!(out.status.success(), "{:?}", out.stderr); let stdout = String::from_utf8(out.stdout).unwrap(); @@ -483,10 +483,7 @@ fn render_mismatch_warns_and_keeps_the_good_paragraph() { stdout.starts_with("**Note.**\nNext sentence here.\nThird one.\n\n"), "{stdout}" ); - assert!( - stdout.ends_with("This is done.** Next starts here.\n"), - "{stdout}" - ); + assert!(stdout.ends_with("A. a0.A.\n"), "{stdout}"); let err = String::from_utf8(out.stderr).unwrap(); assert!( err.contains("render check failed") && err.contains("left lines") && err.contains("3-3"), From 8c529756a26b4b36105b9dda24b99e19524b3109 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 10:58:35 +0000 Subject: [PATCH 06/14] fix(markdown): never start a continuation line with a block marker A sentence split used to leave a bare enumerator or a list marker at column zero. That token now stays on the previous line, so the paragraph is not reread as a new block. --- src/lib.rs | 49 ++++++++++++++++++++ src/reflow.rs | 125 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6328b7cc..423e99ec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -620,6 +620,55 @@ mod region_fallback_tests { } } +#[cfg(test)] +mod markdown_block_marker_tests { + use super::*; + use crate::format::Format; + + fn format_md(input: &str) -> String { + format_text( + input, + &FormatConfig { + format: Format::Markdown, + ..Default::default() + }, + ) + .unwrap() + } + + fn opens_block_line(line: &str) -> bool { + let t = line.trim_start(); + t == "1." + || t.starts_with("1. ") + || t.starts_with("1) ") + || t.starts_with("- ") + || t.starts_with("+ ") + || t.starts_with("* ") + || t.starts_with("# ") + || t.starts_with("> ") + || (!t.is_empty() && (t.bytes().all(|b| b == b'-') || t.bytes().all(|b| b == b'='))) + } + + #[test] + fn continuation_lines_do_not_start_with_block_markers() { + let enumerated = format_md("All set. 1. Then foo happens.\n"); + assert_eq!(enumerated, "All set. 1.\nThen foo happens.\n"); + assert!( + !enumerated.lines().any(|l| l.trim() == "1."), + "{enumerated}" + ); + + let dashed = format_md("Done. - not a list.\n"); + assert!( + dashed.lines().skip(1).all(|l| !opens_block_line(l)), + "{dashed}" + ); + assert!(dashed.contains("Done.")); + assert!(dashed.contains("not a list.")); + assert!(dashed.contains('-')); + } +} + #[cfg(test)] mod lowercase_starts_tests { use super::*; diff --git a/src/reflow.rs b/src/reflow.rs index 31fa15aa..8e43e13e 100644 --- a/src/reflow.rs +++ b/src/reflow.rs @@ -264,8 +264,14 @@ fn reflow_prose( }; let hanging = hang.chars().count(); let mut sentences = splitter.split(text); - if config.format == Format::Markdown && hanging > 0 { - glue_md_same_line_ol_opener(&mut sentences, text); + if config.format == Format::Markdown { + if hanging > 0 { + glue_md_same_line_ol_opener(&mut sentences, text); + } + // A continuation that starts with a block marker (`1.`, `- `, `# `, + // `> `, a setext underline) is reread as a new block. Keep the + // marker on the previous sentence. + glue_md_continuation_block_markers(&mut sentences); } let nsent = sentences.len(); for (i, sentence) in sentences.iter().enumerate() { @@ -984,6 +990,96 @@ fn md_same_line_ol_opener(text: &str) -> bool { && bytes.get(i + 1).is_some_and(|b| !b.is_ascii_whitespace()) } +/// Move a leading block marker off a continuation sentence and onto the +/// previous one. The first sentence of the paragraph is left alone. +fn glue_md_continuation_block_markers(sentences: &mut Vec) { + let mut i = 1usize; + while i < sentences.len() { + let Some((token, rest)) = md_continuation_block_token(sentences[i].trim()) else { + i += 1; + continue; + }; + let token = token.to_string(); + let rest = rest.to_string(); + let prev = sentences[i - 1].trim_end(); + sentences[i - 1] = if prev.is_empty() { + token + } else { + format!("{prev} {token}") + }; + if rest.is_empty() { + sentences.remove(i); + } else { + sentences[i] = rest; + } + } +} + +/// Token that CommonMark would read as a block start if it opened a line. +/// The returned token excludes the following whitespace. The rest is the +/// remainder of the sentence, or empty when the line is only the marker +/// or a setext underline. +fn md_continuation_block_token(sent: &str) -> Option<(&str, &str)> { + let s = sent.trim(); + if s.is_empty() { + return None; + } + if is_setext_underline(s) { + return Some((s, "")); + } + if let Some(end) = ordered_marker_end(s) { + return Some((&s[..end], s[end..].trim_start())); + } + for prefix in ["- ", "+ ", "* ", "-\t", "+\t", "*\t"] { + if let Some(rest) = s.strip_prefix(prefix) { + return Some((&s[..1], rest.trim_start())); + } + } + if matches!(s, "*" | "+") { + return Some((s, "")); + } + let hashes = s.bytes().take_while(|&b| b == b'#').count(); + if (1..=6).contains(&hashes) && matches!(s.as_bytes().get(hashes), Some(b' ' | b'\t')) { + return Some((&s[..hashes], s[hashes..].trim_start())); + } + if let Some(rest) = s.strip_prefix("> ").or_else(|| s.strip_prefix(">\t")) { + return Some((">", rest.trim_start())); + } + if s == ">" { + return Some((">", "")); + } + None +} + +fn is_setext_underline(s: &str) -> bool { + let t = s.trim(); + let Some(&b) = t.as_bytes().first() else { + return false; + }; + (b == b'-' || b == b'=') && t.bytes().all(|c| c == b) +} + +/// End index of a CommonMark ordered-list marker (`1.` / `1)`), when the +/// marker is the whole line or is followed by whitespace. +fn ordered_marker_end(s: &str) -> Option { + let bytes = s.as_bytes(); + let mut i = 0; + while i < bytes.len() && bytes[i].is_ascii_digit() { + i += 1; + } + if i == 0 || i > 9 { + return None; + } + if !matches!(bytes.get(i), Some(b'.') | Some(b')')) { + return None; + } + let end = i + 1; + match bytes.get(end) { + None | Some(b' ' | b'\t') => Some(end), + _ => None, + } +} + /// Join a same-line Markdown `0.` / `1)` opener onto the next sentence so /// sembr does not hang after the opener. fn glue_md_same_line_ol_opener(sentences: &mut Vec, source: &str) { @@ -1349,6 +1445,31 @@ mod tests { reflow(®ions, &UnicodeSentenceSplitter::new(), &config) } + #[test] + fn continuation_block_marker_stays_on_the_previous_sentence() { + let mut enumerated = vec![ + "All set.".to_string(), + "1.".to_string(), + "Then foo happens.".to_string(), + ]; + glue_md_continuation_block_markers(&mut enumerated); + assert_eq!( + enumerated, + vec!["All set. 1.".to_string(), "Then foo happens.".to_string()] + ); + + let mut dashed = vec!["Done.".to_string(), "- not a list.".to_string()]; + glue_md_continuation_block_markers(&mut dashed); + assert_eq!( + dashed, + vec!["Done. -".to_string(), "not a list.".to_string()] + ); + + let mut first = vec!["1. Then foo happens.".to_string()]; + glue_md_continuation_block_markers(&mut first); + assert_eq!(first, vec!["1. Then foo happens.".to_string()]); + } + #[test] fn missing_origin_is_error() { let spanned = vec![crate::parser::SpannedRegion::unspanned(Region::Prose( From 169b9391d7a499071c232d12661a9ddd5341e435 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 11:00:30 +0000 Subject: [PATCH 07/14] fix(pandoc): treat Link and Image inlines as atomic A period inside link or image text was a sentence boundary, so the writer broke the brackets across lines. Those inlines are now masked and emitted whole, and a sentence before them still splits. --- src/parser/pandoc/reflow.rs | 109 ++++++++++++++++++++++++++++++++---- tests/pandoc_ast_backend.rs | 37 ++++++++++++ 2 files changed, 135 insertions(+), 11 deletions(-) diff --git a/src/parser/pandoc/reflow.rs b/src/parser/pandoc/reflow.rs index 3a2a189b..06716843 100644 --- a/src/parser/pandoc/reflow.rs +++ b/src/parser/pandoc/reflow.rs @@ -1,8 +1,9 @@ //! Reflow Para/Plain in a pandoc AST. Other node kinds stay structure. //! //! Sentence (and optional wrap) breaks become `SoftBreak` inlines. The -//! writer (`--wrap=preserve`) turns those into source line breaks. Math -//! and inline Code are masked so their periods cannot end a sentence. +//! writer (`--wrap=preserve`) turns those into source line breaks. Math, +//! inline Code, Link, and Image are masked so their periods cannot end +//! a sentence. use pandoc_ast::{Block, Inline, Pandoc}; @@ -139,15 +140,46 @@ fn mask_into(inlines: &[Inline], out: &mut String) { | Inline::SmallCaps(c) | Inline::Quoted(_, c) | Inline::Span(_, c) - | Inline::Cite(_, c) - | Inline::Link(_, c, _) - | Inline::Image(_, c, _) => mask_into(c, out), + | Inline::Cite(_, c) => mask_into(c, out), + Inline::Link(_, c, _) | Inline::Image(_, c, _) => { + let n = inlines_width(c); + out.extend(std::iter::repeat_n('X', n)); + } Inline::RawInline(_, raw) => out.push_str(raw), Inline::Note(_) => {} } } } +/// Width of inlines in the masked string. Link and Image contribute the +/// width of their children and nothing for the URL, matching [`mask_into`]. +fn inlines_width(inlines: &[Inline]) -> usize { + inlines.iter().map(inline_width).sum() +} + +fn inline_width(inline: &Inline) -> usize { + match inline { + Inline::Str(s) => s.len(), + Inline::Space | Inline::SoftBreak | Inline::LineBreak => 1, + Inline::Code(_, code) => 2 + code.len(), + Inline::Math(ty, body) => format_math(ty, body).len(), + Inline::Emph(c) + | Inline::Strong(c) + | Inline::Underline(c) + | Inline::Strikeout(c) + | Inline::Superscript(c) + | Inline::Subscript(c) + | Inline::SmallCaps(c) + | Inline::Quoted(_, c) + | Inline::Span(_, c) + | Inline::Cite(_, c) + | Inline::Link(_, c, _) + | Inline::Image(_, c, _) => inlines_width(c), + Inline::RawInline(_, raw) => raw.len(), + Inline::Note(_) => 0, + } +} + fn break_offsets(text: &str, splitter: &dyn SentenceSplitter, config: &ReflowConfig) -> Vec { let sentences = splitter.split(text); if sentences.len() < 2 && config.max_width == 0 && !config.clause_breaks { @@ -258,14 +290,12 @@ fn insert_breaks( out.push(Inline::Cite(cite, inner)); } Inline::Link(attr, c, target) => { - let mut inner = Vec::new(); - insert_breaks(c, pos, breaks, bi, &mut inner); - out.push(Inline::Link(attr, inner, target)); + let n = inlines_width(&c); + emit_protected(Inline::Link(attr, c, target), n, pos, breaks, bi, out); } Inline::Image(attr, c, target) => { - let mut inner = Vec::new(); - insert_breaks(c, pos, breaks, bi, &mut inner); - out.push(Inline::Image(attr, inner, target)); + let n = inlines_width(&c); + emit_protected(Inline::Image(attr, c, target), n, pos, breaks, bi, out); } Inline::RawInline(fmt, raw) => { emit_str_at(raw, pos, breaks, bi, out, |s| { @@ -422,6 +452,63 @@ mod tests { } } + fn atomic_inline(image: bool) -> Inline { + let children = vec![ + Inline::Str("the".into()), + Inline::Space, + Inline::Str("docs.".into()), + Inline::Space, + Inline::Str("Really".into()), + ]; + let target = ("https://x.y/a.b".into(), String::new()); + let attr = (String::new(), Vec::new(), Vec::new()); + if image { + Inline::Image(attr, children, target) + } else { + Inline::Link(attr, children, target) + } + } + + fn para(inlines: Vec) -> Pandoc { + Pandoc { + meta: pandoc_ast::Map::new(), + pandoc_api_version: vec![1, 23, 1], + blocks: vec![Block::Para(inlines)], + } + } + + #[test] + fn link_and_image_stay_atomic_across_an_interior_period() { + for image in [false, true] { + let mut doc = para(vec![ + Inline::Str("See".into()), + Inline::Space, + atomic_inline(image), + Inline::Space, + Inline::Str("for".into()), + Inline::Space, + Inline::Str("more.".into()), + ]); + reflow_pandoc(&mut doc, &UnicodeSentenceSplitter::new(), &cfg()); + let Block::Para(inlines) = &doc.blocks[0] else { + panic!("expected Para"); + }; + assert!( + !inlines.iter().any(|i| matches!(i, Inline::SoftBreak)), + "period inside the target must not split the sentence: {inlines:?}" + ); + let inner = inlines.iter().find_map(|i| match i { + Inline::Link(_, c, _) | Inline::Image(_, c, _) => Some(c), + _ => None, + }); + let inner = inner.expect("atomic inline survives"); + assert!( + !inner.iter().any(|i| matches!(i, Inline::SoftBreak)), + "{inner:?}" + ); + } + } + #[test] fn reflow_leaves_header_and_codeblock() { let json = include_str!("../../../tests/fixtures/pandoc_ast/mixed_minimal.json"); diff --git a/tests/pandoc_ast_backend.rs b/tests/pandoc_ast_backend.rs index c50c711b..791de643 100644 --- a/tests/pandoc_ast_backend.rs +++ b/tests/pandoc_ast_backend.rs @@ -539,6 +539,43 @@ fn pandoc_org_verbatim_inner_equals_stays_one_span() { ); } +/// `--use-pandoc` must not break a link or image on a period inside the brackets. +#[test] +fn snapper_cli_use_pandoc_does_not_split_inside_link_or_image() { + if !require_writer() { + return; + } + let bin = env!("CARGO_BIN_EXE_snapper"); + let dir = tempfile::tempdir().expect("tempdir"); + for name in ["link.md", "image.md"] { + let body = if name.starts_with("link") { + "Hello world. See [the docs. Really](https://x.y/a.b) for more.\n" + } else { + "Hello world. See ![the docs. Really](https://x.y/a.b) for more.\n" + }; + std::fs::write(dir.path().join(name), body).unwrap(); + let out = std::process::Command::new(bin) + .args(["--use-pandoc", "--format", "markdown", name]) + .current_dir(dir.path()) + .output() + .expect("spawn snapper"); + assert!( + out.status.success(), + "{name} stderr={}", + String::from_utf8_lossy(&out.stderr) + ); + let stdout = String::from_utf8(out.stdout).unwrap(); + assert!( + stdout.contains("Hello world.\n"), + "{name} must still split the sentence before the target:\n{stdout}" + ); + assert!( + stdout.contains("[the docs. Really](https://x.y/a.b)"), + "{name} split inside the brackets:\n{stdout}" + ); + } +} + /// `snapper --use-pandoc` exits 0. /// Prose that round-trips through gfm is sentence-split. /// A file whose gfm write changes structure (tables, fences) is left unchanged From 0a8331eab52095d79cd11d4d28fbce68df186157 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 11:05:32 +0000 Subject: [PATCH 08/14] fix(sentence): leave a line-leading bullet off the previous sentence A closer run is moved back only when it was split from the same line. A star after a line break is a bullet and stays on its own segment. --- src/sentence/neural.rs | 12 ++++++------ src/sentence/unicode.rs | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/sentence/neural.rs b/src/sentence/neural.rs index 767fb8a4..bcb185a6 100644 --- a/src/sentence/neural.rs +++ b/src/sentence/neural.rs @@ -95,16 +95,16 @@ impl SentenceSplitter for NeuralSentenceSplitter { splits[0] .flatten(0) .into_iter() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .filter(|s| !s.trim().is_empty()) .collect() }; + // Same newline rule as the Unicode path: a line-leading `*` is a bullet. + let raw = crate::sentence::unicode::reattach_leading_closer_runs(raw); let restored = restore_inline_tokens(raw, &placeholders); - crate::sentence::unicode::reattach_leading_closer_runs( - self.post - .split_markup_sentences(self.post.refine_segments(restored)), - ) + self.post + .split_markup_sentences(self.post.refine_segments(restored)) } } diff --git a/src/sentence/unicode.rs b/src/sentence/unicode.rs index 4f9da6a2..1d6789f3 100644 --- a/src/sentence/unicode.rs +++ b/src/sentence/unicode.rs @@ -3164,8 +3164,11 @@ impl SentenceSplitter for UnicodeSentenceSplitter { let expanded = split_before_sentence_starts(&expanded, self.lowercase_starts); let refs: Vec<&str> = expanded.iter().map(String::as_str).collect(); let merged = self.refine_segments_from_strs(&refs); + // Reattach while trailing newlines are still on the segments. + // A `*` after a line break is a bullet, not a closer torn off `.`. + let merged = reattach_leading_closer_runs(merged); let restored = restore_inline_tokens(merged, &placeholders); - reattach_leading_closer_runs(self.split_markup_sentences(restored)) + self.split_markup_sentences(restored) } } @@ -3389,20 +3392,27 @@ fn piece_starts_sentence_after_ticks(piece: &str) -> bool { /// A segment that begins with `*`, `_`, `~`, or backticks and then /// whitespace is a closer that belongs on the previous sentence. +/// A closer after a line break is a bullet (`* item`) and stays put. pub(crate) fn reattach_leading_closer_runs(segments: Vec) -> Vec { let mut out: Vec = Vec::with_capacity(segments.len()); + let mut prev_broke = false; for seg in segments { + let broke = seg.trim_end_matches([' ', '\t']).ends_with('\n'); + let line_before = prev_broke || seg.trim_start_matches([' ', '\t']).starts_with('\n'); let trimmed = seg.trim(); + prev_broke = broke || (trimmed.is_empty() && seg.contains('\n')); if trimmed.is_empty() { continue; } - if let Some((run, rest)) = leading_closer_run(trimmed) { - if let Some(prev) = out.last_mut() { - prev.push_str(run); - if !rest.is_empty() { - out.push(rest.to_string()); + if !line_before { + if let Some((run, rest)) = leading_closer_run(trimmed) { + if let Some(prev) = out.last_mut() { + prev.push_str(run); + if !rest.is_empty() { + out.push(rest.to_string()); + } + continue; } - continue; } } out.push(trimmed.to_string()); @@ -4049,6 +4059,19 @@ mod tests { split("It works.** Next one."), vec!["It works.**", "Next one."] ); + assert_eq!( + split("It works.* Next one."), + vec!["It works.*", "Next one."] + ); + let prose = "Topics include: * Product requirements - what pages exist? What functionality lives on them?\n* Technical requirements"; + assert_eq!( + split(prose), + vec![ + "Topics include: * Product requirements - what pages exist?", + "What functionality lives on them?", + "* Technical requirements", + ] + ); } #[test] From 9039b00d5728c42a9f8b5d05c4ebdd517cb403ef Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 11:11:27 +0000 Subject: [PATCH 09/14] docs: reflow formats.org after per-region fallback A rejected line used to revert the whole file, so this reference stayed on its old breaks. Regions that still match the render check are formatted, and the rejected line is unchanged. --- docs/orgmode/reference/formats.org | 68 ++++++++++-------------------- 1 file changed, 22 insertions(+), 46 deletions(-) diff --git a/docs/orgmode/reference/formats.org b/docs/orgmode/reference/formats.org index 30531ed9..8da85ffc 100644 --- a/docs/orgmode/reference/formats.org +++ b/docs/orgmode/reference/formats.org @@ -19,7 +19,8 @@ The classification depends on the format. :CUSTOM_ID: org-structure :END: - Non-source =#+BEGIN_*= ... -=#+END_*= fences, and the bodies of literal blocks (example, export, comment, verse). An unmatched =#+BEGIN_EXPORT= is a paragraph (org-element export-block-parser) +=#+END_*= fences, and the bodies of literal blocks (example, export, comment, verse). +An unmatched =#+BEGIN_EXPORT= is a paragraph (org-element export-block-parser) - Dynamic blocks (=#+BEGIN: NAME= ... =#+END:=), including the generated body - Quote, center, and special-block (NOTE and other unknown NAME) open/close lines - Verse-block body (org-element verse-lines; leftover opaque walker, GitHub #395) @@ -33,15 +34,19 @@ The classification depends on the format. - Column-0 diary-sexp lines (=%%(...)=; org-element-diary-sexp-parser) - Full headline lines (stars, optional TODO keyword, and title text) - List item markers (=-=, =+=, =1.=, including a tab after the bullet). - An ordered marker is one or more digits. A wrap cut that would park it at column 0 skip-cuts the token onto the previous line. + An ordered marker is one or more digits. + A wrap cut that would park it at column 0 skip-cuts the token onto the previous line. Continuation sentences hang at the marker width. - One blank line stays inside the item. Two blank lines end the plain list. + One blank line stays inside the item. + Two blank lines end the plain list. An item also ends at a line indented less than or equal to its bullet, so a two-space line stays in a column-0 item. A line indented like a child bullet ends that child and stays in the parent -- LaTeX environments (=\begin{equation}= ... =\end{equation}=, =\begin{align}=, etc.). An unmatched =\begin{env}= is a paragraph (org-element latex-environment-parser) +- LaTeX environments (=\begin{equation}= ... =\end{equation}=, =\begin{align}=, etc.). + An unmatched =\begin{env}= is a paragraph (org-element latex-environment-parser) - Display math (=\[= ... =\]=) - Line breaks (org-element-line-break-parser: =\\= plus optional spaces or tabs at end of line) -- A line that is only an export snippet (=@@latex:\newpage@@=). Trailing prose after the closer stays Prose. +- A line that is only an export snippet (=@@latex:\newpage@@=). + Trailing prose after the closer stays Prose. *** Code regions (=#+BEGIN_SRC= ... =#+END_SRC=) :PROPERTIES: @@ -82,7 +87,9 @@ These tokens within prose are not split across lines: - Inline babel call: =call_name(...)= / =call_name[inside](...)[end]= (org-element-inline-babel-call-parser; same =\=, optional =[...]=, then a ={filename}=; a flush following sentence stays on its own line - =\PitonInputFileT= / =\PitonInputFileF= take the same optional args then ={file}= and one extra required brace; =\PitonInputFileTF= takes two extra braces; a flush following sentence stays on its own line - =\tcbinputlisting= takes one required ={keyval}= group; a flush following sentence stays on its own line - =\inputpy= / =\inputpycon= take optional =[...]= then a ={filename}=; a flush following sentence stays on its own line - =\py= / =\pyc= / =\pys= / =\pyb= / =\pyv= / =\pycon= and twins / =\sympy= / =\pylab= and twins take optional =[...]= then a delimiter or ={body}=; a flush following sentence stays on its own line - =\inputpygments= takes optional =[...]=, ={lang}=, then a ={filename}=; =\pygment= takes ={lang}= then a delimiter or ={code}= body; a flush following sentence stays on its own line - =\inputpython= takes ={file}= ={first}= ={last}=; =\inputpythonfile= takes ={file}= then optional =[first]= =[last]=; a flush following sentence stays on its own line - =\pyth= takes a delimiter or ={code}= body; a flush following sentence stays on its own line - =\CatchFileBetweenTags= takes ={macro}= ={file}= ={tag}=; =\CatchFileBetweenDelims= takes ={macro}= ={file}= ={start}= ={end}=; =\ExecuteMetaData= takes optional =[file]= then ={tag}=; a flush following sentence stays on its own line - leftover =\CatchFileDef= / =\CatchFileEdef= take ={macro}= ={file}= ={setup}=; a flush following sentence stays on its own line - =\listinginput= takes optional =[interval]= then ={start}= and ={filename}=; a flush following sentence stays on its own line - =\verbatimtabinput= / =\verbatimtabinput*= take optional =[tabwidth]= then a ={filename}=; a flush following sentence stays on its own line - leftover =\verbatimwrite= / =\verbatimwrite*= take a ={filename}=; a flush following sentence stays on its own line - leftover =\listingcont= takes no args; a flush following sentence stays on its own line - =\sageinput= takes optional =[...]= then a ={filename}=; a flush following sentence stays on its own line - =\sageplot= takes optional =[ltx opts]= =[fmt]= then a ={graphics}=; =\sagestr= takes a ={code}=; a flush following sentence stays on its own line - =\inputsc= takes optional =[...]= then a ={name}=; a flush following sentence stays on its own line - =\pythontexcustomc= takes optional =[begin|end]=, a required ={type}=, then a delimiter or ={code}= body; a flush following sentence stays on its own line - =\Scontents= / =\Scontents*= take optional =[...]= then a standard or verbatim arg; =\typestored= / =\getstored= / =\mergesc= / =\meaningsc= / =\foreachsc= take optional =[...]= then a ={seq}=; a flush following sentence stays on its own line + A width wrap keeps a space inside that delimiter on the same line leftover =\mintinline= / =\mint= take optional =[...]=, ={lang}=, then a delimiter or ={...}= body; a flush following sentence stays on its own line leftover =\SaveVerb= takes optional =[...]=, a ={name}=, then the same delimiter body as =\Verb=; a flush following sentence stays on its own line leftover =\UseVerb= / =\UseVerb*= take optional =[...]= then a ={name}=; leftover =\UseVerbatim= / =\LUseVerbatim= / =\BUseVerbatim= take optional =[...]= then a ={name}=; a flush following sentence stays on its own line leftover =\DefineShortVerb= takes optional =[...]= then a ={char}=; leftover =\UndefineShortVerb= takes a ={char}=; a flush following sentence stays on its own line leftover =\Verb= takes a delimiter like =\verb=; leftover =\EscVerb= takes a delimiter or ={code}=; a flush following sentence stays on its own line leftover =\VerbatimInsertBuffer= / =\InsertBuffer= take optional =[...]=; leftover =\VerbatimClearBuffer= takes no args; leftover =\IterateBuffer= takes optional =[...]= then a ={cmd}=; a flush following sentence stays on its own line leftover =\piton|...|= is verb-like; leftover =\piton{...}= stays one token via the generic command argument; a flush following sentence stays on its own line =\lstinputlisting= / =\lstinputlisting*= take optional =[...]= then a ={filename}=; a flush following sentence stays on its own line =\inputminted= / =\inputminted*= take optional =[...]=, ={lang}=, then a ={filename}=; a flush following sentence stays on its own line =\VerbatimInput= / =\BVerbatimInput= / =\LVerbatimInput= take optional =[...]= then a ={filename}=; a flush following sentence stays on its own line =\verbatiminput= / =\verbatiminput*= take a ={filename}=; a flush following sentence stays on its own line =\PitonInputFile= takes optional =<...>=, optional =[...]=, then a ={filename}=; a flush following sentence stays on its own line =\PitonInputFileT= / =\PitonInputFileF= take the same optional args then ={file}= and one extra required brace; =\PitonInputFileTF= takes two extra braces; a flush following sentence stays on its own line =\tcbinputlisting= takes one required ={keyval}= group; a flush following sentence stays on its own line =\inputpy= / =\inputpycon= take optional =[...]= then a ={filename}=; a flush following sentence stays on its own line =\py= / =\pyc= / =\pys= / =\pyb= / =\pyv= / =\pycon= and twins / =\sympy= / =\pylab= and twins take optional =[...]= then a delimiter or ={body}=; a flush following sentence stays on its own line =\inputpygments= takes optional =[...]=, ={lang}=, then a ={filename}=; =\pygment= takes ={lang}= then a delimiter or ={code}= body; a flush following sentence stays on its own line =\inputpython= takes ={file}= ={first}= ={last}=; =\inputpythonfile= takes ={file}= then optional =[first]= =[last]=; a flush following sentence stays on its own line =\pyth= takes a delimiter or ={code}= body; a flush following sentence stays on its own line =\CatchFileBetweenTags= takes ={macro}= ={file}= ={tag}=; =\CatchFileBetweenDelims= takes ={macro}= ={file}= ={start}= ={end}=; =\ExecuteMetaData= takes optional =[file]= then ={tag}=; a flush following sentence stays on its own line leftover =\CatchFileDef= / =\CatchFileEdef= take ={macro}= ={file}= ={setup}=; a flush following sentence stays on its own line =\listinginput= takes optional =[interval]= then ={start}= and ={filename}=; a flush following sentence stays on its own line =\verbatimtabinput= / =\verbatimtabinput*= take optional =[tabwidth]= then a ={filename}=; a flush following sentence stays on its own line leftover =\verbatimwrite= / =\verbatimwrite*= take a ={filename}=; a flush following sentence stays on its own line leftover =\listingcont= takes no args; a flush following sentence stays on its own line =\sageinput= takes optional =[...]= then a ={filename}=; a flush following sentence stays on its own line =\sageplot= takes optional =[ltx opts]= =[fmt]= then a ={graphics}=; =\sagestr= takes a ={code}=; a flush following sentence stays on its own line =\inputsc= takes optional =[...]= then a ={name}=; a flush following sentence stays on its own line =\pythontexcustomc= takes optional =[begin|end]=, a required ={type}=, then a delimiter or ={code}= body; a flush following sentence stays on its own line =\Scontents= / =\Scontents*= take optional =[...]= then a standard or verbatim arg; =\typestored= / =\getstored= / =\mergesc= / =\meaningsc= / =\foreachsc= take optional =[...]= then a ={seq}=; a flush following sentence stays on its own line *** Prose regions (reflowed) :PROPERTIES: @@ -199,7 +176,8 @@ These tokens within prose are not split across lines: - Setext headings (title line plus a line of one or more = or - characters). A wrap cut that would park a short underline at column 0 escapes it - List item markers (=-=, =*=, =+=, =1.=); continuation sentences hang at the marker width. - A line indented four spaces past the marker stays in an open item paragraph. After a blank line it is indented code. + A line indented four spaces past the marker stays in an open item paragraph. + After a blank line it is indented code. A fence, ATX heading, thematic break, HTML block, or GFM table in the marker remainder stays a block - Empty list markers (including quoted) stay Structure - Definition-list terms and =: = markers (pulldown =ENABLE_DEFINITION_LIST=); the body hangs at the marker width @@ -210,7 +188,8 @@ These tokens within prose are not split across lines: - GFM footnote definitions (=[^label]:=); the marker is Structure and the body hangs - Hard line breaks (two trailing spaces, or a trailing backslash) - HTML comments (==, including multiline); == / == remain pragmas -- Type-6 and type-7 HTML blocks run to the next blank line. A matching end tag does not end the block +- Type-6 and type-7 HTML blocks run to the next blank line. + A matching end tag does not end the block - Void type-6 tags (=
= / == / == / == / == / ==) end on the tag line; they have no closer, so leftover following prose stays Prose even without a blank - Pipe tables @@ -258,14 +237,11 @@ These tokens within prose are not split across lines: - Same-line body after =::= on those containers, leftover =.. parsed-literal::=, =.. header::=, =.. footer::=, =.. |name| replace::=, and =.. meta::= fields hangs and splits. Flush bibliographic fields close a meta leftover - Literal blocks (text after =::= with indented or line-prefix-quoted content). - A paragraph whose text is only =::= is that marker. A two-character title underlined with =::= stays a section + A paragraph whose text is only =::= is that marker. + A two-character title underlined with =::= stays a section - Section titles and underlines (=====, =-----=, etc.). - A wrap cut that would park a solid Docutils =Body.line= adornment at column 0 skip-cuts the token onto the previous line (snapper-7xd3) - A wrap cut that would park a Docutils simple-table border at column 0 skip-cuts the token onto the previous line (snapper-2knx) - A wrap cut that would park a Docutils grid-table top (=+---+=) at column 0 skip-cuts the token onto the previous line - A wrap cut that would park a Jinja statement (=\{% ... %\}=) or a Docutils anonymous target (=__ uri=) at column 0 skip-cuts the token onto the previous line. - The indented link block under that target stays structure until a blank - A wrap cut that would park a leftover plus fragment (=+===+=, =+foo=) at column 0 skip-cuts the token onto the previous line + A wrap cut that would park a solid Docutils =Body.line= adornment at column 0 skip-cuts the token onto the previous line (snapper-7xd3) A wrap cut that would park a Docutils simple-table border at column 0 skip-cuts the token onto the previous line (snapper-2knx) A wrap cut that would park a Docutils grid-table top (=+---+=) at column 0 skip-cuts the token onto the previous line A wrap cut that would park a Jinja statement (=\{% ... %\}=) or a Docutils anonymous target (=__ uri=) at column 0 skip-cuts the token onto the previous line. + The indented link block under that target stays structure until a blank A wrap cut that would park a leftover plus fragment (=+===+=, =+foo=) at column 0 skip-cuts the token onto the previous line - Field lists (=:Author:=, =:Date:=, etc.) - Empty list items, empty doctest openers, and empty field-list markers at EOL stay Structure - Comments (=..= without a directive) From 36a3de87021a5f7dc9c32c334f640350f2a227e0 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 11:29:29 +0000 Subject: [PATCH 10/14] fix(sentence): do not split on a code span or an escaped period A code span that ends in a period was a sentence end before a lowercase word. An escaped period was too, once lowercase starts were on. A capital after a code closer still starts a sentence. --- src/lib.rs | 12 +++ src/sentence/unicode.rs | 222 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 218 insertions(+), 16 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 423e99ec..12f5c583 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -719,4 +719,16 @@ mod lowercase_starts_tests { ] ); } + + #[test] + fn period_in_code_span_or_escape_stays_one_line() { + let pip = "`pip install -e .` and then the rest."; + let into = "`1.` into the list."; + let esc = "1\\. and the rest."; + let seen = "See 1\\. and the rest."; + for text in [pip, into, esc, seen] { + assert_eq!(lines(false, text), vec![text], "default {text:?}"); + assert_eq!(lines(true, text), vec![text], "lowercase {text:?}"); + } + } } diff --git a/src/sentence/unicode.rs b/src/sentence/unicode.rs index 1d6789f3..534ff1dc 100644 --- a/src/sentence/unicode.rs +++ b/src/sentence/unicode.rs @@ -247,7 +247,48 @@ pub fn protect_inline_tokens_with( placeholders.push(token.to_string()); format!("\x00PH{idx}\x00") }); - (protected.into_owned(), placeholders) + // CommonMark `\.` is not a sentence end. Mask it after the other + // tokens so a URL or verb span that already swallowed the escape + // stays one placeholder. + let protected = protect_escaped_terminators(&protected, &mut placeholders); + (protected, placeholders) +} + +/// Hide a backslash-escaped `.` `!` or `?` from UAX and the later scans. +/// An odd run of backslashes escapes the punctuation (`1\.`, `\\\.`). +/// An even run leaves a real terminator (`\\.` is a backslash plus a period). +fn protect_escaped_terminators(text: &str, placeholders: &mut Vec) -> String { + let bytes = text.as_bytes(); + let mut out = String::with_capacity(text.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'\\' { + let mut n = 0; + while i + n < bytes.len() && bytes[i + n] == b'\\' { + n += 1; + } + if n % 2 == 1 { + if let Some(&punct) = bytes.get(i + n) { + if matches!(punct, b'.' | b'!' | b'?') { + let pairs = n - 1; + out.push_str(&text[i..i + pairs]); + let start = i + pairs; + let end = i + n + 1; + push_placeholder(&mut out, placeholders, &text[start..end]); + i = end; + continue; + } + } + } + out.push_str(&text[i..i + n]); + i += n; + continue; + } + let ch = text[i..].chars().next().expect("i is in range"); + out.push(ch); + i += ch.len_utf8(); + } + out } fn dollar_closer_post_context(next: Option) -> bool { @@ -3292,6 +3333,8 @@ fn take_lowercase_proper_noun_sentence(seg: &str) -> Option<(String, String)> { /// opens with a digit, or with a lowercase letter when `lowercase` is set. /// A head with no letter (`1.` opening a list item, `...`) is not a /// sentence, so the scan moves on to the next terminator. +/// A backslash-escaped terminator, or one inside a code span, is not a +/// sentence end. fn split_before_sentence_starts(segments: &[String], lowercase: bool) -> Vec { let mut out = Vec::with_capacity(segments.len()); for seg in segments { @@ -3299,8 +3342,11 @@ fn split_before_sentence_starts(segments: &[String], lowercase: bool) -> Vec Option { + let bytes = head.as_bytes(); + let mut end = head.len(); + loop { + if end == 0 { + return None; + } + if bytes[end - 1] == b')' { + if let Some(open) = head[..end].rfind("](") { + let dest = &head[open + 2..end]; + if dest.ends_with(')') && !dest[..dest.len() - 1].contains(')') { + end = open; + continue; + } + } + } + let ch = head[..end].chars().next_back()?; + if matches!(ch, '*' | '_' | '`' | '~' | '/' | '=' | '+') { + end -= ch.len_utf8(); + continue; + } + break; } - // GitHub #374: `.` plus a capital looks like a markup sentence end, - // but the backticks may sit inside a still-open quote/paren span. - // Same protect + DelimState feed as newlines_respect_delimiter_spans. - if markup_head_leaves_delim_span_open(head) { - return None; + let ch = head[..end].chars().next_back()?; + if matches!(ch, '.' | '!' | '?') { + Some(end - ch.len_utf8()) + } else { + None } - Some((head.to_string(), rest.to_string())) +} + +/// True when `at` is a `.!?` preceded by an odd number of backslashes. +fn escaped_punct(text: &str, at: usize) -> bool { + let bytes = text.as_bytes(); + if at == 0 || at >= bytes.len() { + return false; + } + let mut i = at; + let mut n = 0usize; + while i > 0 && bytes[i - 1] == b'\\' { + n += 1; + i -= 1; + } + n % 2 == 1 +} + +/// True when `rest` opens with a lowercase word. +fn starts_lowercase_word(rest: &str) -> bool { + rest.chars().next().is_some_and(|c| c.is_lowercase()) +} + +/// True when byte `at` sits inside a Markdown backtick span. +/// Emphasis, strike, and Org verbatim stay sentence ends before a capital +/// (`**Bold.** Next`, `` `Code.` Next ``, `=Verbatim.= Next`). +fn punct_inside_code_span(text: &str, at: usize) -> bool { + let bytes = text.as_bytes(); + let mut i = 0; + while i < text.len() && i <= at { + if bytes[i] == b'`' { + if let Some(end) = find_md_code_span(text, i) { + if i < at && at < end { + return true; + } + if end > i { + i = end; + continue; + } + } + } + let ch = text[i..].chars().next().expect("i is in range"); + i += ch.len_utf8(); + } + false } /// True when the word before the closer's period is an abbreviation @@ -4107,6 +4253,50 @@ mod tests { assert_eq!(on.split("**e.g.** foo bar."), vec!["**e.g.** foo bar."]); } + #[test] + fn period_inside_code_span_or_escape_is_not_a_sentence_end() { + let on = UnicodeSentenceSplitter::new().with_lowercase_starts(true); + let cases = [ + "`pip install -e .` and then the rest.", + "`1.` into the list.", + "1\\. and the rest.", + "See 1\\. and the rest.", + "See 1\\. Next stays here.", + ]; + for text in cases { + assert_eq!(split(text), vec![text.to_string()], "default {text:?}"); + assert_eq!(on.split(text), vec![text.to_string()], "lowercase {text:?}"); + } + // A capital after a code span still ends the sentence. The period + // before the closer is the boundary; only a lowercase follower is not. + assert_eq!( + split("`Code sentence.` Next one."), + vec!["`Code sentence.`".to_string(), "Next one.".to_string()] + ); + assert_eq!( + on.split("`pip install -e .` Next stays here."), + vec![ + "`pip install -e .`".to_string(), + "Next stays here.".to_string(), + ] + ); + assert_eq!( + split("See `file`. Next one."), + vec!["See `file`.".to_string(), "Next one.".to_string()] + ); + assert_eq!( + on.split("See `file`. Next one."), + vec!["See `file`.".to_string(), "Next one.".to_string()] + ); + assert_eq!( + on.split("Done. `pip install -e .` and go."), + vec![ + "Done.".to_string(), + "`pip install -e .` and go.".to_string(), + ] + ); + } + #[test] fn a_lowercase_start_opens_a_sentence_only_when_asked() { let text = "Restart it. packsetd reloads the pack."; From 6412db307e0f5130d7b77e4da33481c7b64bdbd0 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 11:30:05 +0000 Subject: [PATCH 11/14] build(binstall): point Windows at the flat release zip The default metadata asks for a tar.xz with a snapper-fmt-/ directory. The Windows asset is a zip with snapper.exe at the archive root. --- Cargo.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index ec287c14..15f9151f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -157,6 +157,13 @@ pkg-url = "{ repo }/releases/download/v{ version }/snapper-fmt-{ target }.tar.xz bin-dir = "snapper-fmt-{ target }/{ bin }{ binary-ext }" pkg-fmt = "txz" +# The Windows release zip is flat (snapper.exe at the archive root). +# Unix tarballs keep the snapper-fmt-/ directory above. +[package.metadata.binstall.overrides.x86_64-pc-windows-msvc] +pkg-url = "{ repo }/releases/download/v{ version }/snapper-fmt-{ target }.zip" +bin-dir = "{ bin }{ binary-ext }" +pkg-fmt = "zip" + [[bench]] name = "reflow_bench" From 2455b8ed58d3a14c0c70f8dd1a62c591dd095a46 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 11:34:07 +0000 Subject: [PATCH 12/14] build(dist): build static musl archives for Linux x86_64 and aarch64 unknown-linux-musl are dist targets. dist plan gives each an Ubuntu runner and installs musl-tools, so release.yml stays as it is. The archives are snapper-fmt-.tar.xz, the layout the default binstall metadata already expects. --- dist-workspace.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist-workspace.toml b/dist-workspace.toml index f89a42f7..3b7fc9ee 100644 --- a/dist-workspace.toml +++ b/dist-workspace.toml @@ -15,4 +15,4 @@ installers = ["shell", "powershell", "homebrew"] tap = "TurtleTech-ehf/homebrew-tap" publish-jobs = ["homebrew"] # Target platforms to build apps for (Rust target-triple syntax) -targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] +targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "aarch64-unknown-linux-musl", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", "x86_64-pc-windows-msvc"] From 90515737839d0d93afcee8b0fc09b9e682f92b4c Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 11:34:47 +0000 Subject: [PATCH 13/14] ci(release): smoke-test cargo-binstall after publish The job dry-runs a prebuilt install for all seven dist targets once the GitHub release exists. Pull requests and other non-publishing runs skip it. --- .github/workflows/release.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 58f8130f..64e1c7b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -360,3 +360,38 @@ jobs: with: persist-credentials: false submodules: recursive + + # After the GitHub Release exists, confirm cargo-binstall resolves a + # prebuilt for every dist target. Skip on pull requests and other + # non-publishing runs. Musl assets are produced by this same release. + binstall-smoke: + needs: + - plan + - host + if: ${{ needs.plan.outputs.publishing == 'true' && needs.host.result == 'success' }} + runs-on: "ubuntu-22.04" + steps: + - name: Install cargo-binstall + run: | + curl --proto '=https' --tlsv1.2 -fsSL -o /tmp/cargo-binstall.tgz \ + https://github.com/cargo-bins/cargo-binstall/releases/download/v1.25.2/cargo-binstall-x86_64-unknown-linux-gnu.tgz + echo "ca1217094690d15606c050e3d085bc03ce6fe12486de9a815d85dae345bb6803 /tmp/cargo-binstall.tgz" | sha256sum -c - + tar -xzf /tmp/cargo-binstall.tgz -C /usr/local/bin + cargo binstall -V + - name: Dry-run prebuilt installs + env: + TAG: ${{ needs.plan.outputs.tag }} + run: | + version="${TAG#v}" + for target in \ + aarch64-apple-darwin \ + aarch64-unknown-linux-gnu \ + aarch64-unknown-linux-musl \ + x86_64-apple-darwin \ + x86_64-unknown-linux-gnu \ + x86_64-unknown-linux-musl \ + x86_64-pc-windows-msvc + do + cargo binstall --dry-run --no-confirm --disable-strategies compile \ + --targets "$target" "snapper-fmt@${version}" + done From b9a81eb41ba1f7c1950e89fe396b635aac51cc1e Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 11 Oct 2026 11:49:49 +0000 Subject: [PATCH 14/14] ci(release): wait for crates.io before the binstall smoke test Quick-install is disabled so only this release's assets and metadata can pass. The job polls crates.io until the new version is served, then dry-runs each target. cargo-binstall is unpacked into the user cargo bin directory. --- .github/workflows/release.yml | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 64e1c7b0..f8773e68 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -376,8 +376,35 @@ jobs: curl --proto '=https' --tlsv1.2 -fsSL -o /tmp/cargo-binstall.tgz \ https://github.com/cargo-bins/cargo-binstall/releases/download/v1.25.2/cargo-binstall-x86_64-unknown-linux-gnu.tgz echo "ca1217094690d15606c050e3d085bc03ce6fe12486de9a815d85dae345bb6803 /tmp/cargo-binstall.tgz" | sha256sum -c - - tar -xzf /tmp/cargo-binstall.tgz -C /usr/local/bin - cargo binstall -V + mkdir -p "$HOME/.cargo/bin" + tar -xzf /tmp/cargo-binstall.tgz -C "$HOME/.cargo/bin" + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + "$HOME/.cargo/bin/cargo-binstall" -V + - name: Wait until crates.io serves this version + env: + TAG: ${{ needs.plan.outputs.tag }} + run: | + version="${TAG#v}" + if [[ -z "$version" ]]; then + echo "release tag is empty, so crates.io has no snapper-fmt version to wait for" >&2 + exit 1 + fi + url="https://crates.io/api/v1/crates/snapper-fmt/${version}" + ua="snapper-fmt/binstall-smoke (+https://github.com/TurtleTech-ehf/snapper)" + deadline=$((SECONDS + 20 * 60)) + while true; do + code="$(curl --proto '=https' --tlsv1.2 -sS -o /tmp/snapper-fmt-crate.json -w '%{http_code}' -A "$ua" "$url")" || code="000" + if [[ "$code" == "200" ]]; then + echo "crates.io is serving snapper-fmt ${version}" + exit 0 + fi + if (( SECONDS >= deadline )); then + echo "snapper-fmt ${version} did not appear on crates.io within 20 minutes (last HTTP status ${code})" >&2 + exit 1 + fi + echo "crates.io does not have snapper-fmt ${version} yet (HTTP ${code}); retrying in 30s" + sleep 30 + done - name: Dry-run prebuilt installs env: TAG: ${{ needs.plan.outputs.tag }} @@ -392,6 +419,6 @@ jobs: x86_64-unknown-linux-musl \ x86_64-pc-windows-msvc do - cargo binstall --dry-run --no-confirm --disable-strategies compile \ + cargo binstall --dry-run --no-confirm --disable-strategies quick-install,compile \ --targets "$target" "snapper-fmt@${version}" done