Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
313 changes: 282 additions & 31 deletions src/linekind.rs

Large diffs are not rendered by default.

42 changes: 7 additions & 35 deletions src/normalize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1172,9 +1172,9 @@ fn group_paragraphs(lines: &[Line]) -> Vec<Paragraph> {
blank_pending = false;
i += 1;
}
kind if is_preformatted_kind(kind) => {
kind if kind.is_preformatted() => {
let start = i;
while i < lines.len() && is_preformatted_kind(lines[i].kind) {
while i < lines.len() && lines[i].kind.is_preformatted() {
i += 1;
}
paragraphs.push(Paragraph {
Expand All @@ -1184,10 +1184,11 @@ fn group_paragraphs(lines: &[Line]) -> Vec<Paragraph> {
});
blank_pending = false;
}
LineKind::Prose
| LineKind::DoxygenTag
| LineKind::ListItem
| LineKind::SetextUnderline => {
// Everything "is_preformatted" answered no to, which the guard
// above has already taken the yes-cases from. A catch-all rather
// than a list, so a new variant lands where its own answer says it
// belongs instead of being frozen by a trailing wildcard.
_ => {
let start = i;
i += 1;
while i < lines.len()
Expand All @@ -1203,40 +1204,11 @@ fn group_paragraphs(lines: &[Line]) -> Vec<Paragraph> {
});
blank_pending = false;
}
_ => {
paragraphs.push(Paragraph {
kind: ParagraphKind::Preformatted,
line_indices: vec![i],
preceded_by_blank: blank_pending,
});
blank_pending = false;
i += 1;
}
}
}
paragraphs
}

fn is_preformatted_kind(k: LineKind) -> bool {
matches!(
k,
LineKind::FenceOpen
| LineKind::FenceContent
| LineKind::FenceClose
| LineKind::DoxyVerbatimOpen
| LineKind::DoxyVerbatimContent
| LineKind::DoxyVerbatimClose
| LineKind::IndentedCode
| LineKind::TableRow
| LineKind::Blockquote
| LineKind::ReferenceLink
| LineKind::Metadata
| LineKind::LabelRow
| LineKind::Banner
| LineKind::Art
)
}

fn starts_new_prose_paragraph(l: &Line) -> bool {
let t = l.text.trim_start();
starts_with_word(t, "Return") || starts_with_word(t, "Returns")
Expand Down
38 changes: 33 additions & 5 deletions src/parse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ use tree_sitter::{Node, Parser, Tree};

use crate::signature::{collect_param_shifts, manpage_relocate_target};
use crate::textline::{
block_is_doc, fence_marker_run, is_art, is_horizontal_rule, is_indented_code, is_table_row,
BookendKind, block_is_doc, bookend_match, fence_marker_run, is_art, is_assignment_line,
is_horizontal_rule, is_indented_code, is_table_row,
};

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
Expand Down Expand Up @@ -547,10 +548,37 @@ fn block_body_lines(text: &str) -> Vec<String> {
}

fn block_merge_has_preformatted(c: &Comment) -> bool {
marker_kind(&c.text) == Some(MarkerKind::PlainBlock)
&& block_body_lines(&c.text)
.iter()
.any(|line| block_line_is_preformatted(line))
if marker_kind(&c.text) != Some(MarkerKind::PlainBlock) {
return false;
}
let body = block_body_lines(&c.text);
body.iter().any(|line| block_line_is_preformatted(line)) || has_assignment_run(&body)
}

/// The merge-stage twin of the classifier's assignment-row rule. A mapping
/// table is preformatted at the classify stage, so merging a neighboring
/// comment into it would pull unrelated prose inside the frozen block.
///
/// This has to answer exactly what the classifier would answer for this
/// comment standing alone, in both directions. Under-blocking merges a comment
/// into a frozen table; over-blocking refuses a merge the classifier would
/// have been happy with, leaving two comments where one belongs. Sharing
/// Sharing "is_assignment_line" is what keeps the two answers identical, over
/// the same
/// two-adjacent-rows threshold the classifier uses; a lone row is a sentence
/// there too, and must not block a merge.
fn has_assignment_run(body: &[String]) -> bool {
// Through the bookend, because "strip_decorative_bookends" runs between
// this stage and the classifier: the classifier will see "base = 0x40"
// where this sees "==== base = 0x40 ====", and answering on the raw line
// lets a prose comment merge into a table that then freezes around it. A
// maximal run of two or more exists exactly when some adjacent pair both
// qualify, so the gate needs no run marking of its own.
let row = |l: &String| match bookend_match(l) {
Some(BookendKind::Labeled(label)) => is_assignment_line(&label),
_ => is_assignment_line(l),
};
body.windows(2).any(|w| row(&w[0]) && row(&w[1]))
}

fn block_line_is_preformatted(line: &str) -> bool {
Expand Down
23 changes: 14 additions & 9 deletions src/reflow.rs
Original file line number Diff line number Diff line change
Expand Up @@ -191,15 +191,20 @@ fn emit_paragraphs(
for &li in &para.line_indices {
let raw = &doc.lines[li].raw;

// A banner row is the one preformatted kind whose bytes are
// ordinary text, not layout: re-emit it behind the
// canonical prefix so a drifted "**" marker and a stripped
// decorative bookend land like every reflowed sibling. The
// body itself still goes out verbatim, unwrapped. Metadata
// keeps raw replay: a license block is not ours to retouch.
if matches!(doc.lines[li].kind, LineKind::LabelRow | LineKind::Banner)
&& !(skip_first_emit_inline_opener && li == 0)
{
// A banner, label, or assignment row is a preformatted kind
// whose bytes are ordinary text, not layout: re-emit it
// behind the canonical prefix so a drifted "**" marker and
// a stripped decorative bookend land like every reflowed
// sibling. The body itself still goes out verbatim,
// unwrapped. Metadata keeps raw replay: a license block is
// not ours to retouch.
//
// Line 0 of an inline-opener block included: raw replay
// would hand back the pre-strip bytes for that one line
// while its siblings keep the strip, and the next pass
// would settle on different ones. See
// "preformatted_label_row_on_an_inline_opener_keeps_its_strip".
if doc.lines[li].kind.emits_canonically() {
out.push(
format!("{prefix}{}", doc.lines[li].text)
.trim_end()
Expand Down
88 changes: 88 additions & 0 deletions src/textline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,80 @@ pub(crate) fn is_table_row(body: &str) -> bool {
t.matches('|').count() >= 2 || (t.contains('\t') && t.split_whitespace().count() >= 3)
}

/// Narrows a row mask to the rows that sit in a maximal run of two or more
/// adjacent ones: the shape that freezes instead of packing into a paragraph.
///
/// Requiring two is what keeps a sentence out. A lone "Note: ..." or
/// "count = zero when the queue drained" is prose that happens to open like a
/// row, and wrapping it is correct. Two or more adjacent rows are not one
/// wrapped sentence, so the run freezes whole and whatever follows is simply
/// the next paragraph -- releasing its last row to a prose tail would split
/// one logical table, which is the damage this rule exists to prevent.
pub(crate) fn keep_complete_row_runs(is_row: &mut [bool]) {
let mut prev = false;
for i in 0..is_row.len() {
let cur = is_row[i];
let next = is_row.get(i + 1).copied().unwrap_or(false);
is_row[i] = cur && (prev || next);
prev = cur;
}
}

/// One "X = Y" mapping row: a compact assignment or formula that documents a
/// mapping rather than stating a sentence, so its one-row layout is the
/// content. "ir->imm = immediate", "flags |= MASK", "imm = value".
///
/// Shared by the classify stage and the merge stage, which must agree byte
/// for byte on what a mapping row is.
///
/// What keeps prose out is the left-hand side: it must be exactly one
/// code-like token. A sentence that happens to contain an equals sign
/// ("the default width = 80 columns") has spaces on the left and is rejected
/// here; a sentence that happens to START with one ("count = zero when the
/// queue drained") is rejected by the run rule at the call site, not here.
pub(crate) fn is_assignment_line(body: &str) -> bool {
let t = body.trim();

// The operator is the first '='. If it turns out to compare rather than
// assign, the '=' it leaves behind on the left fails the token test below,
// so no later '=' could rescue the line anyway.
let Some((lhs, rhs)) = t.split_once('=') else {
return false;
};

// "==" / "===" compare, "=>" is an arrow and "=<" is nothing at all; a
// right side of nothing is not a mapping either. Tested unspaced on
// purpose, so a genuine value like "x = <unknown>" still reads as one.
if rhs.starts_with(['=', '>', '<']) || rhs.trim().is_empty() {
return false;
}

let b = lhs.as_bytes();
let last = b.last().copied();
let prev = b.len().checked_sub(2).map(|j| b[j]);
let left = match last {
// "!=", and bare "<=" / ">=", compare; "<<=" / ">>=" assign.
Some(b'!') => return false,
Some(c @ (b'<' | b'>')) if prev != Some(c) => return false,
Some(b'<' | b'>') => &lhs[..lhs.len() - 2],
// Compound assignment: "+=", "-=", "*=", "/=", "%=", "|=", "&=", "^=".
Some(b'+' | b'-' | b'*' | b'/' | b'%' | b'|' | b'&' | b'^') => &lhs[..lhs.len() - 1],
_ => lhs,
}
.trim_end();

// One token, code-shaped, and short enough to be an identifier rather than
// a clause. Every accepted byte is ASCII, so the byte length is the char
// count; the cap is a sanity bound, not a real limit, and has to clear
// names like "ctx->sub->field[MAX_INDEX]".
!left.is_empty()
&& left.len() <= 48
&& left.bytes().any(|c| c.is_ascii_alphabetic() || c == b'_')
&& left
.bytes()
.all(|c| c.is_ascii_alphanumeric() || b"_.->:[]()*%$".contains(&c))
}

pub(crate) fn is_indented_code(body: &str) -> bool {
// Intentional content alignment: ≥2 leading spaces OR ≥1 leading tab. A
// single leading ASCII space is too common as a wrap artifact (it can
Expand Down Expand Up @@ -135,6 +209,20 @@ pub(crate) fn is_art(body: &str) -> bool {
if has_alpha_word_min4(body) {
return false;
}

// An assignment row's operator is content, not drawing. "=" is a member of
// the art alphabet, so a short mapping row like "a += b" clears the density
// threshold on the very characters that make it a row, and art would claim
// it after the run rule had deliberately declined to freeze it -- stranding
// the tail of the paragraph it belongs to, the damage the run rule exists
// to avoid. Whether a mapping row freezes is the run rule's alone. A
// rule-decorated line is excepted: released to prose, the packer can join
// "x += y ---" to a "--- label" above it and build a line ruled on both
// ends, which the next pass strips. Different bytes each pass, and
// "--check" would never settle. Decoration outranks the mapping reading.
if is_assignment_line(body) && bookend_match(body).is_none() && !one_sided_banner(body) {
return false;
}
if ascii_art_density(body) {
return true;
}
Expand Down
9 changes: 9 additions & 0 deletions tests/convergence.rs
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,15 @@ const VOCAB: &[&str] = &[
"\\param.h",
"@see.com",
"user@example.com",
// Assignment operators, which "textline::is_assignment_line" keys on.
// Without these no token in this vocabulary can spell a mapping row, so
// the packer could never forge one at a line start and the run rule in
// "linekind" would be unreachable from this sweep. "==" is here to be
// rejected: it is the comparison the row rule must not claim.
"=",
"==",
"+=",
"x=1",
// Rule runs, which the bookend and one-sided-banner rules key on. A run
// that lands alone in a paragraph is decoration and freezes; one inside a
// paragraph is an em-dash and must keep reflowing.
Expand Down
Loading