From 1cc32367e3adf84651b058c5b97bb9b52050ff46 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:35:58 +0000 Subject: [PATCH 1/9] feat(formats): add plain text and Markdown input Plain text (always on) goes through the IR: paragraphs on blank lines, hard breaks on single newlines. Markdown (new `markdown` feature, in all_formats) is re-parsed with pulldown-cmark into the IR and rendered back, so raw HTML never passes through. Both decode UTF-8/UTF-16 BOMs, drop control characters and warn on lossy decoding. Neither is sniffed from bytes. Threat model T7, invariant S13. --- .github/workflows/ci.yml | 6 +- CHANGELOG.md | 8 + Cargo.lock | 18 + Cargo.toml | 1 + README.md | 6 +- crates/markkit-cli/Cargo.toml | 1 + crates/markkit-cli/tests/cli_integration.rs | 31 ++ crates/markkit/Cargo.toml | 6 +- crates/markkit/src/extract.rs | 45 +- crates/markkit/src/format.rs | 8 +- crates/markkit/src/lib.rs | 5 + crates/markkit/src/markdown.rs | 505 ++++++++++++++++++++ crates/markkit/src/text.rs | 164 +++++++ crates/markkit/tests/text_integration.rs | 228 +++++++++ knowledge/formats/format-support.md | 37 +- knowledge/log.md | 1 + knowledge/operations/testing.md | 3 +- knowledge/security/security-model.md | 5 + knowledge/security/threat-model.md | 25 +- 19 files changed, 1084 insertions(+), 19 deletions(-) create mode 100644 crates/markkit/src/markdown.rs create mode 100644 crates/markkit/src/text.rs create mode 100644 crates/markkit/tests/text_integration.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 647799b..a70dd81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,9 +78,9 @@ jobs: run: cargo test --workspace --all-features - name: Run tests (office formats, no HTTP client) run: | - cargo test -p markkit --no-default-features --features docx,spreadsheet - if cargo tree -p markkit --no-default-features --features docx,spreadsheet -e normal | grep -E 'reqwest|hyper'; then - echo "HTTP client leaked into the docx,spreadsheet feature set" + cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown + if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown -e normal | grep -E 'reqwest|hyper'; then + echo "HTTP client leaked into the docx,spreadsheet,markdown feature set" exit 1 fi - name: Run examples diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b78190..fd38d68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- Plain text input (`text/plain`, `.txt`), always on: paragraphs and hard breaks through the IR, + UTF-8/UTF-16 BOM decoding, control characters dropped, `invalid_utf8` warning on lossy decode. +- Markdown input (`markdown` feature, in `all_formats`; `text/markdown`, `.md`): re-parsed with + pulldown-cmark and rendered back, so raw HTML never passes through. Front matter `title:` or the + first `# ` heading becomes `Extraction::title`. Available in the CLI, Python and Node bindings. + ### Security - `extract_url` (CLI, Python and Node `extract_url` too) now refuses non-public destinations by default: diff --git a/Cargo.lock b/Cargo.lock index 01303b3..edbaa19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1188,6 +1188,7 @@ dependencies = [ "criterion", "html5ever", "proptest", + "pulldown-cmark", "quick-xml", "reqwest", "scraper", @@ -1591,6 +1592,17 @@ dependencies = [ "unarray", ] +[[package]] +name = "pulldown-cmark" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278e9268c93b9e498a2da46c34d90ba93a3255eb99009646398f34bc73ab36eb" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + [[package]] name = "pyo3" version = "0.30.1" @@ -2525,6 +2537,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + [[package]] name = "unicode-ident" version = "1.0.27" diff --git a/Cargo.toml b/Cargo.toml index a67f998..69bd085 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ calamine = "0.37" # Excel/spreadsheet (XLSX/XLS/ODS) zip = { version = "9", default-features = false, features = ["deflate"] } # OOXML/ODF containers; matches calamine's zip quick-xml = "0.42" # DOCX/ODS XML parsing cfb = "0.15" # XLS (OLE2) container pre-scan +pulldown-cmark = { version = "0.13", default-features = false } # Markdown input # Logging tracing = "0.1" diff --git a/README.md b/README.md index 0d9e55a..1786881 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ a Rust library, a CLI, and native bindings for Python and Node.js. - **HTML** – Web pages and HTML documents; nav, header, footer, scripts, styles and forms stripped by default - **DOCX** – Headings, lists, tables, links, emphasis - **Spreadsheets** – XLSX/XLSM, XLS and ODS workbooks to markdown tables, one section per sheet +- **Plain text / Markdown** – Text to paragraphs; Markdown normalized through the same IR (raw HTML never passed through) - **Format detection** – Content type, then file extension, then magic bytes - **URL fetching** – Fetch and extract in one call (`http_client` feature) - **Configurable** – CSS include/exclude selectors, link/image preservation, resource limits @@ -31,6 +32,8 @@ a Rust library, a CLI, and native bindings for Python and Node.js. | HTML / XHTML | Supported | `html` (default) | | DOCX | Supported | `docx` | | XLSX, XLSM, XLS, ODS | Supported | `spreadsheet` | +| Plain text | Supported | always on | +| Markdown | Supported | `markdown` | | PDF | Planned | `pdf` | | DOC, PPTX, XLSB | Detected, rejected explicitly | – | @@ -154,7 +157,8 @@ let markkit = Markkit::builder() | `docx` | No | DOCX conversion (zip + quick-xml) | | `spreadsheet` | No | XLSX/XLSM, XLS, ODS conversion (calamine) | | `pdf` | No | Placeholder; PDF currently returns `UnsupportedFormat` | -| `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet` | +| `markdown` | No | Markdown normalization (pulldown-cmark) | +| `all_formats` | No | `html`, `pdf`, `docx`, `spreadsheet`, `markdown` | | `logging` | No | Reserved: pulls in `tracing`, emits no events yet | `http_client` is optional. Embedders that fetch content themselves can build without diff --git a/crates/markkit-cli/Cargo.toml b/crates/markkit-cli/Cargo.toml index 69495f0..b7bcc30 100644 --- a/crates/markkit-cli/Cargo.toml +++ b/crates/markkit-cli/Cargo.toml @@ -24,6 +24,7 @@ all_formats = ["markkit/all_formats"] pdf = ["markkit/pdf"] docx = ["markkit/docx"] spreadsheet = ["markkit/spreadsheet"] +markdown = ["markkit/markdown"] [dependencies] markkit = { path = "../markkit", version = "0.1.1", features = ["http_client"] } diff --git a/crates/markkit-cli/tests/cli_integration.rs b/crates/markkit-cli/tests/cli_integration.rs index 2d50944..c3e23cc 100644 --- a/crates/markkit-cli/tests/cli_integration.rs +++ b/crates/markkit-cli/tests/cli_integration.rs @@ -103,6 +103,37 @@ fn extract_from_file() { assert!(stdout.contains("Paragraph"), "output: {stdout}"); } +#[test] +fn extract_text_and_markdown_files() { + let dir = tempfile::tempdir().unwrap(); + let cases = [ + ( + "notes.txt", + "line one\nline two\n", + "line one \nline two\n", + ), + ( + "doc.md", + "Title\n=====\n\n* a\n\n\n", + "# Title\n\n- a\n", + ), + ]; + for (name, input, expected) in cases { + let file_path = dir.path().join(name); + std::fs::write(&file_path, input).unwrap(); + let output = markkit_bin() + .args(["extract", "--file", file_path.to_str().unwrap()]) + .output() + .expect("failed to run markkit"); + assert!( + output.status.success(), + "{name} stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout), expected, "{name}"); + } +} + #[test] fn extract_json_output() { let mut child = markkit_bin() diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index f6f5d96..082fad2 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -19,9 +19,10 @@ html = ["dep:scraper", "dep:html5ever"] pdf = [] docx = ["dep:zip", "dep:quick-xml"] spreadsheet = ["dep:calamine", "dep:zip", "dep:quick-xml", "dep:cfb"] +markdown = ["dep:pulldown-cmark"] http_client = ["dep:reqwest", "tokio/net"] logging = ["dep:tracing"] -all_formats = ["html", "pdf", "docx", "spreadsheet"] +all_formats = ["html", "pdf", "docx", "spreadsheet", "markdown"] [dependencies] # Async @@ -46,6 +47,9 @@ quick-xml = { workspace = true, optional = true } calamine = { workspace = true, optional = true } cfb = { workspace = true, optional = true } +# Markdown (optional) +pulldown-cmark = { workspace = true, optional = true } + # HTTP client (optional) reqwest = { workspace = true, optional = true } diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 06ce32b..f264310 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -10,12 +10,14 @@ use crate::format::{DocumentHint, Format}; pub struct Extraction { /// Rendered Markdown. pub markdown: String, - /// Document title when known (DOCX core properties or `Title` style). + /// Document title when known (DOCX core properties or `Title` style, + /// Markdown front matter `title:` or first `# ` heading). pub title: Option, /// Format the document was parsed as. pub format: Format, /// Non-fatal issues, each prefixed with a stable code - /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`). + /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`, + /// `invalid_utf8:`). pub warnings: Vec, /// Format-specific counters. pub stats: ExtractionStats, @@ -31,9 +33,9 @@ pub struct ExtractionStats { pub sheets: Option, /// Data rows rendered across all sheets (header rows excluded). pub rows: Option, - /// Non-empty paragraphs, headings and list items (DOCX). + /// Non-empty paragraphs, headings and list items (DOCX, plain text, Markdown). pub paragraphs: Option, - /// Tables (DOCX). + /// Tables (DOCX, Markdown). pub tables: Option, } @@ -142,7 +144,8 @@ impl Markkit { .map(|e| e.markdown) } - /// Convert a document (HTML, DOCX, XLSX/XLS/ODS) to Markdown with metadata. + /// Convert a document (HTML, DOCX, XLSX/XLS/ODS, plain text, Markdown) to + /// Markdown with metadata. /// /// The format is detected from `hint.content_type`, then the extension of /// `hint.file_name`, then magic bytes. Parsing is CPU-bound and runs inline; @@ -182,6 +185,8 @@ impl Markkit { Format::Pdf => self.extract_pdf_sync(bytes), Format::Docx => self.extract_docx(bytes), Format::Xlsx | Format::Xls | Format::Ods => self.extract_spreadsheet(bytes), + Format::PlainText => Ok(self.extract_plain_text(bytes)), + Format::Markdown => self.extract_markdown(bytes), Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -195,6 +200,36 @@ impl Markkit { } } + fn extract_plain_text(&self, bytes: &[u8]) -> Extraction { + let decoded = crate::text::decode(bytes); + let (doc, paragraphs) = crate::text::plain_to_ir(&decoded.text); + let mut renderer = crate::render::MarkdownRenderer::new(); + let mut e = Extraction::new(renderer.render(&doc), Format::PlainText); + e.warnings = decoded.warnings; + e.stats.paragraphs = Some(paragraphs); + e + } + + #[cfg(feature = "markdown")] + fn extract_markdown(&self, bytes: &[u8]) -> crate::Result { + let decoded = crate::text::decode(bytes); + let out = crate::markdown::parse(&decoded.text, &self.config); + let mut renderer = crate::render::MarkdownRenderer::new(); + let mut e = Extraction::new(renderer.render(&out.doc), Format::Markdown); + e.title = out.title; + e.warnings = decoded.warnings; + e.stats.paragraphs = Some(out.paragraphs); + e.stats.tables = Some(out.tables); + Ok(e) + } + + #[cfg(not(feature = "markdown"))] + fn extract_markdown(&self, _bytes: &[u8]) -> crate::Result { + Err(Error::UnsupportedFormat( + "Markdown support requires the 'markdown' feature".to_string(), + )) + } + #[cfg(feature = "docx")] fn extract_docx(&self, bytes: &[u8]) -> crate::Result { let out = guarded("DOCX", Error::DocxExtract, || { diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index cf632a7..4c918c3 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -29,7 +29,9 @@ pub enum Format { /// Office Open XML presentation (`.pptx`). Detected but not supported. Pptx, Csv, + /// Plain text (`text/plain`, `.txt`). Never sniffed from bytes. PlainText, + /// Markdown (`text/markdown`, `.md`), passed through normalized. Markdown, Unknown, } @@ -110,7 +112,7 @@ impl Format { Format::Csv } else if ct.contains("text/plain") { Format::PlainText - } else if ct.contains("text/markdown") { + } else if ct.contains("text/markdown") || ct.contains("text/x-markdown") { Format::Markdown } else { Format::Unknown @@ -129,8 +131,8 @@ impl Format { "ods" => Format::Ods, "pptx" | "pptm" | "ppt" => Format::Pptx, "csv" | "tsv" => Format::Csv, - "txt" => Format::PlainText, - "md" | "markdown" => Format::Markdown, + "txt" | "text" => Format::PlainText, + "md" | "markdown" | "mdown" | "mkd" | "mkdn" => Format::Markdown, _ => Format::Unknown, } } diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index d6453e2..bfca7ab 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -34,6 +34,8 @@ //! - **HTML** (`html`, default): Convert HTML to markdown with configurable rules //! - **DOCX** (`docx`): Convert Word documents //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLS and ODS workbooks to tables +//! - **Plain text** (always on): `text/plain`, `.txt` +//! - **Markdown** (`markdown`): `text/markdown`, `.md`, normalized through the IR //! - **PDF** (`pdf`): Planned; currently returns [`Error::UnsupportedFormat`] //! - **URL fetching** (`http_client`, default): Fetch and extract from URLs via reqwest //! @@ -53,9 +55,12 @@ pub mod format; #[cfg(feature = "html")] pub mod html; pub mod ir; +#[cfg(feature = "markdown")] +mod markdown; pub mod render; #[cfg(feature = "spreadsheet")] mod spreadsheet; +mod text; #[cfg(feature = "docx")] mod xml; diff --git a/crates/markkit/src/markdown.rs b/crates/markkit/src/markdown.rs new file mode 100644 index 0000000..cda2327 --- /dev/null +++ b/crates/markkit/src/markdown.rs @@ -0,0 +1,505 @@ +//! Markdown input: parse with pulldown-cmark into the IR, render back. +//! +//! Decisions: +//! - Markdown is re-parsed, not passed through, so every output goes through +//! the same IR and renderer (and the same output-safety rules). Raw HTML is +//! never copied: HTML blocks go through the HTML parser when the `html` +//! feature is on (scripts/styles stripped), otherwise they are dropped; +//! inline HTML tags are dropped and the text between them kept. +//! - Extensions: GFM tables, strikethrough, task lists, YAML front matter. +//! Front matter is not rendered; its `title:` becomes the title, otherwise +//! the first level-1 heading. +//! - Soft line breaks become spaces (paragraphs reflow); hard breaks stay. +//! - pulldown-cmark is iterative; the IR is built with an explicit stack and +//! capped at `MAX_DEPTH` open containers. Deeper content is flattened into +//! the deepest container so the recursive renderer cannot overflow. + +use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, MetadataBlockKind, Options, Parser, Tag}; + +use crate::config::MarkkitBuilder; +use crate::ir::{Document, ListItem, Node}; + +/// Max nested containers (block quotes, lists, emphasis, …) kept as structure. +const MAX_DEPTH: usize = 200; + +/// Parsed Markdown document. +pub(crate) struct ParsedMarkdown { + pub doc: Document, + pub title: Option, + pub paragraphs: usize, + pub tables: usize, +} + +enum Kind { + Root, + Para, + Heading(u8), + Quote, + List { + ordered: bool, + start: Option, + }, + Item, + Code { + language: Option, + }, + Table, + Head, + Row, + Cell, + Em, + Strong, + Strike, + Link { + url: String, + title: Option, + }, + Image { + src: String, + }, + Html, + Meta, + /// Beyond `MAX_DEPTH` or an unsupported container: children go to the parent. + Flat, +} + +struct Frame { + kind: Kind, + nodes: Vec, + items: Vec, + cells: Vec>, + rows: Vec>>, + /// Code, image alt, raw HTML or front matter text. + text: String, +} + +impl Frame { + fn new(kind: Kind) -> Self { + Self { + kind, + nodes: Vec::new(), + items: Vec::new(), + cells: Vec::new(), + rows: Vec::new(), + text: String::new(), + } + } +} + +/// What a finished frame hands to its parent. +enum Out { + Nodes(Vec), + Item(ListItem), + Cell(Vec), + Row(Vec>), + Head(Vec>), +} + +struct Builder<'c> { + #[cfg_attr(not(feature = "html"), allow(dead_code))] + config: &'c MarkkitBuilder, + stack: Vec, + meta_title: Option, + paragraphs: usize, + tables: usize, +} + +/// Parse Markdown text into the IR. +pub(crate) fn parse(text: &str, config: &MarkkitBuilder) -> ParsedMarkdown { + let opts = Options::ENABLE_TABLES + | Options::ENABLE_STRIKETHROUGH + | Options::ENABLE_TASKLISTS + | Options::ENABLE_YAML_STYLE_METADATA_BLOCKS; + let mut b = Builder { + config, + stack: vec![Frame::new(Kind::Root)], + meta_title: None, + paragraphs: 0, + tables: 0, + }; + for event in Parser::new_ext(text, opts) { + b.event(event); + } + while b.stack.len() > 1 { + b.end(); + } + let root = b.stack.pop().map(|f| f.nodes).unwrap_or_default(); + let title = b.meta_title.or_else(|| first_h1(&root)); + ParsedMarkdown { + doc: Document::new(root), + title, + paragraphs: b.paragraphs, + tables: b.tables, + } +} + +impl Builder<'_> { + fn top(&mut self) -> &mut Frame { + self.stack.last_mut().expect("root frame") + } + + fn event(&mut self, event: Event<'_>) { + match event { + Event::Start(tag) => self.start(tag), + Event::End(_) => self.end(), + Event::Text(t) => self.text(&t), + Event::Code(c) => self.push(Node::InlineCode(c.into_string())), + Event::Html(h) | Event::InlineHtml(h) => { + if matches!(self.top().kind, Kind::Html) { + self.top().text.push_str(&h); + } + } + Event::SoftBreak => self.text(" "), + Event::HardBreak => self.push(Node::LineBreak), + Event::Rule => self.push(Node::ThematicBreak), + Event::TaskListMarker(done) => self.text(if done { "[x] " } else { "[ ] " }), + Event::FootnoteReference(r) => self.text(&format!("[^{r}]")), + Event::InlineMath(m) | Event::DisplayMath(m) => { + self.push(Node::InlineCode(m.into_string())) + } + } + } + + fn start(&mut self, tag: Tag<'_>) { + let kind = if self.stack.len() > MAX_DEPTH { + Kind::Flat + } else { + match tag { + Tag::Paragraph => Kind::Para, + Tag::Heading { level, .. } => Kind::Heading(heading_level(level)), + Tag::BlockQuote(_) => Kind::Quote, + Tag::CodeBlock(kind) => Kind::Code { + language: match kind { + CodeBlockKind::Fenced(info) => info + .split_whitespace() + .next() + .map(|s| s.chars().filter(|c| *c != '`').collect::()) + .filter(|s| !s.is_empty()), + CodeBlockKind::Indented => None, + }, + }, + Tag::HtmlBlock => Kind::Html, + Tag::List(start) => Kind::List { + ordered: start.is_some(), + start, + }, + Tag::Item => Kind::Item, + Tag::Table(_) => Kind::Table, + Tag::TableHead => Kind::Head, + Tag::TableRow => Kind::Row, + Tag::TableCell => Kind::Cell, + Tag::Emphasis => Kind::Em, + Tag::Strong => Kind::Strong, + Tag::Strikethrough => Kind::Strike, + Tag::Link { + dest_url, title, .. + } => Kind::Link { + url: dest_url.into_string(), + title: Some(title.into_string()).filter(|t| !t.is_empty()), + }, + Tag::Image { dest_url, .. } => Kind::Image { + src: dest_url.into_string(), + }, + Tag::MetadataBlock(MetadataBlockKind::YamlStyle) => Kind::Meta, + _ => Kind::Flat, + } + }; + self.stack.push(Frame::new(kind)); + } + + fn end(&mut self) { + if self.stack.len() <= 1 { + return; + } + let f = self.stack.pop().expect("non-root frame"); + let out = match f.kind { + Kind::Root | Kind::Flat => Out::Nodes(f.nodes), + Kind::Para => { + self.paragraphs += 1; + Out::Nodes(vec![Node::Paragraph { children: f.nodes }]) + } + Kind::Heading(level) => { + self.paragraphs += 1; + Out::Nodes(vec![Node::Heading { + level, + children: f.nodes, + }]) + } + Kind::Quote => Out::Nodes(vec![Node::BlockQuote { + children: wrap_inline(f.nodes), + }]), + Kind::List { ordered, start } => Out::Nodes(vec![Node::List { + ordered, + start, + items: f.items, + }]), + Kind::Item => { + self.paragraphs += 1; + let has_block = f.nodes.iter().any(is_block); + let children = if has_block { + wrap_inline(f.nodes) + } else { + f.nodes + }; + Out::Item(ListItem::new(children)) + } + Kind::Code { language } => Out::Nodes(vec![Node::CodeBlock { + language, + code: f.text, + }]), + Kind::Table => { + self.tables += 1; + let mut nodes = vec![Node::Table { + headers: f.cells, + rows: f.rows, + }]; + nodes.extend(f.nodes); + Out::Nodes(nodes) + } + Kind::Head => Out::Head(f.cells), + Kind::Row => Out::Row(f.cells), + Kind::Cell => Out::Cell(f.nodes), + Kind::Em => Out::Nodes(vec![Node::Italic { children: f.nodes }]), + Kind::Strong => Out::Nodes(vec![Node::Bold { children: f.nodes }]), + Kind::Strike => Out::Nodes(vec![Node::Strikethrough { children: f.nodes }]), + Kind::Link { url, title } => Out::Nodes(vec![Node::Link { + url, + title, + children: f.nodes, + }]), + Kind::Image { src } => Out::Nodes(vec![Node::Image { src, alt: f.text }]), + Kind::Html => Out::Nodes(self.html_block(&f.text)), + Kind::Meta => { + if self.meta_title.is_none() { + self.meta_title = yaml_title(&f.text); + } + Out::Nodes(Vec::new()) + } + }; + self.accept(out); + } + + /// Hand a finished frame's output to the new top frame. + fn accept(&mut self, out: Out) { + let parent = self.top(); + match (&parent.kind, out) { + (Kind::List { .. }, Out::Item(item)) => parent.items.push(item), + (Kind::List { .. }, Out::Nodes(nodes)) if !nodes.is_empty() => { + parent.items.push(ListItem::new(nodes)) + } + (Kind::Head | Kind::Row, Out::Cell(cell)) => parent.cells.push(cell), + (Kind::Table, Out::Head(cells)) => parent.cells = cells, + (Kind::Table, Out::Row(cells)) => parent.rows.push(cells), + (_, Out::Nodes(nodes) | Out::Cell(nodes)) => self.push_all(nodes), + (_, Out::Item(item)) => self.push_all(item.children), + (_, Out::Row(cells) | Out::Head(cells)) => { + for cell in cells { + self.push_all(cell); + self.push(Node::Text(" ".into())); + } + } + } + } + + fn push_all(&mut self, nodes: Vec) { + for n in nodes { + self.push(n); + } + } + + fn push(&mut self, node: Node) { + let top = self.top(); + match top.kind { + // Alt text, code and front matter take plain text only. + Kind::Image { .. } | Kind::Code { .. } => plain_text(&node, &mut top.text), + Kind::Html | Kind::Meta => {} + Kind::List { .. } => top.items.push(ListItem::new(vec![node])), + Kind::Head | Kind::Row => top.cells.push(vec![node]), + _ => top.nodes.push(node), + } + } + + fn text(&mut self, t: &str) { + let top = self.top(); + match top.kind { + Kind::Image { .. } | Kind::Code { .. } | Kind::Meta => top.text.push_str(t), + Kind::Html => {} + _ => { + if let Some(Node::Text(prev)) = top.nodes.last_mut() { + prev.push_str(t); + } else { + self.push(Node::Text(t.to_string())); + } + } + } + } + + #[cfg(feature = "html")] + fn html_block(&self, html: &str) -> Vec { + crate::html::HtmlParser::new(self.config) + .parse(html) + .map(|d| d.nodes) + .unwrap_or_default() + } + + #[cfg(not(feature = "html"))] + fn html_block(&self, _html: &str) -> Vec { + Vec::new() + } +} + +fn heading_level(level: HeadingLevel) -> u8 { + match level { + HeadingLevel::H1 => 1, + HeadingLevel::H2 => 2, + HeadingLevel::H3 => 3, + HeadingLevel::H4 => 4, + HeadingLevel::H5 => 5, + HeadingLevel::H6 => 6, + } +} + +fn is_block(node: &Node) -> bool { + matches!( + node, + Node::Heading { .. } + | Node::Paragraph { .. } + | Node::List { .. } + | Node::BlockQuote { .. } + | Node::CodeBlock { .. } + | Node::Table { .. } + | Node::ThematicBreak + ) +} + +/// Group runs of inline nodes into paragraphs (for containers mixing both). +fn wrap_inline(nodes: Vec) -> Vec { + let mut out = Vec::with_capacity(nodes.len()); + let mut run = Vec::new(); + for n in nodes { + if is_block(&n) { + if !run.is_empty() { + out.push(Node::Paragraph { + children: std::mem::take(&mut run), + }); + } + out.push(n); + } else { + run.push(n); + } + } + if !run.is_empty() { + out.push(Node::Paragraph { children: run }); + } + out +} + +/// Append the plain text of a node (iteratively; nodes may be nested). +fn plain_text(node: &Node, out: &mut String) { + let mut stack = vec![node]; + while let Some(n) = stack.pop() { + match n { + Node::Text(t) | Node::InlineCode(t) => out.push_str(t), + Node::CodeBlock { code, .. } => out.push_str(code), + Node::Image { alt, .. } => out.push_str(alt), + Node::LineBreak => out.push(' '), + Node::Heading { children, .. } + | Node::Paragraph { children } + | Node::BlockQuote { children } + | Node::Bold { children } + | Node::Italic { children } + | Node::Strikethrough { children } + | Node::Link { children, .. } => stack.extend(children.iter().rev()), + Node::List { items, .. } => { + stack.extend(items.iter().rev().flat_map(|i| i.children.iter().rev())) + } + Node::Table { headers, rows } => { + stack.extend(headers.iter().chain(rows.iter().flatten()).flatten().rev()) + } + Node::ThematicBreak => {} + } + } +} + +fn first_h1(nodes: &[Node]) -> Option { + nodes.iter().find_map(|n| match n { + Node::Heading { level: 1, .. } => { + let mut t = String::new(); + plain_text(n, &mut t); + let t = t.trim(); + (!t.is_empty()).then(|| t.to_string()) + } + _ => None, + }) +} + +/// `title:` from YAML front matter (top-level key, scalar value only). +fn yaml_title(yaml: &str) -> Option { + yaml.lines().find_map(|line| { + let v = line.strip_prefix("title:")?.trim(); + let v = v + .strip_prefix('"') + .and_then(|v| v.strip_suffix('"')) + .or_else(|| v.strip_prefix('\'').and_then(|v| v.strip_suffix('\''))) + .unwrap_or(v) + .trim(); + (!v.is_empty()).then(|| v.to_string()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn md(src: &str) -> String { + let p = parse(src, &MarkkitBuilder::default()); + crate::render::MarkdownRenderer::new().render(&p.doc) + } + + #[test] + fn roundtrip_common_blocks() { + let src = "# T\n\nSome *em* and **strong** and ~~del~~ `code`.\n\n> quote\n\n- a\n- b\n 1. c\n\n```rust\nfn x() {}\n```\n\n---\n\n[link](http://e.com \"t\") ![alt *x*](i.png)\n"; + assert_eq!( + md(src), + "# T\n\nSome *em* and **strong** and ~~del~~ `code`.\n\n> quote\n\n- a\n- b\n 1. c\n\n```rust\nfn x() {}\n```\n\n---\n\n[link](http://e.com \"t\") ![alt x](i.png)\n" + ); + } + + #[test] + fn table_and_tasks() { + assert_eq!( + md("| a | b |\n|---|---|\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n"), + "| a | b |\n| --- | --- |\n| 1 | 2 |\n\n- [x] done\n- [ ] todo\n" + ); + } + + #[test] + fn soft_breaks_reflow_hard_breaks_kept() { + assert_eq!(md("a\nb \nc\\\nd"), "a b \nc \nd\n"); + } + + #[test] + fn titles() { + let p = parse( + "---\ntitle: \"Doc\"\n---\n\n# H\n", + &MarkkitBuilder::default(), + ); + assert_eq!(p.title.as_deref(), Some("Doc")); + let p = parse("## sub\n\n# Real *one*\n", &MarkkitBuilder::default()); + assert_eq!(p.title.as_deref(), Some("Real one")); + assert_eq!(parse("text", &MarkkitBuilder::default()).title, None); + } + + #[test] + fn loose_item_with_nested_list() { + assert_eq!(md("- a **b**\n - c\n"), "- a **b**\n - c\n"); + } + + #[test] + fn deep_nesting_is_flattened() { + let src = format!("{}x", "> ".repeat(MAX_DEPTH * 5)); + let out = md(&src); + assert!(out.trim_end().ends_with('x')); + assert!(out.lines().next().unwrap().matches('>').count() <= MAX_DEPTH + 1); + } +} diff --git a/crates/markkit/src/text.rs b/crates/markkit/src/text.rs new file mode 100644 index 0000000..19e7999 --- /dev/null +++ b/crates/markkit/src/text.rs @@ -0,0 +1,164 @@ +//! Plain text input, plus the text decoding shared with Markdown input. +//! +//! Decisions: +//! - Always compiled (no dependencies, no feature gate); selected only by a +//! content type (`text/plain`) or extension (`.txt`). Bytes are never +//! sniffed as text, so unknown binaries stay `UnsupportedFormat` instead +//! of rendering as mojibake. +//! - Decoding: UTF-8 (BOM stripped), UTF-16 LE/BE when a BOM is present; +//! anything else is decoded lossily with an `invalid_utf8` warning. +//! - C0 control characters other than tab/newline, and DEL, are dropped so +//! terminal escapes and NULs never reach the output; CRLF/CR become LF. +//! - Plain text goes through the IR: blank lines separate paragraphs, single +//! newlines become hard breaks, text stays `Node::Text` so renderer-level +//! escaping applies uniformly. + +use crate::ir::{Document, Node}; + +/// Decoded text plus non-fatal warnings. +pub(crate) struct Decoded { + pub text: String, + pub warnings: Vec, +} + +/// Decode bytes to normalized text (see module docs). +pub(crate) fn decode(bytes: &[u8]) -> Decoded { + let mut warnings = Vec::new(); + let raw = if let Some(rest) = bytes.strip_prefix(b"\xEF\xBB\xBF") { + utf8(rest, &mut warnings) + } else if let Some(rest) = bytes.strip_prefix(b"\xFF\xFE") { + utf16(rest, u16::from_le_bytes, &mut warnings) + } else if let Some(rest) = bytes.strip_prefix(b"\xFE\xFF") { + utf16(rest, u16::from_be_bytes, &mut warnings) + } else { + utf8(bytes, &mut warnings) + }; + Decoded { + text: normalize(&raw), + warnings, + } +} + +fn utf8(bytes: &[u8], warnings: &mut Vec) -> String { + match std::str::from_utf8(bytes) { + Ok(s) => s.to_string(), + Err(_) => { + warnings.push("invalid_utf8: invalid UTF-8 sequences replaced with U+FFFD".into()); + String::from_utf8_lossy(bytes).into_owned() + } + } +} + +fn utf16(bytes: &[u8], read: fn([u8; 2]) -> u16, warnings: &mut Vec) -> String { + let units = bytes.chunks_exact(2).map(|c| read([c[0], c[1]])); + let mut lossy = !bytes.len().is_multiple_of(2); + let s: String = char::decode_utf16(units) + .map(|r| { + r.unwrap_or_else(|_| { + lossy = true; + char::REPLACEMENT_CHARACTER + }) + }) + .collect(); + if lossy { + warnings.push("invalid_utf8: invalid UTF-16 sequences replaced with U+FFFD".into()); + } + s +} + +/// CRLF/CR → LF; drop C0 controls (except `\t`, `\n`) and DEL. +fn normalize(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\r' => { + if chars.peek() == Some(&'\n') { + chars.next(); + } + out.push('\n'); + } + '\t' | '\n' => out.push(c), + c if c.is_ascii_control() => {} + c => out.push(c), + } + } + out +} + +/// Plain text → IR. Returns the document and the paragraph count. +pub(crate) fn plain_to_ir(text: &str) -> (Document, usize) { + let mut nodes = Vec::new(); + let mut para: Vec = Vec::new(); + let flush = |para: &mut Vec, nodes: &mut Vec| { + if !para.is_empty() { + nodes.push(Node::Paragraph { + children: std::mem::take(para), + }); + } + }; + for line in text.split('\n') { + let line = line.trim_end(); + if line.is_empty() { + flush(&mut para, &mut nodes); + continue; + } + if !para.is_empty() { + para.push(Node::LineBreak); + } + para.push(Node::Text(line.to_string())); + } + flush(&mut para, &mut nodes); + let count = nodes.len(); + (Document::new(nodes), count) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_strips_bom_and_controls() { + let d = decode(b"\xEF\xBB\xBFa\r\nb\rc\x00\x1b[31m\x7f\td"); + assert_eq!(d.text, "a\nb\nc[31m\td"); + assert!(d.warnings.is_empty()); + } + + #[test] + fn decode_utf16() { + let mut le = b"\xFF\xFE".to_vec(); + le.extend("hé".encode_utf16().flat_map(u16::to_le_bytes)); + assert_eq!(decode(&le).text, "hé"); + let mut be = b"\xFE\xFF".to_vec(); + be.extend("hé".encode_utf16().flat_map(u16::to_be_bytes)); + assert_eq!(decode(&be).text, "hé"); + // Odd length and lone surrogate are lossy, never panic. + let d = decode(b"\xFF\xFE\x00\xD8a"); + assert!(d.text.contains('\u{FFFD}')); + assert_eq!(d.warnings.len(), 1); + } + + #[test] + fn decode_invalid_utf8_warns() { + let d = decode(b"ok \xFF\xFE!"); + assert_eq!(d.text, "ok \u{FFFD}\u{FFFD}!"); + assert!(d.warnings[0].starts_with("invalid_utf8:")); + } + + #[test] + fn plain_paragraphs_and_breaks() { + let (doc, n) = plain_to_ir("a\nb \n\n\n c\n"); + assert_eq!(n, 2); + assert_eq!( + doc.nodes[0], + Node::Paragraph { + children: vec![ + Node::Text("a".into()), + Node::LineBreak, + Node::Text("b".into()) + ] + } + ); + assert_eq!(plain_to_ir("\n \n").1, 0); + } +} diff --git a/crates/markkit/tests/text_integration.rs b/crates/markkit/tests/text_integration.rs new file mode 100644 index 0000000..d55f846 --- /dev/null +++ b/crates/markkit/tests/text_integration.rs @@ -0,0 +1,228 @@ +//! Plain text and Markdown → Markdown integration and security tests. + +use markkit::format::Format; +use markkit::{DocumentHint, Error, Markkit}; + +fn extract(bytes: &[u8], hint: DocumentHint<'_>) -> markkit::Extraction { + Markkit::builder() + .build() + .extract_document_sync(bytes, hint) + .unwrap() +} + +#[test] +fn plain_text_by_content_type_and_extension() { + let e = extract( + b"Hello\nworld\n\nSecond paragraph\n", + DocumentHint::content_type("text/plain; charset=utf-8"), + ); + assert_eq!(e.format, Format::PlainText); + assert_eq!(e.markdown, "Hello \nworld\n\nSecond paragraph\n"); + assert_eq!(e.stats.paragraphs, Some(2)); + assert_eq!(e.title, None); + assert!(e.warnings.is_empty()); + + let e = extract(b"notes", DocumentHint::file_name("dir/NOTES.TXT")); + assert_eq!(e.format, Format::PlainText); + assert_eq!(e.markdown, "notes\n"); +} + +#[test] +fn plain_text_crlf_and_blank_runs() { + let e = extract( + b"a\r\nb\r\n\r\n\r\n\r\nc \r\n", + DocumentHint::file_name("x.txt"), + ); + assert_eq!(e.markdown, "a \nb\n\nc\n"); +} + +#[test] +fn plain_text_empty() { + let e = extract(b"", DocumentHint::file_name("empty.txt")); + assert_eq!(e.markdown, ""); + assert_eq!(e.stats.paragraphs, Some(0)); +} + +#[cfg(feature = "markdown")] +#[test] +fn markdown_normalized_with_title() { + let src = "\u{feff}---\ntitle: Guide\n---\n\n# Intro\n\nSome *text*\nwrapped.\n\n```rust\nfn main() {}\n```\n\n* item\n\n\n"; + let e = extract(src.as_bytes(), DocumentHint::file_name("guide.md")); + assert_eq!(e.format, Format::Markdown); + assert_eq!(e.title.as_deref(), Some("Guide")); + assert_eq!( + e.markdown, + "# Intro\n\nSome *text* wrapped.\n\n```rust\nfn main() {}\n```\n\n- item\n" + ); + assert_eq!(e.stats.tables, Some(0)); +} + +#[cfg(feature = "markdown")] +#[test] +fn markdown_title_from_heading() { + for ct in ["text/markdown", "text/x-markdown; charset=utf-8"] { + let e = extract(b"# Title\n\nbody", DocumentHint::content_type(ct)); + assert_eq!(e.format, Format::Markdown, "{ct}"); + assert_eq!(e.title.as_deref(), Some("Title")); + assert_eq!(e.markdown, "# Title\n\nbody\n"); + } +} + +#[cfg(not(feature = "markdown"))] +#[test] +fn markdown_requires_feature() { + let m = Markkit::builder().build(); + let err = m + .extract_document_sync(b"# x", DocumentHint::file_name("a.md")) + .unwrap_err(); + assert!(matches!(err, Error::UnsupportedFormat(_)), "{err:?}"); +} + +#[test] +fn text_is_never_sniffed_without_hint() { + let m = Markkit::builder().build(); + let err = m + .extract_document_sync(b"# just text", DocumentHint::default()) + .unwrap_err(); + assert!(matches!(err, Error::UnsupportedFormat(_)), "{err:?}"); +} + +#[tokio::test] +async fn extract_bytes_text_plain() { + let m = Markkit::builder().build(); + let md = m.extract_bytes(b"x\ny", Some("text/plain")).await; + assert_eq!(md.unwrap(), "x \ny\n"); +} + +// ─── Security (threat model T7) ───────────────────────────────────── + +/// File names for every text-like format compiled in. +fn names() -> Vec<&'static str> { + let mut v = vec!["a.txt"]; + if cfg!(feature = "markdown") { + v.push("a.md"); + } + v +} + +fn has_control(s: &str) -> bool { + s.chars() + .any(|c| c.is_ascii_control() && c != '\n' && c != '\t') +} + +#[test] +fn security_control_chars_stripped() { + for name in names() { + let e = extract( + b"safe\x1b]0;pwned\x07\x00text\x7f\x08", + DocumentHint::file_name(name), + ); + assert!(!has_control(&e.markdown), "{name}: {:?}", e.markdown); + assert_eq!(e.markdown, "safe]0;pwnedtext\n"); + } +} + +#[test] +fn security_invalid_utf8_lossy_with_warning() { + let e = extract(b"caf\xE9 \xC3", DocumentHint::file_name("a.txt")); + assert_eq!(e.markdown, "caf\u{FFFD} \u{FFFD}\n"); + assert_eq!(e.warnings.len(), 1); + assert!(e.warnings[0].starts_with("invalid_utf8:")); +} + +#[test] +fn security_utf16_malformed_no_panic() { + for bytes in [ + &b"\xFF\xFE"[..], + b"\xFF\xFE\x41", + b"\xFE\xFF\xD8\x00\xD8\x00", + b"\xFF\xFE\x00\xDC", + ] { + let e = extract(bytes, DocumentHint::file_name("a.txt")); + assert!(e.markdown.len() <= 16, "{bytes:?}"); + } +} + +#[test] +fn security_input_size_limit() { + let m = Markkit::builder().max_input_size(8).build(); + for name in ["a.txt", "a.md"] { + let err = m + .extract_document_sync(b"0123456789", DocumentHint::file_name(name)) + .unwrap_err(); + assert!(matches!(err, Error::ResourceLimit(_)), "{err:?}"); + } +} + +#[test] +fn security_output_bounded_by_input() { + // Each line becomes a hard break (" \n" = +2 bytes per line): output + // stays linear in input size. + let input = "a\n".repeat(100_000); + let e = extract(input.as_bytes(), DocumentHint::file_name("a.txt")); + assert!(e.markdown.len() <= input.len() * 2); + let blanks = "\n".repeat(1_000_000); + for name in names() { + let e = extract(blanks.as_bytes(), DocumentHint::file_name(name)); + assert_eq!(e.markdown, "", "{name}"); + } +} + +#[test] +fn security_long_line() { + let line = "x".repeat(2_000_000); + for name in names() { + let e = extract(line.as_bytes(), DocumentHint::file_name(name)); + assert_eq!(e.markdown.len(), line.len() + 1, "{name}"); + } +} + +#[test] +fn security_arbitrary_bytes_never_panic() { + let mut seed = 0x2545_F491_4F6C_DD1Du64; + for _ in 0..300 { + let len = (seed % 512) as usize; + let bytes: Vec = (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + // Bias towards Markdown syntax so the parser is exercised. + b"#*_`[]()!<>|-~\n \\x\x00\x1b\xff"[(seed % 21) as usize] + }) + .collect(); + for name in names() { + let e = extract(&bytes, DocumentHint::file_name(name)); + assert!(!has_control(&e.markdown), "{name}: {bytes:?}"); + } + } +} + +#[cfg(feature = "markdown")] +#[test] +fn security_markdown_raw_html_never_passes_through() { + let src = b"\n\n
Hi there
\n\npara kept\n\n\n"; + let e = extract(src, DocumentHint::file_name("a.md")); + for bad in [ + " ".repeat(100_000)), + format!("{}x", "- ".repeat(50_000)), + format!("{}x{}", "*".repeat(50_000), "*".repeat(50_000)), + format!("{}x{}", "[".repeat(50_000), "](u)".repeat(50_000)), + ] { + let e = extract(src.as_bytes(), DocumentHint::file_name("a.md")); + assert!(e.markdown.contains('x')); + } +} diff --git a/knowledge/formats/format-support.md b/knowledge/formats/format-support.md index 7e3a825..0dce0ff 100644 --- a/knowledge/formats/format-support.md +++ b/knowledge/formats/format-support.md @@ -1,13 +1,14 @@ --- type: Subsystem Design title: Format Support -description: Supported and planned formats, bytes-in API, format detection, DOCX and spreadsheet mapping. +description: Supported and planned formats, bytes-in API, format detection, DOCX, spreadsheet, plain text and Markdown mapping. tags: - markkit - formats - docx - spreadsheet - detection + - markdown --- # Format Support @@ -16,11 +17,12 @@ tags: ### Tier 1 (Core) - **HTML/XHTML**: Full support via html5ever + scraper (`html` feature, default) -- **Plain text**: Planned pass-through (currently `UnsupportedFormat` via the bytes API) +- **Plain text**: Always on, no dependencies (`text/plain`, `.txt`, `.text`) ### Tier 2 (Feature-gated) - **DOCX** (`docx`): Word documents via a direct `zip` + `quick-xml` reader (no docx-rs) - **XLSX/XLSM, XLS, ODS** (`spreadsheet`): Workbooks to Markdown tables via calamine +- **Markdown** (`markdown`): Re-parsed with pulldown-cmark into the IR and rendered back (`text/markdown`, `text/x-markdown`, `.md`, `.markdown`, `.mdown`, `.mkd`, `.mkdn`) - **PDF** (`pdf`): Planned. Returns `Error::UnsupportedFormat` (never an empty success) ### Detected, not supported @@ -30,7 +32,6 @@ tags: - **EPUB**: E-book extraction - **RTF**: Rich text format - **CSV/TSV**: Tabular data to markdown tables -- **Markdown**: Normalize/clean existing markdown ## Bytes-in API @@ -55,7 +56,7 @@ pub struct Extraction { // #[non_exhaustive] only the Markdown. Parsing is CPU-bound; async callers should use the sync variant in `spawn_blocking`. -Warning codes: `truncated_sheet`, `hidden_sheet_skipped`, `sheet_error`. +Warning codes: `truncated_sheet`, `hidden_sheet_skipped`, `sheet_error`, `invalid_utf8`. ## Format Detection @@ -71,6 +72,9 @@ Priority order (`Format::detect`): - `%PDF-` within the first 1 KiB → PDF. - ``, event handlers) or pathological nesting. + +**Impact**: Medium -- raw HTML passed through re-introduces T1; deep nesting can crash. + +**Mitigations**: +- Text and Markdown are never sniffed; a content type or extension must select them +- Decoding is total: invalid UTF-8/UTF-16 is replaced (U+FFFD) with an `invalid_utf8` warning +- C0 controls (except tab/newline) and DEL are dropped before parsing +- Markdown is re-parsed (pulldown-cmark) into the IR; raw HTML is never copied. HTML blocks + go through the HTML parser (T1 stripping) or are dropped; inline tags are dropped +- Explicit-stack IR builder, nesting capped at 200 (deeper content flattened) + +**Test Coverage** (`text_integration`): +- `security_control_chars_stripped`, `security_arbitrary_bytes_never_panic` +- `security_invalid_utf8_lossy_with_warning`, `security_utf16_malformed_no_panic` +- `security_markdown_raw_html_never_passes_through` +- `security_markdown_deep_nesting_no_stack_overflow`, `security_long_line` +- `security_input_size_limit`, `security_output_bounded_by_input` + ### T5: Information Disclosure **Description**: Error messages or output leak sensitive information. @@ -239,7 +261,7 @@ process, or smuggling content past a human reviewer. | Processing time | 60 s | `max_processing_secs(u64)` | Whole `extract_url` fetch (parsing itself TBD) | | Redirects (`extract_url`) | 5 | `max_redirects(usize)` | Every redirect hop | | Fetched body (`extract_url`) | = input size | `max_input_size(usize)` | `Content-Length` and while streaming | -| Nesting depth | 200 | Hardcoded `MAX_DEPTH` | During HTML tree walk | +| Nesting depth | 200 | Hardcoded `MAX_DEPTH` | During HTML tree walk and Markdown IR build | | XML nesting depth (DOCX) | 256 | Hardcoded `MAX_XML_DEPTH` | While building the XML tree | | ZIP entries | 10 000 | `max_archive_entries(usize)` | On archive open | | ZIP entry size (decompressed) | 64 MiB | `max_archive_entry_size(u64)` | While decompressing | @@ -285,6 +307,7 @@ process, or smuggling content past a human reviewer. | S10 | Spreadsheet parsing never allocates a grid beyond `max_spreadsheet_cells` | `document_security::xls_*_rejected`, `ods_repeated_rows_and_columns_rejected` | | S11 | Unsupported formats fail explicitly (no empty success) | `document_detection::pdf_is_explicitly_unsupported_not_empty` | | S12 | Default `extract_url` never connects to a non-public address | `url_security::*_blocked*`, `fetch::tests::non_public_addresses_forbidden` | +| S13 | Text and Markdown output never contains C0 controls (except tab/newline) or raw HTML from the input | `text_integration::security_*` | ## See also From 7e3a23392805788ed4f335a30fc434e0f9413388 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:37:16 +0000 Subject: [PATCH 2/9] fix(text): use as_chunks for UTF-16 decoding (clippy) --- crates/markkit/src/text.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/markkit/src/text.rs b/crates/markkit/src/text.rs index 19e7999..0d33cb3 100644 --- a/crates/markkit/src/text.rs +++ b/crates/markkit/src/text.rs @@ -50,7 +50,7 @@ fn utf8(bytes: &[u8], warnings: &mut Vec) -> String { } fn utf16(bytes: &[u8], read: fn([u8; 2]) -> u16, warnings: &mut Vec) -> String { - let units = bytes.chunks_exact(2).map(|c| read([c[0], c[1]])); + let units = bytes.as_chunks::<2>().0.iter().map(|c| read(*c)); let mut lossy = !bytes.len().is_multiple_of(2); let s: String = char::decode_utf16(units) .map(|r| { From ba3006272c6a28dcf926e3c60f6480e3c5462b9f Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:51:54 +0000 Subject: [PATCH 3/9] chore(knowledge): scope S13 to markup copying, note text escaping residual --- knowledge/security/threat-model.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 1fb8377..9f075ed 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -233,6 +233,10 @@ raw HTML (`\n", "# Title\n\n- a\n", ), + ( + "data.csv", + "a;b\n1;2\n", + "| a | b |\n| --- | --- |\n| 1 | 2 |\n", + ), ]; for (name, input, expected) in cases { let file_path = dir.path().join(name); diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs index 84483c9..347c897 100644 --- a/crates/markkit-js/src/lib.rs +++ b/crates/markkit-js/src/lib.rs @@ -90,7 +90,7 @@ pub struct Metadata { pub struct Extraction { pub markdown: String, pub title: Option, - /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`, `text`, `markdown`). + /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`, `text`, `markdown`, `csv`, `tsv`). pub format: String, pub warnings: Vec, pub stats: ExtractionStats, diff --git a/crates/markkit-wasm/README.md b/crates/markkit-wasm/README.md index dcc03bd..9459c4d 100644 --- a/crates/markkit-wasm/README.md +++ b/crates/markkit-wasm/README.md @@ -1,8 +1,8 @@ # @everruns/markkit-wasm Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX, -spreadsheets (XLSX/XLSM, XLS, ODS), plain text and Markdown in the **browser -and any JavaScript runtime**: +spreadsheets (XLSX/XLSM, XLS, ODS), CSV/TSV, plain text and Markdown in the +**browser and any JavaScript runtime**: edge workers (Cloudflare Workers, Vercel Edge, Deno Deploy), Node, Deno and Bun. Single-threaded `wasm-bindgen` build: no `SharedArrayBuffer`, no `COOP`/`COEP` diff --git a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs index 0458d6e..85800ad 100644 --- a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs +++ b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs @@ -85,6 +85,15 @@ test("extractDocument reads plain text and Markdown", () => { assert.ok(!md.markdown.includes("script")); }); +test("extractDocument reads CSV and TSV", () => { + const mk = new Markkit(); + const csv = mk.extractDocument(new TextEncoder().encode("a;b\n1;2\n"), { fileName: "t.csv" }); + assert.equal(csv.format, "csv"); + assert.equal(csv.markdown, "| a | b |\n| --- | --- |\n| 1 | 2 |\n"); + const tsv = mk.extractDocument(new TextEncoder().encode("x\ty\n"), { fileName: "t.tsv" }); + assert.equal(tsv.format, "tsv"); +}); + test("extractBytes and unsupported formats", () => { const mk = new Markkit(); assert.equal(mk.extractBytes(new TextEncoder().encode("

x

"), "text/html"), "x\n"); diff --git a/crates/markkit/src/csv.rs b/crates/markkit/src/csv.rs new file mode 100644 index 0000000..4c648bc --- /dev/null +++ b/crates/markkit/src/csv.rs @@ -0,0 +1,257 @@ +//! CSV / TSV input → one GFM table. +//! +//! Decisions: +//! - Always compiled, hand-written RFC 4180 reader (no dependency). Like plain +//! text, selected only by content type or extension, never sniffed. +//! - Delimiter: tab for TSV; for CSV the candidate (`,` `;` `\t` `|`) seen most +//! often outside quotes in the first record, default `,`. +//! - Quoted fields may span lines and use `""` escapes. A stray quote inside an +//! unquoted field is literal; an unterminated quote takes the rest of the +//! input as one field and emits `malformed_csv`. +//! - First non-empty record is the header; fully empty records are dropped; +//! short rows are padded. Cells are trimmed and kept raw in the IR; the +//! renderer escapes pipes and flattens line breaks in table context (S17). +//! - Bounded by `max_sheet_rows` / `max_sheet_cols`: the reader stops after the +//! row cap and never stores columns past the column cap (`truncated_table`). +//! Each record ticks the call's `timeout` budget. + +use crate::config::MarkkitBuilder; +use crate::ir::{Document, Node, TableCell}; +use crate::limits::Budget; + +/// Parsed CSV/TSV document. +pub(crate) struct CsvOutput { + pub doc: Document, + pub warnings: Vec, + /// Data rows (header excluded). + pub rows: usize, + /// 1 when a table was produced, else 0. + pub tables: usize, +} + +/// Parse delimited text. `tab` forces the TSV delimiter. +pub(crate) fn parse( + text: &str, + tab: bool, + config: &MarkkitBuilder, + budget: &Budget, +) -> crate::Result { + let delim = if tab { '\t' } else { sniff_delimiter(text) }; + let max_cols = config.limits.max_sheet_cols.max(1); + let max_rows = config.limits.max_sheet_rows; + let mut warnings = Vec::new(); + + let mut reader = Reader::new(text, delim, max_cols); + let mut records: Vec> = Vec::new(); + let mut truncated_rows = false; + while let Some(rec) = reader.next_record() { + if budget.expired() { + return Err(budget.timeout_error()); + } + if rec.iter().all(|f| f.trim().is_empty()) { + continue; + } + // Header + `max_rows` data rows. + if records.len() > max_rows { + truncated_rows = true; + break; + } + records.push(rec); + } + if reader.unterminated { + warnings.push("malformed_csv: unterminated quoted field".into()); + } + if truncated_rows { + warnings.push(format!( + "truncated_table: table truncated to {max_rows} rows" + )); + } + if reader.truncated_cols { + warnings.push(format!( + "truncated_table: table truncated to {max_cols} columns" + )); + } + + let width = records.iter().map(Vec::len).max().unwrap_or(0); + let mut rows = records.into_iter().map(|rec| { + let mut cells: Vec = rec + .iter() + .map(|f| { + let f = f.trim(); + TableCell::new(if f.is_empty() { + vec![] + } else { + vec![Node::Text(f.to_string())] + }) + }) + .collect(); + cells.resize_with(width, || TableCell::new(vec![])); + cells + }); + let Some(headers) = rows.next() else { + return Ok(CsvOutput { + doc: Document::new(vec![]), + warnings, + rows: 0, + tables: 0, + }); + }; + let rows: Vec<_> = rows.collect(); + let n = rows.len(); + Ok(CsvOutput { + doc: Document::new(vec![Node::Table { + alignments: Vec::new(), + headers, + rows, + source: None, + }]), + warnings, + rows: n, + tables: 1, + }) +} + +fn sniff_delimiter(text: &str) -> char { + const CANDIDATES: [char; 4] = [',', ';', '\t', '|']; + let mut counts = [0usize; 4]; + let mut quoted = false; + let mut seen = false; + for c in text.chars() { + match c { + '"' => quoted = !quoted, + '\n' if !quoted => { + if seen { + break; + } + } + _ if !quoted => { + if let Some(i) = CANDIDATES.iter().position(|d| *d == c) { + counts[i] += 1; + } + if !c.is_whitespace() { + seen = true; + } + } + _ => {} + } + } + // First maximum wins, so ties prefer `,`. + let mut best = 0; + for i in 1..CANDIDATES.len() { + if counts[i] > counts[best] { + best = i; + } + } + CANDIDATES[best] +} + +struct Reader<'a> { + chars: std::iter::Peekable>, + delim: char, + max_cols: usize, + unterminated: bool, + truncated_cols: bool, +} + +impl<'a> Reader<'a> { + fn new(text: &'a str, delim: char, max_cols: usize) -> Self { + Self { + chars: text.chars().peekable(), + delim, + max_cols, + unterminated: false, + truncated_cols: false, + } + } + + /// Next record, or `None` at end of input. Input is LF-normalized. + fn next_record(&mut self) -> Option> { + self.chars.peek()?; + let mut rec = Vec::new(); + let mut field = String::new(); + let mut quoted = false; + let mut at_start = true; + loop { + let Some(c) = self.chars.next() else { + if quoted { + self.unterminated = true; + } + self.push_field(&mut rec, &mut field); + return Some(rec); + }; + if quoted { + if c == '"' { + if self.chars.peek() == Some(&'"') { + self.chars.next(); + field.push('"'); + } else { + quoted = false; + } + } else { + field.push(c); + } + continue; + } + if c == self.delim { + self.push_field(&mut rec, &mut field); + at_start = true; + } else if c == '\n' { + self.push_field(&mut rec, &mut field); + return Some(rec); + } else if c == '"' && at_start && field.trim().is_empty() { + field.clear(); + quoted = true; + at_start = false; + } else { + field.push(c); + if !c.is_whitespace() { + at_start = false; + } + } + } + } + + fn push_field(&mut self, rec: &mut Vec, field: &mut String) { + if rec.len() < self.max_cols { + rec.push(std::mem::take(field)); + } else { + self.truncated_cols = true; + field.clear(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn records(text: &str, delim: char) -> Vec> { + let mut r = Reader::new(text, delim, 100); + std::iter::from_fn(|| r.next_record()).collect() + } + + #[test] + fn reader_quotes_and_escapes() { + assert_eq!( + records("a,\"b,c\",\"d \"\"q\"\"\"\n\"multi\nline\",x\"y,\n", ','), + vec![vec!["a", "b,c", "d \"q\""], vec!["multi\nline", "x\"y", ""],] + ); + } + + #[test] + fn reader_unterminated_quote() { + let mut r = Reader::new("a,\"open\nrest", ',', 100); + assert_eq!(r.next_record().unwrap(), vec!["a", "open\nrest"]); + assert!(r.unterminated); + assert!(r.next_record().is_none()); + } + + #[test] + fn delimiter_sniffing() { + assert_eq!(sniff_delimiter("a;b;c\n1,5;2;3"), ';'); + assert_eq!(sniff_delimiter("\"x;y\",b\n"), ','); + assert_eq!(sniff_delimiter("a\tb\n"), '\t'); + assert_eq!(sniff_delimiter("a|b|c\n"), '|'); + assert_eq!(sniff_delimiter("\n\nsingle\n"), ','); + } +} diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index ae904ee..0e33efd 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -22,7 +22,7 @@ pub struct Extraction { pub format: Format, /// Non-fatal issues, each prefixed with a stable code /// (`truncated_sheet:`, `hidden_sheet_skipped:`, `sheet_error:`, - /// `invalid_utf8:`). + /// `invalid_utf8:`, `malformed_csv:`, `truncated_table:`). pub warnings: Vec, /// Format-specific counters. pub stats: ExtractionStats, @@ -65,11 +65,11 @@ pub struct ExtractionStats { pub pages: Option, /// Sheets rendered (spreadsheets; empty and skipped sheets excluded). pub sheets: Option, - /// Data rows rendered across all sheets (header rows excluded). + /// Data rows rendered across all sheets / the CSV table (header rows excluded). pub rows: Option, /// Non-empty paragraphs, headings and list items (DOCX, plain text, Markdown). pub paragraphs: Option, - /// Tables (DOCX, Markdown). + /// Tables (DOCX, Markdown, CSV/TSV). pub tables: Option, } @@ -224,7 +224,8 @@ impl Markkit { .map(|e| e.markdown) } - /// Convert a document (HTML, DOCX, XLSX/XLS/ODS, plain text, Markdown) to Markdown with metadata. + /// Convert a document (HTML, DOCX, XLSX/XLS/ODS, plain text, Markdown, CSV/TSV) to + /// Markdown with metadata. /// /// The format is detected from `hint.content_type`, then the extension of /// `hint.file_name`, then magic bytes. Parsing is CPU-bound and runs inline; @@ -292,7 +293,8 @@ impl Markkit { ) -> crate::Result { self.config.limits.check_input(bytes.len())?; - match Format::detect(bytes, hint) { + let format = Format::detect(bytes, hint); + match format { Format::Html => self.parse_html_doc(bytes, budget), Format::Pdf => self.parse_pdf(bytes), Format::Docx => self.parse_docx(bytes, budget), @@ -301,6 +303,7 @@ impl Markkit { } Format::PlainText => self.parse_plain_text(bytes, budget), Format::Markdown => self.parse_markdown(bytes, budget), + Format::Csv | Format::Tsv => self.parse_csv(bytes, format, budget), Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -343,6 +346,24 @@ impl Markkit { Ok(p) } + fn parse_csv( + &self, + bytes: &[u8], + format: Format, + budget: &Budget, + ) -> crate::Result { + let decoded = crate::text::decode(bytes); + let out = crate::csv::parse(&decoded.text, format == Format::Tsv, &self.config, budget)?; + let mut doc = out.doc; + self.config.transforms.apply(&mut doc)?; + let mut p = ParsedDocument::new(doc, format); + p.warnings = decoded.warnings; + p.warnings.extend(out.warnings); + p.stats.rows = Some(out.rows); + p.stats.tables = Some(out.tables); + Ok(p) + } + #[cfg(feature = "markdown")] fn parse_markdown(&self, bytes: &[u8], budget: &Budget) -> crate::Result { let decoded = crate::text::decode(bytes); diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index fea7b95..614c363 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -30,8 +30,11 @@ pub enum Format { Ods, /// Office Open XML presentation (`.pptx`). Detected but not supported. Pptx, + /// Comma (or `;` / `|`) separated values (`.csv`). Csv, - /// Plain text (`text/plain`, `.txt`). Never sniffed from bytes. + /// Tab separated values (`.tsv`, `.tab`). + Tsv, + /// Plain text (`text/plain`, `.txt`). Never sniffed from bytes (nor CSV/TSV/Markdown). /// Serialized as `"text"`, same as `as_str`. #[serde(rename = "text")] PlainText, @@ -87,6 +90,7 @@ impl Format { Format::Ods => "ods", Format::Pptx => "pptx", Format::Csv => "csv", + Format::Tsv => "tsv", Format::PlainText => "text", Format::Markdown => "markdown", Format::Unknown => "unknown", @@ -115,7 +119,9 @@ impl Format { Format::Ods } else if ct.contains("presentationml") || ct.contains("ms-powerpoint") { Format::Pptx - } else if ct.contains("text/csv") { + } else if ct.contains("tab-separated-values") { + Format::Tsv + } else if ct.contains("text/csv") || ct.contains("application/csv") { Format::Csv } else if ct.contains("text/plain") { Format::PlainText @@ -138,7 +144,8 @@ impl Format { "xls" | "xlt" => Format::Xls, "ods" => Format::Ods, "pptx" | "pptm" | "ppt" => Format::Pptx, - "csv" | "tsv" => Format::Csv, + "csv" => Format::Csv, + "tsv" | "tab" => Format::Tsv, "txt" | "text" => Format::PlainText, "md" | "markdown" | "mdown" | "mkd" | "mkdn" => Format::Markdown, _ => Format::Unknown, diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index afb874d..bcf8fe4 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -35,6 +35,7 @@ //! - **DOCX** (`docx`): Convert Word documents //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLS and ODS workbooks to tables //! - **Plain text** (always on): `text/plain`, `.txt` +//! - **CSV / TSV** (always on): `text/csv`, `text/tab-separated-values`, `.csv`, `.tsv`, one table //! - **Markdown** (`markdown`): `text/markdown`, `.md`, normalized through the IR //! - **PDF** (`pdf`): Planned; currently returns [`Error::UnsupportedFormat`] //! - **URL fetching** (`http_client`, opt-in): Fetch and extract from URLs via reqwest @@ -49,6 +50,7 @@ #[cfg(any(feature = "docx", feature = "spreadsheet"))] mod archive; pub mod config; +mod csv; #[cfg(feature = "docx")] mod docx; pub mod error; diff --git a/crates/markkit/tests/csv_integration.rs b/crates/markkit/tests/csv_integration.rs new file mode 100644 index 0000000..846da61 --- /dev/null +++ b/crates/markkit/tests/csv_integration.rs @@ -0,0 +1,208 @@ +//! CSV / TSV → Markdown table integration and security tests. + +use markkit::format::Format; +use markkit::{DocumentHint, Error, Markkit}; + +fn extract_with(m: &Markkit, bytes: &[u8], hint: DocumentHint<'_>) -> markkit::Extraction { + m.extract_document_sync(bytes, hint).unwrap() +} + +fn extract(bytes: &[u8], hint: DocumentHint<'_>) -> markkit::Extraction { + extract_with(&Markkit::builder().build(), bytes, hint) +} + +#[test] +fn csv_basic_table() { + let e = extract( + b"name,qty,note\nApple,3,\"red, sweet\"\nPear,10\n", + DocumentHint::file_name("fruit.csv"), + ); + assert_eq!(e.format, Format::Csv); + assert_eq!( + e.markdown, + "| name | qty | note |\n| --- | --- | --- |\n| Apple | 3 | red, sweet |\n| Pear | 10 | |\n" + ); + assert_eq!(e.stats.rows, Some(2)); + assert_eq!(e.stats.tables, Some(1)); + assert!(e.warnings.is_empty()); +} + +#[test] +fn tsv_by_extension_and_content_type() { + for hint in [ + DocumentHint::file_name("data.tsv"), + DocumentHint::content_type("text/tab-separated-values"), + ] { + let e = extract(b"a\tb,c\n1\t2\n", hint); + assert_eq!(e.format, Format::Tsv); + assert_eq!(e.markdown, "| a | b,c |\n| --- | --- |\n| 1 | 2 |\n"); + } +} + +#[test] +fn csv_content_types() { + for ct in ["text/csv; charset=utf-8", "application/csv"] { + let e = extract(b"a\n1", DocumentHint::content_type(ct)); + assert_eq!(e.format, Format::Csv, "{ct}"); + } +} + +#[test] +fn csv_semicolon_delimiter_sniffed() { + let e = extract( + b"\xEF\xBB\xBFcity;price\r\nKyiv;1,5\r\n", + DocumentHint::file_name("eu.csv"), + ); + assert_eq!( + e.markdown, + "| city | price |\n| --- | --- |\n| Kyiv | 1,5 |\n" + ); +} + +#[test] +fn csv_cells_escaped_and_flattened() { + let e = extract( + b"h\n\"a|b\nc\"\n\n,,\n spaced \n", + DocumentHint::file_name("x.csv"), + ); + assert_eq!(e.markdown, "| h |\n| --- |\n| a\\|b c |\n| spaced |\n"); +} + +#[test] +fn csv_empty_input() { + for input in [&b""[..], b"\n\n", b",,,\n , \n"] { + let e = extract(input, DocumentHint::file_name("e.csv")); + assert_eq!(e.markdown, ""); + assert_eq!(e.stats.tables, Some(0)); + assert_eq!(e.stats.rows, Some(0)); + } +} + +#[test] +fn csv_never_sniffed_without_hint() { + let err = Markkit::builder() + .build() + .extract_document_sync(b"a,b\n1,2\n", DocumentHint::default()) + .unwrap_err(); + assert!(matches!(err, Error::UnsupportedFormat(_)), "{err:?}"); +} + +// ─── Security (threat model T7) ───────────────────────────────────── + +#[test] +fn security_row_cap_stops_reading() { + let m = Markkit::builder().max_sheet_rows(3).build(); + let input = "h\n".to_string() + &"r\n".repeat(1_000_000); + let e = extract_with(&m, input.as_bytes(), DocumentHint::file_name("a.csv")); + assert_eq!(e.stats.rows, Some(3)); + assert_eq!(e.markdown.lines().count(), 5); + assert_eq!( + e.warnings, + vec!["truncated_table: table truncated to 3 rows"] + ); +} + +#[test] +fn security_column_cap_never_stores_extra_fields() { + let m = Markkit::builder().max_sheet_cols(2).build(); + let input = ",".repeat(5_000_000); + let e = extract_with( + &m, + format!("a{input}x").as_bytes(), + DocumentHint::file_name("a.csv"), + ); + assert_eq!(e.markdown, "| a | |\n| --- | --- |\n"); + assert_eq!( + e.warnings, + vec!["truncated_table: table truncated to 2 columns"] + ); +} + +#[test] +fn security_default_caps_bound_output() { + // 300 columns x 20 000 rows: defaults keep 256 columns and 10 000 rows. + let row = "x,".repeat(299) + "x\n"; + let input = row.repeat(20_001); + let e = extract(input.as_bytes(), DocumentHint::file_name("big.csv")); + assert_eq!(e.stats.rows, Some(10_000)); + let header = e.markdown.lines().next().unwrap(); + assert_eq!(header.matches(" x ").count(), 256); + assert_eq!(e.warnings.len(), 2); +} + +#[test] +fn security_unterminated_quote_warns() { + let input = format!("a,\"{}", "x\n".repeat(100_000)); + let e = extract(input.as_bytes(), DocumentHint::file_name("a.csv")); + assert_eq!(e.markdown.lines().count(), 2); + assert_eq!(e.warnings, vec!["malformed_csv: unterminated quoted field"]); +} + +#[test] +fn security_input_size_limit() { + let m = Markkit::builder().max_input_size(4).build(); + let err = m + .extract_document_sync(b"a,b,c", DocumentHint::file_name("a.csv")) + .unwrap_err(); + assert!(matches!(err, Error::ResourceLimit(_)), "{err:?}"); +} + +#[test] +fn security_control_chars_and_invalid_utf8() { + let e = extract( + b"h\x1b[2J\x00\n\xff\x07v\n", + DocumentHint::file_name("a.tsv"), + ); + assert_eq!(e.markdown, "| h\\[2J |\n| --- |\n| \u{FFFD}v |\n"); + assert!(e.warnings[0].starts_with("invalid_utf8:")); +} + +#[test] +fn security_arbitrary_bytes_never_panic() { + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + for _ in 0..300 { + let len = (seed % 512) as usize; + let bytes: Vec = (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + const ALPHABET: &[u8] = b",;\t|\"\n\r x\\\x00\xff"; + ALPHABET[(seed % ALPHABET.len() as u64) as usize] + }) + .collect(); + for name in ["a.csv", "a.tsv"] { + let e = extract(&bytes, DocumentHint::file_name(name)); + // Every table line is a well-formed row. + assert!( + e.markdown + .lines() + .all(|l| l.starts_with('|') && l.ends_with('|')), + "{name}: {bytes:?} -> {:?}", + e.markdown + ); + } + } +} + +#[test] +fn security_cell_markup_is_escaped() { + let e = extract( + b"h\n\n[x](javascript:alert(1))\n", + DocumentHint::file_name("a.csv"), + ); + assert!(e.markdown.contains("\\`, event handlers) or pathological nesting. **Impact**: Medium -- raw HTML passed through re-introduces T1; deep nesting can crash. **Mitigations**: -- Text and Markdown are never sniffed; a content type or extension must select them +- CSV/TSV reading stops at `max_sheet_rows`; fields past `max_sheet_cols` are never stored + (a 5 MB row of commas allocates two cells) +- Text, Markdown and CSV are never sniffed; a content type or extension must select them - Decoding is total: invalid UTF-8/UTF-16 is replaced (U+FFFD) with an `invalid_utf8` warning - C0 controls (except tab/newline) and DEL are dropped before parsing - Markdown is re-parsed (pulldown-cmark) into the IR; raw HTML is never copied. HTML blocks @@ -323,6 +325,8 @@ in a `.txt`) is emitted as `Node::Text` and escaped by the renderer (S17), like - `security_input_size_limit`, `security_output_bounded_by_input` - `security_text_and_markdown_share_limits` (timeout, `max_nodes`) - `security_plain_text_markup_is_escaped` +- `csv_integration::security_*`: row/column caps, unterminated quotes, control characters, + random bytes always yield well-formed table rows ### T5: Information Disclosure @@ -353,7 +357,7 @@ in a `.txt`) is emitted as `Node::Text` and escaped by the renderer (S17), like | ZIP entry size (decompressed) | 64 MiB | `max_archive_entry_size(u64)` | While decompressing | | ZIP total size (decompressed) | 256 MiB | `max_archive_total_size(u64)` | While decompressing | | Spreadsheet grid (XLS/ODS) | 4 000 000 cells | `max_spreadsheet_cells(u64)` | Pre-scan before calamine | -| Rows / columns per sheet | 10 000 / 256 | `max_sheet_rows`, `max_sheet_cols` | While collecting cells (truncate + warn) | +| Rows / columns per sheet or CSV table | 10 000 / 256 | `max_sheet_rows`, `max_sheet_cols` | While collecting cells (truncate + warn) | ## Malicious Input Categories @@ -400,7 +404,7 @@ in a `.txt`) is emitted as `Node::Text` and escaped by the renderer (S17), like | S15 | No configuration of `max_nesting_depth` can overflow the stack | `limits::*hard_ceiling_survives_small_stack` | | S16 | Extraction stops with `ResourceLimit` once `timeout` passes (checked at least after each non-interruptible third-party step) | `limits::*zero_timeout_rejects`, `limits::tests::*` | | S17 | Rendered markdown has no raw HTML, no non-allowlisted link/image scheme, and document text cannot create Markdown structure | `output_safety::*` (pulldown-cmark oracle + proptests) | -| S18 | Text and Markdown output never contains C0 controls (except tab/newline); Markdown raw HTML (blocks, inline tags) is never copied | `text_integration::security_*` | +| S18 | Text, Markdown and CSV/TSV output never contains C0 controls (except tab/newline); Markdown raw HTML (blocks, inline tags) is never copied | `text_integration::security_*`, `csv_integration::security_*` | ## See also From 911360a072d0d266da839898b0df84fcb90316e6 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:51:15 +0000 Subject: [PATCH 5/9] feat(formats): add Jupyter notebook input nbformat 4 notebooks render Markdown cells through the Markdown path, code cells as fenced blocks tagged with the kernel language, and text outputs and tracebacks (ANSI stripped) as text blocks; rich outputs are skipped. Adds Format::Notebook, Error::NotebookExtract and ExtractionStats::cells. --- CHANGELOG.md | 4 + README.md | 2 + crates/markkit-js/src/lib.rs | 2 +- crates/markkit-wasm/README.md | 4 +- .../__test__/markkit-wasm.test.mjs | 8 + crates/markkit/src/error.rs | 3 + crates/markkit/src/extract.rs | 19 +- crates/markkit/src/format.rs | 9 + crates/markkit/src/lib.rs | 2 + crates/markkit/src/notebook.rs | 272 ++++++++++++++++++ crates/markkit/tests/notebook_integration.rs | 209 ++++++++++++++ knowledge/formats/format-support.md | 24 +- knowledge/log.md | 1 + knowledge/operations/testing.md | 2 +- knowledge/security/security-model.md | 2 +- knowledge/security/threat-model.md | 12 +- 16 files changed, 562 insertions(+), 13 deletions(-) create mode 100644 crates/markkit/src/notebook.rs create mode 100644 crates/markkit/tests/notebook_integration.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 75e9801..badb3d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - `markkit mcp`: Model Context Protocol server over stdio exposing the `markkit` tool (`tools/list`, `tools/call`). URL fetching keeps the SSRF guard; `--no-url` keeps it offline, the `extract` limit flags and `--max-output-chars` bound each call. +- Jupyter notebook input (`.ipynb`, `application/x-ipynb+json`, nbformat 4), always on: Markdown + cells, code cells as fenced blocks tagged with the kernel language, text outputs and ANSI-free + tracebacks; images/HTML outputs skipped. New `Format::Notebook`, `Error::NotebookExtract` and + `ExtractionStats::cells`. - CSV and TSV input (`text/csv`, `text/tab-separated-values`, `.csv`, `.tsv`), always on: one GFM table, RFC 4180 quoting, `;`/`|`/tab delimiter detection, bounded by `max_sheet_rows` / `max_sheet_cols` (`truncated_table` warning). New `Format::Tsv`. diff --git a/README.md b/README.md index dbfb1f6..596404a 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ package for browsers and edge runtimes. - **HTML** – Web pages and HTML documents; nav, header, footer, scripts, styles and forms stripped by default - **DOCX** – Headings, lists, tables, links, emphasis - **Spreadsheets** – XLSX/XLSM, XLSB, XLS and ODS workbooks to markdown tables, one section per sheet +- **Jupyter notebooks** – Markdown cells, code cells with language, text outputs and tracebacks - **CSV / TSV** – One markdown table; `;` and `|` delimiters detected, RFC 4180 quoting - **Plain text / Markdown** – Text to paragraphs; Markdown normalized through the same IR (raw HTML never passed through) - **Format detection** – Content type, then file extension, then magic bytes @@ -36,6 +37,7 @@ package for browsers and edge runtimes. | XLSX, XLSM, XLSB, XLS, ODS | Supported | `spreadsheet` | | Plain text | Supported | always on | | CSV, TSV | Supported | always on | +| Jupyter notebooks (ipynb) | Supported | always on | | Markdown | Supported | `markdown` | | PDF (text layer) | Supported | `pdf` | | DOC, PPTX | Detected, rejected explicitly | – | diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs index 7fe7fbc..355ff29 100644 --- a/crates/markkit-js/src/lib.rs +++ b/crates/markkit-js/src/lib.rs @@ -93,7 +93,7 @@ pub struct Metadata { pub struct Extraction { pub markdown: String, pub title: Option, - /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`, `text`, `markdown`, `csv`, `tsv`). + /// Detected format (`html`, `docx`, `xlsx`, `xls`, `ods`, `text`, `markdown`, `csv`, `tsv`, `ipynb`). pub format: String, pub warnings: Vec, pub stats: ExtractionStats, diff --git a/crates/markkit-wasm/README.md b/crates/markkit-wasm/README.md index 9459c4d..05dc404 100644 --- a/crates/markkit-wasm/README.md +++ b/crates/markkit-wasm/README.md @@ -1,8 +1,8 @@ # @everruns/markkit-wasm Markkit compiled to WebAssembly: extract clean Markdown from HTML, DOCX, -spreadsheets (XLSX/XLSM, XLS, ODS), CSV/TSV, plain text and Markdown in the -**browser and any JavaScript runtime**: +spreadsheets (XLSX/XLSM, XLS, ODS), CSV/TSV, Jupyter notebooks, plain text and +Markdown in the **browser and any JavaScript runtime**: edge workers (Cloudflare Workers, Vercel Edge, Deno Deploy), Node, Deno and Bun. Single-threaded `wasm-bindgen` build: no `SharedArrayBuffer`, no `COOP`/`COEP` diff --git a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs index 85800ad..ccb3f2e 100644 --- a/crates/markkit-wasm/__test__/markkit-wasm.test.mjs +++ b/crates/markkit-wasm/__test__/markkit-wasm.test.mjs @@ -94,6 +94,14 @@ test("extractDocument reads CSV and TSV", () => { assert.equal(tsv.format, "tsv"); }); +test("extractDocument reads Jupyter notebooks", () => { + const nb = { nbformat: 4, metadata: {}, cells: [{ cell_type: "markdown", source: "# NB" }, { cell_type: "code", source: "1+1", outputs: [] }] }; + const out = new Markkit().extractDocument(new TextEncoder().encode(JSON.stringify(nb)), { fileName: "a.ipynb" }); + assert.equal(out.format, "ipynb"); + assert.equal(out.title, "NB"); + assert.ok(out.markdown.includes("1+1")); +}); + test("extractBytes and unsupported formats", () => { const mk = new Markkit(); assert.equal(mk.extractBytes(new TextEncoder().encode("

x

"), "text/html"), "x\n"); diff --git a/crates/markkit/src/error.rs b/crates/markkit/src/error.rs index 9e3f80b..fbbd180 100644 --- a/crates/markkit/src/error.rs +++ b/crates/markkit/src/error.rs @@ -14,6 +14,9 @@ pub enum Error { #[error("Spreadsheet extraction error: {0}")] SpreadsheetExtract(String), + #[error("Notebook extraction error: {0}")] + NotebookExtract(String), + #[error("HTTP error: {0}")] Http(String), diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 9cab775..80169b6 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -72,6 +72,8 @@ pub struct ExtractionStats { pub paragraphs: Option, /// Tables (DOCX, Markdown, CSV/TSV). pub tables: Option, + /// Non-empty cells rendered (Jupyter notebooks). + pub cells: Option, } /// Result of [`Markkit::parse_document`]: the IR plus metadata, not rendered. @@ -231,8 +233,8 @@ impl Markkit { .map(|e| e.markdown) } - /// Convert a document (HTML, PDF, DOCX, XLSX/XLSB/XLS/ODS, plain text, Markdown, CSV/TSV) - /// to Markdown with metadata. + /// Convert a document (HTML, PDF, DOCX, XLSX/XLSB/XLS/ODS, plain text, Markdown, CSV/TSV, + /// Jupyter notebooks) to Markdown with metadata. /// /// The format is detected from `hint.content_type`, then the extension of /// `hint.file_name`, then magic bytes. Parsing is CPU-bound and runs inline; @@ -323,6 +325,7 @@ impl Markkit { Format::PlainText => self.parse_plain_text(bytes, budget), Format::Markdown => self.parse_markdown(bytes, budget), Format::Csv | Format::Tsv => self.parse_csv(bytes, format, budget), + Format::Notebook => self.parse_notebook(bytes, budget), Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -383,6 +386,18 @@ impl Markkit { Ok(p) } + fn parse_notebook(&self, bytes: &[u8], budget: &Budget) -> crate::Result { + let decoded = crate::text::decode(bytes); + let out = crate::notebook::parse(&decoded.text, &self.config, budget)?; + let mut doc = out.doc; + self.config.transforms.apply(&mut doc)?; + let mut p = ParsedDocument::new(doc, Format::Notebook); + p.title = out.title; + p.warnings = decoded.warnings; + p.stats.cells = Some(out.cells); + Ok(p) + } + #[cfg(feature = "markdown")] fn parse_markdown(&self, bytes: &[u8], budget: &Budget) -> crate::Result { let decoded = crate::text::decode(bytes); diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index 614c363..035aa75 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -34,6 +34,9 @@ pub enum Format { Csv, /// Tab separated values (`.tsv`, `.tab`). Tsv, + /// Jupyter notebook (`.ipynb`, nbformat 4). Serialized as `"ipynb"`. + #[serde(rename = "ipynb")] + Notebook, /// Plain text (`text/plain`, `.txt`). Never sniffed from bytes (nor CSV/TSV/Markdown). /// Serialized as `"text"`, same as `as_str`. #[serde(rename = "text")] @@ -91,6 +94,7 @@ impl Format { Format::Pptx => "pptx", Format::Csv => "csv", Format::Tsv => "tsv", + Format::Notebook => "ipynb", Format::PlainText => "text", Format::Markdown => "markdown", Format::Unknown => "unknown", @@ -119,6 +123,8 @@ impl Format { Format::Ods } else if ct.contains("presentationml") || ct.contains("ms-powerpoint") { Format::Pptx + } else if ct.contains("x-ipynb") { + Format::Notebook } else if ct.contains("tab-separated-values") { Format::Tsv } else if ct.contains("text/csv") || ct.contains("application/csv") { @@ -146,6 +152,7 @@ impl Format { "pptx" | "pptm" | "ppt" => Format::Pptx, "csv" => Format::Csv, "tsv" | "tab" => Format::Tsv, + "ipynb" => Format::Notebook, "txt" | "text" => Format::PlainText, "md" | "markdown" | "mdown" | "mkd" | "mkdn" => Format::Markdown, _ => Format::Unknown, @@ -268,6 +275,8 @@ mod tests { Format::Ods, Format::Pptx, Format::Csv, + Format::Tsv, + Format::Notebook, Format::PlainText, Format::Markdown, Format::Unknown, diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index ab35a0f..63a52e8 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -35,6 +35,7 @@ //! - **DOCX** (`docx`): Convert Word documents //! - **Spreadsheet** (`spreadsheet`): Convert XLSX/XLSM, XLSB, XLS and ODS workbooks to tables //! - **Plain text** (always on): `text/plain`, `.txt` +//! - **Jupyter notebooks** (always on): `.ipynb`, cells and text outputs //! - **CSV / TSV** (always on): `text/csv`, `text/tab-separated-values`, `.csv`, `.tsv`, one table //! - **Markdown** (`markdown`): `text/markdown`, `.md`, normalized through the IR //! - **PDF** (`pdf`): Text layer of PDF documents (no OCR); encrypted PDFs are @@ -65,6 +66,7 @@ pub mod ir; pub mod limits; #[cfg(feature = "markdown")] mod markdown; +mod notebook; #[cfg(feature = "pdf")] mod pdf; pub mod render; diff --git a/crates/markkit/src/notebook.rs b/crates/markkit/src/notebook.rs new file mode 100644 index 0000000..e1ec480 --- /dev/null +++ b/crates/markkit/src/notebook.rs @@ -0,0 +1,272 @@ +//! Jupyter notebook (`.ipynb`, nbformat 4) input. +//! +//! Decisions: +//! - Always compiled (serde_json is already a core dependency). Selected by +//! extension or `application/x-ipynb+json`, never sniffed. +//! - Markdown cells go through the Markdown parser when the `markdown` feature +//! is on (same IR, raw HTML never copied), otherwise they render as plain +//! text paragraphs. Code cells become fenced blocks tagged with the kernel +//! language; raw cells become untagged blocks. +//! - Outputs are kept because they often carry the result a reader needs: +//! `stream` and `text/plain` as ```` ```text ```` blocks, `text/markdown` +//! through the Markdown path, errors as `ename: evalue` plus traceback. +//! Images, HTML and widget outputs are skipped. ANSI escape sequences (common +//! in tracebacks) are stripped before the usual control-character pass. +//! - nbformat < 4 and non-notebook JSON are `Error::NotebookExtract`. serde_json +//! caps nesting at 128, so hostile JSON fails instead of overflowing. +//! - Shares the call's `Budget`: each cell ticks the deadline and counts +//! against `max_nodes`; Markdown cells use the same budget. + +use serde_json::Value; + +use crate::config::MarkkitBuilder; +use crate::error::Error; +use crate::ir::{Document, Node}; +use crate::limits::Budget; + +/// Parsed notebook. +pub(crate) struct NotebookOutput { + pub doc: Document, + pub title: Option, + /// Cells rendered (empty cells excluded). + pub cells: usize, +} + +/// Parse decoded notebook JSON into the IR. +pub(crate) fn parse( + json: &str, + config: &MarkkitBuilder, + budget: &Budget, +) -> Result { + let nb: Value = serde_json::from_str(json) + .map_err(|e| Error::NotebookExtract(format!("invalid JSON: {e}")))?; + let major = nb.get("nbformat").and_then(Value::as_u64).unwrap_or(0); + if major < 4 { + return Err(Error::NotebookExtract(format!( + "unsupported nbformat {major} (only 4.x is supported)" + ))); + } + let cells = nb + .get("cells") + .and_then(Value::as_array) + .ok_or_else(|| Error::NotebookExtract("missing 'cells' array".into()))?; + let meta = nb.get("metadata"); + let language = meta + .and_then(|m| m.pointer("/kernelspec/language")) + .or_else(|| meta.and_then(|m| m.pointer("/language_info/name"))) + .and_then(Value::as_str) + .map(|l| { + l.chars() + .filter(|c| c.is_ascii_alphanumeric() || "+-_#.".contains(*c)) + .collect::() + }) + .filter(|l| !l.is_empty()); + + let mut nodes = Vec::new(); + let mut count = 0; + for cell in cells { + if budget.expired() { + return Err(budget.timeout_error()); + } + budget.add_nodes(1)?; + let source = multiline(cell.get("source")); + let before = nodes.len(); + match cell.get("cell_type").and_then(Value::as_str) { + Some("markdown") => nodes.extend(markdown_nodes(&clean(&source), config, budget)?), + Some("code") => { + let code = clean(&source); + if !code.trim().is_empty() { + nodes.push(code_block(language.clone(), code)); + } + if let Some(outputs) = cell.get("outputs").and_then(Value::as_array) { + for out in outputs { + nodes.extend(output_nodes(out, config, budget)?); + } + } + } + Some("raw") => { + let raw = clean(&source); + if !raw.trim().is_empty() { + nodes.push(code_block(None, raw)); + } + } + _ => {} + } + if nodes.len() > before { + count += 1; + } + } + + let title = meta + .and_then(|m| m.get("title")) + .and_then(Value::as_str) + .map(|t| clean(t).trim().to_string()) + .filter(|t| !t.is_empty()) + .or_else(|| first_h1(&nodes)); + Ok(NotebookOutput { + doc: Document::new(nodes), + title, + cells: count, + }) +} + +/// nbformat stores text either as a string or a list of lines. +fn multiline(v: Option<&Value>) -> String { + match v { + Some(Value::String(s)) => s.clone(), + Some(Value::Array(parts)) => parts.iter().filter_map(Value::as_str).collect(), + _ => String::new(), + } +} + +/// Strip ANSI escape sequences, then apply the shared text normalization. +fn clean(s: &str) -> String { + crate::text::decode(strip_ansi(s).as_bytes()).text +} + +fn strip_ansi(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c != '\u{1b}' { + out.push(c); + continue; + } + match chars.peek() { + // CSI: ESC [ params… final byte in @..~ + Some('[') => { + chars.next(); + for c in chars.by_ref() { + if ('@'..='~').contains(&c) { + break; + } + } + } + // OSC: ESC ] … BEL or ESC \ + Some(']') => { + chars.next(); + while let Some(c) = chars.next() { + if c == '\u{7}' { + break; + } + if c == '\u{1b}' && chars.peek() == Some(&'\\') { + chars.next(); + break; + } + } + } + // Charset designation: ESC ( B and friends. + Some('(' | ')' | '*' | '+') => { + chars.next(); + chars.next(); + } + // Two-character escape. + Some(_) => { + chars.next(); + } + None => {} + } + } + out +} + +fn code_block(language: Option, code: String) -> Node { + Node::CodeBlock { + language, + code: code.trim_end_matches('\n').to_string(), + source: None, + } +} + +fn text_block(text: String) -> Option { + (!text.trim().is_empty()).then(|| code_block(Some("text".into()), text)) +} + +#[cfg(feature = "markdown")] +fn markdown_nodes(src: &str, config: &MarkkitBuilder, budget: &Budget) -> Result, Error> { + Ok(crate::markdown::parse(src, config, budget)?.doc.nodes) +} + +#[cfg(not(feature = "markdown"))] +fn markdown_nodes( + src: &str, + _config: &MarkkitBuilder, + budget: &Budget, +) -> Result, Error> { + Ok(crate::text::plain_to_ir(src, budget)?.0.nodes) +} + +fn output_nodes(out: &Value, config: &MarkkitBuilder, budget: &Budget) -> Result, Error> { + Ok(match out.get("output_type").and_then(Value::as_str) { + Some("stream") => text_block(clean(&multiline(out.get("text")))) + .into_iter() + .collect(), + Some("execute_result" | "display_data") => { + let data = out.get("data"); + if let Some(md) = data.and_then(|d| d.get("text/markdown")) { + markdown_nodes(&clean(&multiline(Some(md))), config, budget)? + } else if let Some(txt) = data.and_then(|d| d.get("text/plain")) { + text_block(clean(&multiline(Some(txt)))) + .into_iter() + .collect() + } else { + Vec::new() + } + } + Some("error") => { + let field = |k| out.get(k).and_then(Value::as_str).unwrap_or(""); + let mut text = format!("{}: {}", field("ename"), field("evalue")); + if let Some(tb) = out.get("traceback").and_then(Value::as_array) { + for line in tb.iter().filter_map(Value::as_str) { + text.push('\n'); + text.push_str(line); + } + } + text_block(clean(&text)).into_iter().collect() + } + _ => Vec::new(), + }) +} + +fn first_h1(nodes: &[Node]) -> Option { + nodes.iter().find_map(|n| match n { + Node::Heading { + level: 1, children, .. + } => { + let t: String = children + .iter() + .filter_map(|c| match c { + Node::Text(t) | Node::InlineCode(t) => Some(t.as_str()), + _ => None, + }) + .collect(); + let t = t.trim(); + (!t.is_empty()).then(|| t.to_string()) + } + _ => None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strips_ansi_sequences() { + assert_eq!( + strip_ansi("\u{1b}[0;31mValueError\u{1b}[0m: x\u{1b}]0;title\u{7}!\u{1b}(B"), + "ValueError: x!" + ); + assert_eq!(strip_ansi("trailing \u{1b}"), "trailing "); + assert_eq!(strip_ansi("ok\u{1b}[12;"), "ok"); + } + + #[test] + fn multiline_forms() { + let v: Value = serde_json::json!(["a\n", "b"]); + assert_eq!(multiline(Some(&v)), "a\nb"); + assert_eq!(multiline(Some(&Value::String("x".into()))), "x"); + assert_eq!(multiline(Some(&Value::Null)), ""); + assert_eq!(multiline(None), ""); + } +} diff --git a/crates/markkit/tests/notebook_integration.rs b/crates/markkit/tests/notebook_integration.rs new file mode 100644 index 0000000..e07f233 --- /dev/null +++ b/crates/markkit/tests/notebook_integration.rs @@ -0,0 +1,209 @@ +//! Jupyter notebook (.ipynb) → Markdown integration and security tests. + +use markkit::format::Format; +use markkit::{DocumentHint, Error, Markkit}; +use serde_json::json; + +fn extract(nb: &serde_json::Value) -> markkit::Extraction { + Markkit::builder() + .build() + .extract_document_sync( + nb.to_string().as_bytes(), + DocumentHint::file_name("nb.ipynb"), + ) + .unwrap() +} + +fn err(bytes: &[u8]) -> Error { + Markkit::builder() + .build() + .extract_document_sync(bytes, DocumentHint::file_name("nb.ipynb")) + .unwrap_err() +} + +fn notebook(cells: serde_json::Value) -> serde_json::Value { + json!({ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": {"kernelspec": {"name": "python3", "language": "python"}}, + "cells": cells + }) +} + +#[test] +fn cells_and_outputs() { + let nb = notebook(json!([ + {"cell_type": "markdown", "source": ["# Analysis\n", "\n", "Load the data."]}, + {"cell_type": "code", "source": "import pandas as pd\ndf = pd.read_csv('x.csv')\ndf.shape", + "outputs": [ + {"output_type": "stream", "name": "stdout", "text": ["loading\n"]}, + {"output_type": "execute_result", "data": {"text/plain": ["(3, 2)"], "image/png": "iVBOR"}}, + {"output_type": "display_data", "data": {"image/png": "iVBOR"}} + ]}, + {"cell_type": "code", "source": [], "outputs": []}, + {"cell_type": "raw", "source": "raw text"} + ])); + let e = extract(&nb); + assert_eq!(e.format, Format::Notebook); + if cfg!(feature = "markdown") { + assert_eq!(e.title.as_deref(), Some("Analysis")); + assert!(e.markdown.starts_with("# Analysis\n\nLoad the data.\n")); + } else { + // Without the parser, Markdown cells are plain text paragraphs. + assert_eq!(e.title, None); + } + assert_eq!(e.stats.cells, Some(3)); + assert!(e.markdown.ends_with( + "```python\nimport pandas as pd\ndf = pd.read_csv('x.csv')\ndf.shape\n```\n\n```text\nloading\n```\n\n```text\n(3, 2)\n```\n\n```\nraw text\n```\n" + ), "{}", e.markdown); + assert!(!e.markdown.contains("iVBOR")); +} + +#[test] +fn error_output_strips_ansi() { + let nb = notebook(json!([ + {"cell_type": "code", "source": "1/0", "outputs": [{ + "output_type": "error", "ename": "ZeroDivisionError", "evalue": "division by zero", + "traceback": ["\u{1b}[0;31m---------\u{1b}[0m", "\u{1b}[0;31mZeroDivisionError\u{1b}[0m: division by zero"] + }]} + ])); + let e = extract(&nb); + assert!( + e.markdown.contains("```text\nZeroDivisionError: division by zero\n---------\nZeroDivisionError: division by zero\n```"), + "{}", + e.markdown + ); + assert!(!e.markdown.contains('\u{1b}') && !e.markdown.contains("[0;31m")); +} + +#[test] +fn metadata_title_and_language_info() { + let nb = json!({ + "nbformat": 4, "nbformat_minor": 2, + "metadata": {"title": "My NB", "language_info": {"name": "julia"}}, + "cells": [{"cell_type": "code", "source": "1 + 1", "outputs": []}] + }); + let e = extract(&nb); + assert_eq!(e.title.as_deref(), Some("My NB")); + assert_eq!(e.markdown, "```julia\n1 + 1\n```\n"); +} + +#[test] +fn detection() { + assert_eq!(Format::from_extension("IPYNB"), Format::Notebook); + assert_eq!( + Format::from_content_type("application/x-ipynb+json"), + Format::Notebook + ); + // Never sniffed: plain JSON bytes without a hint stay unsupported. + let e = Markkit::builder() + .build() + .extract_document_sync(br#"{"nbformat":4,"cells":[]}"#, DocumentHint::default()) + .unwrap_err(); + assert!(matches!(e, Error::UnsupportedFormat(_))); +} + +#[test] +fn empty_notebook() { + let e = extract(¬ebook(json!([]))); + assert_eq!(e.markdown, ""); + assert_eq!(e.stats.cells, Some(0)); +} + +// ─── Security (threat model T7) ───────────────────────────────────── + +#[test] +fn security_malformed_json_is_error() { + for bytes in [ + &b""[..], + b"not json", + b"{\"nbformat\": 4", + b"[1,2,3]", + b"{\"nbformat\": 4, \"cells\": 7}", + b"{\"nbformat\": 3, \"worksheets\": []}", + b"\xff\xfe\x00", + ] { + assert!(matches!(err(bytes), Error::NotebookExtract(_)), "{bytes:?}"); + } +} + +#[test] +fn security_deep_json_nesting_is_error_not_overflow() { + let deep = format!( + "{{\"nbformat\":4,\"cells\":[],\"metadata\":{}{}}}", + "[".repeat(100_000), + "]".repeat(100_000) + ); + assert!(matches!(err(deep.as_bytes()), Error::NotebookExtract(_))); +} + +#[test] +fn security_wrong_types_ignored() { + let nb = json!({ + "nbformat": 4, + "metadata": {"kernelspec": {"language": "py`thon\n```"}, "title": 5}, + "cells": [ + {"cell_type": 1, "source": "x"}, + {"cell_type": "code", "source": {"a": 1}, "outputs": "nope"}, + {"cell_type": "code", "source": ["ok", 3, null], "outputs": [7, {"output_type": "stream", "text": 9}]}, + "not a cell" + ] + }); + let e = extract(&nb); + assert_eq!(e.markdown, "```python\nok\n```\n"); + assert_eq!(e.title, None); +} + +#[cfg(feature = "markdown")] +#[test] +fn security_markdown_cells_and_outputs_drop_raw_html() { + let nb = notebook(json!([ + {"cell_type": "markdown", "source": "hi \n\n"}, + {"cell_type": "code", "source": "x", "outputs": [ + {"output_type": "display_data", "data": {"text/html": "", "text/markdown": "**ok**