From 038e4423d1545bed697360d7e7810fd7df598ed4 Mon Sep 17 00:00:00 2001 From: Illia Kripaka Date: Mon, 3 Aug 2026 16:22:21 +0300 Subject: [PATCH 1/4] Extend capabilities of lexer and parser for formatting * add `fmt` feature * edit logic of lexer to include other characters (restrict behaviour to be consistent across different lexers) * add FmtTokens, which stores comments, newlines and whitespaces data * add `#[non_exhaustive]` to Token, TriviaKind and FmtToken * separate tests with modules for different behaviour (lexer + parser) * add additional spans for appropriate structs in parser * add integration test for lossless lexer, which checks that lossless lexer didn't lose anything from context (relies onto the all .simf examples in this repo) --- Cargo.toml | 1 + src/ast.rs | 180 ++++- src/lexer.rs | 891 ++++++++++++++++++---- src/parse.rs | 1271 +++++++++++++++++++++++++------- tests/lex_lossless_coverage.rs | 103 +++ 5 files changed, 1992 insertions(+), 454 deletions(-) create mode 100644 tests/lex_lossless_coverage.rs diff --git a/Cargo.toml b/Cargo.toml index 233268b4..5d0b11ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ default = [ "serde" ] serde = ["dep:serde", "dep:serde_json"] docs = [] external-jets = [] +fmt = [] [dependencies] base64 = "0.21.2" diff --git a/src/ast.rs b/src/ast.rs index 8d4a0c93..971e36e2 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -343,10 +343,11 @@ impl PartialEq for CallName { } /// Definition of a custom function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct CustomFunction { params: Arc<[FunctionParam]>, body: Arc, + span: Span, } impl CustomFunction { @@ -360,6 +361,11 @@ impl CustomFunction { &self.body } + /// Access the span of the complete function declaration. + pub fn span(&self) -> &Span { + &self.span + } + /// Return a pattern for the parameters of the function. pub fn params_pattern(&self) -> Pattern { Pattern::tuple( @@ -372,11 +378,14 @@ impl CustomFunction { } } +impl_eq_hash!(CustomFunction; params, body); + /// Parameter of a function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct FunctionParam { identifier: Identifier, ty: ResolvedType, + span: Span, } impl FunctionParam { @@ -389,8 +398,15 @@ impl FunctionParam { pub fn ty(&self) -> &ResolvedType { &self.ty } + + /// Access the span of the complete parameter declaration. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(FunctionParam; identifier, ty); + /// Match expression. #[derive(Clone, Debug)] pub struct Match { @@ -461,6 +477,7 @@ pub struct EnumMatchArm { /// variants. pattern: Pattern, body: Arc, + span: Span, } impl EnumMatchArm { @@ -473,6 +490,11 @@ impl EnumMatchArm { pub fn body(&self) -> &Expression { &self.body } + + /// Access the span of the complete enum match arm. + pub fn span(&self) -> &Span { + &self.span + } } impl_eq_hash!(EnumMatchArm; pattern, body); @@ -514,10 +536,11 @@ impl AsRef for EnumMatch { } /// Arm of a [`Match`] expression. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct MatchArm { pattern: MatchPattern, expression: Arc, + span: Span, } impl MatchArm { @@ -530,8 +553,15 @@ impl MatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the span of the complete match arm. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(MatchArm; pattern, expression); + #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] pub enum ExprTree<'a> { Expression(&'a Expression), @@ -1379,7 +1409,11 @@ impl AbstractSyntaxTree for Function { .map(|param| { let identifier = param.identifier().clone(); let ty = scope.resolve(param.ty())?; - Ok(FunctionParam { identifier, ty }) + Ok(FunctionParam { + identifier, + ty, + span: *param.span(), + }) }) .collect::, Error>>() .with_span(from)?; @@ -1398,7 +1432,11 @@ impl AbstractSyntaxTree for Function { scope.exit_block(); debug_assert!(scope.is_outside_function()); - let function = CustomFunction { params, body }; + let function = CustomFunction { + params, + body, + span: *from.span(), + }; scope .insert_function(from.name().clone(), from.visibility().clone(), function) .with_span(from)?; @@ -1945,10 +1983,11 @@ impl AbstractSyntaxTree for EnumMatch { .into_iter() .zip(info.variants()) .map(|(arm, variant)| { - let pattern = analyze_enum_arm_bindings(arm, variant, scope, span)?; + let arm_span = *arm.span(); + let pattern = analyze_enum_arm_bindings(arm, variant, scope, arm_span)?; scope.enter_block(); let payload_ty = variant.payload_type(); - let typed_variables = pattern.is_of_type(payload_ty).with_span(span)?; + let typed_variables = pattern.is_of_type(payload_ty).with_span(arm_span)?; for (identifier, variable_ty) in typed_variables { scope.insert_variable(identifier, variable_ty); } @@ -1957,6 +1996,7 @@ impl AbstractSyntaxTree for EnumMatch { Ok(EnumMatchArm { pattern, body: body?, + span: arm_span, }) }) .collect::, Diagnostic>>()?; @@ -2347,8 +2387,8 @@ impl AbstractSyntaxTree for Match { scope.enter_block(); if let Some((pat_l, ty_l)) = from.left().pattern().as_typed_pattern() { - let ty_l = scope.resolve(ty_l).with_span(from)?; - let typed_variables = pat_l.is_of_type(&ty_l).with_span(from)?; + let ty_l = scope.resolve(ty_l).with_span(from.left())?; + let typed_variables = pat_l.is_of_type(&ty_l).with_span(from.left())?; for (identifier, ty) in typed_variables { scope.insert_variable(identifier, ty); } @@ -2357,8 +2397,8 @@ impl AbstractSyntaxTree for Match { scope.exit_block(); scope.enter_block(); if let Some((pat_r, ty_r)) = from.right().pattern().as_typed_pattern() { - let ty_r = scope.resolve(ty_r).with_span(from)?; - let typed_variables = pat_r.is_of_type(&ty_r).with_span(from)?; + let ty_r = scope.resolve(ty_r).with_span(from.right())?; + let typed_variables = pat_r.is_of_type(&ty_r).with_span(from.right())?; for (identifier, ty) in typed_variables { scope.insert_variable(identifier, ty); } @@ -2371,10 +2411,12 @@ impl AbstractSyntaxTree for Match { left: MatchArm { pattern: from.left().pattern().clone(), expression: ast_l, + span: *from.left().span(), }, right: MatchArm { pattern: from.right().pattern().clone(), expression: ast_r, + span: *from.right().span(), }, span: *from.as_ref(), }) @@ -2387,6 +2429,18 @@ impl AsRef for Assignment { } } +impl AsRef for FunctionParam { + fn as_ref(&self) -> &Span { + &self.span + } +} + +impl AsRef for CustomFunction { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for Expression { fn as_ref(&self) -> &Span { &self.span @@ -2411,6 +2465,110 @@ impl AsRef for Match { } } +impl AsRef for MatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + +impl AsRef for EnumMatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + +#[cfg(test)] +mod span_tests { + use crate::parse::ParseFromStr; + + use super::*; + + #[test] + fn analyzed_custom_function_preserves_declaration_and_parameter_spans() { + let source = "fn helper(value: u8) -> u8 { value }"; + let parsed = parse::Function::parse_from_str(source).expect("function parses"); + let mut scope = Scope::new(Box::new(ElementsJetHinter)); + + Function::analyze(&parsed, &ResolvedType::unit(), &mut scope).expect("function analyzes"); + let function = scope + .get_function(parsed.name()) + .expect("function is registered in scope"); + + assert_eq!(function.span().to_slice(source), Some(source)); + assert_eq!( + function.params()[0].span().to_slice(source), + Some("value: u8") + ); + } + + #[test] + fn analyzed_match_arms_preserve_their_parsed_spans() { + let source = r#"fn main() { + let input: Either = Left(1); + match input { + Left(left: u8) => {}, + Right(right: u8) => {}, + } +}"#; + let parsed = parse::Program::parse_from_str(source).expect("program parses"); + let program = + Program::analyze(&parsed, Box::new(ElementsJetHinter)).expect("program analyzes"); + + let ExpressionInner::Block(_, Some(last)) = program.main().inner() else { + panic!("main body should end in a match"); + }; + let ExpressionInner::Single(single) = last.inner() else { + panic!("match should be a single expression"); + }; + let SingleExpressionInner::Match(match_) = single.inner() else { + panic!("expected a binary match"); + }; + + assert_eq!( + match_.left().span().to_slice(source), + Some("Left(left: u8) => {},") + ); + assert_eq!( + match_.right().span().to_slice(source), + Some("Right(right: u8) => {},") + ); + } + + #[test] + fn analyzed_enum_match_arms_preserve_their_parsed_spans() { + let source = r#"enum Choice { First, Second, } +fn main() { + let input: Choice = Choice::First; + match input { + Choice::First => {}, + Choice::Second => {}, + } +}"#; + let parsed = parse::Program::parse_from_str(source).expect("program parses"); + let program = + Program::analyze(&parsed, Box::new(ElementsJetHinter)).expect("program analyzes"); + + let ExpressionInner::Block(_, Some(last)) = program.main().inner() else { + panic!("main body should end in an enum match"); + }; + let ExpressionInner::Single(single) = last.inner() else { + panic!("enum match should be a single expression"); + }; + let SingleExpressionInner::EnumMatch(match_) = single.inner() else { + panic!("expected an enum match"); + }; + + assert_eq!( + match_.arms()[0].span().to_slice(source), + Some("Choice::First => {},") + ); + assert_eq!( + match_.arms()[1].span().to_slice(source), + Some("Choice::Second => {},") + ); + } +} + #[cfg(test)] mod scope_resolution_tests { use super::{ElementsJetHinter, Program}; diff --git a/src/lexer.rs b/src/lexer.rs index 862195e1..116e112a 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -11,7 +11,11 @@ use crate::version::SIMC_STR; pub type Spanned = (T, SimpleSpan); pub type Tokens<'src> = Vec<(Token<'src>, crate::error::Span)>; +#[cfg(feature = "fmt")] +pub type FmtTokens<'src> = Vec<(FmtToken<'src>, crate::error::Span)>; + #[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] pub enum Token<'src> { // Keywords Pub, @@ -67,12 +71,81 @@ pub enum Token<'src> { // Built-in functions Macro(&'src str), +} - // Comments and block comments - // - // We would discard them for the compiler, but they are needed, for example, for the formatter. - Comment, +#[cfg(feature = "fmt")] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum TriviaKind { + LineComment, BlockComment, + Newline, + Whitespace, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LineEnding { + /// Windows carriage-return/line-feed (`\r\n`). + CrLf, + /// Unix line-feed (`\n`). + Lf, + /// Classic Mac OS carriage-return (`\r`). + Cr, +} + +impl LineEnding { + pub const fn as_str(self) -> &'static str { + match self { + Self::CrLf => "\r\n", + Self::Lf => "\n", + Self::Cr => "\r", + } + } +} + +#[cfg(feature = "fmt")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Trivia<'src> { + LineComment(&'src str), + BlockComment(&'src str), + Newline(LineEnding), + Whitespace(&'src str), +} + +#[cfg(feature = "fmt")] +impl<'src> Trivia<'src> { + pub const fn line_comment(text: &'src str) -> Self { + Self::LineComment(text) + } + + pub const fn block_comment(text: &'src str) -> Self { + Self::BlockComment(text) + } + + pub const fn newline(line_ending: LineEnding) -> Self { + Self::Newline(line_ending) + } + + pub const fn whitespace(text: &'src str) -> Self { + Self::Whitespace(text) + } + + pub const fn kind(&self) -> TriviaKind { + match self { + Self::LineComment(_) => TriviaKind::LineComment, + Self::BlockComment(_) => TriviaKind::BlockComment, + Self::Newline(_) => TriviaKind::Newline, + Self::Whitespace(_) => TriviaKind::Whitespace, + } + } +} + +#[cfg(feature = "fmt")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum FmtToken<'src> { + Token(Token<'src>), + Trivia(Trivia<'src>), } impl<'src> fmt::Display for Token<'src> { @@ -120,9 +193,21 @@ impl<'src> fmt::Display for Token<'src> { Token::Param(s) => write!(f, "param::{}", s), Token::Bool(b) => write!(f, "{}", b), + } + } +} - Token::Comment => write!(f, "comment"), - Token::BlockComment => write!(f, "block_comment"), +#[cfg(feature = "fmt")] +impl<'src> fmt::Display for FmtToken<'src> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FmtToken::Token(t) => { + write!(f, "{}", t) + } + FmtToken::Trivia( + Trivia::LineComment(text) | Trivia::BlockComment(text) | Trivia::Whitespace(text), + ) => write!(f, "{text}"), + FmtToken::Trivia(Trivia::Newline(line_ending)) => write!(f, "{}", line_ending.as_str()), } } } @@ -130,11 +215,40 @@ impl<'src> fmt::Display for Token<'src> { /// Recognizer for a `// ...` line comment. fn line_comment<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + let newline = line_ending(); + just("//") - .then(any().and_is(just('\n').not()).repeated()) + .ignore_then(any().and_is(newline.not()).repeated()) .ignored() } +/// Recognizer for different newline encodings (`Windows`: `\r\n`, `Unix`: `\n`, `Mac`: `\r`). +fn line_ending<'src>( +) -> impl Parser<'src, &'src str, LineEnding, extra::Err>> + Clone { + choice(( + just("\r\n").to(LineEnding::CrLf), + just("\n").to(LineEnding::Lf), + just("\r").to(LineEnding::Cr), + )) +} + +/// Recognizer for whitespace. +#[cfg(feature = "fmt")] +fn whitespace<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + any() + .filter(|c: &char| c.is_whitespace() && *c != '\n' && *c != '\r') + .repeated() + .at_least(1) + .ignored() +} + +/// Recognizer for whitespace or a newline. +fn whitespace_or_newline<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + any().filter(|c: &char| c.is_whitespace()).ignored() +} + /// Recognizer for a (possibly nested) `/* ... */` block comment; an unterminated /// comment is reported and swallows the rest of the input. fn block_comment<'src>( @@ -152,23 +266,23 @@ fn block_comment<'src>( }) } +/// One non-empty trivia item. Keeping this separate from [`trivia`] lets the +/// ordinary lexer include trivia in the same recovery boundary as tokens. +fn trivia_item<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + choice((line_comment(), block_comment(), whitespace_or_newline())) +} + /// Trivia — whitespace and comments — shared with the version-directive scanner /// (`version::SimcDirective::scan`) so the lexer and the scanner agree on comment /// syntax. pub(crate) fn trivia<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> { - choice(( - line_comment(), - block_comment(), - any().filter(|c: &char| c.is_whitespace()).ignored(), - )) - .repeated() - .ignored() + trivia_item().repeated().ignored() } -pub fn lexer<'src>( -) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> -{ +fn to_token<'src>( +) -> impl Parser<'src, &'src str, Token<'src>, extra::Err>> { let digits_with_underscore = |radix: u32| { any() .filter(move |c: &char| c.is_digit(radix)) @@ -243,25 +357,46 @@ pub fn lexer<'src>( just(">").to(Token::RAngle), )); - let comment = line_comment().to(Token::Comment); - let block_comment = block_comment().to(Token::BlockComment); - let token = choice(( - comment, - block_comment, - jet, - witness, - param, - macros, - keyword, - hex, - bin, - num, - op, - )); + choice((jet, witness, param, macros, keyword, hex, bin, num, op)) +} - token - .map_with(|tok, e| (tok, e.span())) - .padded() +pub fn lexer<'src>( +) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> +{ + let lexeme = choice((trivia_item().to(None), to_token().map(Some))) + .map_with(|token, e| (token, e.span())) + .recover_with(skip_then_retry_until(any().ignored(), end())); + + lexeme.repeated().collect::>().map(|lexemes| { + lexemes + .into_iter() + .filter_map(|(token, span)| token.map(|token| (token, span))) + .collect() + }) +} + +#[cfg(feature = "fmt")] +pub fn lexer_lossless<'src>( +) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> +{ + let token = to_token().map(FmtToken::Token); + + let newline = line_ending().map(Trivia::newline).map(FmtToken::Trivia); + let whitespace = whitespace() + .to_slice() + .map(Trivia::whitespace) + .map(FmtToken::Trivia); + let line_comment = line_comment() + .to_slice() + .map(Trivia::line_comment) + .map(FmtToken::Trivia); + let block_comment = block_comment() + .to_slice() + .map(Trivia::block_comment) + .map(FmtToken::Trivia); + + choice((line_comment, block_comment, newline, whitespace, token)) + .map_with(|lexeme, e| (lexeme, e.span())) .recover_with(skip_then_retry_until(any().ignored(), end())) .repeated() .collect() @@ -271,7 +406,7 @@ pub fn lexer<'src>( /// offset `start` — the end of the version directive per `SimcDirective::prescan`, /// or `0`. Spans are reported relative to the full input. /// -/// All comments in the input code are discarded. +/// All comments, newlines, and spaces in the input code are discarded. pub fn lex( file_id: usize, input: &str, @@ -280,26 +415,59 @@ pub fn lex( let (tokens, lex_errors) = lexer().parse(&input[start..]).into_output_errors(); let shift = |span| Span::from_chumsky(file_id, span, start); - let mut diagnostics: Vec = Vec::new(); + let mut diagnostics: Vec = lex_errors + .into_iter() + .map(|err| { + Diagnostic::new( + Error::CannotParse { + msg: err.reason().to_string(), + }, + shift(*err.span()), + ) + }) + .collect(); + + let tokens = tokens.map(|vec| { + vec.into_iter() + .filter_map(|(tok, span)| filter_token(tok, span, &mut diagnostics, shift)) + .collect() + }); + + (tokens, diagnostics) +} - diagnostics.extend(lex_errors.into_iter().map(|err| { - Diagnostic::new( - Error::CannotParse { - msg: err.reason().to_string(), - }, - shift(*err.span()), - ) - })); +/// Lexes an input string into a lossles stream of tokens with spans, beginning at byte +/// offset `start` — the end of the version directive per `SimcDirective::prescan`, +/// or `0`. Spans are reported relative to the full input. +/// +/// All comments, newlines, and spaces in the input code are remained. +#[cfg(feature = "fmt")] +pub fn lex_lossless( + file_id: usize, + input: &str, + start: usize, +) -> (Option>, Vec) { + let (tokens, lex_errors) = lexer_lossless().parse(&input[start..]).into_output_errors(); + let shift = |span: SimpleSpan| Span::from_chumsky(file_id, span, start); + + let mut diagnostics: Vec = lex_errors + .into_iter() + .map(|err| { + Diagnostic::new( + Error::CannotParse { + msg: err.reason().to_string(), + }, + shift(*err.span()), + ) + }) + .collect(); let tokens = tokens.map(|vec| { vec.into_iter() - .filter_map(|(tok, span)| match tok { - Token::Comment | Token::BlockComment => None, - Token::Simc => { - diagnostics.push(Diagnostic::new(Error::ReservedSimcKeyword, shift(span))); - None - } - tok => Some((tok, shift(span))), + .filter_map(|(fmt_tok, span)| match fmt_tok { + FmtToken::Token(tok) => filter_token(tok, span, &mut diagnostics, shift) + .map(|(t, s)| (FmtToken::Token(t), s)), + FmtToken::Trivia(t) => Some((FmtToken::Trivia(t), shift(span))), }) .collect() }); @@ -307,6 +475,26 @@ pub fn lex( (tokens, diagnostics) } +fn filter_token<'src, F: Fn(SimpleSpan) -> Span>( + tok: Token<'src>, + span: SimpleSpan, + errors: &mut Vec, + convert_span: F, +) -> Option<(Token<'src>, Span)> { + match tok { + // The reserved keyword is a sentinel: the prescan consumed the one + // legitimate directive before lexing, so any occurrence is misplaced. + Token::Simc => { + errors.push(Diagnostic::new( + Error::ReservedSimcKeyword, + convert_span(span), + )); + None + } + tok => Some((tok, convert_span(span))), + } +} + /// A list of all reserved keywords. pub const KEYWORDS: &[&str] = &[ "pub", "use", "as", "fn", "let", "type", "mod", "const", "match", "enum", CRATE_STR, SIMC_STR, @@ -324,148 +512,525 @@ mod tests { use super::*; - fn lex<'src>( - input: &'src str, - ) -> (Option>>, Vec>) { - let (tokens, errors) = lexer().parse(input).into_output_errors(); - let tokens = tokens.map(|vec| vec.iter().map(|(tok, _)| tok.clone()).collect::>()); - (tokens, errors) - } + mod lexer { + use super::*; + + fn lex<'src>( + input: &'src str, + ) -> (Option>>, Vec>) { + let (tokens, errors) = lexer().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(tok, _)| tok.clone()) + .collect::>() + }); + (tokens, errors) + } + #[test] + fn test_block_comment_simple() { + let input = "/* hello world */"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![]), + "Should produce a single block comment token" + ); + } - #[test] - fn test_block_comment_simple() { - let input = "/* hello world */"; - let (tokens, errors) = lex(input); - - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!( - tokens, - Some(vec![Token::BlockComment]), - "Should produce a single block comment token" - ); - } + #[test] + fn test_block_comment_nested() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_nested() { - let input = "/* outer /* inner */ outer */"; - let (tokens, errors) = lex(input); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_deeply_nested() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_deeply_nested() { - let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; - let (tokens, errors) = lex(input); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_multiline() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_multiline() { - let input = "/* \n line 1 \n /* inner \n line */ \n */"; - let (tokens, errors) = lex(input); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_unclosed() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_unclosed() { - let input = "/* unclosed comment start"; - let (tokens, errors) = lex(input); + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); - assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); - let err = &errors[0]; - assert_eq!(err.span().start, 0); - assert_eq!(err.span().end, 2); - assert_eq!(err.to_string(), "Unclosed block comment"); + assert_eq!(tokens, Some(vec![])); + } - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_partial_nesting_unclosed() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_partial_nesting_unclosed() { - let input = "/* outer /* inner */"; - let (tokens, errors) = lex(input); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!(tokens, Some(vec![])); + } - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].span().start, 0); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_double_unclosed() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex(input); + + assert_eq!(errors.len(), 2); + + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); + + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); + + assert_eq!(tokens, Some(vec![])); + } + + #[test] + fn test_spaces_resolution() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } + + #[test] + fn test_ignoring_tokens_after_comment_with_incorrect_symbol() { + let input = "fn main() {} @// fn hello(){} simc"; + let (tokens, errors) = lex(input); + + assert_eq!( + errors.len(), + 1, + "comment contents must not be retried as code" + ); + assert!(errors[0].to_string().contains("found '@' expected")); + assert_eq!( + tokens, + Some(vec![ + Token::Fn, + Token::Ident("main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + ]) + ); - #[test] - fn test_block_comment_double_unclosed() { - let input = "/* outer /* inner"; - let (tokens, errors) = lex(input); + let (_tokens, diagnostics) = super::lex(0, input, 0); + assert_eq!(diagnostics.len(), 1); + assert!(matches!(diagnostics[0].error(), Error::CannotParse { .. })); + } - assert_eq!(errors.len(), 2); + #[test] + fn test_ignoring_tokens_after_comment() { + let input = "fn main() {} /* fn hello(){} \n simc 0.6.0; */ \n\ + // enum Name {} match true {} \n fn other_main() {} "; + let (tokens, errors) = lex(input); + + assert!(dbg!(errors).is_empty()); + assert_eq!( + tokens, + Some(vec![ + Token::Fn, + Token::Ident("main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + Token::Fn, + Token::Ident("other_main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + ]) + ); + } - assert_eq!(errors[0].span().start, 9); - assert_eq!(errors[0].to_string(), "Unclosed block comment"); + #[test] + fn simc_is_reserved() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, Token::Simc)), + "the sentinel must not reach the token stream for {src:?}" + ); + } + + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); + } - assert_eq!(errors[1].span().start, 0); - assert_eq!(errors[1].to_string(), "Unclosed block comment"); + #[test] + fn test_enum_token() { + let (tokens, errors) = lex("enum Path { Inherit, ColdSpend }"); + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![ + Token::Enum, + Token::Ident("Path"), + Token::LBrace, + Token::Ident("Inherit"), + Token::Comma, + Token::Ident("ColdSpend"), + Token::RBrace, + ]) + ); + } + + #[test] + fn lexer_test() { + use chumsky::prelude::*; + + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + let (tokens, lex_errs) = lexer().parse(src).into_output_errors(); + let _ = tokens.unwrap(); + + assert!(lex_errs.is_empty()); + } } - #[test] - fn simc_is_reserved() { - // The prescan consumes the one legitimate leading directive before lexing, - // so `lex` reports any `simc` it sees and drops the sentinel token. - for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { - let (tokens, errors) = super::lex(0, src, 0); - assert!( - errors.iter().any(|e| e.to_string().contains("reserved")), - "expected a reserved-keyword error for {src:?}, got: {errors:?}" + #[cfg(feature = "fmt")] + mod fmt_lexer { + use super::*; + + fn lex_lossless<'src>( + input: &'src str, + ) -> ( + Option>>, + Vec>, + ) { + let (tokens, errors) = lexer_lossless().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(fmt_tok, _)| fmt_tok) + .collect::>() + }); + (tokens, errors) + } + + fn fmt_trivia<'src>(kind: TriviaKind, text: &'src str) -> FmtToken<'src> { + let trivia = match kind { + TriviaKind::LineComment => Trivia::line_comment(text), + TriviaKind::BlockComment => Trivia::block_comment(text), + TriviaKind::Newline => Trivia::newline(match text { + "\r\n" => LineEnding::CrLf, + "\n" => LineEnding::Lf, + "\r" => LineEnding::Cr, + _ => panic!("invalid newline spelling: {text:?}"), + }), + TriviaKind::Whitespace => Trivia::whitespace(text), + }; + + FmtToken::Trivia(trivia) + } + + #[test] + fn test_block_comment_simple_fmt() { + let input = "/* hello world */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]), + "Should produce a single block comment token" + ); + } + + #[test] + fn lossless_trivia_display_preserves_source_text() { + let input = "// comment\r\n\t"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let rendered: String = tokens + .expect("lossless lexing succeeds") + .iter() + .map(ToString::to_string) + .collect(); + + assert_eq!(rendered, input); + } + + #[test] + fn lossless_lexer_keeps_each_newline_kind_with_its_span() { + let input = "first\r\nsecond\rthird\nfourth"; + let (tokens, errors) = super::lex_lossless(0, input, 0); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let tokens = tokens.expect("lossless lexing succeeds"); + let line_endings: Vec<_> = tokens + .iter() + .filter_map(|(token, _)| match token { + FmtToken::Trivia(Trivia::Newline(line_ending)) => Some(*line_ending), + _ => None, + }) + .collect(); + let newlines: Vec<_> = tokens + .iter() + .filter(|(token, _)| { + matches!(token, FmtToken::Trivia(trivia) if trivia.kind() == TriviaKind::Newline) + }) + .map(|(_, span)| span.to_slice(input)) + .collect(); + + assert_eq!( + line_endings, + vec![LineEnding::CrLf, LineEnding::Cr, LineEnding::Lf] + ); + assert_eq!(newlines, vec![Some("\r\n"), Some("\r"), Some("\n")]); + } + + #[test] + fn test_block_comment_nested_fmt() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_deeply_nested_fmt() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_multiline_fmt() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_unclosed_fmt() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); + + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_partial_nesting_unclosed_fmt() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_double_unclosed_fmt() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 2); + + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); + + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); + + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_spaces_resolution_fmt() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![ + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r"), + ]) ); + } + + #[test] + fn test_ignoring_tokens_after_comment_with_incorrect_symbol() { + let input = "fn main() {} @// fn hello(){} simc"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1, "only the invalid character is an error"); + assert!(errors[0].to_string().contains("found '@' expected")); + + let tokens = tokens.expect("recovery keeps the lossless stream"); + assert!(matches!( + tokens.last(), + Some(FmtToken::Trivia(Trivia::LineComment(_))) + )); assert!( tokens - .expect("recovery keeps the stream") .iter() - .all(|(tok, _)| !matches!(tok, Token::Simc)), - "the sentinel must not reach the token stream for {src:?}" + .all(|token| !matches!(token, FmtToken::Token(Token::Simc))), + "comment contents must not become semantic tokens" ); } - // Identifiers merely starting with `simc` are ordinary identifiers. - let (tokens, errors) = lex("simcfoo"); - assert!(errors.is_empty(), "unexpected: {errors:?}"); - assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); - } + #[test] + fn test_not_ignoring_tokens_after_comment() { + let input = "fn main() {} /* fn hello(){} \n simc 0.6.0; */ \n\ + // enum Name {} match true {} \n fn other_main() {} "; + let (tokens, errors) = lex_lossless(input); + + assert!(dbg!(errors).is_empty()); + assert_eq!( + tokens, + Some(vec![ + FmtToken::Token(Token::Fn), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Ident("main")), + FmtToken::Token(Token::LParen), + FmtToken::Token(Token::RParen), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::LBrace), + FmtToken::Token(Token::RBrace), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia( + TriviaKind::BlockComment, + "/* fn hello(){} \n simc 0.6.0; */" + ), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::LineComment, "// enum Name {} match true {} "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Fn), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Ident("other_main")), + FmtToken::Token(Token::LParen), + FmtToken::Token(Token::RParen), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::LBrace), + FmtToken::Token(Token::RBrace), + fmt_trivia(TriviaKind::Whitespace, " "), + ]) + ); + } - #[test] - fn test_enum_token() { - let (tokens, errors) = lex("enum Path { Inherit, ColdSpend }"); - assert!(errors.is_empty()); - assert_eq!( - tokens, - Some(vec![ - Token::Enum, - Token::Ident("Path"), - Token::LBrace, - Token::Ident("Inherit"), - Token::Comma, - Token::Ident("ColdSpend"), - Token::RBrace, - ]) - ); - } + #[test] + fn simc_is_reserved_fmt() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex_lossless(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, FmtToken::Token(Token::Simc))), + "the sentinel must not reach the token stream for {src:?}" + ); + } + + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex_lossless("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![FmtToken::Token(Token::Ident("simcfoo"))])); + } - #[test] - fn lexer_test() { - use chumsky::prelude::*; + #[test] + fn lossless_lexer_test() { + use chumsky::prelude::*; - // Check if the lexer parses the example file without errors. - let src = include_str!("../examples/last_will.simf"); + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); - let (tokens, lex_errs) = lexer().parse(src).into_output_errors(); - let _ = tokens.unwrap(); + let (tokens, lex_errs) = lexer_lossless().parse(src).into_output_errors(); + let _ = tokens.unwrap(); - assert!(lex_errs.is_empty()); + assert!(lex_errs.is_empty()); + } } } diff --git a/src/parse.rs b/src/parse.rs index a627adeb..60c522f1 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -22,7 +22,7 @@ use crate::driver::{CRATE_STR, MAIN_MODULE}; use crate::error::DiagnosticManager; use crate::error::{Diagnostic, Error, Span}; use crate::impl_eq_hash; -use crate::lexer::Token; +use crate::lexer::{Token, Tokens}; use crate::num::NonZeroPow2Usize; use crate::pattern::Pattern; use crate::str::{ @@ -30,9 +30,12 @@ use crate::str::{ SymbolName, WitnessName, }; use crate::types::{AliasedType, BuiltinAlias, TypeConstructible, UIntType}; -use crate::unstable::{impl_require_feature, UnstableFeature, UnstableFeatures}; +use crate::unstable::{impl_require_feature, RequireFeature, UnstableFeature, UnstableFeatures}; use crate::version::SimcDirective; +#[cfg(feature = "fmt")] +use crate::lexer::{FmtToken, FmtTokens}; + /// A program is a sequence of items. #[derive(Clone, Debug)] pub struct Program { @@ -40,6 +43,40 @@ pub struct Program { span: Span, } +/// Source-aware parse result used by formatters. +/// +/// The compiler keeps using [`Program`] directly. Formatters need the same +/// semantic tree plus the lossless trivia stream that the grammar intentionally +/// does not consume. +#[cfg(feature = "fmt")] +#[derive(Clone, Debug)] +pub struct ParsedSource<'src> { + program: Program, + tokens: FmtTokens<'src>, + prefix: Span, +} + +#[cfg(feature = "fmt")] +impl<'src> ParsedSource<'src> { + /// Access the semantic program used for formatting. + pub fn program(&self) -> &Program { + &self.program + } + + /// Access the lossless token stream in source order. + pub fn tokens(&self) -> &FmtTokens<'src> { + &self.tokens + } + + /// Access the source prefix skipped by the version-directive prescan. + /// + /// A formatter should preserve this range verbatim until the directive gains + /// its own formatting grammar. + pub fn prefix(&self) -> Span { + self.prefix + } +} + impl Program { // Need for driver usage pub(crate) fn new(items: &[Item], span: Span) -> Self { @@ -53,6 +90,50 @@ impl Program { pub fn items(&self) -> &[Item] { &self.items } + + /// Parse source for formatting while retaining all comments and whitespace. + #[cfg(feature = "fmt")] + pub fn parse_with_errors_for_fmt<'src>( + file_id: usize, + source: &'src str, + unstable_features: &UnstableFeatures, + diagnostics: &mut DiagnosticManager, + ) -> Option> { + let before = diagnostics.error_count(); + + let start = pipeline::directive_prescan(source, file_id, diagnostics)?; + + let (tokens, lex_errors) = crate::lexer::lex_lossless(file_id, source, start); + let lex_ok = pipeline::is_lex_ok(lex_errors, diagnostics)?; + + let tokens = tokens?; + + let semantic_tokens = tokens + .iter() + .filter_map(|(token, span)| match token { + FmtToken::Token(token) => Some((token.clone(), *span)), + FmtToken::Trivia(_) => None, + }) + .collect::>(); + + let (program, parse_ok) = + pipeline::parse_ast(file_id, source, semantic_tokens, diagnostics); + + if parse_ok && lex_ok { + pipeline::post_check(unstable_features, program.as_ref(), diagnostics); + } + + if diagnostics.error_count() > before { + None + } else { + let program = program?; + Some(ParsedSource { + program, + tokens, + prefix: Span::new(file_id, 0..start), + }) + } + } } impl_eq_hash!(Program; items); @@ -80,6 +161,22 @@ pub enum Item { Ignored, } +impl Item { + /// Access the source span when this item was parsed successfully. + /// + /// Error-recovery placeholders have no source node to decorate. + pub fn span(&self) -> Option<&Span> { + match self { + Self::TypeAlias(alias) => Some(alias.span()), + Self::Function(function) => Some(function.span()), + Self::Use(use_decl) => Some(use_decl.span()), + Self::EnumDeclaration(declaration) => Some(declaration.span()), + Self::Module(module) => Some(module.span()), + Self::Ignored => None, + } + } +} + impl_require_feature!(Item { variants: TypeAlias(alias), @@ -277,11 +374,12 @@ impl_eq_hash!(Function; visibility, name, params, ret, body); impl_require_feature!(Function { recurse: params, ret, body; }); /// Parameter of a function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] pub struct FunctionParam { identifier: Identifier, ty: AliasedType, + span: Span, } impl FunctionParam { @@ -294,8 +392,15 @@ impl FunctionParam { pub fn ty(&self) -> &AliasedType { &self.ty } + + /// Access the source span of the complete parameter. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(FunctionParam; identifier, ty); + impl_require_feature!(FunctionParam { recurse: ty; }); /// A statement is a component of a block expression. @@ -307,6 +412,19 @@ pub enum Statement { Expression(Expression), } +impl Statement { + /// Access the span of the statement contents. + /// + /// The terminating semicolon is deliberately not part of this span: it is a + /// separator owned by the containing block's layout. + pub fn span(&self) -> &Span { + match self { + Self::Assignment(assignment) => assignment.span(), + Self::Expression(expression) => expression.span(), + } + } +} + impl_require_feature!(Statement { variants: Assignment(assignment), @@ -502,6 +620,11 @@ impl EnumVariant { pub fn payload(&self) -> &[AliasedType] { &self.payload } + + /// Access the source span of the complete variant. + pub fn span(&self) -> &Span { + &self.span + } } impl_eq_hash!(EnumVariant; name, payload); @@ -533,6 +656,11 @@ impl EnumDeclaration { pub fn variants(&self) -> &[EnumVariant] { &self.variants } + + /// Access the source span of the complete declaration. + pub fn span(&self) -> &Span { + &self.span + } } impl_require_feature!(EnumVariant { recurse: payload; }); @@ -829,6 +957,7 @@ pub struct EnumMatchArm { variant: Identifier, bindings: Arc<[(Pattern, AliasedType)]>, expression: Arc, + span: Span, } impl EnumMatchArm { @@ -860,6 +989,11 @@ impl EnumMatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the source span of the complete arm, including its optional comma. + pub fn span(&self) -> &Span { + &self.span + } } impl_eq_hash!(EnumMatchArm; enum_path, variant, bindings, expression); @@ -942,10 +1076,11 @@ impl AsRef for EnumConstruction { } /// Arm of a match expression. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct MatchArm { pattern: MatchPattern, expression: Arc, + span: Span, } impl MatchArm { @@ -958,8 +1093,15 @@ impl MatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the source span of the complete arm, including its optional comma. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(MatchArm; pattern, expression); + impl_require_feature!(MatchArm {recurse: pattern, expression; }); /// Pattern of a match arm. @@ -1520,6 +1662,10 @@ impl_parse_wrapped_string!(WitnessName, "witness name"); impl_parse_wrapped_string!(AliasName, "alias name"); impl_parse_wrapped_string!(ModuleName, "module name"); +trait AstNode: ChumskyParse + crate::unstable::RequireFeature + std::fmt::Debug {} + +impl AstNode for T where T: ChumskyParse + crate::unstable::RequireFeature + std::fmt::Debug {} + /// Copy of [`FromStr`] that internally uses the `chumsky` parser. pub trait ParseFromStr: Sized { /// Parse a value from the string `s`. @@ -1540,6 +1686,73 @@ pub trait ParseFromStrWithErrors: Sized { ) -> Option; } +mod pipeline { + use super::*; + /// Handle the `simc` directive before lexing: an incompatible or malformed + /// directive is reported as the only diagnostic (the rest is noise), and + /// lexing starts right after a valid one, so the lexer and grammar never + /// see it. + pub fn directive_prescan( + content: &str, + file_id: usize, + diagnostics: &mut DiagnosticManager, + ) -> Option { + match SimcDirective::prescan(content, file_id) { + Ok(start) => Some(start), + Err((err, span)) => { + diagnostics.push(Diagnostic::new(err, span)); + None + } + } + } + + pub fn is_lex_ok( + mut lex_errs: Vec, + diagnostics: &mut DiagnosticManager, + ) -> Option { + // A stray `simc` makes every other diagnostic noise — its `"";` remnant + // does not lex — so the reserved-keyword errors are reported alone. + if lex_errs + .iter() + .any(|e| matches!(e.error(), Error::ReservedSimcKeyword)) + { + lex_errs.retain(|e| matches!(e.error(), Error::ReservedSimcKeyword)); + diagnostics.extend(lex_errs); + None + } else { + let lex_ok = lex_errs.is_empty(); + diagnostics.extend(lex_errs); + Some(lex_ok) + } + } + + pub fn parse_ast( + file_id: usize, + src: &str, + tokens: Tokens<'_>, + diagnostics: &mut DiagnosticManager, + ) -> (Option, bool) { + let eoi = Span::eof(file_id, src.len()); + let (ast, parse_errs) = T::parser() + .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) + .into_output_errors(); + + let parse_ok = parse_errs.is_empty(); + diagnostics.extend(parse_errs); + (ast, parse_ok) + } + + pub fn post_check( + unstable_features: &UnstableFeatures, + program: Option<&T>, + diagnostics: &mut DiagnosticManager, + ) { + if let Some(ast) = program { + unstable_features.check_program(ast, diagnostics); + } + } +} + /// Trait for generating parsers of themselves. /// /// Replacement for previous `PestParse` trait. @@ -1594,9 +1807,7 @@ impl ParseFromStr for A { } } -impl ParseFromStrWithErrors - for A -{ +impl ParseFromStrWithErrors for A { fn parse_from_str_with_errors( file_id: usize, content: &str, @@ -1605,42 +1816,18 @@ impl ParseF ) -> Option { let before = diagnostics.error_count(); - // Handle the `simc` directive before lexing: an incompatible or malformed - // directive is reported as the only diagnostic (the rest is noise), and - // lexing starts right after a valid one, so the lexer and grammar never - // see it. - let start = match SimcDirective::prescan(content, file_id) { - Ok(start) => start, - Err((err, span)) => { - diagnostics.push(Diagnostic::new(err, span)); - return None; - } - }; + let start = pipeline::directive_prescan(content, file_id, diagnostics)?; + + let (tokens, lex_errs) = crate::lexer::lex(file_id, content, start); + + let lex_ok = pipeline::is_lex_ok(lex_errs, diagnostics)?; - let (tokens, mut lex_errs) = crate::lexer::lex(file_id, content, start); - // A stray `simc` makes every other diagnostic noise — its `"";` remnant - // does not lex — so the reserved-keyword errors are reported alone. - if lex_errs - .iter() - .any(|e| matches!(e.error(), Error::ReservedSimcKeyword)) - { - lex_errs.retain(|e| matches!(e.error(), Error::ReservedSimcKeyword)); - diagnostics.extend(lex_errs); - return None; - } - let lex_ok = lex_errs.is_empty(); - diagnostics.extend(lex_errs); let tokens = tokens?; - let eoi = Span::eof(file_id, content.len()); - let (ast, parse_errs) = A::parser() - .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) - .into_output_errors(); - let parse_ok = parse_errs.is_empty(); - diagnostics.extend(parse_errs); + let (ast, parse_status) = pipeline::parse_ast::(file_id, content, tokens, diagnostics); - if let (Some(ast), true) = (&ast, lex_ok && parse_ok) { - unstable_features.check_program(ast, diagnostics); + if lex_ok && parse_status { + let () = pipeline::post_check(unstable_features, ast.as_ref(), diagnostics); } // TODO: We should return parsed result if we found errors, but because analyzing in `ast` module @@ -2034,7 +2221,11 @@ impl ChumskyParse for FunctionParam { identifier .then_ignore(just(Token::Colon)) .then(ty) - .map(|(identifier, ty)| Self { identifier, ty }) + .map_with(|(identifier, ty), e| Self { + identifier, + ty, + span: e.span(), + }) } } @@ -2613,20 +2804,20 @@ struct EnumArmHead { } impl EnumArmHead { - fn into_arm(self, expression: Arc) -> EnumMatchArm { + fn into_arm(self, expression: Arc, span: Span) -> EnumMatchArm { EnumMatchArm { enum_path: self.enum_path, variant: self.variant, bindings: self.bindings, expression, + span, } } } /// One parsed match arm. An enum variant head or a built-in pattern, -/// plus the arm body. The head classification decides below whether the -/// whole match becomes an [`EnumMatch`] or a binary [`Match`]. -type ParsedMatchArm = (Either, Arc); +/// plus the arm body and its source boundaries. +type ParsedMatchArm = (Either, Arc, Span); /// Parser for the head of an enum match arm: `EnumName::Variant` with /// optional payload bindings `(pattern: Type, ...)`. @@ -2709,7 +2900,7 @@ where .with_span(e.span()), ); } - (head, expression) + (head, expression, e.span()) }) } @@ -2719,6 +2910,7 @@ fn placeholder_match(scrutinee: Arc, span: Span) -> SingleExpression let fallback_arm = MatchArm { expression: Arc::new(Expression::empty(Span::DUMMY)), pattern: MatchPattern::False, + span: Span::DUMMY, }; SingleExpressionInner::Match(Match { scrutinee, @@ -2765,11 +2957,12 @@ fn assemble_match_arms( // reports missing variants. The binary arm-count error below would be misleading there. let (enum_arms, builtin_arms): (Vec, Vec) = arms .into_iter() - .partition_map(|(head, expression)| match head { - Either::Left(head) => Either::Left(head.into_arm(expression)), + .partition_map(|(head, expression, arm_span)| match head { + Either::Left(head) => Either::Left(head.into_arm(expression, arm_span)), Either::Right(pattern) => Either::Right(MatchArm { pattern, expression, + span: arm_span, }), }); @@ -2950,6 +3143,12 @@ impl AsRef for Function { } } +impl AsRef for FunctionParam { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for Assignment { fn as_ref(&self) -> &Span { &self.span @@ -2986,6 +3185,18 @@ impl AsRef for Match { } } +impl AsRef for MatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + +impl AsRef for EnumMatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for UseDecl { fn as_ref(&self) -> &Span { &self.span @@ -3306,10 +3517,12 @@ impl crate::ArbitraryRec for Match { left: MatchArm { pattern: pat_l, expression: expr_l, + span: Span::DUMMY, }, right: MatchArm { pattern: pat_r, expression: expr_r, + span: Span::DUMMY, }, span: Span::DUMMY, }) @@ -3334,172 +3547,531 @@ mod test { } } - #[test] - fn test_reject_redefined_builtin_type() { - let ty = TypeAlias::parse_from_str("type Ctx8 = u32") - .expect_err("Redefining built-in alias should be rejected"); + mod type_alias { + use super::*; - assert!(ty - .error() - .to_string() - .contains("Type alias `Ctx8` is already exists as built-in alias")); - } + #[test] + fn test_reject_redefined_builtin_type() { + let ty = TypeAlias::parse_from_str("type Ctx8 = u32") + .expect_err("Redefining built-in alias should be rejected"); + + assert!(ty + .error() + .to_string() + .contains("Type alias `Ctx8` is already exists as built-in alias")); + } - #[test] - fn test_fragment_rejects_version_directive() { - let err = TypeAlias::parse_from_str("simc \"*\"; type Ctx8 = u32") - .expect_err("a version directive must not be accepted in a fragment"); + #[test] + fn test_fragment_rejects_version_directive() { + let err = TypeAlias::parse_from_str("simc \"*\"; type Ctx8 = u32") + .expect_err("a version directive must not be accepted in a fragment"); - assert!(matches!(err.error(), Error::ReservedSimcKeyword)); + assert!(matches!(err.error(), Error::ReservedSimcKeyword)); + } } - #[test] - fn test_double_colon() { - let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; - let mut diagnostics = DiagnosticManager::new(); + mod regular_parsing { + use super::*; - let parsed_program = Program::parse_from_str_with_errors( - MAIN_MODULE, - input, - &UnstableFeatures::all(), - &mut diagnostics, - ); + #[test] + fn test_double_colon() { + let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; + let mut diagnostics = DiagnosticManager::new(); - assert!(parsed_program.is_none()); - assert!(diagnostics.to_string().contains("Expected '::', found ':'")); - } + let parsed_program = Program::parse_from_str_with_errors( + MAIN_MODULE, + input, + &UnstableFeatures::all(), + &mut diagnostics, + ); - #[test] - fn test_double_double_colon() { - let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; - let mut diagnostics = DiagnosticManager::new(); + assert!(parsed_program.is_none()); + assert!(diagnostics.to_string().contains("Expected '::', found ':'")); + } - let parsed_program = Program::parse_from_str_with_errors( - MAIN_MODULE, - input, - &UnstableFeatures::all(), - &mut diagnostics, - ); + #[test] + fn test_double_double_colon() { + let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; + let mut diagnostics = DiagnosticManager::new(); - assert!(parsed_program.is_none()); - assert!( - diagnostics - .to_string() - .contains("Expected identifier, found ::"), - "the second :: should be reported as the error site" - ); - } + let parsed_program = Program::parse_from_str_with_errors( + MAIN_MODULE, + input, + &UnstableFeatures::all(), + &mut diagnostics, + ); - /// Parse `input` and return whether it was rejected and the collected error text. - fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { - let mut diagnostics = DiagnosticManager::new(); - let program = - Program::parse_from_str_with_errors(MAIN_MODULE, input, features, &mut diagnostics); + assert!(parsed_program.is_none()); + assert!( + diagnostics + .to_string() + .contains("Expected identifier, found ::"), + "the second :: should be reported as the error site" + ); + } - let rejected = program.is_none(); - let text = diagnostics.to_string(); + /// Parse `input` and return whether it was rejected and the collected error text. + fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { + let mut diagnostics = DiagnosticManager::new(); + let program = + Program::parse_from_str_with_errors(MAIN_MODULE, input, features, &mut diagnostics); - (rejected, text) - } + let rejected = program.is_none(); + let text = diagnostics.to_string(); - #[test] - fn inverted_empty_span_from_token_gap_does_not_panic() { - // Fuzz-found (compile_text). - // - // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. - // The only path that requests an empty span at a lex-error gap and the - // NUL after `enum` creates that gap. - // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted - // across the gap and panicked the strict `Span` constructor. - // - // Simplifying any part (even NUL to a space) loses the crash. - let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; - let mut diagnostics = DiagnosticManager::new(); - let program = Program::parse_from_str_with_errors( - MAIN_MODULE, - src, - &UnstableFeatures::all(), - &mut diagnostics, - ); - - assert!(program.is_none(), "garbage input must be rejected"); - assert!(diagnostics.has_errors()); - } - - #[test] - fn recovery_synchronizes_at_enum_declaration() { - // Discriminating setup: an invalid prefix followed by a malformed - // enum. Without `Token::Enum` in the synchronization set the whole - // enum is swallowed into the prefix's recovery span and only one - // error is reported; with it, the enum parses as its own item and - // its malformation is reported as a second error. - let mut diagnostics = DiagnosticManager::new(); - let program = Program::parse_from_str_with_errors( - MAIN_MODULE, - "let invalid = 1;\nenum Action { A B }\nfn main() {}", - &UnstableFeatures::all(), - &mut diagnostics, - ); - assert!(program.is_none(), "the invalid program must be rejected"); - assert_eq!( - 2, - diagnostics.error_count(), - "the invalid prefix and the malformed enum must each report an error" - ); - } - - #[test] - fn recovery_after_malformed_enum_reaches_next_item() { - let (rejected, _text) = parse_with( - "enum Action { A B }\nfn main() {}", - &UnstableFeatures::all(), - ); - assert!(rejected, "a malformed enum must fail compilation"); - } - - #[test] - fn malformed_construct_containing_enum_keyword_reports_errors() { - // `enum` inside a badly malformed nested construct may be mistaken - // for an item boundary; compilation must still fail cleanly. - let (rejected, _text) = parse_with( - "fn broken() { let x = (enum; }\nfn main() {}", - &UnstableFeatures::all(), - ); - assert!(rejected, "a malformed construct must fail compilation"); - } - - #[test] - fn test_gated_syntax_is_rejected_without_features() { - // Real `use`/`mod` syntax: rejected under none() (naming the feature + a - // -Z hint), accepted under all(). Delete when `imports` stabilizes. - for input in [ - "use crate::foo::bar;\nfn main() { }", - "mod inner { }\nfn main() { }", - ] { + (rejected, text) + } + + #[test] + fn inverted_empty_span_from_token_gap_does_not_panic() { + // Fuzz-found (compile_text). + // + // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. + // The only path that requests an empty span at a lex-error gap and the + // NUL after `enum` creates that gap. + // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted + // across the gap and panicked the strict `Span` constructor. + // + // Simplifying any part (even NUL to a space) loses the crash. + let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; + let mut diagnostics = DiagnosticManager::new(); + let program = Program::parse_from_str_with_errors( + MAIN_MODULE, + src, + &UnstableFeatures::all(), + &mut diagnostics, + ); + + assert!(program.is_none(), "garbage input must be rejected"); + assert!(diagnostics.has_errors()); + } + + #[test] + fn recovery_synchronizes_at_enum_declaration() { + // Discriminating setup: an invalid prefix followed by a malformed + // enum. Without `Token::Enum` in the synchronization set the whole + // enum is swallowed into the prefix's recovery span and only one + // error is reported; with it, the enum parses as its own item and + // its malformation is reported as a second error. + let mut diagnostics = DiagnosticManager::new(); + let program = Program::parse_from_str_with_errors( + MAIN_MODULE, + "let invalid = 1;\nenum Action { A B }\nfn main() {}", + &UnstableFeatures::all(), + &mut diagnostics, + ); + assert!(program.is_none(), "the invalid program must be rejected"); + assert_eq!( + 2, + diagnostics.error_count(), + "the invalid prefix and the malformed enum must each report an error" + ); + } + + #[test] + fn recovery_after_malformed_enum_reaches_next_item() { + let (rejected, _text) = parse_with( + "enum Action { A B }\nfn main() {}", + &UnstableFeatures::all(), + ); + assert!(rejected, "a malformed enum must fail compilation"); + } + + #[test] + fn malformed_construct_containing_enum_keyword_reports_errors() { + // `enum` inside a badly malformed nested construct may be mistaken + // for an item boundary; compilation must still fail cleanly. + let (rejected, _text) = parse_with( + "fn broken() { let x = (enum; }\nfn main() {}", + &UnstableFeatures::all(), + ); + assert!(rejected, "a malformed construct must fail compilation"); + } + + #[test] + fn test_gated_syntax_is_rejected_without_features() { + // Real `use`/`mod` syntax: rejected under none() (naming the feature + a + // -Z hint), accepted under all(). Delete when `imports` stabilizes. + for input in [ + "use crate::foo::bar;\nfn main() { }", + "mod inner { }\nfn main() { }", + ] { + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + rejected, + "gated syntax must be rejected without features (if this feature was \ + just stabilized, delete this test):\n{input}" + ); + assert!( + error.contains("imports") && error.contains("-Z"), + "rejection should name the feature and suggest -Z, got:\n{error}" + ); + + let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + assert!( + !rejected, + "the same syntax must parse with all features enabled:\n{error}" + ); + } + } + + #[test] + fn test_type_heavy_program_needs_no_features() { + // Types are traversed by `RequireFeature` but not gated, so a type-heavy, + // import-free program must parse with no features (no false-positive gates). + let input = r#"type Alias = u32; +fn pick(e: Either) -> u32 { + match e { + Left(a: u32) => a, + Right(b: u32) => b, + } +} +fn main() { + let casted: u32 = <(u16, u16)>::into((0xbeef, 0xbabe)); + let chosen: Alias = pick(Left(casted)); + assert!(jet::eq_32(chosen, chosen)); +} +"#; + // `parse_from_str_with_errors` returns `Some` only when no errors were + // collected, so a non-rejection already proves there were no gate errors. let (rejected, error) = parse_with(input, &UnstableFeatures::none()); assert!( - rejected, - "gated syntax must be rejected without features (if this feature was \ - just stabilized, delete this test):\n{input}" + !rejected, + "type-heavy but import-free program should parse with no features:\n{error}" ); + } + + #[test] + fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { + // The gate check skips error-recovered ASTs, so a program with + // both a syntax error and gated syntax reports only the syntax error. + let input = "use crate::foo;\nfn main( {"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "broken program must be rejected"); assert!( - error.contains("imports") && error.contains("-Z"), - "rejection should name the feature and suggest -Z, got:\n{error}" + !error.contains("-Z"), + "syntax error should not produce a spurious feature-gate hint, got:\n{error}" ); + } - let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + #[test] + fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { + // Lexer-side companion to the case above: the stray `@` lexes with an + // error but recovers a cleanly-parsing `use` AST, and a program that + // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. + let input = "use crate@::foo;\nfn main() { }"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "program with a lexer error must be rejected"); assert!( - !rejected, - "the same syntax must parse with all features enabled:\n{error}" + !error.contains("-Z"), + "lexer error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_use_decl_display_round_trips_with_aliases() { + for input in [ + "use lib::A::foo;", + "use lib::A::foo as bar;", + "use lib::A::{foo, bar as baz};", + ] { + let program = parse::Program::parse_from_str(input).expect("parsing works"); + assert_eq!(program.to_string(), format!("{input}\n")); + } + } + + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; + let program = Program::parse_from_str(input).expect("program parses"); + + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; + + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } + + fn parse_item(input: &str) -> Item { + let program = parse::Program::parse_from_str(input).expect("parsing should succeed"); + program.items().first().expect("expected one item").clone() + } + + #[test] + fn test_enum_declaration_basic() { + let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration, got {item:?}"); + }; + assert_eq!(decl.name().as_inner(), "Path"); + assert_eq!(decl.variants().len(), 3); + assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); + assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); + } + + #[test] + fn test_enum_declaration_pub() { + let item = parse_item("pub enum Color { Red, Green, Blue, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!(decl.visibility(), &Visibility::Public); + assert_eq!(decl.name().as_inner(), "Color"); + } + + #[test] + fn test_enum_declaration_display_round_trip() { + let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; + let item = parse_item(input); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!( + decl.to_string(), + "enum Path { Inherit, ColdSpend, RefreshSpend, }" ); } + + #[test] + fn test_enum_declaration_reserved_name() { + for reserved in RESERVED_PATTERN_NAMES { + let result = Program::parse_from_str(&format!("enum {reserved} {{ A, B, }}")); + let error = result.expect_err(&format!("enum name {reserved} is reserved")); + assert!( + error.to_string().contains("reserved"), + "error should say the name is reserved: {error}" + ); + } + } } - #[test] - fn test_type_heavy_program_needs_no_features() { - // Types are traversed by `RequireFeature` but not gated, so a type-heavy, - // import-free program must parse with no features (no false-positive gates). - let input = r#"type Alias = u32; + #[cfg(feature = "fmt")] + mod fmt_parsing { + use super::*; + + /// Parse `input`, appending any formatter diagnostics to `diagnostics`. + fn parse_with_diagnostics<'src>( + input: &'src str, + features: &UnstableFeatures, + diagnostics: &mut DiagnosticManager, + ) -> Option> { + Program::parse_with_errors_for_fmt(MAIN_MODULE, input, features, diagnostics) + } + + /// Parse `input` and return whether it was rejected and the collected error text. + fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); + + let rejected = program.is_none(); + let text = diagnostics.to_string(); + + (rejected, text) + } + + /// Parse `input` and return whether it was rejected and the collected error in `DiagnosticManager`. + fn parse_with_diagnosis( + input: &str, + features: &UnstableFeatures, + ) -> (bool, DiagnosticManager) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); + + let rejected = program.is_none(); + + (rejected, diagnostics) + } + + /// Parse `input` and return whether it was rejected and the collected error text together with `ParsedSource`. + fn parse_with_extended_out<'src>( + input: &'src str, + features: &UnstableFeatures, + ) -> (bool, String, Option>) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); + + let rejected = program.is_none(); + let text = diagnostics.to_string(); + + (rejected, text, program) + } + + fn assert_formatter_matches_regular_parser(input: &str, features: &UnstableFeatures) { + let mut regular_diagnostics = DiagnosticManager::new(); + let regular = Program::parse_from_str_with_errors( + MAIN_MODULE, + input, + features, + &mut regular_diagnostics, + ); + + let mut formatting_diagnostics = DiagnosticManager::new(); + let formatting = parse_with_diagnostics(input, features, &mut formatting_diagnostics); + + assert_eq!( + formatting.as_ref().map(ParsedSource::program), + regular.as_ref(), + "formatter and regular parsers must produce the same AST for {input:?}" + ); + assert_eq!( + formatting_diagnostics.error_count(), + regular_diagnostics.error_count(), + "formatter and regular parsers must report the same number of errors for {input:?}, \n\ + fmt errors: \n\t '{formatting_diagnostics}, \n regular errors: \n\t '{regular_diagnostics}" + ); + } + + fn assert_lossless_source(input: &str, parsed: &ParsedSource<'_>) { + let prefix = parsed.prefix(); + assert_eq!(prefix.file_id, MAIN_MODULE); + + let mut reconstructed = prefix + .to_slice(input) + .expect("the directive prefix must be a valid source slice") + .to_owned(); + let mut cursor = prefix.end; + + for (_, span) in parsed.tokens() { + assert_eq!(span.file_id, MAIN_MODULE); + assert!( + span.start >= prefix.end, + "formatter tokens must not overlap the preserved directive prefix" + ); + assert_eq!( + span.start, cursor, + "formatter token spans must be contiguous and in source order" + ); + + let text = span + .to_slice(input) + .expect("formatter token spans must be valid source slices"); + assert!( + !text.is_empty(), + "the formatter token stream must not contain empty source slices" + ); + reconstructed.push_str(text); + cursor = span.end; + } + + assert_eq!(cursor, input.len(), "formatter tokens must reach EOF"); + assert_eq!(reconstructed, input, "formatter input must be lossless"); + } + + #[test] + fn test_double_colon() { + let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; + + let (rejected, text) = parse_with(input, &UnstableFeatures::all()); + + assert!(rejected); + assert!(text.contains("Expected '::', found ':'")); + } + + #[test] + fn test_double_double_colon() { + let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; + + let (rejected, text) = parse_with(input, &UnstableFeatures::all()); + + assert!(rejected); + assert!( + text.contains("Expected identifier, found ::"), + "the second :: should be reported as the error site" + ); + } + + #[test] + fn inverted_empty_span_from_token_gap_does_not_panic() { + // Fuzz-found (compile_text). + // + // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. + // The only path that requests an empty span at a lex-error gap and the + // NUL after `enum` creates that gap. + // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted + // across the gap and panicked the strict `Span` constructor. + // + // Simplifying any part (even NUL to a space) loses the crash. + let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; + let (rejected, text) = parse_with(src, &UnstableFeatures::all()); + + assert!(rejected, "garbage input must be rejected"); + assert!(!text.is_empty()); + } + + #[test] + fn recovery_synchronizes_at_enum_declaration() { + // Discriminating setup: an invalid prefix followed by a malformed + // enum. Without `Token::Enum` in the synchronization set the whole + // enum is swallowed into the prefix's recovery span and only one + // error is reported; with it, the enum parses as its own item and + // its malformation is reported as a second error. + + let src = "let invalid = 1;\nenum Action { A B }\nfn main() {}"; + let (rejected, diagnostics) = parse_with_diagnosis(src, &UnstableFeatures::all()); + + assert!(rejected, "the invalid program must be rejected"); + assert_eq!( + 2, + diagnostics.error_count(), + "the invalid prefix and the malformed enum must each report an error" + ); + } + + #[test] + fn recovery_after_malformed_enum_reaches_next_item() { + let src = "enum Action { A B }\nfn main() {}"; + let (rejected, _text) = parse_with(src, &UnstableFeatures::all()); + assert!(rejected, "a malformed enum must fail compilation"); + } + + #[test] + fn malformed_construct_containing_enum_keyword_reports_errors() { + // `enum` inside a badly malformed nested construct may be mistaken + // for an item boundary; compilation must still fail cleanly. + let src = "fn broken() { let x = (enum; }\nfn main() {}"; + let (rejected, _text) = parse_with(src, &UnstableFeatures::all()); + assert!(rejected, "a malformed construct must fail compilation"); + } + + #[test] + fn test_gated_syntax_is_rejected_without_features() { + // Real `use`/`mod` syntax: rejected under none() (naming the feature + a + // -Z hint), accepted under all(). Delete when `imports` stabilizes. + for input in [ + "use crate::foo::bar;\nfn main() { }", + "mod inner { }\nfn main() { }", + ] { + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + rejected, + "gated syntax must be rejected without features (if this feature was \ + just stabilized, delete this test):\n{input}" + ); + assert!( + error.contains("imports") && error.contains("-Z"), + "rejection should name the feature and suggest -Z, got:\n{error}" + ); + + let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + assert!( + !rejected, + "the same syntax must parse with all features enabled:\n{error}" + ); + } + } + + #[test] + fn test_type_heavy_program_needs_no_features() { + // Types are traversed by `RequireFeature` but not gated, so a type-heavy, + // import-free program must parse with no features (no false-positive gates). + let input = r#"type Alias = u32; fn pick(e: Either) -> u32 { match e { Left(a: u32) => a, @@ -3512,102 +4084,241 @@ fn main() { assert!(jet::eq_32(chosen, chosen)); } "#; - // `parse_from_str_with_errors` returns `Some` only when no errors were - // collected, so a non-rejection already proves there were no gate errors. - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!( - !rejected, - "type-heavy but import-free program should parse with no features:\n{error}" - ); - } - - #[test] - fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { - // The gate check skips error-recovered ASTs, so a program with - // both a syntax error and gated syntax reports only the syntax error. - let input = "use crate::foo;\nfn main( {"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "broken program must be rejected"); - assert!( - !error.contains("-Z"), - "syntax error should not produce a spurious feature-gate hint, got:\n{error}" - ); - } - - #[test] - fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { - // Lexer-side companion to the case above: the stray `@` lexes with an - // error but recovers a cleanly-parsing `use` AST, and a program that - // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. - let input = "use crate@::foo;\nfn main() { }"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "program with a lexer error must be rejected"); - assert!( - !error.contains("-Z"), - "lexer error should not produce a spurious feature-gate hint, got:\n{error}" - ); - } - - #[test] - fn test_use_decl_display_round_trips_with_aliases() { - for input in [ - "use lib::A::foo;", - "use lib::A::foo as bar;", - "use lib::A::{foo, bar as baz};", - ] { - let program = parse::Program::parse_from_str(input).expect("parsing works"); - assert_eq!(program.to_string(), format!("{input}\n")); - } - } - - fn parse_item(input: &str) -> Item { - let program = parse::Program::parse_from_str(input).expect("parsing should succeed"); - program.items().first().expect("expected one item").clone() - } - - #[test] - fn test_enum_declaration_basic() { - let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration, got {item:?}"); - }; - assert_eq!(decl.name().as_inner(), "Path"); - assert_eq!(decl.variants().len(), 3); - assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); - assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); - } + // `parse_from_str_with_errors` returns `Some` only when no errors were + // collected, so a non-rejection already proves there were no gate errors. + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + !rejected, + "type-heavy but import-free program should parse with no features:\n{error}" + ); + } - #[test] - fn test_enum_declaration_pub() { - let item = parse_item("pub enum Color { Red, Green, Blue, }"); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration"); - }; - assert_eq!(decl.visibility(), &Visibility::Public); - assert_eq!(decl.name().as_inner(), "Color"); - } + #[test] + fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { + // The gate check skips error-recovered ASTs, so a program with + // both a syntax error and gated syntax reports only the syntax error. + let input = "use crate::foo;\nfn main( {"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "broken program must be rejected"); + assert!( + !error.contains("-Z"), + "syntax error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } - #[test] - fn test_enum_declaration_display_round_trip() { - let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; - let item = parse_item(input); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration"); - }; - assert_eq!( - decl.to_string(), - "enum Path { Inherit, ColdSpend, RefreshSpend, }" - ); - } + #[test] + fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { + // Lexer-side companion to the case above: the stray `@` lexes with an + // error but recovers a cleanly-parsing `use` AST, and a program that + // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. + let input = "use crate@::foo;\nfn main() { }"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "program with a lexer error must be rejected"); + assert!( + !error.contains("-Z"), + "lexer error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_use_decl_display_round_trips_with_aliases() { + for input in [ + "use lib::A::foo;", + "use lib::A::foo as bar;", + "use lib::A::{foo, bar as baz};", + ] { + let (rejected, _, program) = + parse_with_extended_out(input, &UnstableFeatures::all()); + assert!(!rejected, "parsing works"); + assert_eq!(program.unwrap().program.to_string(), format!("{input}\n")); + } + } + + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; + + let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::none()); + assert!(!rejected, "parsing works"); + let program = program.unwrap().program; + + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; + + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } + + fn parse_item(input: &str) -> Item { + let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::all()); + assert!(!rejected, "parsing works"); + let program = program.unwrap().program; + + program.items().first().expect("expected one item").clone() + } + + #[test] + fn test_enum_declaration_basic() { + let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration, got {item:?}"); + }; + assert_eq!(decl.name().as_inner(), "Path"); + assert_eq!(decl.variants().len(), 3); + assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); + assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); + } + + #[test] + fn test_enum_declaration_pub() { + let item = parse_item("pub enum Color { Red, Green, Blue, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!(decl.visibility(), &Visibility::Public); + assert_eq!(decl.name().as_inner(), "Color"); + } + + #[test] + fn test_enum_declaration_display_round_trip() { + let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; + let item = parse_item(input); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!( + decl.to_string(), + "enum Path { Inherit, ColdSpend, RefreshSpend, }" + ); + } + + #[test] + fn test_enum_declaration_reserved_name() { + for reserved in RESERVED_PATTERN_NAMES { + let input = format!("enum {reserved} {{ A, B, }}"); + let (rejected, error) = parse_with(&input, &UnstableFeatures::none()); + assert!(rejected, "enum name {reserved} is reserved"); + assert!( + error.contains("reserved"), + "error should say the name is reserved: {error}" + ); + } + } + + #[test] + fn formatting_parse_is_lossless_after_directive_prescan() { + let input = "// header\r\nsimc \"*\";\r\n\t/* block */ fn main() { // body\r\n\t}\r"; + + let (_rejected, _error, parsed) = + parse_with_extended_out(input, &UnstableFeatures::none()); + let parsed = parsed.expect("formatting parse should succeed"); + + assert_eq!( + parsed.prefix().to_slice(input), + Some("// header\r\nsimc \"*\";") + ); + for trivia in [ + crate::lexer::TriviaKind::BlockComment, + crate::lexer::TriviaKind::LineComment, + crate::lexer::TriviaKind::Newline, + crate::lexer::TriviaKind::Whitespace, + ] { + assert!( + parsed.tokens().iter().any( + |(token, _)| matches!(token, FmtToken::Trivia(token_trivia) if token_trivia.kind() == trivia) + ), + "formatter token stream should retain {trivia:?}" + ); + } + assert_lossless_source(input, &parsed); + assert_eq!(parsed.program().items().len(), 1); + } + + #[test] + fn formatting_parse_matches_regular_parser_pipeline() { + let no_features = UnstableFeatures::none(); + let all_features = UnstableFeatures::all(); + + for (input, features) in [ + ( + "/* header */\r\nfn\tmain() { // comment\r\n}\r", + &no_features, + ), + ("// version\r\nsimc \"*\";\r\nfn main() {}", &no_features), + ("use crate::foo::bar;\nfn main() {}", &no_features), + ("use crate::foo::bar;\nfn main() {}", &all_features), + ("fn main( {", &no_features), + ("fn main() { @// comment }", &no_features), + ] { + assert_formatter_matches_regular_parser(input, features); + } + } + + #[test] + fn formatting_parse_ignores_preexisting_errors() { + let input = "fn main() {}"; + let mut diagnostics = DiagnosticManager::new(); + diagnostics.push(Diagnostic::global(Error::CannotParse { + msg: "an error from an earlier source file".to_owned(), + })); + let before = diagnostics.error_count(); + + let parsed = parse_with_diagnostics(input, &UnstableFeatures::none(), &mut diagnostics); + + assert!( + parsed.is_some(), + "valid source must still produce ParsedSource" + ); + assert_eq!(diagnostics.error_count(), before); + assert_eq!(diagnostics.diagnostics().len(), before); + } + + #[test] + fn formatting_parse_stops_after_invalid_directive() { + for (input, is_malformed) in [ + ("simc \"*\"\nfn broken( @", true), + ("simc \">99.0.0\"; @", false), + ] { + let (rejected, diagnostics) = + parse_with_diagnosis(input, &UnstableFeatures::none()); + + assert!(rejected, "invalid directive must abort formatting parse"); + assert_eq!( + diagnostics.error_count(), + 1, + "the invalid body must not add diagnostics after prescan aborts" + ); + if is_malformed { + assert!(matches!( + diagnostics.diagnostics()[0].error(), + Error::MalformedSimcDirective + )); + } else { + assert!(matches!( + diagnostics.diagnostics()[0].error(), + Error::SimcVersionMismatch { .. } + )); + } + } + } + + #[test] + fn formatting_parse_reports_missing_match_arm_comma() { + let input = "fn main() { match true { false => () true => (), } }"; + + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - #[test] - fn test_enum_declaration_reserved_name() { - for reserved in RESERVED_PATTERN_NAMES { - let result = Program::parse_from_str(&format!("enum {reserved} {{ A, B, }}")); - let error = result.expect_err(&format!("enum name {reserved} is reserved")); + assert!(rejected); assert!( - error.to_string().contains("reserved"), - "error should say the name is reserved: {error}" + error.contains("Missing ',' after a match arm that isn't block expression"), + "expected the missing-comma diagnostic, got {error:?}" ); } } diff --git a/tests/lex_lossless_coverage.rs b/tests/lex_lossless_coverage.rs new file mode 100644 index 00000000..954ae5e2 --- /dev/null +++ b/tests/lex_lossless_coverage.rs @@ -0,0 +1,103 @@ +#[cfg(feature = "fmt")] +mod lossless_test { + use simplicityhl::lexer::{lex_lossless, FmtTokens}; + + fn get_input(input_file: &str) -> String { + use std::fs; + use std::path::Path; + + let tests_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("examples"); + let input_path = tests_dir.join(input_file); + fs::read_to_string(&input_path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", input_path.display())) + } + + fn assert_lossless_fixture(input_file: &str) { + const DEFAULT_FILE_ID: usize = 0; + + let original_input = get_input(input_file); + + let (tokens, diagnostics) = lex_lossless(DEFAULT_FILE_ID, &original_input, 0); + assert!( + diagnostics.is_empty(), + "Diagnostic errors aren't empty: {diagnostics:?}" + ); + let tokens = tokens.expect("lossless lexing succeeds"); + let restored_input = restore_input_by_tokens(&tokens); + assert_eq!( + restored_input, original_input, + "lossless tokens must reproduce the input" + ); + } + + fn restore_input_by_tokens(tokens: &FmtTokens<'_>) -> String { + let mut buf = String::new(); + + let mut prev_span_end = None; + for (t, s) in tokens { + match prev_span_end { + None => { + let _ = prev_span_end.insert(s.end); + } + Some(x) => { + assert_eq!(x, s.start); + let _ = prev_span_end.insert(s.end); + } + } + buf.push_str(&t.to_string()); + } + buf + } + + macro_rules! test_lex_coverage { + ($($lossless_name:ident: $input:literal,)+) => { + $( + #[test] + fn $lossless_name() { + assert_lossless_fixture($input); + } + )+ + }; +} + + test_lex_coverage! { + array_fold_2n_lossless: "array_fold_2n.simf", + array_fold_lossless: "array_fold.simf", + cat_lossless: "cat.simf", + ctv_lossless: "ctv.simf", + escrow_with_delay_lossless: "escrow_with_delay.simf", + hash_loop_lossless: "hash_loop.simf", + hodl_vault_lossless: "hodl_vault.simf", + htlc_lossless: "htlc.simf", + last_will_lossless: "last_will.simf", + modules_lossless: "modules.simf", + non_interactive_fee_bump_lossless: "non_interactive_fee_bump.simf", + p2ms_lossless: "p2ms.simf", + p2pk_lossless: "p2pk.simf", + p2pkh_lossless: "p2pkh.simf", + pattern_matching_lossless: "pattern_matching.simf", + presigned_vault_lossless: "presigned_vault.simf", + reveal_collision_lossless: "reveal_collision.simf", + reveal_fix_point_lossless: "reveal_fix_point.simf", + sighash_all_anyonecanpay_lossless: "sighash_all_anyonecanpay.simf", + sighash_all_anyprevout_lossless: "sighash_all_anyprevout.simf", + sighash_all_anyprevoutanyscript_lossless: "sighash_all_anyprevoutanyscript.simf", + sighash_none_lossless: "sighash_none.simf", + sighash_single_lossless: "sighash_single.simf", + transfer_with_timeout_lossless: "transfer_with_timeout.simf", + local_crate_main_lossless: "local_crate/main.simf", + local_crate_math_lossless: "local_crate/math.simf", + simple_multidep_hashes_lossless: "simple_multidep/crypto/hashes.simf", + simple_multidep_arithmetic_lossless: "simple_multidep/math/arithmetic.simf", + simple_multidep_flattened_lossless: "simple_multidep/flattened.simf", + simple_multidep_main_lossless: "simple_multidep/main.simf", + single_dep_flattened_lossless: "single_dep/flattened.simf", + single_dep_main_lossless: "single_dep/main.simf", + single_dep_funcs_lossless: "single_dep/temp/funcs.simf", + single_dep_utils_lossless: "single_dep/temp/constants/utils.simf", + multiple_deps_main_lossless: "multiple_deps/main.simf", + multiple_deps_flattened_lossless: "multiple_deps/flattened.simf", + multiple_deps_simple_op_lossless: "multiple_deps/math/simple_op.simf", + multiple_deps_build_root_lossless: "multiple_deps/merkle/build_root.simf", + } +} From da16f18b5fad302a46c1b10e176345da78b95868 Mon Sep 17 00:00:00 2001 From: Illia Kripaka Date: Wed, 5 Aug 2026 15:13:30 +0300 Subject: [PATCH 2/4] Add possiblity to scrap original underscored values from digit literals * add tests to test such behaviour * separate digits parsers, reject digits with leading undersrores in decimal variant * process whitespace in lexer by chunks instead of one-by-one * featuregate logic with including underscores --- src/ast.rs | 42 ++++++++++++++++++ src/lexer.rs | 118 ++++++++++++++++++++++++++++++++++++++++++++------- src/parse.rs | 28 ++++++++++-- src/str.rs | 40 +++++++++++++++-- src/types.rs | 4 +- src/value.rs | 26 +++++++++--- 6 files changed, 228 insertions(+), 30 deletions(-) diff --git a/src/ast.rs b/src/ast.rs index 971e36e2..e3a084cb 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -3604,3 +3604,45 @@ mod enum_tests { assert!(result.is_err(), "enum syntax is gated behind -Z enums"); } } + +#[cfg(feature = "fmt")] +#[cfg(test)] +mod literal_tests { + use crate::parse::ParseFromStr; + use crate::value::{UIntValue, Value}; + + use super::*; + + #[test] + fn analyzed_numeric_literals_accept_digit_separators() { + let cases = [ + ("1_337", UIntType::U16, Value::from(UIntValue::U16(1_337))), + ( + "0b1010_0101", + UIntType::U8, + Value::from(UIntValue::U8(0b1010_0101)), + ), + ( + "0xDE_AD_BE_EF", + UIntType::U32, + Value::from(UIntValue::U32(0xdead_beef)), + ), + ]; + + for (source, integer_type, expected) in cases { + let parsed = parse::Expression::parse_from_str(source).expect("literal parses"); + let analyzed = + Expression::analyze_const(&parsed, &integer_type.into()).expect("literal analyzes"); + + let ExpressionInner::Single(single) = analyzed.inner() else { + panic!("expected a single expression") + }; + let SingleExpressionInner::Constant(value) = single.inner() else { + panic!("expected a constant expression") + }; + + assert_eq!(value, &expected, "unexpected value for {source:?}"); + assert_eq!(single.span().to_slice(source), Some(source)); + } + } +} diff --git a/src/lexer.rs b/src/lexer.rs index 116e112a..8715af1e 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -246,7 +246,11 @@ fn whitespace<'src>( /// Recognizer for whitespace or a newline. fn whitespace_or_newline<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { - any().filter(|c: &char| c.is_whitespace()).ignored() + any() + .filter(|c: &char| c.is_whitespace()) + .repeated() + .at_least(1) + .ignored() } /// Recognizer for a (possibly nested) `/* ... */` block comment; an unterminated @@ -281,9 +285,21 @@ pub(crate) fn trivia<'src>( trivia_item().repeated().ignored() } -fn to_token<'src>( -) -> impl Parser<'src, &'src str, Token<'src>, extra::Err>> { - let digits_with_underscore = |radix: u32| { +/// Parses the digit body of a numeric literal. +fn digits_with_underscores<'src>( + radix: u32, +) -> impl Parser<'src, &'src str, &'src str, extra::Err>> { + #[cfg(feature = "fmt")] + { + let digit = any().filter(move |c: &char| c.is_digit(radix)); + just('_') + .repeated() + .then(digit) + .then(choice((digit, just('_'))).repeated()) + .to_slice() + } + #[cfg(not(feature = "fmt"))] + { any() .filter(move |c: &char| c.is_digit(radix)) .then( @@ -292,18 +308,19 @@ fn to_token<'src>( .repeated(), ) .to_slice() - }; + } +} - let num = digits_with_underscore(10) - .map(|s: &str| Token::DecLiteral(Decimal::from_str_unchecked(s.replace('_', "").as_str()))); +fn to_token<'src>( +) -> impl Parser<'src, &'src str, Token<'src>, extra::Err>> { + let num = digits_with_underscores(10) + .map(|s: &str| Token::DecLiteral(Decimal::from_str_unchecked(s))); let hex = just("0x") - .ignore_then(digits_with_underscore(16)) - .map(|s: &str| { - Token::HexLiteral(Hexadecimal::from_str_unchecked(s.replace('_', "").as_str())) - }); + .ignore_then(digits_with_underscores(16)) + .map(|s: &str| Token::HexLiteral(Hexadecimal::from_str_unchecked(s))); let bin = just("0b") - .ignore_then(digits_with_underscore(2)) - .map(|s: &str| Token::BinLiteral(Binary::from_str_unchecked(s.replace('_', "").as_str()))); + .ignore_then(digits_with_underscores(2)) + .map(|s: &str| Token::BinLiteral(Binary::from_str_unchecked(s))); let macros = choice((just("assert!"), just("panic!"), just("dbg!"), just("list!"))).map(Token::Macro); @@ -357,7 +374,7 @@ fn to_token<'src>( just(">").to(Token::RAngle), )); - choice((jet, witness, param, macros, keyword, hex, bin, num, op)) + choice((jet, witness, param, macros, hex, bin, num, keyword, op)) } pub fn lexer<'src>( @@ -650,7 +667,7 @@ mod tests { // enum Name {} match true {} \n fn other_main() {} "; let (tokens, errors) = lex(input); - assert!(dbg!(errors).is_empty()); + assert!(errors.is_empty()); assert_eq!( tokens, Some(vec![ @@ -713,6 +730,18 @@ mod tests { ); } + #[test] + fn leading_whitespaces_before_main() { + let input = " fn main(){}"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty()); + assert!(tokens.is_some()); + + let tokens = tokens.unwrap(); + assert_eq!(tokens[0], Token::Fn); + } + #[test] fn lexer_test() { use chumsky::prelude::*; @@ -959,7 +988,7 @@ mod tests { // enum Name {} match true {} \n fn other_main() {} "; let (tokens, errors) = lex_lossless(input); - assert!(dbg!(errors).is_empty()); + assert!(errors.is_empty()); assert_eq!( tokens, Some(vec![ @@ -1020,6 +1049,63 @@ mod tests { assert_eq!(tokens, Some(vec![FmtToken::Token(Token::Ident("simcfoo"))])); } + #[test] + fn numeric_literals_preserve_underscores() { + let input = "1_234_567_89 0xDEAD_BEEF 0b__1010_0101_ 0b1010_0101 _1_234__567___890____000_____ 0x_DEAD_BEEF__BEEF_DEAD_"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + assert!(tokens.is_some(), "lexing must succeed"); + let tokens = tokens + .unwrap() + .into_iter() + .filter(|x| !matches!(x, FmtToken::Trivia(Trivia::Whitespace(_)))) + .collect::>(); + assert_eq!( + tokens, + vec![ + FmtToken::Token(Token::DecLiteral(Decimal::from_str_unchecked( + "1_234_567_89" + ))), + FmtToken::Token(Token::HexLiteral(Hexadecimal::from_str_unchecked( + "DEAD_BEEF" + ))), + FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked( + "__1010_0101_" + ))), + FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked("1010_0101"))), + FmtToken::Token(Token::DecLiteral(Decimal::from_str_unchecked( + "_1_234__567___890____000_____" + ))), + FmtToken::Token(Token::HexLiteral(Hexadecimal::from_str_unchecked( + "_DEAD_BEEF__BEEF_DEAD_" + ))), + ] + ); + + assert_eq!( + tokens + .iter() + .map(ToString::to_string) + .collect::>() + .join(" "), + input + ); + } + + #[test] + fn leading_whitespaces_before_main() { + let input = " fn main(){}"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert!(tokens.is_some()); + + let tokens = dbg!(tokens).unwrap(); + assert!(matches!(tokens[0], FmtToken::Trivia(Trivia::Whitespace(_)))); + assert!(matches!(tokens[1], FmtToken::Token(Token::Fn))); + } + #[test] fn lossless_lexer_test() { use chumsky::prelude::*; diff --git a/src/parse.rs b/src/parse.rs index 60c522f1..32f6bd6a 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -1973,7 +1973,12 @@ impl ChumskyParse for AliasedType { .then_ignore(parse_token_with_recovery(Token::Semi)) .then(num.clone()) .map(|(ty, size)| { - AliasedType::array(ty, usize::from_str(size.as_inner()).unwrap_or_default()) + #[cfg(feature = "fmt")] + let digits = size.strip_digit_separators(); + #[cfg(not(feature = "fmt"))] + let digits = std::borrow::Cow::from(size.as_inner()); + + AliasedType::array(ty, usize::from_str(digits.as_ref()).unwrap_or_default()) }), Token::LBracket, Token::RBracket, @@ -1990,7 +1995,12 @@ impl ChumskyParse for AliasedType { .ignore_then(delimited_with_recovery( ty.then_ignore(parse_token_with_recovery(Token::Comma)) .then(num.clone().validate(|num, e, emit| -> NonZeroPow2Usize { - match NonZeroPow2Usize::from_str(num.as_inner()) { + #[cfg(feature = "fmt")] + let digits = num.strip_digit_separators(); + #[cfg(not(feature = "fmt"))] + let digits = std::borrow::Cow::from(num.as_inner()); + + match NonZeroPow2Usize::from_str(digits.as_ref()) { Ok(number) => number, Err(err) => { emit.emit( @@ -2371,7 +2381,12 @@ impl ChumskyParse for CallName { .then(select! { Token::DecLiteral(s) => s }.labelled("list size")) .then_ignore(generics_close.clone()) .validate(|(func, bound_str), e, emit| { - let bound = match bound_str.as_inner().parse::() { + #[cfg(feature = "fmt")] + let digits = bound_str.strip_digit_separators(); + #[cfg(not(feature = "fmt"))] + let digits = std::borrow::Cow::from(bound_str.as_inner()); + + let bound = match digits.parse::() { Ok(num) => match NonZeroPow2Usize::new(num) { Some(val) => val, None => { @@ -2400,7 +2415,12 @@ impl ChumskyParse for CallName { .then(select! { Token::DecLiteral(s) => s }.labelled("array size")) .then_ignore(generics_close.clone()) .validate(|(func, size_str), e, emit| { - let size = match size_str.as_inner().parse::() { + #[cfg(feature = "fmt")] + let digits = size_str.strip_digit_separators(); + #[cfg(not(feature = "fmt"))] + let digits = std::borrow::Cow::from(size_str.as_inner()); + + let size = match digits.parse::() { Ok(0) => { emit.emit(Error::ArraySizeNonZero { size: 0 }.with_span(e.span())); NonZeroUsize::new(1).unwrap() diff --git a/src/str.rs b/src/str.rs index 95b1e1f9..2b9b1505 100644 --- a/src/str.rs +++ b/src/str.rs @@ -254,7 +254,7 @@ impl<'a> arbitrary::Arbitrary<'a> for AliasName { } } -/// A string of decimal digits. +/// A string of decimal digits, optionally separated by underscores (requires the `"fmt"` feature). #[derive(Clone, Ord, PartialOrd, Eq, PartialEq, Hash)] pub struct Decimal(Arc); @@ -273,7 +273,7 @@ impl<'a> arbitrary::Arbitrary<'a> for Decimal { } } -/// A string of binary digits. +/// A string of binary digits, optionally separated by underscores (requires the `"fmt"` feature). #[derive(Clone, Ord, PartialOrd, Eq, PartialEq, Hash)] pub struct Binary(Arc); @@ -293,12 +293,46 @@ impl<'a> arbitrary::Arbitrary<'a> for Binary { } } -/// A string of hexadecimal digits. +/// A string of hexadecimal digits, optionally separated by underscores (requires the `"fmt"` feature). #[derive(Clone, Ord, PartialOrd, Eq, PartialEq, Hash)] pub struct Hexadecimal(Arc); wrapped_string!(Hexadecimal, "hexadecimal string"); +#[cfg(feature = "fmt")] +mod underscore_parsing { + use super::*; + use std::borrow::Cow; + + /// Remove digit separators from a numeric literal while avoiding an allocation + /// when the literal contains no separators. + fn strip_digit_separators(input: &str) -> Cow<'_, str> { + if input.contains('_') { + Cow::Owned(input.replace('_', "")) + } else { + Cow::Borrowed(input) + } + } + + impl Decimal { + pub fn strip_digit_separators(&self) -> Cow<'_, str> { + strip_digit_separators(self.0.as_ref()) + } + } + + impl Binary { + pub fn strip_digit_separators(&self) -> Cow<'_, str> { + strip_digit_separators(self.0.as_ref()) + } + } + + impl Hexadecimal { + pub fn strip_digit_separators(&self) -> Cow<'_, str> { + strip_digit_separators(self.0.as_ref()) + } + } +} + #[cfg(feature = "arbitrary")] impl<'a> arbitrary::Arbitrary<'a> for Hexadecimal { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { diff --git a/src/types.rs b/src/types.rs index fefa298a..b2d67f67 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1145,13 +1145,13 @@ impl TreeLike for StructuralType { impl fmt::Debug for StructuralType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", &self.0) + write!(f, "{}", self.0) } } impl fmt::Display for StructuralType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", &self.0) + write!(f, "{}", self.0) } } diff --git a/src/value.rs b/src/value.rs index 49ebbc5a..825b452d 100644 --- a/src/value.rs +++ b/src/value.rs @@ -122,7 +122,12 @@ impl UIntValue { /// Create an integer from a `decimal` string and type. pub fn parse_decimal(decimal: &Decimal, ty: UIntType) -> Result { - let s = decimal.as_inner(); + #[cfg(feature = "fmt")] + let digits = decimal.strip_digit_separators(); + #[cfg(not(feature = "fmt"))] + let digits = std::borrow::Cow::from(decimal.as_inner()); + + let s = digits.as_ref(); match ty { UIntType::U1 => s.parse::().map_err(Error::from).and_then(Self::u1), UIntType::U2 => s.parse::().map_err(Error::from).and_then(Self::u2), @@ -138,7 +143,12 @@ impl UIntValue { /// Create an integer from a `binary` string and type. pub fn parse_binary(binary: &Binary, ty: UIntType) -> Result { - let s = binary.as_inner(); + #[cfg(feature = "fmt")] + let digits = binary.strip_digit_separators(); + #[cfg(not(feature = "fmt"))] + let digits = std::borrow::Cow::from(binary.as_inner()); + + let s = digits.as_ref(); let bit_len = Pow2Usize::new(s.len()).ok_or(Error::BitStringPow2 { len: s.len() })?; let bit_ty = UIntType::from_bit_width(bit_len).ok_or(Error::BitStringPow2 { len: s.len() })?; @@ -655,7 +665,13 @@ impl Value { TypeInner::Array(inner, len) if inner.as_integer() == Some(UIntType::U8) => *len, _ => return Err(Error::ExpressionUnexpectedType { ty: ty.clone() }), }; - let s = hexadecimal.as_inner(); + + #[cfg(feature = "fmt")] + let digits = hexadecimal.strip_digit_separators(); + #[cfg(not(feature = "fmt"))] + let digits = std::borrow::Cow::from(hexadecimal.as_inner()); + + let s = digits.as_ref(); if s.len() % 2 != 0 || s.len() != expected_byte_len * 2 { return Err(Error::ExpressionUnexpectedType { ty: ty.clone() }); } @@ -981,13 +997,13 @@ impl TreeLike for StructuralValue { impl fmt::Debug for StructuralValue { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", &self.0) + write!(f, "{}", self.0) } } impl fmt::Display for StructuralValue { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", &self.0) + write!(f, "{}", self.0) } } From 11c8937b3a3a34c0f575b2c6280fcfb228ee2e45 Mon Sep 17 00:00:00 2001 From: Illia Kripaka Date: Wed, 5 Aug 2026 16:28:19 +0300 Subject: [PATCH 3/4] Flatten tests in parse.rs and lexer.rs --- src/lexer.rs | 1009 +++++++++++++++++++-------------------- src/parse.rs | 1299 +++++++++++++++++++++++++------------------------- 2 files changed, 1147 insertions(+), 1161 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 8715af1e..9a573125 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -524,316 +524,312 @@ pub fn is_keyword(s: &str) -> bool { } #[cfg(test)] -mod tests { - use chumsky::error::Rich; - +mod original_lexer { use super::*; - mod lexer { - use super::*; - - fn lex<'src>( - input: &'src str, - ) -> (Option>>, Vec>) { - let (tokens, errors) = lexer().parse(input).into_output_errors(); - let tokens = tokens.map(|vec| { - vec.into_iter() - .map(|(tok, _)| tok.clone()) - .collect::>() - }); - (tokens, errors) - } - #[test] - fn test_block_comment_simple() { - let input = "/* hello world */"; - let (tokens, errors) = lex(input); - - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!( - tokens, - Some(vec![]), - "Should produce a single block comment token" - ); - } - - #[test] - fn test_block_comment_nested() { - let input = "/* outer /* inner */ outer */"; - let (tokens, errors) = lex(input); - - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!(tokens, Some(vec![])); - } + fn lex<'src>( + input: &'src str, + ) -> (Option>>, Vec>) { + let (tokens, errors) = lexer().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(tok, _)| tok.clone()) + .collect::>() + }); + (tokens, errors) + } + #[test] + fn test_block_comment_simple() { + let input = "/* hello world */"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![]), + "Should produce a single block comment token" + ); + } - #[test] - fn test_block_comment_deeply_nested() { - let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; - let (tokens, errors) = lex(input); + #[test] + fn test_block_comment_nested() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex(input); - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!(tokens, Some(vec![])); - } + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - #[test] - fn test_block_comment_multiline() { - let input = "/* \n line 1 \n /* inner \n line */ \n */"; - let (tokens, errors) = lex(input); + #[test] + fn test_block_comment_deeply_nested() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex(input); - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!(tokens, Some(vec![])); - } + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - #[test] - fn test_block_comment_unclosed() { - let input = "/* unclosed comment start"; - let (tokens, errors) = lex(input); + #[test] + fn test_block_comment_multiline() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex(input); - assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - let err = &errors[0]; - assert_eq!(err.span().start, 0); - assert_eq!(err.span().end, 2); - assert_eq!(err.to_string(), "Unclosed block comment"); + #[test] + fn test_block_comment_unclosed() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex(input); - assert_eq!(tokens, Some(vec![])); - } + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); - #[test] - fn test_block_comment_partial_nesting_unclosed() { - let input = "/* outer /* inner */"; - let (tokens, errors) = lex(input); + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].span().start, 0); - assert_eq!(tokens, Some(vec![])); - } + assert_eq!(tokens, Some(vec![])); + } - #[test] - fn test_block_comment_double_unclosed() { - let input = "/* outer /* inner"; - let (tokens, errors) = lex(input); + #[test] + fn test_block_comment_partial_nesting_unclosed() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex(input); - assert_eq!(errors.len(), 2); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!(tokens, Some(vec![])); + } - assert_eq!(errors[0].span().start, 9); - assert_eq!(errors[0].to_string(), "Unclosed block comment"); + #[test] + fn test_block_comment_double_unclosed() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex(input); - assert_eq!(errors[1].span().start, 0); - assert_eq!(errors[1].to_string(), "Unclosed block comment"); + assert_eq!(errors.len(), 2); - assert_eq!(tokens, Some(vec![])); - } + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); - #[test] - fn test_spaces_resolution() { - let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; - let (tokens, errors) = lex(input); + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!(tokens, Some(vec![])); - } + assert_eq!(tokens, Some(vec![])); + } - #[test] - fn test_ignoring_tokens_after_comment_with_incorrect_symbol() { - let input = "fn main() {} @// fn hello(){} simc"; - let (tokens, errors) = lex(input); + #[test] + fn test_spaces_resolution() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex(input); - assert_eq!( - errors.len(), - 1, - "comment contents must not be retried as code" - ); - assert!(errors[0].to_string().contains("found '@' expected")); - assert_eq!( - tokens, - Some(vec![ - Token::Fn, - Token::Ident("main"), - Token::LParen, - Token::RParen, - Token::LBrace, - Token::RBrace, - ]) - ); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - let (_tokens, diagnostics) = super::lex(0, input, 0); - assert_eq!(diagnostics.len(), 1); - assert!(matches!(diagnostics[0].error(), Error::CannotParse { .. })); - } + #[test] + fn test_ignoring_tokens_after_comment_with_incorrect_symbol() { + let input = "fn main() {} @// fn hello(){} simc"; + let (tokens, errors) = lex(input); + + assert_eq!( + errors.len(), + 1, + "comment contents must not be retried as code" + ); + assert!(errors[0].to_string().contains("found '@' expected")); + assert_eq!( + tokens, + Some(vec![ + Token::Fn, + Token::Ident("main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + ]) + ); + + let (_tokens, diagnostics) = super::lex(0, input, 0); + assert_eq!(diagnostics.len(), 1); + assert!(matches!(diagnostics[0].error(), Error::CannotParse { .. })); + } - #[test] - fn test_ignoring_tokens_after_comment() { - let input = "fn main() {} /* fn hello(){} \n simc 0.6.0; */ \n\ + #[test] + fn test_ignoring_tokens_after_comment() { + let input = "fn main() {} /* fn hello(){} \n simc 0.6.0; */ \n\ // enum Name {} match true {} \n fn other_main() {} "; - let (tokens, errors) = lex(input); - - assert!(errors.is_empty()); - assert_eq!( - tokens, - Some(vec![ - Token::Fn, - Token::Ident("main"), - Token::LParen, - Token::RParen, - Token::LBrace, - Token::RBrace, - Token::Fn, - Token::Ident("other_main"), - Token::LParen, - Token::RParen, - Token::LBrace, - Token::RBrace, - ]) + let (tokens, errors) = lex(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![ + Token::Fn, + Token::Ident("main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + Token::Fn, + Token::Ident("other_main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + ]) + ); + } + + #[test] + fn simc_is_reserved() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, Token::Simc)), + "the sentinel must not reach the token stream for {src:?}" ); } - #[test] - fn simc_is_reserved() { - // The prescan consumes the one legitimate leading directive before lexing, - // so `lex` reports any `simc` it sees and drops the sentinel token. - for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { - let (tokens, errors) = super::lex(0, src, 0); - assert!( - errors.iter().any(|e| e.to_string().contains("reserved")), - "expected a reserved-keyword error for {src:?}, got: {errors:?}" - ); - assert!( - tokens - .expect("recovery keeps the stream") - .iter() - .all(|(tok, _)| !matches!(tok, Token::Simc)), - "the sentinel must not reach the token stream for {src:?}" - ); - } - - // Identifiers merely starting with `simc` are ordinary identifiers. - let (tokens, errors) = lex("simcfoo"); - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); - } + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); + } - #[test] - fn test_enum_token() { - let (tokens, errors) = lex("enum Path { Inherit, ColdSpend }"); - assert!(errors.is_empty()); - assert_eq!( - tokens, - Some(vec![ - Token::Enum, - Token::Ident("Path"), - Token::LBrace, - Token::Ident("Inherit"), - Token::Comma, - Token::Ident("ColdSpend"), - Token::RBrace, - ]) - ); - } + #[test] + fn test_enum_token() { + let (tokens, errors) = lex("enum Path { Inherit, ColdSpend }"); + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![ + Token::Enum, + Token::Ident("Path"), + Token::LBrace, + Token::Ident("Inherit"), + Token::Comma, + Token::Ident("ColdSpend"), + Token::RBrace, + ]) + ); + } - #[test] - fn leading_whitespaces_before_main() { - let input = " fn main(){}"; - let (tokens, errors) = lex(input); + #[test] + fn leading_whitespaces_before_main() { + let input = " fn main(){}"; + let (tokens, errors) = lex(input); - assert!(errors.is_empty()); - assert!(tokens.is_some()); + assert!(errors.is_empty()); + assert!(tokens.is_some()); - let tokens = tokens.unwrap(); - assert_eq!(tokens[0], Token::Fn); - } + let tokens = tokens.unwrap(); + assert_eq!(tokens[0], Token::Fn); + } - #[test] - fn lexer_test() { - use chumsky::prelude::*; + #[test] + fn lexer_test() { + use chumsky::prelude::*; - // Check if the lexer parses the example file without errors. - let src = include_str!("../examples/last_will.simf"); + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); - let (tokens, lex_errs) = lexer().parse(src).into_output_errors(); - let _ = tokens.unwrap(); + let (tokens, lex_errs) = lexer().parse(src).into_output_errors(); + let _ = tokens.unwrap(); - assert!(lex_errs.is_empty()); - } + assert!(lex_errs.is_empty()); } +} - #[cfg(feature = "fmt")] - mod fmt_lexer { - use super::*; - - fn lex_lossless<'src>( - input: &'src str, - ) -> ( - Option>>, - Vec>, - ) { - let (tokens, errors) = lexer_lossless().parse(input).into_output_errors(); - let tokens = tokens.map(|vec| { - vec.into_iter() - .map(|(fmt_tok, _)| fmt_tok) - .collect::>() - }); - (tokens, errors) - } - - fn fmt_trivia<'src>(kind: TriviaKind, text: &'src str) -> FmtToken<'src> { - let trivia = match kind { - TriviaKind::LineComment => Trivia::line_comment(text), - TriviaKind::BlockComment => Trivia::block_comment(text), - TriviaKind::Newline => Trivia::newline(match text { - "\r\n" => LineEnding::CrLf, - "\n" => LineEnding::Lf, - "\r" => LineEnding::Cr, - _ => panic!("invalid newline spelling: {text:?}"), - }), - TriviaKind::Whitespace => Trivia::whitespace(text), - }; - - FmtToken::Trivia(trivia) - } +#[cfg(feature = "fmt")] +#[cfg(test)] +mod fmt_lexer { + use super::*; - #[test] - fn test_block_comment_simple_fmt() { - let input = "/* hello world */"; - let (tokens, errors) = lex_lossless(input); + fn lex_lossless<'src>( + input: &'src str, + ) -> ( + Option>>, + Vec>, + ) { + let (tokens, errors) = lexer_lossless().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(fmt_tok, _)| fmt_tok) + .collect::>() + }); + (tokens, errors) + } - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!( - tokens, - Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]), - "Should produce a single block comment token" - ); - } + fn fmt_trivia<'src>(kind: TriviaKind, text: &'src str) -> FmtToken<'src> { + let trivia = match kind { + TriviaKind::LineComment => Trivia::line_comment(text), + TriviaKind::BlockComment => Trivia::block_comment(text), + TriviaKind::Newline => Trivia::newline(match text { + "\r\n" => LineEnding::CrLf, + "\n" => LineEnding::Lf, + "\r" => LineEnding::Cr, + _ => panic!("invalid newline spelling: {text:?}"), + }), + TriviaKind::Whitespace => Trivia::whitespace(text), + }; + + FmtToken::Trivia(trivia) + } - #[test] - fn lossless_trivia_display_preserves_source_text() { - let input = "// comment\r\n\t"; - let (tokens, errors) = lex_lossless(input); + #[test] + fn test_block_comment_simple_fmt() { + let input = "/* hello world */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]), + "Should produce a single block comment token" + ); + } - assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); - let rendered: String = tokens - .expect("lossless lexing succeeds") - .iter() - .map(ToString::to_string) - .collect(); + #[test] + fn lossless_trivia_display_preserves_source_text() { + let input = "// comment\r\n\t"; + let (tokens, errors) = lex_lossless(input); - assert_eq!(rendered, input); - } + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let rendered: String = tokens + .expect("lossless lexing succeeds") + .iter() + .map(ToString::to_string) + .collect(); - #[test] - fn lossless_lexer_keeps_each_newline_kind_with_its_span() { - let input = "first\r\nsecond\rthird\nfourth"; - let (tokens, errors) = super::lex_lossless(0, input, 0); + assert_eq!(rendered, input); + } - assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); - let tokens = tokens.expect("lossless lexing succeeds"); - let line_endings: Vec<_> = tokens - .iter() - .filter_map(|(token, _)| match token { - FmtToken::Trivia(Trivia::Newline(line_ending)) => Some(*line_ending), - _ => None, - }) - .collect(); - let newlines: Vec<_> = tokens + #[test] + fn lossless_lexer_keeps_each_newline_kind_with_its_span() { + let input = "first\r\nsecond\rthird\nfourth"; + let (tokens, errors) = super::lex_lossless(0, input, 0); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let tokens = tokens.expect("lossless lexing succeeds"); + let line_endings: Vec<_> = tokens + .iter() + .filter_map(|(token, _)| match token { + FmtToken::Trivia(Trivia::Newline(line_ending)) => Some(*line_ending), + _ => None, + }) + .collect(); + let newlines: Vec<_> = tokens .iter() .filter(|(token, _)| { matches!(token, FmtToken::Trivia(trivia) if trivia.kind() == TriviaKind::Newline) @@ -841,282 +837,281 @@ mod tests { .map(|(_, span)| span.to_slice(input)) .collect(); - assert_eq!( - line_endings, - vec![LineEnding::CrLf, LineEnding::Cr, LineEnding::Lf] - ); - assert_eq!(newlines, vec![Some("\r\n"), Some("\r"), Some("\n")]); - } + assert_eq!( + line_endings, + vec![LineEnding::CrLf, LineEnding::Cr, LineEnding::Lf] + ); + assert_eq!(newlines, vec![Some("\r\n"), Some("\r"), Some("\n")]); + } - #[test] - fn test_block_comment_nested_fmt() { - let input = "/* outer /* inner */ outer */"; - let (tokens, errors) = lex_lossless(input); + #[test] + fn test_block_comment_nested_fmt() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex_lossless(input); - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!( - tokens, - Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) - ); - } + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } - #[test] - fn test_block_comment_deeply_nested_fmt() { - let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; - let (tokens, errors) = lex_lossless(input); + #[test] + fn test_block_comment_deeply_nested_fmt() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex_lossless(input); - assert!(errors.is_empty()); - assert_eq!( - tokens, - Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) - ); - } + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } - #[test] - fn test_block_comment_multiline_fmt() { - let input = "/* \n line 1 \n /* inner \n line */ \n */"; - let (tokens, errors) = lex_lossless(input); + #[test] + fn test_block_comment_multiline_fmt() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex_lossless(input); - assert!(errors.is_empty()); - assert_eq!( - tokens, - Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) - ); - } + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } - #[test] - fn test_block_comment_unclosed_fmt() { - let input = "/* unclosed comment start"; - let (tokens, errors) = lex_lossless(input); + #[test] + fn test_block_comment_unclosed_fmt() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex_lossless(input); - assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); - let err = &errors[0]; - assert_eq!(err.span().start, 0); - assert_eq!(err.span().end, 2); - assert_eq!(err.to_string(), "Unclosed block comment"); + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); - assert_eq!( - tokens, - Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) - ); - } + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } - #[test] - fn test_block_comment_partial_nesting_unclosed_fmt() { - let input = "/* outer /* inner */"; - let (tokens, errors) = lex_lossless(input); + #[test] + fn test_block_comment_partial_nesting_unclosed_fmt() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].span().start, 0); - assert_eq!( - tokens, - Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) - ); - } + #[test] + fn test_block_comment_double_unclosed_fmt() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex_lossless(input); - #[test] - fn test_block_comment_double_unclosed_fmt() { - let input = "/* outer /* inner"; - let (tokens, errors) = lex_lossless(input); + assert_eq!(errors.len(), 2); - assert_eq!(errors.len(), 2); + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); - assert_eq!(errors[0].span().start, 9); - assert_eq!(errors[0].to_string(), "Unclosed block comment"); + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); - assert_eq!(errors[1].span().start, 0); - assert_eq!(errors[1].to_string(), "Unclosed block comment"); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } - assert_eq!( - tokens, - Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) - ); - } + #[test] + fn test_spaces_resolution_fmt() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![ + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r"), + ]) + ); + } - #[test] - fn test_spaces_resolution_fmt() { - let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; - let (tokens, errors) = lex_lossless(input); - - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!( - tokens, - Some(vec![ - fmt_trivia(TriviaKind::Newline, "\r\n"), - fmt_trivia(TriviaKind::Newline, "\n"), - fmt_trivia(TriviaKind::Newline, "\r"), - fmt_trivia(TriviaKind::Newline, "\r"), - fmt_trivia(TriviaKind::Newline, "\r"), - fmt_trivia(TriviaKind::Newline, "\r"), - fmt_trivia(TriviaKind::Newline, "\r\n"), - fmt_trivia(TriviaKind::Newline, "\r\n"), - fmt_trivia(TriviaKind::Whitespace, " "), - fmt_trivia(TriviaKind::Newline, "\n"), - fmt_trivia(TriviaKind::Newline, "\n"), - fmt_trivia(TriviaKind::Newline, "\r\n"), - fmt_trivia(TriviaKind::Newline, "\n"), - fmt_trivia(TriviaKind::Newline, "\r"), - ]) - ); - } + #[test] + fn test_ignoring_tokens_after_comment_with_incorrect_symbol() { + let input = "fn main() {} @// fn hello(){} simc"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1, "only the invalid character is an error"); + assert!(errors[0].to_string().contains("found '@' expected")); + + let tokens = tokens.expect("recovery keeps the lossless stream"); + assert!(matches!( + tokens.last(), + Some(FmtToken::Trivia(Trivia::LineComment(_))) + )); + assert!( + tokens + .iter() + .all(|token| !matches!(token, FmtToken::Token(Token::Simc))), + "comment contents must not become semantic tokens" + ); + } - #[test] - fn test_ignoring_tokens_after_comment_with_incorrect_symbol() { - let input = "fn main() {} @// fn hello(){} simc"; - let (tokens, errors) = lex_lossless(input); + #[test] + fn test_not_ignoring_tokens_after_comment() { + let input = "fn main() {} /* fn hello(){} \n simc 0.6.0; */ \n\ + // enum Name {} match true {} \n fn other_main() {} "; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![ + FmtToken::Token(Token::Fn), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Ident("main")), + FmtToken::Token(Token::LParen), + FmtToken::Token(Token::RParen), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::LBrace), + FmtToken::Token(Token::RBrace), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia( + TriviaKind::BlockComment, + "/* fn hello(){} \n simc 0.6.0; */" + ), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::LineComment, "// enum Name {} match true {} "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Fn), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Ident("other_main")), + FmtToken::Token(Token::LParen), + FmtToken::Token(Token::RParen), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::LBrace), + FmtToken::Token(Token::RBrace), + fmt_trivia(TriviaKind::Whitespace, " "), + ]) + ); + } - assert_eq!(errors.len(), 1, "only the invalid character is an error"); - assert!(errors[0].to_string().contains("found '@' expected")); + #[test] + fn simc_is_reserved_fmt() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex_lossless(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); - let tokens = tokens.expect("recovery keeps the lossless stream"); - assert!(matches!( - tokens.last(), - Some(FmtToken::Trivia(Trivia::LineComment(_))) - )); assert!( tokens + .expect("recovery keeps the stream") .iter() - .all(|token| !matches!(token, FmtToken::Token(Token::Simc))), - "comment contents must not become semantic tokens" + .all(|(tok, _)| !matches!(tok, FmtToken::Token(Token::Simc))), + "the sentinel must not reach the token stream for {src:?}" ); } - #[test] - fn test_not_ignoring_tokens_after_comment() { - let input = "fn main() {} /* fn hello(){} \n simc 0.6.0; */ \n\ - // enum Name {} match true {} \n fn other_main() {} "; - let (tokens, errors) = lex_lossless(input); - - assert!(errors.is_empty()); - assert_eq!( - tokens, - Some(vec![ - FmtToken::Token(Token::Fn), - fmt_trivia(TriviaKind::Whitespace, " "), - FmtToken::Token(Token::Ident("main")), - FmtToken::Token(Token::LParen), - FmtToken::Token(Token::RParen), - fmt_trivia(TriviaKind::Whitespace, " "), - FmtToken::Token(Token::LBrace), - FmtToken::Token(Token::RBrace), - fmt_trivia(TriviaKind::Whitespace, " "), - fmt_trivia( - TriviaKind::BlockComment, - "/* fn hello(){} \n simc 0.6.0; */" - ), - fmt_trivia(TriviaKind::Whitespace, " "), - fmt_trivia(TriviaKind::Newline, "\n"), - fmt_trivia(TriviaKind::LineComment, "// enum Name {} match true {} "), - fmt_trivia(TriviaKind::Newline, "\n"), - fmt_trivia(TriviaKind::Whitespace, " "), - FmtToken::Token(Token::Fn), - fmt_trivia(TriviaKind::Whitespace, " "), - FmtToken::Token(Token::Ident("other_main")), - FmtToken::Token(Token::LParen), - FmtToken::Token(Token::RParen), - fmt_trivia(TriviaKind::Whitespace, " "), - FmtToken::Token(Token::LBrace), - FmtToken::Token(Token::RBrace), - fmt_trivia(TriviaKind::Whitespace, " "), - ]) - ); - } - - #[test] - fn simc_is_reserved_fmt() { - // The prescan consumes the one legitimate leading directive before lexing, - // so `lex` reports any `simc` it sees and drops the sentinel token. - for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { - let (tokens, errors) = super::lex_lossless(0, src, 0); - assert!( - errors.iter().any(|e| e.to_string().contains("reserved")), - "expected a reserved-keyword error for {src:?}, got: {errors:?}" - ); - - assert!( - tokens - .expect("recovery keeps the stream") - .iter() - .all(|(tok, _)| !matches!(tok, FmtToken::Token(Token::Simc))), - "the sentinel must not reach the token stream for {src:?}" - ); - } - - // Identifiers merely starting with `simc` are ordinary identifiers. - let (tokens, errors) = lex_lossless("simcfoo"); - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!(tokens, Some(vec![FmtToken::Token(Token::Ident("simcfoo"))])); - } + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex_lossless("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![FmtToken::Token(Token::Ident("simcfoo"))])); + } - #[test] - fn numeric_literals_preserve_underscores() { - let input = "1_234_567_89 0xDEAD_BEEF 0b__1010_0101_ 0b1010_0101 _1_234__567___890____000_____ 0x_DEAD_BEEF__BEEF_DEAD_"; - let (tokens, errors) = lex_lossless(input); - - assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); - assert!(tokens.is_some(), "lexing must succeed"); - let tokens = tokens - .unwrap() - .into_iter() - .filter(|x| !matches!(x, FmtToken::Trivia(Trivia::Whitespace(_)))) - .collect::>(); - assert_eq!( - tokens, - vec![ - FmtToken::Token(Token::DecLiteral(Decimal::from_str_unchecked( - "1_234_567_89" - ))), - FmtToken::Token(Token::HexLiteral(Hexadecimal::from_str_unchecked( - "DEAD_BEEF" - ))), - FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked( - "__1010_0101_" - ))), - FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked("1010_0101"))), - FmtToken::Token(Token::DecLiteral(Decimal::from_str_unchecked( - "_1_234__567___890____000_____" - ))), - FmtToken::Token(Token::HexLiteral(Hexadecimal::from_str_unchecked( - "_DEAD_BEEF__BEEF_DEAD_" - ))), - ] - ); + #[test] + fn numeric_literals_preserve_underscores() { + let input = "1_234_567_89 0xDEAD_BEEF 0b__1010_0101_ 0b1010_0101 _1_234__567___890____000_____ 0x_DEAD_BEEF__BEEF_DEAD_"; + let (tokens, errors) = lex_lossless(input); - assert_eq!( - tokens - .iter() - .map(ToString::to_string) - .collect::>() - .join(" "), - input - ); - } + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + assert!(tokens.is_some(), "lexing must succeed"); + let tokens = tokens + .unwrap() + .into_iter() + .filter(|x| !matches!(x, FmtToken::Trivia(Trivia::Whitespace(_)))) + .collect::>(); + assert_eq!( + tokens, + vec![ + FmtToken::Token(Token::DecLiteral(Decimal::from_str_unchecked( + "1_234_567_89" + ))), + FmtToken::Token(Token::HexLiteral(Hexadecimal::from_str_unchecked( + "DEAD_BEEF" + ))), + FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked( + "__1010_0101_" + ))), + FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked("1010_0101"))), + FmtToken::Token(Token::DecLiteral(Decimal::from_str_unchecked( + "_1_234__567___890____000_____" + ))), + FmtToken::Token(Token::HexLiteral(Hexadecimal::from_str_unchecked( + "_DEAD_BEEF__BEEF_DEAD_" + ))), + ] + ); + + assert_eq!( + tokens + .iter() + .map(ToString::to_string) + .collect::>() + .join(" "), + input + ); + } - #[test] - fn leading_whitespaces_before_main() { - let input = " fn main(){}"; - let (tokens, errors) = lex_lossless(input); + #[test] + fn leading_whitespaces_before_main() { + let input = " fn main(){}"; + let (tokens, errors) = lex_lossless(input); - assert!(errors.is_empty()); - assert!(tokens.is_some()); + assert!(errors.is_empty()); + assert!(tokens.is_some()); - let tokens = dbg!(tokens).unwrap(); - assert!(matches!(tokens[0], FmtToken::Trivia(Trivia::Whitespace(_)))); - assert!(matches!(tokens[1], FmtToken::Token(Token::Fn))); - } + let tokens = dbg!(tokens).unwrap(); + assert!(matches!(tokens[0], FmtToken::Trivia(Trivia::Whitespace(_)))); + assert!(matches!(tokens[1], FmtToken::Token(Token::Fn))); + } - #[test] - fn lossless_lexer_test() { - use chumsky::prelude::*; + #[test] + fn lossless_lexer_test() { + use chumsky::prelude::*; - // Check if the lexer parses the example file without errors. - let src = include_str!("../examples/last_will.simf"); + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); - let (tokens, lex_errs) = lexer_lossless().parse(src).into_output_errors(); - let _ = tokens.unwrap(); + let (tokens, lex_errs) = lexer_lossless().parse(src).into_output_errors(); + let _ = tokens.unwrap(); - assert!(lex_errs.is_empty()); - } + assert!(lex_errs.is_empty()); } } diff --git a/src/parse.rs b/src/parse.rs index 32f6bd6a..e0ff9daa 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -3550,10 +3550,33 @@ impl crate::ArbitraryRec for Match { } #[cfg(test)] -mod test { - use crate::parse; +mod type_alias { + use super::*; + + #[test] + fn test_reject_redefined_builtin_type() { + let ty = TypeAlias::parse_from_str("type Ctx8 = u32") + .expect_err("Redefining built-in alias should be rejected"); + + assert!(ty + .error() + .to_string() + .contains("Type alias `Ctx8` is already exists as built-in alias")); + } + + #[test] + fn test_fragment_rejects_version_directive() { + let err = TypeAlias::parse_from_str("simc \"*\"; type Ctx8 = u32") + .expect_err("a version directive must not be accepted in a fragment"); + + assert!(matches!(err.error(), Error::ReservedSimcKeyword)); + } +} +#[cfg(test)] +mod regular_parsing { use super::*; + use crate::parse; impl UseDecl { /// Creates a dummy `UseDecl` specifically for testing `DependencyMap` resolution. @@ -3567,179 +3590,153 @@ mod test { } } - mod type_alias { - use super::*; - - #[test] - fn test_reject_redefined_builtin_type() { - let ty = TypeAlias::parse_from_str("type Ctx8 = u32") - .expect_err("Redefining built-in alias should be rejected"); + #[test] + fn test_double_colon() { + let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; + let mut diagnostics = DiagnosticManager::new(); - assert!(ty - .error() - .to_string() - .contains("Type alias `Ctx8` is already exists as built-in alias")); - } + let parsed_program = Program::parse_from_str_with_errors( + MAIN_MODULE, + input, + &UnstableFeatures::all(), + &mut diagnostics, + ); - #[test] - fn test_fragment_rejects_version_directive() { - let err = TypeAlias::parse_from_str("simc \"*\"; type Ctx8 = u32") - .expect_err("a version directive must not be accepted in a fragment"); - - assert!(matches!(err.error(), Error::ReservedSimcKeyword)); - } + assert!(parsed_program.is_none()); + assert!(diagnostics.to_string().contains("Expected '::', found ':'")); } - mod regular_parsing { - use super::*; + #[test] + fn test_double_double_colon() { + let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; + let mut diagnostics = DiagnosticManager::new(); - #[test] - fn test_double_colon() { - let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; - let mut diagnostics = DiagnosticManager::new(); + let parsed_program = Program::parse_from_str_with_errors( + MAIN_MODULE, + input, + &UnstableFeatures::all(), + &mut diagnostics, + ); - let parsed_program = Program::parse_from_str_with_errors( - MAIN_MODULE, - input, - &UnstableFeatures::all(), - &mut diagnostics, - ); + assert!(parsed_program.is_none()); + assert!( + diagnostics + .to_string() + .contains("Expected identifier, found ::"), + "the second :: should be reported as the error site" + ); + } - assert!(parsed_program.is_none()); - assert!(diagnostics.to_string().contains("Expected '::', found ':'")); - } + /// Parse `input` and return whether it was rejected and the collected error text. + fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { + let mut diagnostics = DiagnosticManager::new(); + let program = + Program::parse_from_str_with_errors(MAIN_MODULE, input, features, &mut diagnostics); - #[test] - fn test_double_double_colon() { - let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; - let mut diagnostics = DiagnosticManager::new(); + let rejected = program.is_none(); + let text = diagnostics.to_string(); - let parsed_program = Program::parse_from_str_with_errors( - MAIN_MODULE, - input, - &UnstableFeatures::all(), - &mut diagnostics, - ); + (rejected, text) + } - assert!(parsed_program.is_none()); + #[test] + fn inverted_empty_span_from_token_gap_does_not_panic() { + // Fuzz-found (compile_text). + // + // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. + // The only path that requests an empty span at a lex-error gap and the + // NUL after `enum` creates that gap. + // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted + // across the gap and panicked the strict `Span` constructor. + // + // Simplifying any part (even NUL to a space) loses the crash. + let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; + let mut diagnostics = DiagnosticManager::new(); + let program = Program::parse_from_str_with_errors( + MAIN_MODULE, + src, + &UnstableFeatures::all(), + &mut diagnostics, + ); + + assert!(program.is_none(), "garbage input must be rejected"); + assert!(diagnostics.has_errors()); + } + + #[test] + fn recovery_synchronizes_at_enum_declaration() { + // Discriminating setup: an invalid prefix followed by a malformed + // enum. Without `Token::Enum` in the synchronization set the whole + // enum is swallowed into the prefix's recovery span and only one + // error is reported; with it, the enum parses as its own item and + // its malformation is reported as a second error. + let mut diagnostics = DiagnosticManager::new(); + let program = Program::parse_from_str_with_errors( + MAIN_MODULE, + "let invalid = 1;\nenum Action { A B }\nfn main() {}", + &UnstableFeatures::all(), + &mut diagnostics, + ); + assert!(program.is_none(), "the invalid program must be rejected"); + assert_eq!( + 2, + diagnostics.error_count(), + "the invalid prefix and the malformed enum must each report an error" + ); + } + + #[test] + fn recovery_after_malformed_enum_reaches_next_item() { + let (rejected, _text) = parse_with( + "enum Action { A B }\nfn main() {}", + &UnstableFeatures::all(), + ); + assert!(rejected, "a malformed enum must fail compilation"); + } + + #[test] + fn malformed_construct_containing_enum_keyword_reports_errors() { + // `enum` inside a badly malformed nested construct may be mistaken + // for an item boundary; compilation must still fail cleanly. + let (rejected, _text) = parse_with( + "fn broken() { let x = (enum; }\nfn main() {}", + &UnstableFeatures::all(), + ); + assert!(rejected, "a malformed construct must fail compilation"); + } + + #[test] + fn test_gated_syntax_is_rejected_without_features() { + // Real `use`/`mod` syntax: rejected under none() (naming the feature + a + // -Z hint), accepted under all(). Delete when `imports` stabilizes. + for input in [ + "use crate::foo::bar;\nfn main() { }", + "mod inner { }\nfn main() { }", + ] { + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); assert!( - diagnostics - .to_string() - .contains("Expected identifier, found ::"), - "the second :: should be reported as the error site" - ); - } - - /// Parse `input` and return whether it was rejected and the collected error text. - fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { - let mut diagnostics = DiagnosticManager::new(); - let program = - Program::parse_from_str_with_errors(MAIN_MODULE, input, features, &mut diagnostics); - - let rejected = program.is_none(); - let text = diagnostics.to_string(); - - (rejected, text) - } - - #[test] - fn inverted_empty_span_from_token_gap_does_not_panic() { - // Fuzz-found (compile_text). - // - // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. - // The only path that requests an empty span at a lex-error gap and the - // NUL after `enum` creates that gap. - // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted - // across the gap and panicked the strict `Span` constructor. - // - // Simplifying any part (even NUL to a space) loses the crash. - let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; - let mut diagnostics = DiagnosticManager::new(); - let program = Program::parse_from_str_with_errors( - MAIN_MODULE, - src, - &UnstableFeatures::all(), - &mut diagnostics, - ); - - assert!(program.is_none(), "garbage input must be rejected"); - assert!(diagnostics.has_errors()); - } - - #[test] - fn recovery_synchronizes_at_enum_declaration() { - // Discriminating setup: an invalid prefix followed by a malformed - // enum. Without `Token::Enum` in the synchronization set the whole - // enum is swallowed into the prefix's recovery span and only one - // error is reported; with it, the enum parses as its own item and - // its malformation is reported as a second error. - let mut diagnostics = DiagnosticManager::new(); - let program = Program::parse_from_str_with_errors( - MAIN_MODULE, - "let invalid = 1;\nenum Action { A B }\nfn main() {}", - &UnstableFeatures::all(), - &mut diagnostics, - ); - assert!(program.is_none(), "the invalid program must be rejected"); - assert_eq!( - 2, - diagnostics.error_count(), - "the invalid prefix and the malformed enum must each report an error" + rejected, + "gated syntax must be rejected without features (if this feature was \ + just stabilized, delete this test):\n{input}" ); - } - - #[test] - fn recovery_after_malformed_enum_reaches_next_item() { - let (rejected, _text) = parse_with( - "enum Action { A B }\nfn main() {}", - &UnstableFeatures::all(), + assert!( + error.contains("imports") && error.contains("-Z"), + "rejection should name the feature and suggest -Z, got:\n{error}" ); - assert!(rejected, "a malformed enum must fail compilation"); - } - #[test] - fn malformed_construct_containing_enum_keyword_reports_errors() { - // `enum` inside a badly malformed nested construct may be mistaken - // for an item boundary; compilation must still fail cleanly. - let (rejected, _text) = parse_with( - "fn broken() { let x = (enum; }\nfn main() {}", - &UnstableFeatures::all(), + let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + assert!( + !rejected, + "the same syntax must parse with all features enabled:\n{error}" ); - assert!(rejected, "a malformed construct must fail compilation"); - } - - #[test] - fn test_gated_syntax_is_rejected_without_features() { - // Real `use`/`mod` syntax: rejected under none() (naming the feature + a - // -Z hint), accepted under all(). Delete when `imports` stabilizes. - for input in [ - "use crate::foo::bar;\nfn main() { }", - "mod inner { }\nfn main() { }", - ] { - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!( - rejected, - "gated syntax must be rejected without features (if this feature was \ - just stabilized, delete this test):\n{input}" - ); - assert!( - error.contains("imports") && error.contains("-Z"), - "rejection should name the feature and suggest -Z, got:\n{error}" - ); - - let (rejected, error) = parse_with(input, &UnstableFeatures::all()); - assert!( - !rejected, - "the same syntax must parse with all features enabled:\n{error}" - ); - } } + } - #[test] - fn test_type_heavy_program_needs_no_features() { - // Types are traversed by `RequireFeature` but not gated, so a type-heavy, - // import-free program must parse with no features (no false-positive gates). - let input = r#"type Alias = u32; + #[test] + fn test_type_heavy_program_needs_no_features() { + // Types are traversed by `RequireFeature` but not gated, so a type-heavy, + // import-free program must parse with no features (no false-positive gates). + let input = r#"type Alias = u32; fn pick(e: Either) -> u32 { match e { Left(a: u32) => a, @@ -3752,346 +3749,344 @@ fn main() { assert!(jet::eq_32(chosen, chosen)); } "#; - // `parse_from_str_with_errors` returns `Some` only when no errors were - // collected, so a non-rejection already proves there were no gate errors. - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!( - !rejected, - "type-heavy but import-free program should parse with no features:\n{error}" - ); - } - - #[test] - fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { - // The gate check skips error-recovered ASTs, so a program with - // both a syntax error and gated syntax reports only the syntax error. - let input = "use crate::foo;\nfn main( {"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "broken program must be rejected"); - assert!( - !error.contains("-Z"), - "syntax error should not produce a spurious feature-gate hint, got:\n{error}" - ); - } - - #[test] - fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { - // Lexer-side companion to the case above: the stray `@` lexes with an - // error but recovers a cleanly-parsing `use` AST, and a program that - // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. - let input = "use crate@::foo;\nfn main() { }"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "program with a lexer error must be rejected"); - assert!( - !error.contains("-Z"), - "lexer error should not produce a spurious feature-gate hint, got:\n{error}" - ); + // `parse_from_str_with_errors` returns `Some` only when no errors were + // collected, so a non-rejection already proves there were no gate errors. + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + !rejected, + "type-heavy but import-free program should parse with no features:\n{error}" + ); + } + + #[test] + fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { + // The gate check skips error-recovered ASTs, so a program with + // both a syntax error and gated syntax reports only the syntax error. + let input = "use crate::foo;\nfn main( {"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "broken program must be rejected"); + assert!( + !error.contains("-Z"), + "syntax error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { + // Lexer-side companion to the case above: the stray `@` lexes with an + // error but recovers a cleanly-parsing `use` AST, and a program that + // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. + let input = "use crate@::foo;\nfn main() { }"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "program with a lexer error must be rejected"); + assert!( + !error.contains("-Z"), + "lexer error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_use_decl_display_round_trips_with_aliases() { + for input in [ + "use lib::A::foo;", + "use lib::A::foo as bar;", + "use lib::A::{foo, bar as baz};", + ] { + let program = parse::Program::parse_from_str(input).expect("parsing works"); + assert_eq!(program.to_string(), format!("{input}\n")); } + } - #[test] - fn test_use_decl_display_round_trips_with_aliases() { - for input in [ - "use lib::A::foo;", - "use lib::A::foo as bar;", - "use lib::A::{foo, bar as baz};", - ] { - let program = parse::Program::parse_from_str(input).expect("parsing works"); - assert_eq!(program.to_string(), format!("{input}\n")); - } - } + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; + let program = Program::parse_from_str(input).expect("program parses"); - #[test] - fn statement_spans_exclude_terminating_semicolons() { - let input = "fn main() { let value: u8 = 0; value; }"; - let program = Program::parse_from_str(input).expect("program parses"); + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; - let Item::Function(function) = &program.items()[0] else { - panic!("expected a function item"); - }; - let ExpressionInner::Block(statements, None) = function.body().inner() else { - panic!("expected a block containing only statements"); - }; + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } - assert_eq!( - statements[0].span().to_slice(input), - Some("let value: u8 = 0") - ); - assert_eq!(statements[1].span().to_slice(input), Some("value")); - } + fn parse_item(input: &str) -> Item { + let program = parse::Program::parse_from_str(input).expect("parsing should succeed"); + program.items().first().expect("expected one item").clone() + } - fn parse_item(input: &str) -> Item { - let program = parse::Program::parse_from_str(input).expect("parsing should succeed"); - program.items().first().expect("expected one item").clone() - } + #[test] + fn test_enum_declaration_basic() { + let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration, got {item:?}"); + }; + assert_eq!(decl.name().as_inner(), "Path"); + assert_eq!(decl.variants().len(), 3); + assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); + assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); + } - #[test] - fn test_enum_declaration_basic() { - let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration, got {item:?}"); - }; - assert_eq!(decl.name().as_inner(), "Path"); - assert_eq!(decl.variants().len(), 3); - assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); - assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); - } + #[test] + fn test_enum_declaration_pub() { + let item = parse_item("pub enum Color { Red, Green, Blue, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!(decl.visibility(), &Visibility::Public); + assert_eq!(decl.name().as_inner(), "Color"); + } - #[test] - fn test_enum_declaration_pub() { - let item = parse_item("pub enum Color { Red, Green, Blue, }"); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration"); - }; - assert_eq!(decl.visibility(), &Visibility::Public); - assert_eq!(decl.name().as_inner(), "Color"); - } + #[test] + fn test_enum_declaration_display_round_trip() { + let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; + let item = parse_item(input); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!( + decl.to_string(), + "enum Path { Inherit, ColdSpend, RefreshSpend, }" + ); + } - #[test] - fn test_enum_declaration_display_round_trip() { - let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; - let item = parse_item(input); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration"); - }; - assert_eq!( - decl.to_string(), - "enum Path { Inherit, ColdSpend, RefreshSpend, }" + #[test] + fn test_enum_declaration_reserved_name() { + for reserved in RESERVED_PATTERN_NAMES { + let result = Program::parse_from_str(&format!("enum {reserved} {{ A, B, }}")); + let error = result.expect_err(&format!("enum name {reserved} is reserved")); + assert!( + error.to_string().contains("reserved"), + "error should say the name is reserved: {error}" ); } - - #[test] - fn test_enum_declaration_reserved_name() { - for reserved in RESERVED_PATTERN_NAMES { - let result = Program::parse_from_str(&format!("enum {reserved} {{ A, B, }}")); - let error = result.expect_err(&format!("enum name {reserved} is reserved")); - assert!( - error.to_string().contains("reserved"), - "error should say the name is reserved: {error}" - ); - } - } } +} - #[cfg(feature = "fmt")] - mod fmt_parsing { - use super::*; - - /// Parse `input`, appending any formatter diagnostics to `diagnostics`. - fn parse_with_diagnostics<'src>( - input: &'src str, - features: &UnstableFeatures, - diagnostics: &mut DiagnosticManager, - ) -> Option> { - Program::parse_with_errors_for_fmt(MAIN_MODULE, input, features, diagnostics) - } +#[cfg(test)] +#[cfg(feature = "fmt")] +mod fmt_parsing { + use super::*; + + /// Parse `input`, appending any formatter diagnostics to `diagnostics`. + fn parse_with_diagnostics<'src>( + input: &'src str, + features: &UnstableFeatures, + diagnostics: &mut DiagnosticManager, + ) -> Option> { + Program::parse_with_errors_for_fmt(MAIN_MODULE, input, features, diagnostics) + } - /// Parse `input` and return whether it was rejected and the collected error text. - fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { - let mut diagnostics = DiagnosticManager::new(); - let program = parse_with_diagnostics(input, features, &mut diagnostics); + /// Parse `input` and return whether it was rejected and the collected error text. + fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); - let rejected = program.is_none(); - let text = diagnostics.to_string(); + let rejected = program.is_none(); + let text = diagnostics.to_string(); - (rejected, text) - } + (rejected, text) + } - /// Parse `input` and return whether it was rejected and the collected error in `DiagnosticManager`. - fn parse_with_diagnosis( - input: &str, - features: &UnstableFeatures, - ) -> (bool, DiagnosticManager) { - let mut diagnostics = DiagnosticManager::new(); - let program = parse_with_diagnostics(input, features, &mut diagnostics); + /// Parse `input` and return whether it was rejected and the collected error in `DiagnosticManager`. + fn parse_with_diagnosis(input: &str, features: &UnstableFeatures) -> (bool, DiagnosticManager) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); - let rejected = program.is_none(); + let rejected = program.is_none(); - (rejected, diagnostics) - } + (rejected, diagnostics) + } - /// Parse `input` and return whether it was rejected and the collected error text together with `ParsedSource`. - fn parse_with_extended_out<'src>( - input: &'src str, - features: &UnstableFeatures, - ) -> (bool, String, Option>) { - let mut diagnostics = DiagnosticManager::new(); - let program = parse_with_diagnostics(input, features, &mut diagnostics); + /// Parse `input` and return whether it was rejected and the collected error text together with `ParsedSource`. + fn parse_with_extended_out<'src>( + input: &'src str, + features: &UnstableFeatures, + ) -> (bool, String, Option>) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); - let rejected = program.is_none(); - let text = diagnostics.to_string(); + let rejected = program.is_none(); + let text = diagnostics.to_string(); - (rejected, text, program) - } + (rejected, text, program) + } - fn assert_formatter_matches_regular_parser(input: &str, features: &UnstableFeatures) { - let mut regular_diagnostics = DiagnosticManager::new(); - let regular = Program::parse_from_str_with_errors( - MAIN_MODULE, - input, - features, - &mut regular_diagnostics, - ); + fn assert_formatter_matches_regular_parser(input: &str, features: &UnstableFeatures) { + let mut regular_diagnostics = DiagnosticManager::new(); + let regular = Program::parse_from_str_with_errors( + MAIN_MODULE, + input, + features, + &mut regular_diagnostics, + ); - let mut formatting_diagnostics = DiagnosticManager::new(); - let formatting = parse_with_diagnostics(input, features, &mut formatting_diagnostics); + let mut formatting_diagnostics = DiagnosticManager::new(); + let formatting = parse_with_diagnostics(input, features, &mut formatting_diagnostics); - assert_eq!( - formatting.as_ref().map(ParsedSource::program), - regular.as_ref(), - "formatter and regular parsers must produce the same AST for {input:?}" - ); - assert_eq!( + assert_eq!( + formatting.as_ref().map(ParsedSource::program), + regular.as_ref(), + "formatter and regular parsers must produce the same AST for {input:?}" + ); + assert_eq!( formatting_diagnostics.error_count(), regular_diagnostics.error_count(), "formatter and regular parsers must report the same number of errors for {input:?}, \n\ fmt errors: \n\t '{formatting_diagnostics}, \n regular errors: \n\t '{regular_diagnostics}" ); - } - - fn assert_lossless_source(input: &str, parsed: &ParsedSource<'_>) { - let prefix = parsed.prefix(); - assert_eq!(prefix.file_id, MAIN_MODULE); - - let mut reconstructed = prefix - .to_slice(input) - .expect("the directive prefix must be a valid source slice") - .to_owned(); - let mut cursor = prefix.end; - - for (_, span) in parsed.tokens() { - assert_eq!(span.file_id, MAIN_MODULE); - assert!( - span.start >= prefix.end, - "formatter tokens must not overlap the preserved directive prefix" - ); - assert_eq!( - span.start, cursor, - "formatter token spans must be contiguous and in source order" - ); - - let text = span - .to_slice(input) - .expect("formatter token spans must be valid source slices"); - assert!( - !text.is_empty(), - "the formatter token stream must not contain empty source slices" - ); - reconstructed.push_str(text); - cursor = span.end; - } - - assert_eq!(cursor, input.len(), "formatter tokens must reach EOF"); - assert_eq!(reconstructed, input, "formatter input must be lossless"); - } - - #[test] - fn test_double_colon() { - let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; - - let (rejected, text) = parse_with(input, &UnstableFeatures::all()); + } - assert!(rejected); - assert!(text.contains("Expected '::', found ':'")); - } + fn assert_lossless_source(input: &str, parsed: &ParsedSource<'_>) { + let prefix = parsed.prefix(); + assert_eq!(prefix.file_id, MAIN_MODULE); - #[test] - fn test_double_double_colon() { - let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; + let mut reconstructed = prefix + .to_slice(input) + .expect("the directive prefix must be a valid source slice") + .to_owned(); + let mut cursor = prefix.end; - let (rejected, text) = parse_with(input, &UnstableFeatures::all()); + for (_, span) in parsed.tokens() { + assert_eq!(span.file_id, MAIN_MODULE); + assert!( + span.start >= prefix.end, + "formatter tokens must not overlap the preserved directive prefix" + ); + assert_eq!( + span.start, cursor, + "formatter token spans must be contiguous and in source order" + ); - assert!(rejected); + let text = span + .to_slice(input) + .expect("formatter token spans must be valid source slices"); assert!( - text.contains("Expected identifier, found ::"), - "the second :: should be reported as the error site" + !text.is_empty(), + "the formatter token stream must not contain empty source slices" ); + reconstructed.push_str(text); + cursor = span.end; } - #[test] - fn inverted_empty_span_from_token_gap_does_not_panic() { - // Fuzz-found (compile_text). - // - // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. - // The only path that requests an empty span at a lex-error gap and the - // NUL after `enum` creates that gap. - // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted - // across the gap and panicked the strict `Span` constructor. - // - // Simplifying any part (even NUL to a space) loses the crash. - let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; - let (rejected, text) = parse_with(src, &UnstableFeatures::all()); - - assert!(rejected, "garbage input must be rejected"); - assert!(!text.is_empty()); - } + assert_eq!(cursor, input.len(), "formatter tokens must reach EOF"); + assert_eq!(reconstructed, input, "formatter input must be lossless"); + } - #[test] - fn recovery_synchronizes_at_enum_declaration() { - // Discriminating setup: an invalid prefix followed by a malformed - // enum. Without `Token::Enum` in the synchronization set the whole - // enum is swallowed into the prefix's recovery span and only one - // error is reported; with it, the enum parses as its own item and - // its malformation is reported as a second error. + #[test] + fn test_double_colon() { + let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; - let src = "let invalid = 1;\nenum Action { A B }\nfn main() {}"; - let (rejected, diagnostics) = parse_with_diagnosis(src, &UnstableFeatures::all()); + let (rejected, text) = parse_with(input, &UnstableFeatures::all()); - assert!(rejected, "the invalid program must be rejected"); - assert_eq!( - 2, - diagnostics.error_count(), - "the invalid prefix and the malformed enum must each report an error" - ); - } + assert!(rejected); + assert!(text.contains("Expected '::', found ':'")); + } - #[test] - fn recovery_after_malformed_enum_reaches_next_item() { - let src = "enum Action { A B }\nfn main() {}"; - let (rejected, _text) = parse_with(src, &UnstableFeatures::all()); - assert!(rejected, "a malformed enum must fail compilation"); - } + #[test] + fn test_double_double_colon() { + let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; - #[test] - fn malformed_construct_containing_enum_keyword_reports_errors() { - // `enum` inside a badly malformed nested construct may be mistaken - // for an item boundary; compilation must still fail cleanly. - let src = "fn broken() { let x = (enum; }\nfn main() {}"; - let (rejected, _text) = parse_with(src, &UnstableFeatures::all()); - assert!(rejected, "a malformed construct must fail compilation"); - } + let (rejected, text) = parse_with(input, &UnstableFeatures::all()); + + assert!(rejected); + assert!( + text.contains("Expected identifier, found ::"), + "the second :: should be reported as the error site" + ); + } - #[test] - fn test_gated_syntax_is_rejected_without_features() { - // Real `use`/`mod` syntax: rejected under none() (naming the feature + a - // -Z hint), accepted under all(). Delete when `imports` stabilizes. - for input in [ - "use crate::foo::bar;\nfn main() { }", - "mod inner { }\nfn main() { }", - ] { - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!( - rejected, - "gated syntax must be rejected without features (if this feature was \ + #[test] + fn inverted_empty_span_from_token_gap_does_not_panic() { + // Fuzz-found (compile_text). + // + // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. + // The only path that requests an empty span at a lex-error gap and the + // NUL after `enum` creates that gap. + // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted + // across the gap and panicked the strict `Span` constructor. + // + // Simplifying any part (even NUL to a space) loses the crash. + let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; + let (rejected, text) = parse_with(src, &UnstableFeatures::all()); + + assert!(rejected, "garbage input must be rejected"); + assert!(!text.is_empty()); + } + + #[test] + fn recovery_synchronizes_at_enum_declaration() { + // Discriminating setup: an invalid prefix followed by a malformed + // enum. Without `Token::Enum` in the synchronization set the whole + // enum is swallowed into the prefix's recovery span and only one + // error is reported; with it, the enum parses as its own item and + // its malformation is reported as a second error. + + let src = "let invalid = 1;\nenum Action { A B }\nfn main() {}"; + let (rejected, diagnostics) = parse_with_diagnosis(src, &UnstableFeatures::all()); + + assert!(rejected, "the invalid program must be rejected"); + assert_eq!( + 2, + diagnostics.error_count(), + "the invalid prefix and the malformed enum must each report an error" + ); + } + + #[test] + fn recovery_after_malformed_enum_reaches_next_item() { + let src = "enum Action { A B }\nfn main() {}"; + let (rejected, _text) = parse_with(src, &UnstableFeatures::all()); + assert!(rejected, "a malformed enum must fail compilation"); + } + + #[test] + fn malformed_construct_containing_enum_keyword_reports_errors() { + // `enum` inside a badly malformed nested construct may be mistaken + // for an item boundary; compilation must still fail cleanly. + let src = "fn broken() { let x = (enum; }\nfn main() {}"; + let (rejected, _text) = parse_with(src, &UnstableFeatures::all()); + assert!(rejected, "a malformed construct must fail compilation"); + } + + #[test] + fn test_gated_syntax_is_rejected_without_features() { + // Real `use`/`mod` syntax: rejected under none() (naming the feature + a + // -Z hint), accepted under all(). Delete when `imports` stabilizes. + for input in [ + "use crate::foo::bar;\nfn main() { }", + "mod inner { }\nfn main() { }", + ] { + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + rejected, + "gated syntax must be rejected without features (if this feature was \ just stabilized, delete this test):\n{input}" - ); - assert!( - error.contains("imports") && error.contains("-Z"), - "rejection should name the feature and suggest -Z, got:\n{error}" - ); + ); + assert!( + error.contains("imports") && error.contains("-Z"), + "rejection should name the feature and suggest -Z, got:\n{error}" + ); - let (rejected, error) = parse_with(input, &UnstableFeatures::all()); - assert!( - !rejected, - "the same syntax must parse with all features enabled:\n{error}" - ); - } + let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + assert!( + !rejected, + "the same syntax must parse with all features enabled:\n{error}" + ); } + } - #[test] - fn test_type_heavy_program_needs_no_features() { - // Types are traversed by `RequireFeature` but not gated, so a type-heavy, - // import-free program must parse with no features (no false-positive gates). - let input = r#"type Alias = u32; + #[test] + fn test_type_heavy_program_needs_no_features() { + // Types are traversed by `RequireFeature` but not gated, so a type-heavy, + // import-free program must parse with no features (no false-positive gates). + let input = r#"type Alias = u32; fn pick(e: Either) -> u32 { match e { Left(a: u32) => a, @@ -4104,242 +4099,238 @@ fn main() { assert!(jet::eq_32(chosen, chosen)); } "#; - // `parse_from_str_with_errors` returns `Some` only when no errors were - // collected, so a non-rejection already proves there were no gate errors. - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!( - !rejected, - "type-heavy but import-free program should parse with no features:\n{error}" - ); - } - - #[test] - fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { - // The gate check skips error-recovered ASTs, so a program with - // both a syntax error and gated syntax reports only the syntax error. - let input = "use crate::foo;\nfn main( {"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "broken program must be rejected"); - assert!( - !error.contains("-Z"), - "syntax error should not produce a spurious feature-gate hint, got:\n{error}" - ); - } - - #[test] - fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { - // Lexer-side companion to the case above: the stray `@` lexes with an - // error but recovers a cleanly-parsing `use` AST, and a program that - // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. - let input = "use crate@::foo;\nfn main() { }"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "program with a lexer error must be rejected"); - assert!( - !error.contains("-Z"), - "lexer error should not produce a spurious feature-gate hint, got:\n{error}" - ); + // `parse_from_str_with_errors` returns `Some` only when no errors were + // collected, so a non-rejection already proves there were no gate errors. + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + !rejected, + "type-heavy but import-free program should parse with no features:\n{error}" + ); + } + + #[test] + fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { + // The gate check skips error-recovered ASTs, so a program with + // both a syntax error and gated syntax reports only the syntax error. + let input = "use crate::foo;\nfn main( {"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "broken program must be rejected"); + assert!( + !error.contains("-Z"), + "syntax error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { + // Lexer-side companion to the case above: the stray `@` lexes with an + // error but recovers a cleanly-parsing `use` AST, and a program that + // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. + let input = "use crate@::foo;\nfn main() { }"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "program with a lexer error must be rejected"); + assert!( + !error.contains("-Z"), + "lexer error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_use_decl_display_round_trips_with_aliases() { + for input in [ + "use lib::A::foo;", + "use lib::A::foo as bar;", + "use lib::A::{foo, bar as baz};", + ] { + let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::all()); + assert!(!rejected, "parsing works"); + assert_eq!(program.unwrap().program.to_string(), format!("{input}\n")); } + } - #[test] - fn test_use_decl_display_round_trips_with_aliases() { - for input in [ - "use lib::A::foo;", - "use lib::A::foo as bar;", - "use lib::A::{foo, bar as baz};", - ] { - let (rejected, _, program) = - parse_with_extended_out(input, &UnstableFeatures::all()); - assert!(!rejected, "parsing works"); - assert_eq!(program.unwrap().program.to_string(), format!("{input}\n")); - } - } + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; - #[test] - fn statement_spans_exclude_terminating_semicolons() { - let input = "fn main() { let value: u8 = 0; value; }"; + let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::none()); + assert!(!rejected, "parsing works"); + let program = program.unwrap().program; - let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::none()); - assert!(!rejected, "parsing works"); - let program = program.unwrap().program; - - let Item::Function(function) = &program.items()[0] else { - panic!("expected a function item"); - }; - let ExpressionInner::Block(statements, None) = function.body().inner() else { - panic!("expected a block containing only statements"); - }; + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; - assert_eq!( - statements[0].span().to_slice(input), - Some("let value: u8 = 0") - ); - assert_eq!(statements[1].span().to_slice(input), Some("value")); - } + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } - fn parse_item(input: &str) -> Item { - let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::all()); - assert!(!rejected, "parsing works"); - let program = program.unwrap().program; + fn parse_item(input: &str) -> Item { + let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::all()); + assert!(!rejected, "parsing works"); + let program = program.unwrap().program; - program.items().first().expect("expected one item").clone() - } + program.items().first().expect("expected one item").clone() + } - #[test] - fn test_enum_declaration_basic() { - let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration, got {item:?}"); - }; - assert_eq!(decl.name().as_inner(), "Path"); - assert_eq!(decl.variants().len(), 3); - assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); - assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); - } + #[test] + fn test_enum_declaration_basic() { + let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration, got {item:?}"); + }; + assert_eq!(decl.name().as_inner(), "Path"); + assert_eq!(decl.variants().len(), 3); + assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); + assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); + } - #[test] - fn test_enum_declaration_pub() { - let item = parse_item("pub enum Color { Red, Green, Blue, }"); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration"); - }; - assert_eq!(decl.visibility(), &Visibility::Public); - assert_eq!(decl.name().as_inner(), "Color"); - } + #[test] + fn test_enum_declaration_pub() { + let item = parse_item("pub enum Color { Red, Green, Blue, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!(decl.visibility(), &Visibility::Public); + assert_eq!(decl.name().as_inner(), "Color"); + } - #[test] - fn test_enum_declaration_display_round_trip() { - let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; - let item = parse_item(input); - let Item::EnumDeclaration(decl) = item else { - panic!("expected EnumDeclaration"); - }; - assert_eq!( - decl.to_string(), - "enum Path { Inherit, ColdSpend, RefreshSpend, }" + #[test] + fn test_enum_declaration_display_round_trip() { + let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; + let item = parse_item(input); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!( + decl.to_string(), + "enum Path { Inherit, ColdSpend, RefreshSpend, }" + ); + } + + #[test] + fn test_enum_declaration_reserved_name() { + for reserved in RESERVED_PATTERN_NAMES { + let input = format!("enum {reserved} {{ A, B, }}"); + let (rejected, error) = parse_with(&input, &UnstableFeatures::none()); + assert!(rejected, "enum name {reserved} is reserved"); + assert!( + error.contains("reserved"), + "error should say the name is reserved: {error}" ); } + } - #[test] - fn test_enum_declaration_reserved_name() { - for reserved in RESERVED_PATTERN_NAMES { - let input = format!("enum {reserved} {{ A, B, }}"); - let (rejected, error) = parse_with(&input, &UnstableFeatures::none()); - assert!(rejected, "enum name {reserved} is reserved"); - assert!( - error.contains("reserved"), - "error should say the name is reserved: {error}" - ); - } - } - - #[test] - fn formatting_parse_is_lossless_after_directive_prescan() { - let input = "// header\r\nsimc \"*\";\r\n\t/* block */ fn main() { // body\r\n\t}\r"; + #[test] + fn formatting_parse_is_lossless_after_directive_prescan() { + let input = "// header\r\nsimc \"*\";\r\n\t/* block */ fn main() { // body\r\n\t}\r"; - let (_rejected, _error, parsed) = - parse_with_extended_out(input, &UnstableFeatures::none()); - let parsed = parsed.expect("formatting parse should succeed"); + let (_rejected, _error, parsed) = parse_with_extended_out(input, &UnstableFeatures::none()); + let parsed = parsed.expect("formatting parse should succeed"); - assert_eq!( - parsed.prefix().to_slice(input), - Some("// header\r\nsimc \"*\";") - ); - for trivia in [ - crate::lexer::TriviaKind::BlockComment, - crate::lexer::TriviaKind::LineComment, - crate::lexer::TriviaKind::Newline, - crate::lexer::TriviaKind::Whitespace, - ] { - assert!( + assert_eq!( + parsed.prefix().to_slice(input), + Some("// header\r\nsimc \"*\";") + ); + for trivia in [ + crate::lexer::TriviaKind::BlockComment, + crate::lexer::TriviaKind::LineComment, + crate::lexer::TriviaKind::Newline, + crate::lexer::TriviaKind::Whitespace, + ] { + assert!( parsed.tokens().iter().any( |(token, _)| matches!(token, FmtToken::Trivia(token_trivia) if token_trivia.kind() == trivia) ), "formatter token stream should retain {trivia:?}" ); - } - assert_lossless_source(input, &parsed); - assert_eq!(parsed.program().items().len(), 1); } + assert_lossless_source(input, &parsed); + assert_eq!(parsed.program().items().len(), 1); + } - #[test] - fn formatting_parse_matches_regular_parser_pipeline() { - let no_features = UnstableFeatures::none(); - let all_features = UnstableFeatures::all(); + #[test] + fn formatting_parse_matches_regular_parser_pipeline() { + let no_features = UnstableFeatures::none(); + let all_features = UnstableFeatures::all(); - for (input, features) in [ - ( - "/* header */\r\nfn\tmain() { // comment\r\n}\r", - &no_features, - ), - ("// version\r\nsimc \"*\";\r\nfn main() {}", &no_features), - ("use crate::foo::bar;\nfn main() {}", &no_features), - ("use crate::foo::bar;\nfn main() {}", &all_features), - ("fn main( {", &no_features), - ("fn main() { @// comment }", &no_features), - ] { - assert_formatter_matches_regular_parser(input, features); - } + for (input, features) in [ + ( + "/* header */\r\nfn\tmain() { // comment\r\n}\r", + &no_features, + ), + ("// version\r\nsimc \"*\";\r\nfn main() {}", &no_features), + ("use crate::foo::bar;\nfn main() {}", &no_features), + ("use crate::foo::bar;\nfn main() {}", &all_features), + ("fn main( {", &no_features), + ("fn main() { @// comment }", &no_features), + ] { + assert_formatter_matches_regular_parser(input, features); } + } - #[test] - fn formatting_parse_ignores_preexisting_errors() { - let input = "fn main() {}"; - let mut diagnostics = DiagnosticManager::new(); - diagnostics.push(Diagnostic::global(Error::CannotParse { - msg: "an error from an earlier source file".to_owned(), - })); - let before = diagnostics.error_count(); + #[test] + fn formatting_parse_ignores_preexisting_errors() { + let input = "fn main() {}"; + let mut diagnostics = DiagnosticManager::new(); + diagnostics.push(Diagnostic::global(Error::CannotParse { + msg: "an error from an earlier source file".to_owned(), + })); + let before = diagnostics.error_count(); - let parsed = parse_with_diagnostics(input, &UnstableFeatures::none(), &mut diagnostics); + let parsed = parse_with_diagnostics(input, &UnstableFeatures::none(), &mut diagnostics); - assert!( - parsed.is_some(), - "valid source must still produce ParsedSource" - ); - assert_eq!(diagnostics.error_count(), before); - assert_eq!(diagnostics.diagnostics().len(), before); - } + assert!( + parsed.is_some(), + "valid source must still produce ParsedSource" + ); + assert_eq!(diagnostics.error_count(), before); + assert_eq!(diagnostics.diagnostics().len(), before); + } - #[test] - fn formatting_parse_stops_after_invalid_directive() { - for (input, is_malformed) in [ - ("simc \"*\"\nfn broken( @", true), - ("simc \">99.0.0\"; @", false), - ] { - let (rejected, diagnostics) = - parse_with_diagnosis(input, &UnstableFeatures::none()); - - assert!(rejected, "invalid directive must abort formatting parse"); - assert_eq!( - diagnostics.error_count(), - 1, - "the invalid body must not add diagnostics after prescan aborts" - ); - if is_malformed { - assert!(matches!( - diagnostics.diagnostics()[0].error(), - Error::MalformedSimcDirective - )); - } else { - assert!(matches!( - diagnostics.diagnostics()[0].error(), - Error::SimcVersionMismatch { .. } - )); - } + #[test] + fn formatting_parse_stops_after_invalid_directive() { + for (input, is_malformed) in [ + ("simc \"*\"\nfn broken( @", true), + ("simc \">99.0.0\"; @", false), + ] { + let (rejected, diagnostics) = parse_with_diagnosis(input, &UnstableFeatures::none()); + + assert!(rejected, "invalid directive must abort formatting parse"); + assert_eq!( + diagnostics.error_count(), + 1, + "the invalid body must not add diagnostics after prescan aborts" + ); + if is_malformed { + assert!(matches!( + diagnostics.diagnostics()[0].error(), + Error::MalformedSimcDirective + )); + } else { + assert!(matches!( + diagnostics.diagnostics()[0].error(), + Error::SimcVersionMismatch { .. } + )); } } + } - #[test] - fn formatting_parse_reports_missing_match_arm_comma() { - let input = "fn main() { match true { false => () true => (), } }"; + #[test] + fn formatting_parse_reports_missing_match_arm_comma() { + let input = "fn main() { match true { false => () true => (), } }"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected); - assert!( - error.contains("Missing ',' after a match arm that isn't block expression"), - "expected the missing-comma diagnostic, got {error:?}" - ); - } + assert!(rejected); + assert!( + error.contains("Missing ',' after a match arm that isn't block expression"), + "expected the missing-comma diagnostic, got {error:?}" + ); } } From 9a1a4d1d1f25d1c77810f1dfc327b7986f64cdab Mon Sep 17 00:00:00 2001 From: Illia Kripaka Date: Thu, 6 Aug 2026 10:01:22 +0300 Subject: [PATCH 4/4] Align previous behaviour with new logic * add additional tests to test correctness of numbers with underlines --- src/lexer.rs | 139 ++++++++++++++++++++++++++++++++++++--------------- src/parse.rs | 25 ++++----- src/str.rs | 24 +-------- src/value.rs | 15 ++---- 4 files changed, 113 insertions(+), 90 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 9a573125..f71375a2 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -289,38 +289,42 @@ pub(crate) fn trivia<'src>( fn digits_with_underscores<'src>( radix: u32, ) -> impl Parser<'src, &'src str, &'src str, extra::Err>> { - #[cfg(feature = "fmt")] - { - let digit = any().filter(move |c: &char| c.is_digit(radix)); - just('_') - .repeated() - .then(digit) - .then(choice((digit, just('_'))).repeated()) - .to_slice() - } - #[cfg(not(feature = "fmt"))] - { - any() - .filter(move |c: &char| c.is_digit(radix)) - .then( - any() - .filter(move |c: &char| c.is_digit(radix) || *c == '_') - .repeated(), - ) - .to_slice() + any() + .filter(move |c: &char| c.is_digit(radix)) + .then( + any() + .filter(move |c: &char| c.is_digit(radix) || *c == '_') + .repeated(), + ) + .to_slice() +} + +fn digit_literal_text(input: &str) -> std::borrow::Cow<'_, str> { + if PRESERVE || !input.contains('_') { + std::borrow::Cow::Borrowed(input) + } else { + std::borrow::Cow::Owned(input.replace('_', "")) } } -fn to_token<'src>( +fn to_token<'src, const PRESERVE: bool>( ) -> impl Parser<'src, &'src str, Token<'src>, extra::Err>> { - let num = digits_with_underscores(10) - .map(|s: &str| Token::DecLiteral(Decimal::from_str_unchecked(s))); + let num = digits_with_underscores(10).map(|s: &str| { + let text = digit_literal_text::(s); + Token::DecLiteral(Decimal::from_str_unchecked(text.as_ref())) + }); let hex = just("0x") .ignore_then(digits_with_underscores(16)) - .map(|s: &str| Token::HexLiteral(Hexadecimal::from_str_unchecked(s))); + .map(|s: &str| { + let text = digit_literal_text::(s); + Token::HexLiteral(Hexadecimal::from_str_unchecked(text.as_ref())) + }); let bin = just("0b") .ignore_then(digits_with_underscores(2)) - .map(|s: &str| Token::BinLiteral(Binary::from_str_unchecked(s))); + .map(|s: &str| { + let text = digit_literal_text::(s); + Token::BinLiteral(Binary::from_str_unchecked(text.as_ref())) + }); let macros = choice((just("assert!"), just("panic!"), just("dbg!"), just("list!"))).map(Token::Macro); @@ -380,9 +384,14 @@ fn to_token<'src>( pub fn lexer<'src>( ) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> { - let lexeme = choice((trivia_item().to(None), to_token().map(Some))) - .map_with(|token, e| (token, e.span())) - .recover_with(skip_then_retry_until(any().ignored(), end())); + const REMOVE_SEPARATORS: bool = false; + + let lexeme = choice(( + trivia_item().to(None), + to_token::().map(Some), + )) + .map_with(|token, e| (token, e.span())) + .recover_with(skip_then_retry_until(any().ignored(), end())); lexeme.repeated().collect::>().map(|lexemes| { lexemes @@ -396,7 +405,9 @@ pub fn lexer<'src>( pub fn lexer_lossless<'src>( ) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> { - let token = to_token().map(FmtToken::Token); + const PRESERVE_SEPARATORS: bool = true; + + let token = to_token::().map(FmtToken::Token); let newline = line_ending().map(Trivia::newline).map(FmtToken::Trivia); let whitespace = whitespace() @@ -725,6 +736,56 @@ mod original_lexer { ); } + #[test] + fn numeric_literals_with_underscores_and_underscore_digit_identifiers() { + let cases = [ + ( + "[u8; 1_6]", + vec![ + Token::LBracket, + Token::Ident("u8"), + Token::Semi, + Token::DecLiteral(Decimal::from_str_unchecked("16")), + Token::RBracket, + ], + ), + ( + "List", + vec![ + Token::Ident("List"), + Token::LAngle, + Token::Ident("u8"), + Token::Comma, + Token::DecLiteral(Decimal::from_str_unchecked("16")), + Token::RAngle, + ], + ), + ( + "1_3_3_7", + vec![Token::DecLiteral(Decimal::from_str_unchecked("1337"))], + ), + ( + "0x1_", + vec![Token::HexLiteral(Hexadecimal::from_str_unchecked("1"))], + ), + ( + "0b1010_0101", + vec![Token::BinLiteral(Binary::from_str_unchecked("10100101"))], + ), + ("_34", vec![Token::Ident("_34")]), + ("_34foo", vec![Token::Ident("_34foo")]), + ]; + + for (input, expected) in cases { + let (tokens, errors) = lex(input); + assert!( + errors.is_empty(), + "Expected no errors for {input:?}: {errors:?}" + ); + assert_eq!(tokens, Some(expected), "Unexpected tokens for {input:?}"); + } + } + #[test] fn leading_whitespaces_before_main() { let input = " fn main(){}"; @@ -830,12 +891,12 @@ mod fmt_lexer { }) .collect(); let newlines: Vec<_> = tokens - .iter() - .filter(|(token, _)| { - matches!(token, FmtToken::Trivia(trivia) if trivia.kind() == TriviaKind::Newline) - }) - .map(|(_, span)| span.to_slice(input)) - .collect(); + .iter() + .filter(|(token, _)| { + matches!(token, FmtToken::Trivia(trivia) if trivia.kind() == TriviaKind::Newline) + }) + .map(|(_, span)| span.to_slice(input)) + .collect(); assert_eq!( line_endings, @@ -1047,7 +1108,7 @@ mod fmt_lexer { #[test] fn numeric_literals_preserve_underscores() { - let input = "1_234_567_89 0xDEAD_BEEF 0b__1010_0101_ 0b1010_0101 _1_234__567___890____000_____ 0x_DEAD_BEEF__BEEF_DEAD_"; + let input = "1_234_567_89 0xDEAD_BEEF 0b1010_0101_ 0b1010_0101 1_234__567___890____000_____ 0xDEAD_BEEF__BEEF_DEAD_"; let (tokens, errors) = lex_lossless(input); assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); @@ -1066,15 +1127,13 @@ mod fmt_lexer { FmtToken::Token(Token::HexLiteral(Hexadecimal::from_str_unchecked( "DEAD_BEEF" ))), - FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked( - "__1010_0101_" - ))), + FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked("1010_0101_"))), FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked("1010_0101"))), FmtToken::Token(Token::DecLiteral(Decimal::from_str_unchecked( - "_1_234__567___890____000_____" + "1_234__567___890____000_____" ))), FmtToken::Token(Token::HexLiteral(Hexadecimal::from_str_unchecked( - "_DEAD_BEEF__BEEF_DEAD_" + "DEAD_BEEF__BEEF_DEAD_" ))), ] ); diff --git a/src/parse.rs b/src/parse.rs index e0ff9daa..547af341 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -1973,10 +1973,8 @@ impl ChumskyParse for AliasedType { .then_ignore(parse_token_with_recovery(Token::Semi)) .then(num.clone()) .map(|(ty, size)| { - #[cfg(feature = "fmt")] - let digits = size.strip_digit_separators(); - #[cfg(not(feature = "fmt"))] - let digits = std::borrow::Cow::from(size.as_inner()); + let digits = + crate::str::underscore_parsing::strip_digit_separators(size.as_inner()); AliasedType::array(ty, usize::from_str(digits.as_ref()).unwrap_or_default()) }), @@ -1995,10 +1993,9 @@ impl ChumskyParse for AliasedType { .ignore_then(delimited_with_recovery( ty.then_ignore(parse_token_with_recovery(Token::Comma)) .then(num.clone().validate(|num, e, emit| -> NonZeroPow2Usize { - #[cfg(feature = "fmt")] - let digits = num.strip_digit_separators(); - #[cfg(not(feature = "fmt"))] - let digits = std::borrow::Cow::from(num.as_inner()); + let digits = crate::str::underscore_parsing::strip_digit_separators( + num.as_inner(), + ); match NonZeroPow2Usize::from_str(digits.as_ref()) { Ok(number) => number, @@ -2381,10 +2378,8 @@ impl ChumskyParse for CallName { .then(select! { Token::DecLiteral(s) => s }.labelled("list size")) .then_ignore(generics_close.clone()) .validate(|(func, bound_str), e, emit| { - #[cfg(feature = "fmt")] - let digits = bound_str.strip_digit_separators(); - #[cfg(not(feature = "fmt"))] - let digits = std::borrow::Cow::from(bound_str.as_inner()); + let digits = + crate::str::underscore_parsing::strip_digit_separators(bound_str.as_inner()); let bound = match digits.parse::() { Ok(num) => match NonZeroPow2Usize::new(num) { @@ -2415,10 +2410,8 @@ impl ChumskyParse for CallName { .then(select! { Token::DecLiteral(s) => s }.labelled("array size")) .then_ignore(generics_close.clone()) .validate(|(func, size_str), e, emit| { - #[cfg(feature = "fmt")] - let digits = size_str.strip_digit_separators(); - #[cfg(not(feature = "fmt"))] - let digits = std::borrow::Cow::from(size_str.as_inner()); + let digits = + crate::str::underscore_parsing::strip_digit_separators(size_str.as_inner()); let size = match digits.parse::() { Ok(0) => { diff --git a/src/str.rs b/src/str.rs index 2b9b1505..1fb95cc6 100644 --- a/src/str.rs +++ b/src/str.rs @@ -299,38 +299,18 @@ pub struct Hexadecimal(Arc); wrapped_string!(Hexadecimal, "hexadecimal string"); -#[cfg(feature = "fmt")] -mod underscore_parsing { - use super::*; +pub(crate) mod underscore_parsing { use std::borrow::Cow; /// Remove digit separators from a numeric literal while avoiding an allocation /// when the literal contains no separators. - fn strip_digit_separators(input: &str) -> Cow<'_, str> { + pub(crate) fn strip_digit_separators(input: &str) -> Cow<'_, str> { if input.contains('_') { Cow::Owned(input.replace('_', "")) } else { Cow::Borrowed(input) } } - - impl Decimal { - pub fn strip_digit_separators(&self) -> Cow<'_, str> { - strip_digit_separators(self.0.as_ref()) - } - } - - impl Binary { - pub fn strip_digit_separators(&self) -> Cow<'_, str> { - strip_digit_separators(self.0.as_ref()) - } - } - - impl Hexadecimal { - pub fn strip_digit_separators(&self) -> Cow<'_, str> { - strip_digit_separators(self.0.as_ref()) - } - } } #[cfg(feature = "arbitrary")] diff --git a/src/value.rs b/src/value.rs index 825b452d..2dd1ca22 100644 --- a/src/value.rs +++ b/src/value.rs @@ -122,10 +122,7 @@ impl UIntValue { /// Create an integer from a `decimal` string and type. pub fn parse_decimal(decimal: &Decimal, ty: UIntType) -> Result { - #[cfg(feature = "fmt")] - let digits = decimal.strip_digit_separators(); - #[cfg(not(feature = "fmt"))] - let digits = std::borrow::Cow::from(decimal.as_inner()); + let digits = crate::str::underscore_parsing::strip_digit_separators(decimal.as_inner()); let s = digits.as_ref(); match ty { @@ -143,10 +140,7 @@ impl UIntValue { /// Create an integer from a `binary` string and type. pub fn parse_binary(binary: &Binary, ty: UIntType) -> Result { - #[cfg(feature = "fmt")] - let digits = binary.strip_digit_separators(); - #[cfg(not(feature = "fmt"))] - let digits = std::borrow::Cow::from(binary.as_inner()); + let digits = crate::str::underscore_parsing::strip_digit_separators(binary.as_inner()); let s = digits.as_ref(); let bit_len = Pow2Usize::new(s.len()).ok_or(Error::BitStringPow2 { len: s.len() })?; @@ -666,10 +660,7 @@ impl Value { _ => return Err(Error::ExpressionUnexpectedType { ty: ty.clone() }), }; - #[cfg(feature = "fmt")] - let digits = hexadecimal.strip_digit_separators(); - #[cfg(not(feature = "fmt"))] - let digits = std::borrow::Cow::from(hexadecimal.as_inner()); + let digits = crate::str::underscore_parsing::strip_digit_separators(hexadecimal.as_inner()); let s = digits.as_ref(); if s.len() % 2 != 0 || s.len() != expected_byte_len * 2 {