From 0084ccc487e071606a16a76c9ff5635eff97a1a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 25 Aug 2026 11:33:44 +0200 Subject: [PATCH 1/8] Lex tokens into a flat arena To avoid unnecessary allocations during lexing. --- compiler/rustc_ast/src/lib.rs | 1 + compiler/rustc_ast/src/tokenarena.rs | 136 +++++++++++++++++++ compiler/rustc_parse/src/lexer/mod.rs | 8 +- compiler/rustc_parse/src/lexer/tokentrees.rs | 40 +++--- 4 files changed, 164 insertions(+), 21 deletions(-) create mode 100644 compiler/rustc_ast/src/tokenarena.rs diff --git a/compiler/rustc_ast/src/lib.rs b/compiler/rustc_ast/src/lib.rs index 3b01eb6eefa7d..6613377871ef0 100644 --- a/compiler/rustc_ast/src/lib.rs +++ b/compiler/rustc_ast/src/lib.rs @@ -31,6 +31,7 @@ pub mod format; pub mod mut_visit; pub mod node_id; pub mod token; +pub mod tokenarena; pub mod tokenstream; pub mod visit; diff --git a/compiler/rustc_ast/src/tokenarena.rs b/compiler/rustc_ast/src/tokenarena.rs new file mode 100644 index 0000000000000..536cfc9ca6443 --- /dev/null +++ b/compiler/rustc_ast/src/tokenarena.rs @@ -0,0 +1,136 @@ +use rustc_macros::{Decodable, Encodable, StableHash}; + +use crate::token::{Delimiter, Token}; +use crate::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; + +/// Part of a `TokenArena`. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] +pub enum ArenaTokenTree { + /// A single token. Should never be `OpenDelim` or `CloseDelim`, because + /// delimiters are implicitly represented by `Delimited`. + Token(Token, Spacing), + /// A delimited sequence of token trees. + Delimited(DelimitedBounds, DelimitedData), +} + +#[derive(Debug, Default, PartialEq, Eq, Hash, Encodable, Decodable)] +pub struct TokenArena { + tokens: Vec, +} + +impl TokenArena { + pub fn new(tokens: Vec) -> Self { + Self { tokens } + } + + pub fn push(&mut self, token: ArenaTokenTree) { + self.tokens.push(token); + } + + pub fn start_delimited(&mut self) -> OpenDelimited { + let index = self.length(); + self.tokens.push(ArenaTokenTree::Delimited( + DelimitedBounds { start: index as u32, length: 0 }, + DelimitedData { + span: DelimSpan { open: Default::default(), close: Default::default() }, + spacing: DelimSpacing { open: Spacing::Alone, close: Spacing::Alone }, + delimiter: Delimiter::Parenthesis, + }, + )); + OpenDelimited { start: index } + } + + pub fn finish_delimited(&mut self, open: OpenDelimited, delimited_data: DelimitedData) { + let length = self.length(); + match &mut self.tokens[open.start] { + ArenaTokenTree::Token(_, _) => unreachable!("Called finish_delimited on a token"), + ArenaTokenTree::Delimited(bounds, data) => { + let len = length.saturating_sub(open.start); + bounds.length = len as u32; + *data = delimited_data; + } + } + } + + pub fn get_item_at(&self, index: usize) -> Option<&ArenaTokenTree> { + self.tokens.get(index) + } + + pub fn length(&self) -> usize { + self.tokens.len() + } + + pub fn from_stream(stream: &TokenStream) -> Self { + let mut arena = TokenArena { tokens: Vec::with_capacity(stream.len()) }; + arena.fill(stream); + arena + } + + fn fill(&mut self, stream: &TokenStream) { + for item in stream.iter() { + match item { + TokenTree::Token(token, spacing) => { + self.tokens.push(ArenaTokenTree::Token(*token, *spacing)); + } + TokenTree::Delimited(span, spacing, delimiter, stream) => { + let start = self.start_delimited(); + self.fill(stream); + self.finish_delimited( + start, + DelimitedData { span: *span, spacing: *spacing, delimiter: *delimiter }, + ); + } + } + } + } + + pub fn to_token_stream(&self) -> TokenStream { + fn to_token_stream(arena: &TokenArena, start: usize, length: usize) -> TokenStream { + let mut tokens = Vec::new(); + let mut index = start; + let end = start + length; + while index < end { + match &arena.tokens[index] { + ArenaTokenTree::Token(a, b) => { + tokens.push(TokenTree::Token(*a, *b)); + index += 1; + } + ArenaTokenTree::Delimited(bounds, data) => { + let tokenstream = to_token_stream( + arena, + (bounds.start + 1) as usize, + (bounds.length as usize).saturating_sub(1), + ); + tokens.push(TokenTree::Delimited( + data.span, + data.spacing, + data.delimiter, + tokenstream, + )); + index += bounds.length as usize; + } + } + } + + TokenStream::new(tokens) + } + to_token_stream(&self, 0, self.tokens.len()) + } +} + +pub struct OpenDelimited { + start: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] +pub struct DelimitedBounds { + pub start: u32, + pub length: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] +pub struct DelimitedData { + pub span: DelimSpan, + pub spacing: DelimSpacing, + pub delimiter: Delimiter, +} diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index 47366c210243d..fcb56ce65d3bf 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -1,6 +1,7 @@ use diagnostics::make_errors_for_mismatched_closing_delims; use rustc_ast::ast::{self, AttrStyle}; use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Token, TokenKind}; +use rustc_ast::tokenarena::TokenArena; use rustc_ast::tokenstream::TokenStream; use rustc_ast::util::unicode::{TEXT_FLOW_CONTROL_CHARS, contains_text_flow_control_chars}; use rustc_errors::codes::*; @@ -97,15 +98,16 @@ pub(crate) fn lex_token_trees<'psess, 'src>( token: Token::dummy(), diag_info: TokenTreeDiagInfo::default(), }; - let res = lexer.lex_token_trees(/* is_delimited */ false); + let mut arena = TokenArena::new(Vec::new()); + let res = lexer.lex_token_trees(&mut arena, /* is_delimited */ false); let mut unmatched_closing_delims: Vec<_> = make_errors_for_mismatched_closing_delims(&lexer.diag_info.unmatched_delims, psess); match res { - Ok((_open_spacing, stream)) => { + Ok(_) => { if unmatched_closing_delims.is_empty() { - Ok(stream) + Ok(arena.to_token_stream()) } else { // Return error if there are unmatched delimiters or unclosed delimiters. Err(unmatched_closing_delims) diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index 3455947471503..d6c4e1a1fc207 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -1,5 +1,6 @@ use rustc_ast::token::{self, Delimiter, Token}; -use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; +use rustc_ast::tokenarena::{ArenaTokenTree, DelimitedData, TokenArena}; +use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing}; use rustc_ast_pretty::pprust::token_to_string; use rustc_errors::Diag; @@ -13,48 +14,47 @@ impl<'psess, 'src> Lexer<'psess, 'src> { // opening delimiter. pub(super) fn lex_token_trees( &mut self, + arena: &mut TokenArena, is_delimited: bool, - ) -> Result<(Spacing, TokenStream), Diag<'psess>> { + ) -> Result> { // Move past the opening delimiter. let open_spacing = self.bump_minimal(); - let mut buf = Vec::new(); loop { if let Some(delim) = self.token.kind.open_delim() { // Invisible delimiters cannot occur here because `TokenTreesReader` parses // code directly from strings, with no macro expansion involved. debug_assert!(!matches!(delim, Delimiter::Invisible(_))); - buf.push(match self.lex_token_tree_open_delim(delim) { - Ok(val) => val, + let delimited = arena.start_delimited(); + let value = match self.lex_token_tree_open_delim(arena, delim) { + Ok(value) => value, Err(errs) => return Err(errs), - }) + }; + arena.finish_delimited(delimited, value); } else if let Some(delim) = self.token.kind.close_delim() { // Invisible delimiters cannot occur here because `TokenTreesReader` parses // code directly from strings, with no macro expansion involved. debug_assert!(!matches!(delim, Delimiter::Invisible(_))); return if is_delimited { - Ok((open_spacing, TokenStream::new(buf))) + Ok(open_spacing) } else { Err(self.close_delim_err(delim)) }; } else if self.token.kind == token::Eof { - return if is_delimited { - Err(self.eof_err()) - } else { - Ok((open_spacing, TokenStream::new(buf))) - }; + return if is_delimited { Err(self.eof_err()) } else { Ok(open_spacing) }; } else { // Get the next normal token. let (this_tok, this_spacing) = self.bump(); - buf.push(TokenTree::Token(this_tok, this_spacing)); + arena.push(ArenaTokenTree::Token(this_tok, this_spacing)); } } } fn lex_token_tree_open_delim( &mut self, + arena: &mut TokenArena, open_delim: Delimiter, - ) -> Result> { + ) -> Result> { // The span for beginning of the delimited section. let pre_span = self.token.span; @@ -63,7 +63,11 @@ impl<'psess, 'src> Lexer<'psess, 'src> { // Lex the token trees within the delimiters. // We stop at any delimiter so we can try to recover if the user // uses an incorrect delimiter. - let (open_spacing, tts) = self.lex_token_trees(/* is_delimited */ true)?; + + // We remember where we were in the arena, so that we can check how many trees were parsed + let index = arena.length(); + let open_spacing = self.lex_token_trees(arena, /* is_delimited */ true)?; + let lexed_trees = arena.length() - index; // Expand to cover the entire delimited token tree. let delim_span = DelimSpan::from_pair(pre_span, self.token.span); @@ -75,7 +79,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { self.diag_info.open_delimiters.pop().unwrap(); let close_delimiter_span = self.token.span; - if tts.is_empty() && close_delim == Delimiter::Brace { + if lexed_trees == 0 && close_delim == Delimiter::Brace { let empty_block_span = pre_span.to(close_delimiter_span); if !sm.is_multiline(empty_block_span) { // Only track if the block is in the form of `{}`, otherwise it is @@ -93,7 +97,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { // A brace-delimited block whose first token is `&&`/`||` usually means // the user meant to continue an if-let chain, e.g. `if let P = e { && cond {`. if Delimiter::Brace == open_delim - && let Some(TokenTree::Token(tok, _)) = tts.iter().next() + && let Some(ArenaTokenTree::Token(tok, _)) = arena.get_item_at(index) && matches!(tok.kind, token::AndAnd | token::OrOr) { self.diag_info.if_let_chain_hint_spans.push(tok.span); @@ -159,7 +163,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { let spacing = DelimSpacing::new(open_spacing, close_spacing); - Ok(TokenTree::Delimited(delim_span, spacing, open_delim, tts)) + Ok(DelimitedData { span: delim_span, spacing, delimiter: open_delim }) } // Move on to the next token, returning the current token and its spacing. From c7b22a98ab97388aac87ec48d70af652a9027405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 25 Aug 2026 13:43:50 +0200 Subject: [PATCH 2/8] Pass `TokenArena` to `Parser` --- compiler/rustc_ast/src/tokenarena.rs | 4 ++++ compiler/rustc_ast/src/tokenstream.rs | 4 +++- .../rustc_attr_parsing/src/attributes/cfg.rs | 3 ++- compiler/rustc_attr_parsing/src/parser.rs | 7 ++++--- .../rustc_attr_parsing/src/validate_attr.rs | 6 ++++-- compiler/rustc_builtin_macros/src/cfg_eval.rs | 4 +++- compiler/rustc_expand/src/base.rs | 3 ++- compiler/rustc_expand/src/expand.rs | 5 +++++ compiler/rustc_expand/src/mbe/macro_rules.rs | 8 +++++--- compiler/rustc_expand/src/proc_macro.rs | 7 ++++++- .../rustc_expand/src/proc_macro_server.rs | 8 +++++++- compiler/rustc_parse/src/lexer/mod.rs | 5 ++--- compiler/rustc_parse/src/lib.rs | 19 ++++++++++--------- compiler/rustc_parse/src/parser/mod.rs | 5 +++-- 14 files changed, 60 insertions(+), 28 deletions(-) diff --git a/compiler/rustc_ast/src/tokenarena.rs b/compiler/rustc_ast/src/tokenarena.rs index 536cfc9ca6443..f4904b7ec0089 100644 --- a/compiler/rustc_ast/src/tokenarena.rs +++ b/compiler/rustc_ast/src/tokenarena.rs @@ -60,6 +60,10 @@ impl TokenArena { self.tokens.len() } + pub fn is_empty(&self) -> bool { + self.tokens.is_empty() + } + pub fn from_stream(stream: &TokenStream) -> Self { let mut arena = TokenArena { tokens: Vec::with_capacity(stream.len()) }; arena.fill(stream); diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index e860ef61a5332..9c3c76a1825fd 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -20,6 +20,7 @@ use thin_vec::ThinVec; use crate::ast::AttrStyle; use crate::ast_traits::HasTokens; use crate::token::{self, Delimiter, Token, TokenKind}; +use crate::tokenarena::TokenArena; use crate::{AttrVec, Attribute}; #[cfg(test)] @@ -949,7 +950,8 @@ pub struct TokenCursor { impl TokenCursor { #[inline] - pub fn new(stream: TokenStream) -> Self { + pub fn new(arena: TokenArena) -> Self { + let stream = arena.to_token_stream(); TokenCursor { curr: TokenTreeCursor::new(stream), stack: vec![] } } diff --git a/compiler/rustc_attr_parsing/src/attributes/cfg.rs b/compiler/rustc_attr_parsing/src/attributes/cfg.rs index d7f2243faaab9..8af43d059bfe6 100644 --- a/compiler/rustc_attr_parsing/src/attributes/cfg.rs +++ b/compiler/rustc_attr_parsing/src/attributes/cfg.rs @@ -1,6 +1,7 @@ use std::convert::identity; use rustc_ast::token::Delimiter; +use rustc_ast::tokenarena::TokenArena; use rustc_ast::tokenstream::{DelimSpan, WithTokens}; use rustc_ast::{AttrItem, Attribute, LitKind, ast, token}; use rustc_attr_ir::target::Target; @@ -312,7 +313,7 @@ pub fn parse_cfg_attr( match &cfg_attr.get_normal_item().args { ast::AttrArgs::Delimited(ast::DelimArgs { dspan, delim, tokens }) if !tokens.is_empty() => { check_cfg_attr_bad_delim(&sess.psess, *dspan, *delim); - match parse_in(&sess.psess, tokens.clone(), "`cfg_attr` input", |p| { + match parse_in(&sess.psess, TokenArena::from_stream(tokens), "`cfg_attr` input", |p| { parse_cfg_attr_internal(p, sess, features, lint_node_id, cfg_attr) }) { Ok(r) => return Some(r), diff --git a/compiler/rustc_attr_parsing/src/parser.rs b/compiler/rustc_attr_parsing/src/parser.rs index 8efe5bf4f1f90..d57a3ddf316e8 100644 --- a/compiler/rustc_attr_parsing/src/parser.rs +++ b/compiler/rustc_attr_parsing/src/parser.rs @@ -14,6 +14,7 @@ use std::fmt::{Debug, Display}; use std::sync::atomic::{AtomicBool, Ordering}; use rustc_ast::token::{self, Delimiter, MetaVarKind}; +use rustc_ast::tokenarena::TokenArena; use rustc_ast::tokenstream::TokenStream; use rustc_ast::{ AttrArgs, Expr, ExprKind, LitKind, MetaItemLit, Path, PathSegment, StmtKind, UnOp, @@ -721,13 +722,13 @@ impl<'a, 'sess> MetaItemListParserContext<'a, 'sess> { } fn parse( - tokens: TokenStream, + arena: TokenArena, psess: &'sess ParseSess, span: Span, should_emit: ShouldEmit, allow_expr_metavar: AllowExprMetavar, ) -> PResult<'sess, MetaItemListParser> { - let mut parser = Parser::new(psess, tokens, None); + let mut parser = Parser::new(psess, arena, None); if let ShouldEmit::ErrorsAndLints { recovery } = should_emit { parser = parser.recovery(recovery); } @@ -764,7 +765,7 @@ impl MetaItemListParser { allow_expr_metavar: AllowExprMetavar, ) -> Result> { MetaItemListParserContext::parse( - tokens.clone(), + TokenArena::from_stream(tokens), psess, span, should_emit, diff --git a/compiler/rustc_attr_parsing/src/validate_attr.rs b/compiler/rustc_attr_parsing/src/validate_attr.rs index f225458ebc0e6..48e5969cce2bc 100644 --- a/compiler/rustc_attr_parsing/src/validate_attr.rs +++ b/compiler/rustc_attr_parsing/src/validate_attr.rs @@ -4,6 +4,7 @@ use std::convert::identity; use std::slice; use rustc_ast::token::Delimiter; +use rustc_ast::tokenarena::TokenArena; use rustc_ast::tokenstream::DelimSpan; use rustc_ast::{ self as ast, AttrArgs, AttrKind, Attribute, DelimArgs, MetaItem, MetaItemInner, MetaItemKind, @@ -75,8 +76,9 @@ pub fn parse_meta<'a>(psess: &'a ParseSess, attr: &Attribute) -> PResult<'a, Met AttrArgs::Empty => MetaItemKind::Word, AttrArgs::Delimited(DelimArgs { dspan, delim, tokens }) => { check_meta_bad_delim(psess, *dspan, *delim); - let nmis = - parse_in(psess, tokens.clone(), "meta list", |p| p.parse_meta_seq_top())?; + let nmis = parse_in(psess, TokenArena::from_stream(tokens), "meta list", |p| { + p.parse_meta_seq_top() + })?; MetaItemKind::List(nmis) } AttrArgs::Eq { expr, .. } => { diff --git a/compiler/rustc_builtin_macros/src/cfg_eval.rs b/compiler/rustc_builtin_macros/src/cfg_eval.rs index 34ddd9427cdde..0cb583c77ff52 100644 --- a/compiler/rustc_builtin_macros/src/cfg_eval.rs +++ b/compiler/rustc_builtin_macros/src/cfg_eval.rs @@ -2,6 +2,7 @@ use core::ops::ControlFlow; use rustc_ast as ast; use rustc_ast::mut_visit::MutVisitor; +use rustc_ast::tokenarena::TokenArena; use rustc_ast::visit::{AssocCtxt, Visitor}; use rustc_ast::{Attribute, HasTokens, NodeId, mut_visit, visit}; use rustc_errors::PResult; @@ -105,7 +106,8 @@ impl CfgEval<'_> { // // After that we have our re-parsed `AttrTokenStream`, recursively configuring // our attribute target will correctly configure the tokens as well. - let mut parser = Parser::new(&self.0.sess.psess, orig_tokens, None); + let mut parser = + Parser::new(&self.0.sess.psess, TokenArena::from_stream(&orig_tokens), None); parser.capture_cfg = true; let res: PResult<'_, Option> = try { match &annotatable { diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index fda75319b087b..d48fb3bdacb06 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -6,6 +6,7 @@ use std::rc::Rc; use std::sync::Arc; use rustc_ast::attr::MarkedAttrs; +use rustc_ast::tokenarena::TokenArena; use rustc_ast::tokenstream::TokenStream; use rustc_ast::visit::{AssocCtxt, Visitor}; use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind, Safety}; @@ -1258,7 +1259,7 @@ impl<'a> ExtCtxt<'a> { expand::MacroExpander::new(self, true) } pub fn new_parser_from_tts(&self, stream: TokenStream) -> Parser<'a> { - Parser::new(&self.sess.psess, stream, MACRO_ARGUMENTS) + Parser::new(&self.sess.psess, TokenArena::from_stream(&stream), MACRO_ARGUMENTS) } pub fn source_map(&self) -> &'a SourceMap { self.sess.psess.source_map() diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 3a51b758a427b..0546dc9c37bce 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -777,6 +777,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { // we are invoking it on an out-of-line module or crate. Annotatable::Crate(krate) => { rustc_parse::fake_token_stream_for_crate(&self.cx.sess.psess, krate) + .to_token_stream() } Annotatable::Item(item_inner) if matches!(attr.style, AttrStyle::Inner) @@ -795,6 +796,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { item_inner, Some(&attr), ) + .to_token_stream() } Annotatable::Item(item_inner) if item_inner.tokens.is_none() => { rustc_parse::fake_token_stream_for_item( @@ -802,6 +804,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { item_inner, None, ) + .to_token_stream() } // When a function has EII implementations attached (via `eii_impl`), // use fake tokens so the pretty-printer re-emits the EII attribute @@ -818,12 +821,14 @@ impl<'a, 'b> MacroExpander<'a, 'b> { item_inner, None, ) + .to_token_stream() } Annotatable::ForeignItem(item_inner) if item_inner.tokens.is_none() => { rustc_parse::fake_token_stream_for_foreign_item( &self.cx.sess.psess, item_inner, ) + .to_token_stream() } _ => item.to_tokens(), }; diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index b268b8b767327..d8018ec15d392 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -7,6 +7,7 @@ use ast::token::IdentIsRaw; use rustc_ast::token::NtPatKind::*; use rustc_ast::token::TokenKind::*; use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind}; +use rustc_ast::tokenarena::TokenArena; use rustc_ast::tokenstream::{self, DelimSpan, TokenStream}; use rustc_ast::{self as ast, DUMMY_NODE_ID, NodeId, Safety}; use rustc_ast_pretty::pprust; @@ -132,7 +133,7 @@ impl<'a, 'b> ParserAnyMacro<'a, 'b> { matched_rule_bindings: &'b [MatcherLoc], ) -> Self { Self { - parser: Parser::new(&cx.sess.psess, tts, None), + parser: Parser::new(&cx.sess.psess, TokenArena::from_stream(&tts), None), // Pass along the original expansion site and the name of the macro // so we can print a useful error message if the parse of the expanded @@ -796,7 +797,7 @@ pub fn compile_declarative_macro( let macro_rules = macro_def.macro_rules; let exp_sep = if macro_rules { exp!(Semi) } else { exp!(Comma) }; - let body = macro_def.body.tokens.clone(); + let body = TokenArena::from_stream(¯o_def.body.tokens); let mut p = Parser::new(&sess.psess, body, rustc_parse::MACRO_ARGUMENTS); // Don't abort iteration early, so that multiple errors can be reported. We only abort early on @@ -1869,5 +1870,6 @@ pub(super) fn parser_from_cx( recovery: Recovery, ) -> Parser<'_> { tts.desugar_doc_comments(); - Parser::new(psess, tts, rustc_parse::MACRO_ARGUMENTS).recovery(recovery) + Parser::new(psess, TokenArena::from_stream(&tts), rustc_parse::MACRO_ARGUMENTS) + .recovery(recovery) } diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 105d2d796aa80..0cb0abf7593e6 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -1,4 +1,5 @@ use rustc_ast as ast; +use rustc_ast::tokenarena::TokenArena; use rustc_ast::tokenstream::TokenStream; use rustc_data_structures::profiling::TimingGuard; use rustc_errors::ErrorGuaranteed; @@ -131,7 +132,11 @@ impl MultiItemModifier for DeriveProcMacro { }; let error_count_before = ecx.dcx().err_count(); - let mut parser = Parser::new(&ecx.sess.psess, output, Some("proc-macro derive")); + let mut parser = Parser::new( + &ecx.sess.psess, + TokenArena::from_stream(&output), + Some("proc-macro derive"), + ); let mut items = vec![]; loop { diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 7b52a5600ae2a..6c34ce883165d 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -2,6 +2,7 @@ use std::ops::{Bound, Range}; use rustc_ast as ast; use rustc_ast::token as tk; +use rustc_ast::tokenarena::TokenArena; use rustc_ast::tokenstream::{self, DelimSpacing, Spacing, TokenStream}; use rustc_ast::util::literal::escape_byte_str_symbol; use rustc_ast_pretty::pprust; @@ -580,6 +581,7 @@ impl server::Server for Rustc<'_, '_> { src.to_string(), Some(self.call_site), ) + .map(|arena| arena.to_token_stream()) }) .map_err(|_| String::from("failed to parse to tokenstream"))? .map_err(cancel_diags_into_string) @@ -592,7 +594,11 @@ impl server::Server for Rustc<'_, '_> { fn ts_expand_expr(&mut self, stream: &Self::TokenStream) -> Result { // Parse the expression from our tokenstream. let expr = try { - let mut p = Parser::new(self.psess(), stream.clone(), Some("proc_macro expand expr")); + let mut p = Parser::new( + self.psess(), + TokenArena::from_stream(stream), + Some("proc_macro expand expr"), + ); let expr = p.parse_expr()?; if p.token != tk::Eof { p.unexpected()?; diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index fcb56ce65d3bf..2738440ebafff 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -2,7 +2,6 @@ use diagnostics::make_errors_for_mismatched_closing_delims; use rustc_ast::ast::{self, AttrStyle}; use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Token, TokenKind}; use rustc_ast::tokenarena::TokenArena; -use rustc_ast::tokenstream::TokenStream; use rustc_ast::util::unicode::{TEXT_FLOW_CONTROL_CHARS, contains_text_flow_control_chars}; use rustc_errors::codes::*; use rustc_errors::{Applicability, Diag, DiagCtxtHandle, Diagnostic, StashKey}; @@ -69,7 +68,7 @@ pub(crate) fn lex_token_trees<'psess, 'src>( mut start_pos: BytePos, override_span: Option, strip_tokens: StripTokens, -) -> Result>> { +) -> Result>> { match strip_tokens { StripTokens::Shebang | StripTokens::ShebangAndFrontmatter => { if let Some(shebang_len) = rustc_lexer::strip_shebang(src) { @@ -107,7 +106,7 @@ pub(crate) fn lex_token_trees<'psess, 'src>( match res { Ok(_) => { if unmatched_closing_delims.is_empty() { - Ok(arena.to_token_stream()) + Ok(arena) } else { // Return error if there are unmatched delimiters or unclosed delimiters. Err(unmatched_closing_delims) diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 539c15f18a9a4..1f0416f8dc102 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -29,6 +29,7 @@ pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments"); #[macro_use] pub mod parser; use parser::Parser; +use rustc_ast::tokenarena::TokenArena; use crate::lexer::StripTokens; @@ -245,7 +246,7 @@ pub fn source_str_to_stream( name: FileName, source: String, override_span: Option, -) -> Result>> { +) -> Result>> { let source_file = psess.source_map().new_source_file(name, source); // FIXME(frontmatter): Consider stripping frontmatter in a future edition. We can't strip them // in the current edition since that would be breaking. @@ -262,7 +263,7 @@ fn source_file_to_stream<'psess>( source_file: Arc, override_span: Option, strip_tokens: StripTokens, -) -> Result>> { +) -> Result>> { let src = source_file.src.as_ref().unwrap_or_else(|| { psess.dcx().bug(format!( "cannot lex `source_file` without source: {}", @@ -276,11 +277,11 @@ fn source_file_to_stream<'psess>( /// Runs the given subparser `f` on the tokens of the given `attr`'s item. pub fn parse_in<'a, T>( psess: &'a ParseSess, - tts: TokenStream, + arena: TokenArena, name: &'static str, mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, ) -> PResult<'a, T> { - let mut parser = Parser::new(psess, tts, Some(name)); + let mut parser = Parser::new(psess, arena, Some(name)); let result = f(&mut parser)?; if parser.token != token::Eof { parser.unexpected()?; @@ -292,9 +293,9 @@ pub fn fake_token_stream_for_item( psess: &ParseSess, item: &ast::Item, attr_to_exclude: Option<&ast::Attribute>, -) -> TokenStream { +) -> TokenArena { if let Some(tokens) = fake_token_stream_for_file_mod(psess, item, attr_to_exclude) { - return tokens; + return TokenArena::from_stream(&tokens); } let source = pprust::item_to_string(item); @@ -350,7 +351,7 @@ fn lex_token_trees_for_span( ) -> Option> { let src = psess.source_map().span_to_snippet(span).ok()?; let stream = match lexer::lex_token_trees(psess, &src, span.lo(), None, StripTokens::Nothing) { - Ok(stream) => stream, + Ok(arena) => arena.to_token_stream(), Err(errs) => { errs.into_iter().for_each(|err| err.cancel()); return None; @@ -362,13 +363,13 @@ fn lex_token_trees_for_span( pub fn fake_token_stream_for_foreign_item( psess: &ParseSess, item: &ast::ForeignItem, -) -> TokenStream { +) -> TokenArena { let source = pprust::foreign_item_to_string(item); let filename = FileName::macro_expansion_source_code(&source); unwrap_or_emit_fatal(source_str_to_stream(psess, filename, source, Some(item.span))) } -pub fn fake_token_stream_for_crate(psess: &ParseSess, krate: &ast::Crate) -> TokenStream { +pub fn fake_token_stream_for_crate(psess: &ParseSess, krate: &ast::Crate) -> TokenArena { let source = pprust::crate_to_string_for_macros(krate); let filename = FileName::macro_expansion_source_code(&source); unwrap_or_emit_fatal(source_str_to_stream( diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 80c1eeb4ef041..e6d4d177fbb9c 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -29,6 +29,7 @@ pub use path::PathStyle; use rustc_ast::token::{ self, IdentIsRaw, InvisibleOrigin, MetaVarKind, NtExprKind, NtPatKind, Token, TokenKind, }; +use rustc_ast::tokenarena::TokenArena; use rustc_ast::tokenstream::{ ParserRange, ParserReplacement, Spacing, TokenCursor, TokenStream, TokenTree, WithTokens, }; @@ -342,12 +343,12 @@ pub fn token_descr(token: &Token) -> String { impl<'a> Parser<'a> { pub fn new( psess: &'a ParseSess, - stream: TokenStream, + arena: TokenArena, subparser_name: Option<&'static str>, ) -> Self { let mut parser = Parser { psess, - token_cursor: TokenCursor::new(stream), + token_cursor: TokenCursor::new(arena), subparser_name, capture_state: CaptureState { capturing: Capturing::No, From 8500a24628cfa2b8cbffde4c33f15978a69bb1df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 26 Aug 2026 12:46:56 +0200 Subject: [PATCH 3/8] Explicitly store delimited sequence end markers in the token arena To make it easier to track delimited sequences. --- compiler/rustc_ast/src/tokenarena.rs | 29 +++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_ast/src/tokenarena.rs b/compiler/rustc_ast/src/tokenarena.rs index f4904b7ec0089..62b273bc374f4 100644 --- a/compiler/rustc_ast/src/tokenarena.rs +++ b/compiler/rustc_ast/src/tokenarena.rs @@ -1,3 +1,4 @@ +use rustc_index::static_assert_size; use rustc_macros::{Decodable, Encodable, StableHash}; use crate::token::{Delimiter, Token}; @@ -7,12 +8,15 @@ use crate::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTre #[derive(Debug, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] pub enum ArenaTokenTree { /// A single token. Should never be `OpenDelim` or `CloseDelim`, because - /// delimiters are implicitly represented by `Delimited`. + /// delimiters are implicitly represented by `DelimitedStart`/`DelimitedEnd`. Token(Token, Spacing), /// A delimited sequence of token trees. - Delimited(DelimitedBounds, DelimitedData), + DelimitedStart(DelimitedBounds, DelimitedData), + DelimitedEnd(DelimitedBounds, DelimitedData), } +static_assert_size!(ArenaTokenTree, 36); + #[derive(Debug, Default, PartialEq, Eq, Hash, Encodable, Decodable)] pub struct TokenArena { tokens: Vec, @@ -29,7 +33,7 @@ impl TokenArena { pub fn start_delimited(&mut self) -> OpenDelimited { let index = self.length(); - self.tokens.push(ArenaTokenTree::Delimited( + self.tokens.push(ArenaTokenTree::DelimitedStart( DelimitedBounds { start: index as u32, length: 0 }, DelimitedData { span: DelimSpan { open: Default::default(), close: Default::default() }, @@ -42,9 +46,15 @@ impl TokenArena { pub fn finish_delimited(&mut self, open: OpenDelimited, delimited_data: DelimitedData) { let length = self.length(); + self.tokens.push(ArenaTokenTree::DelimitedEnd( + DelimitedBounds { start: open.start as u32, length: length as u32 }, + delimited_data, + )); match &mut self.tokens[open.start] { - ArenaTokenTree::Token(_, _) => unreachable!("Called finish_delimited on a token"), - ArenaTokenTree::Delimited(bounds, data) => { + tree @ (ArenaTokenTree::Token(..) | ArenaTokenTree::DelimitedEnd(..)) => { + unreachable!("Called finish_delimited on an invalid tree type {tree:?}") + } + ArenaTokenTree::DelimitedStart(bounds, data) => { let len = length.saturating_sub(open.start); bounds.length = len as u32; *data = delimited_data; @@ -99,7 +109,7 @@ impl TokenArena { tokens.push(TokenTree::Token(*a, *b)); index += 1; } - ArenaTokenTree::Delimited(bounds, data) => { + ArenaTokenTree::DelimitedStart(bounds, data) => { let tokenstream = to_token_stream( arena, (bounds.start + 1) as usize, @@ -113,6 +123,9 @@ impl TokenArena { )); index += bounds.length as usize; } + ArenaTokenTree::DelimitedEnd(..) => { + index += 1; + } } } @@ -129,10 +142,12 @@ pub struct OpenDelimited { #[derive(Debug, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] pub struct DelimitedBounds { pub start: u32, + /// The length includes both the start and the end token. + /// So an empty delimited sequence has length 2. pub length: u32, } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] pub struct DelimitedData { pub span: DelimSpan, pub spacing: DelimSpacing, From ef138521ac232e12916fae948ed6c224ad764f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 26 Aug 2026 14:01:36 +0200 Subject: [PATCH 4/8] Reimplement the parser's token cursor to work on top of the arena --- compiler/rustc_ast/src/tokenarena.rs | 123 ++++++++----- compiler/rustc_ast/src/tokenstream.rs | 181 +++++++++---------- compiler/rustc_parse/src/lexer/tokentrees.rs | 2 +- compiler/rustc_parse/src/parser/function.rs | 17 +- compiler/rustc_parse/src/parser/item.rs | 5 +- compiler/rustc_parse/src/parser/mod.rs | 19 +- 6 files changed, 190 insertions(+), 157 deletions(-) diff --git a/compiler/rustc_ast/src/tokenarena.rs b/compiler/rustc_ast/src/tokenarena.rs index 62b273bc374f4..02c0a905ef397 100644 --- a/compiler/rustc_ast/src/tokenarena.rs +++ b/compiler/rustc_ast/src/tokenarena.rs @@ -5,14 +5,29 @@ use crate::token::{Delimiter, Token}; use crate::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; /// Part of a `TokenArena`. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] pub enum ArenaTokenTree { /// A single token. Should never be `OpenDelim` or `CloseDelim`, because /// delimiters are implicitly represented by `DelimitedStart`/`DelimitedEnd`. Token(Token, Spacing), /// A delimited sequence of token trees. DelimitedStart(DelimitedBounds, DelimitedData), - DelimitedEnd(DelimitedBounds, DelimitedData), + // TODO: get rid of this and represent it implicitly + DelimitedEnd, +} + +impl ArenaTokenTree { + /// Convert an arena token tree to the tree-shaped token tree. + pub fn to_token_tree(&self, arena: &TokenArena) -> TokenTree { + match self { + ArenaTokenTree::Token(token, spacing) => TokenTree::Token(*token, *spacing), + ArenaTokenTree::DelimitedStart(bounds, data) => { + let tts = arena.iter_delimited(bounds).map(|tt| tt.to_token_tree(arena)).collect(); + TokenTree::Delimited(data.span, data.spacing, data.delimiter, TokenStream::new(tts)) + } + ArenaTokenTree::DelimitedEnd => unreachable!(), + } + } } static_assert_size!(ArenaTokenTree, 36); @@ -31,6 +46,51 @@ impl TokenArena { self.tokens.push(token); } + /// Iter top-level token trees of a delimited token sequence. + pub fn iter_delimited(&self, bounds: &DelimitedBounds) -> impl Iterator { + let mut index = (bounds.start + 1) as usize; + let end = bounds.index_of_next_token_tree().saturating_sub(1); + std::iter::from_fn(move || { + if index >= end { + return None; + } + let item = self.get_innermost_elem_at(index)?; + match item { + token @ ArenaTokenTree::Token(..) => { + index += 1; + Some(*token) + } + tree @ ArenaTokenTree::DelimitedStart(bounds, _) => { + index = bounds.index_of_next_token_tree(); + Some(*tree) + } + ArenaTokenTree::DelimitedEnd => unreachable!(), + } + }) + } + + pub fn iter_top_level_trees(&self) -> impl Iterator { + let mut index = 0; + let end = self.tokens.len(); + std::iter::from_fn(move || { + if index >= end { + return None; + } + let item = self.get_innermost_elem_at(index)?; + match item { + token @ ArenaTokenTree::Token(..) => { + index += 1; + Some(*token) + } + tree @ ArenaTokenTree::DelimitedStart(bounds, _) => { + index = bounds.index_of_next_token_tree(); + Some(*tree) + } + ArenaTokenTree::DelimitedEnd => unreachable!(), + } + }) + } + pub fn start_delimited(&mut self) -> OpenDelimited { let index = self.length(); self.tokens.push(ArenaTokenTree::DelimitedStart( @@ -45,13 +105,10 @@ impl TokenArena { } pub fn finish_delimited(&mut self, open: OpenDelimited, delimited_data: DelimitedData) { + self.tokens.push(ArenaTokenTree::DelimitedEnd); let length = self.length(); - self.tokens.push(ArenaTokenTree::DelimitedEnd( - DelimitedBounds { start: open.start as u32, length: length as u32 }, - delimited_data, - )); match &mut self.tokens[open.start] { - tree @ (ArenaTokenTree::Token(..) | ArenaTokenTree::DelimitedEnd(..)) => { + tree @ (ArenaTokenTree::Token(..) | ArenaTokenTree::DelimitedEnd) => { unreachable!("Called finish_delimited on an invalid tree type {tree:?}") } ArenaTokenTree::DelimitedStart(bounds, data) => { @@ -62,7 +119,7 @@ impl TokenArena { } } - pub fn get_item_at(&self, index: usize) -> Option<&ArenaTokenTree> { + pub fn get_innermost_elem_at(&self, index: usize) -> Option<&ArenaTokenTree> { self.tokens.get(index) } @@ -99,39 +156,11 @@ impl TokenArena { } pub fn to_token_stream(&self) -> TokenStream { - fn to_token_stream(arena: &TokenArena, start: usize, length: usize) -> TokenStream { - let mut tokens = Vec::new(); - let mut index = start; - let end = start + length; - while index < end { - match &arena.tokens[index] { - ArenaTokenTree::Token(a, b) => { - tokens.push(TokenTree::Token(*a, *b)); - index += 1; - } - ArenaTokenTree::DelimitedStart(bounds, data) => { - let tokenstream = to_token_stream( - arena, - (bounds.start + 1) as usize, - (bounds.length as usize).saturating_sub(1), - ); - tokens.push(TokenTree::Delimited( - data.span, - data.spacing, - data.delimiter, - tokenstream, - )); - index += bounds.length as usize; - } - ArenaTokenTree::DelimitedEnd(..) => { - index += 1; - } - } - } - - TokenStream::new(tokens) + let mut tokens = vec![]; + for tt in self.iter_top_level_trees() { + tokens.push(tt.to_token_tree(self)); } - to_token_stream(&self, 0, self.tokens.len()) + TokenStream::new(tokens) } } @@ -139,7 +168,7 @@ pub struct OpenDelimited { start: usize, } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] pub struct DelimitedBounds { pub start: u32, /// The length includes both the start and the end token. @@ -147,6 +176,18 @@ pub struct DelimitedBounds { pub length: u32, } +impl DelimitedBounds { + /// Return the index of the next token tree that follows this delimited token sequence. + pub fn index_of_next_token_tree(&self) -> usize { + (self.start + self.length) as usize + } + + /// Return the index of the closing delimiter of this token sequence. + pub fn index_of_closing_delimiter(&self) -> usize { + self.index_of_next_token_tree().saturating_sub(1) + } +} + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] pub struct DelimitedData { pub span: DelimSpan, diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 9c3c76a1825fd..35900d5e3b198 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -20,7 +20,7 @@ use thin_vec::ThinVec; use crate::ast::AttrStyle; use crate::ast_traits::HasTokens; use crate::token::{self, Delimiter, Token, TokenKind}; -use crate::tokenarena::TokenArena; +use crate::tokenarena::{ArenaTokenTree, DelimitedBounds, DelimitedData, TokenArena}; use crate::{AttrVec, Attribute}; #[cfg(test)] @@ -881,78 +881,20 @@ impl<'t> Iterator for TokenStreamIter<'t> { } } -#[derive(Clone, Debug)] -struct TokenTreeCursor { - stream: TokenStream, - /// Points to the next token tree (or one past the end of the stream). - next_idx: usize, -} - -impl TokenTreeCursor { - #[inline] - fn new(stream: TokenStream) -> Self { - TokenTreeCursor { stream, next_idx: 0 } - } - - /// Gets the current token tree within this cursor. In a debug build it panics on a cursor that - /// hasn't been bumped; in a release build it will return `None`. - #[inline] - fn curr(&self) -> Option<&TokenTree> { - debug_assert!(self.next_idx > 0); - self.stream.get(self.next_idx - 1) - } - - /// Gets the next token tree without advancing. - #[inline] - fn next(&self) -> Option<&TokenTree> { - self.stream.get(self.next_idx) - } - - /// Gets the token tree `n` ahead. `look_ahead(1)` is equivalent to `next()`. `look_ahead(0)` - /// isn't allowed and will panic. - #[inline] - fn look_ahead(&self, n: usize) -> Option<&TokenTree> { - assert_ne!(n, 0); - self.stream.get(self.next_idx + (n - 1)) - } - - /// Move the cursor to the next token tree. - #[inline] - fn bump(&mut self) { - self.next_idx += 1; - } - - /// For skipping ahead in rare circumstances. - #[inline] - fn bump_to_end(&mut self) { - self.next_idx = self.stream.len(); - } -} - -/// A `TokenStream` cursor that produces `Token`s. It's a bit odd that -/// we (a) lex tokens into a nice tree structure (`TokenStream`), and then (b) -/// use this type to emit them as a linear sequence. But a linear sequence is -/// what the parser expects, for the most part. +/// A `TokenArena` cursor that produces `Token`s. #[derive(Clone, Debug)] pub struct TokenCursor { - // Cursor for the current (innermost) token stream. The `next_idx` within the - // cursor can point to any token tree in the stream (or one past the end). - // The delimiters for this token stream are found in the current token tree - // in `self.stack.last()`; if that is `None` we are in the outermost token - // stream which never has delimiters. - curr: TokenTreeCursor, - - // Token streams surrounding the current one. The `next_idx` within each cursor - // is always greater than zero and always points one past the current - // `TokenTree::Delimited`. - stack: Vec, + pub arena: Arc, + /// Global index into the token arena. + index: usize, + /// The current delimited sequences that we are inside of. + stack: Vec<(DelimitedBounds, DelimitedData)>, } impl TokenCursor { #[inline] pub fn new(arena: TokenArena) -> Self { - let stream = arena.to_token_stream(); - TokenCursor { curr: TokenTreeCursor::new(stream), stack: vec![] } + TokenCursor { arena: Arc::new(arena), index: 0, stack: vec![] } } /// Gets the next token and advances the cursor by one. @@ -961,30 +903,61 @@ impl TokenCursor { } /// An `n` of 1 is the next token tree in the current token stream; won't look outside the - /// current token stream. `look_ahead(0)` isn't allowed and will panic. + /// current delimited sequence. `look_ahead(0)` isn't allowed and will panic. #[inline] - pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { - self.curr.look_ahead(n) + pub fn look_ahead(&self, n: usize) -> Option<&ArenaTokenTree> { + assert_ne!(n, 0); + let mut index = self.index; + for _ in 0..n.saturating_sub(1) { + let elem = self.arena.get_innermost_elem_at(index); + match elem { + Some(ArenaTokenTree::Token(..)) => { + index += 1; + } + Some(ArenaTokenTree::DelimitedStart(bounds, ..)) => { + // Skip the whole delimited sequence + index = bounds.index_of_next_token_tree(); + } + Some(ArenaTokenTree::DelimitedEnd) => { + // We reached the end of the current delimited sequence + return None; + } + None => { + // We reached the end of the arena + return None; + } + } + } + match self.arena.get_innermost_elem_at(index) { + None | Some(ArenaTokenTree::DelimitedEnd) => None, + Some(t) => Some(t), + } } /// Returns the first token tree (if there is one) past the close delimiter of the enclosing /// delimited sequence. Panics if we are not within a delimited sequence. #[inline] - pub fn look_ahead_past_close_delim(&self) -> Option<&TokenTree> { - self.stack.last().unwrap().next() + pub fn look_ahead_past_close_delim(&self) -> Option<&ArenaTokenTree> { + let (bounds, _) = self.stack.last().unwrap(); + self.arena.get_innermost_elem_at(bounds.index_of_next_token_tree()) } /// Clones the `TokenTree::Delimited` that we are currently within. Panics if we are not within /// a delimited sequence. #[inline] - pub fn clone_enclosing_delim(&self) -> TokenTree { - self.stack.last().unwrap().curr().unwrap().clone() + pub fn clone_enclosing_delim(&self) -> ArenaTokenTree { + let &(bounds, data) = self.stack.last().unwrap(); + ArenaTokenTree::DelimitedStart(bounds, data) } /// For skipping to the end of the current sequence, in rare circumstances. #[inline] pub fn bump_to_end(&mut self) { - self.curr.bump_to_end() + if let Some((bounds, _)) = self.stack.last() { + self.index = bounds.index_of_closing_delimiter(); + } else { + self.index = self.arena.length(); + } } /// Note: the outermost stream has depth of 0. @@ -996,10 +969,8 @@ impl TokenCursor { /// Returns details about the parent delimited sequence, if there is one. #[inline] pub fn parent_delim_and_span(&self) -> Option<(Delimiter, DelimSpan)> { - if let Some(last) = self.stack.last() - && let Some(TokenTree::Delimited(span, _, delim, _)) = last.curr() - { - Some((*delim, *span)) + if let Some((_, data)) = self.stack.last() { + Some((data.delimiter, data.span)) } else { None } @@ -1012,35 +983,47 @@ impl TokenCursor { // FIXME: we currently don't return `Delimiter::Invisible` open/close delims. To fix // #67062 we will need to, whereupon the `delim != Delimiter::Invisible` conditions // below can be removed. - if let Some(tree) = self.curr.next() { + if let Some(tree) = self.arena.get_innermost_elem_at(self.index) { match tree { - &TokenTree::Token(token, spacing) => { + &ArenaTokenTree::Token(token, spacing) => { debug_assert!(!token.kind.is_delim()); - let res = (token, spacing); - self.curr.bump(); - return res; + self.index += 1; + return (token, spacing); } - &TokenTree::Delimited(sp, spacing, delim, ref tts) => { - let trees = TokenTreeCursor::new(tts.clone()); - self.curr.bump(); // move past the `Delimited` - self.stack.push(mem::replace(&mut self.curr, trees)); - if !delim.skip() { - return (Token::new(delim.as_open_token_kind(), sp.open), spacing.open); + &ArenaTokenTree::DelimitedStart(bounds, data) => { + self.stack.push((bounds, data)); + self.index += 1; + if !data.delimiter.skip() { + return ( + Token::new(data.delimiter.as_open_token_kind(), data.span.open), + data.spacing.open, + ); } // No open delimiter to return; continue on to the next iteration. } - }; - } else if let Some(parent) = self.stack.pop() { - // We have exhausted this token stream. Move back to its parent token stream. - let Some(&TokenTree::Delimited(span, spacing, delim, _)) = parent.curr() else { - panic!("parent should be Delimited") - }; - self.curr = parent; - if !delim.skip() { - return (Token::new(delim.as_close_token_kind(), span.close), spacing.close); + &ArenaTokenTree::DelimitedEnd => { + // Pop the stack + self.index += 1; + let (_, data) = self.stack.pop().unwrap(); + if !data.delimiter.skip() { + return ( + Token::new(data.delimiter.as_close_token_kind(), data.span.close), + data.spacing.close, + ); + } + } } - // No close delimiter to return; continue on to the next iteration. } else { + // self.index += 1; + // let (_, data) = self.stack.pop().unwrap(); + // if !data.delimiter.skip() { + // return ( + // Token::new(data.delimiter.as_close_token_kind(), data.span.close), + // data.spacing.close, + // ); + // } + assert!(self.stack.is_empty()); + // We have exhausted the outermost token stream. The use of // `Spacing::Alone` is arbitrary and immaterial, because the // `Eof` token's spacing is never used. diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index d6c4e1a1fc207..c6c2186258ac0 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -97,7 +97,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { // A brace-delimited block whose first token is `&&`/`||` usually means // the user meant to continue an if-let chain, e.g. `if let P = e { && cond {`. if Delimiter::Brace == open_delim - && let Some(ArenaTokenTree::Token(tok, _)) = arena.get_item_at(index) + && let Some(ArenaTokenTree::Token(tok, _)) = arena.get_innermost_elem_at(index) && matches!(tok.kind, token::AndAnd | token::OrOr) { self.diag_info.if_let_chain_hint_spans.push(tok.span); diff --git a/compiler/rustc_parse/src/parser/function.rs b/compiler/rustc_parse/src/parser/function.rs index c078afee832d6..389c4b98ed462 100644 --- a/compiler/rustc_parse/src/parser/function.rs +++ b/compiler/rustc_parse/src/parser/function.rs @@ -2,7 +2,7 @@ use ast::token::IdentIsRaw; use rustc_ast as ast; use rustc_ast::ast::*; use rustc_ast::token::{self, InvisibleOrigin, MetaVarKind, TokenKind}; -use rustc_ast::tokenstream::TokenTree; +use rustc_ast::tokenarena::ArenaTokenTree; use rustc_ast::util::case::Case; use rustc_ast_pretty::pprust; use rustc_errors::{Applicability, PResult}; @@ -356,8 +356,9 @@ impl<'a> Parser<'a> { && self.look_ahead(1, |t| t.can_begin_string_literal()) && (self.tree_look_ahead(2, |tt| { match tt { - TokenTree::Token(t, _) => t.is_keyword_case(kw::Fn, case), - TokenTree::Delimited(..) => false, + ArenaTokenTree::Token(t, _) => t.is_keyword_case(kw::Fn, case), + ArenaTokenTree::DelimitedStart(..) => false, + _ => unreachable!() } }) == Some(true) || // This branch is only for better diagnostics; `pub`, `unsafe`, etc. are not @@ -365,17 +366,19 @@ impl<'a> Parser<'a> { (self.may_recover() && self.tree_look_ahead(2, |tt| { match tt { - TokenTree::Token(t, _) => + ArenaTokenTree::Token(t, _) => ALL_QUALS.iter().any(|exp| { t.is_keyword(exp.kw) }), - TokenTree::Delimited(..) => false, + ArenaTokenTree::DelimitedStart(..) => false, + _ => unreachable!() } }) == Some(true) && self.tree_look_ahead(3, |tt| { match tt { - TokenTree::Token(t, _) => t.is_keyword_case(kw::Fn, case), - TokenTree::Delimited(..) => false, + ArenaTokenTree::Token(t, _) => t.is_keyword_case(kw::Fn, case), + ArenaTokenTree::DelimitedStart(..) => false, + _ => unreachable!() } }) == Some(true) ) diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 44ce647568d00..e4a70d28225bc 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -5,6 +5,7 @@ use ast::token::IdentIsRaw; use rustc_ast as ast; use rustc_ast::ast::*; use rustc_ast::token::{self, Delimiter, MetaVarKind, TokenKind}; +use rustc_ast::tokenarena::ArenaTokenTree; use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_ast::util::case::Case; use rustc_ast_pretty::pprust; @@ -1116,7 +1117,7 @@ impl<'a> Parser<'a> { SUFFIXES.iter().any(|suffix| { suffix.iter().enumerate().all(|(i, kw)| { self.tree_look_ahead(i + 2, |t| { - if let TokenTree::Token(token, _) = t { + if let ArenaTokenTree::Token(token, _) = t { token.is_keyword(*kw) } else { false @@ -1661,7 +1662,7 @@ impl<'a> Parser<'a> { // might be a metavariable i.e. an invisible-delimited sequence, and // `tree_look_ahead` will consider that a single element when looking // ahead. - self.tree_look_ahead(n, |t| matches!(t, TokenTree::Delimited(_, _, Delimiter::Brace, _))) + self.tree_look_ahead(n, |t| matches!(t, ArenaTokenTree::DelimitedStart(_, data) if matches!(data.delimiter, Delimiter::Brace))) == Some(true) } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index e6d4d177fbb9c..670a1c23922a0 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -29,7 +29,7 @@ pub use path::PathStyle; use rustc_ast::token::{ self, IdentIsRaw, InvisibleOrigin, MetaVarKind, NtExprKind, NtPatKind, Token, TokenKind, }; -use rustc_ast::tokenarena::TokenArena; +use rustc_ast::tokenarena::{ArenaTokenTree, TokenArena}; use rustc_ast::tokenstream::{ ParserRange, ParserReplacement, Spacing, TokenCursor, TokenStream, TokenTree, WithTokens, }; @@ -504,7 +504,7 @@ impl<'a> Parser<'a> { fn check_noexpect_past_close_delim(&self, tok: &TokenKind) -> bool { matches!( self.token_cursor.look_ahead_past_close_delim(), - Some(TokenTree::Token(token::Token { kind, .. }, _)) if kind == tok + Some(ArenaTokenTree::Token(token::Token { kind, .. }, _)) if kind == tok ) } @@ -1157,12 +1157,16 @@ impl<'a> Parser<'a> { Some(tree) => { // Indexing stayed within the current token tree. match tree { - TokenTree::Token(token, _) => return looker(token), - &TokenTree::Delimited(dspan, _, delim, _) => { - if !delim.skip() { - return looker(&Token::new(delim.as_open_token_kind(), dspan.open)); + ArenaTokenTree::Token(token, _) => return looker(token), + &ArenaTokenTree::DelimitedStart(_, data) => { + if !data.delimiter.skip() { + return looker(&Token::new( + data.delimiter.as_open_token_kind(), + data.span.open, + )); } } + _ => unreachable!(), } } None => { @@ -1201,7 +1205,7 @@ impl<'a> Parser<'a> { pub fn tree_look_ahead( &self, dist: usize, - looker: impl FnOnce(&TokenTree) -> R, + looker: impl FnOnce(&ArenaTokenTree) -> R, ) -> Option { self.token_cursor.look_ahead(dist).map(looker) } @@ -1390,6 +1394,7 @@ impl<'a> Parser<'a> { // Clone the `TokenTree::Delimited` that we are currently // within. That's what we are going to return. let tree = self.token_cursor.clone_enclosing_delim(); + let tree = tree.to_token_tree(&self.token_cursor.arena); debug_assert_matches!(tree, TokenTree::Delimited(..)); // Advance the token cursor through the entire delimited From 681e12d3356c73e4d593b004628314f82eba1d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 26 Aug 2026 14:17:10 +0200 Subject: [PATCH 5/8] Push arena token trees further down the stack --- compiler/rustc_ast/src/tokenarena.rs | 30 +++++++++++ compiler/rustc_expand/src/mbe/macro_rules.rs | 52 ++++++++++++------- compiler/rustc_parse/src/parser/cfg_select.rs | 11 ++-- compiler/rustc_parse/src/parser/item.rs | 6 ++- compiler/rustc_parse/src/parser/mod.rs | 21 +++++--- .../rustc_parse/src/parser/nonterminal.rs | 4 +- 6 files changed, 95 insertions(+), 29 deletions(-) diff --git a/compiler/rustc_ast/src/tokenarena.rs b/compiler/rustc_ast/src/tokenarena.rs index 02c0a905ef397..afbe9289cfc92 100644 --- a/compiler/rustc_ast/src/tokenarena.rs +++ b/compiler/rustc_ast/src/tokenarena.rs @@ -1,5 +1,6 @@ use rustc_index::static_assert_size; use rustc_macros::{Decodable, Encodable, StableHash}; +use rustc_span::Span; use crate::token::{Delimiter, Token}; use crate::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; @@ -28,6 +29,31 @@ impl ArenaTokenTree { ArenaTokenTree::DelimitedEnd => unreachable!(), } } + + /// Retrieves the `TokenTree`'s span. + pub fn span(&self) -> Span { + match self { + Self::Token(token, _) => token.span, + Self::DelimitedStart(_, data) => data.span.entire(), + _ => unreachable!(), + } + } + + pub fn to_delimited_data(&self) -> Option<&DelimitedData> { + match self { + ArenaTokenTree::Token(_, _) => None, + ArenaTokenTree::DelimitedStart(_, data) => Some(data), + ArenaTokenTree::DelimitedEnd => unreachable!(), + } + } + + pub fn to_delimited_bounds(&self) -> Option<&DelimitedBounds> { + match self { + ArenaTokenTree::Token(_, _) => None, + ArenaTokenTree::DelimitedStart(bounds, _) => Some(bounds), + ArenaTokenTree::DelimitedEnd => unreachable!(), + } + } } static_assert_size!(ArenaTokenTree, 36); @@ -186,6 +212,10 @@ impl DelimitedBounds { pub fn index_of_closing_delimiter(&self) -> usize { self.index_of_next_token_tree().saturating_sub(1) } + + pub fn is_empty(&self) -> bool { + self.length == 2 + } } #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable, StableHash)] diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index d8018ec15d392..a1f8624deebc2 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -7,8 +7,8 @@ use ast::token::IdentIsRaw; use rustc_ast::token::NtPatKind::*; use rustc_ast::token::TokenKind::*; use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind}; -use rustc_ast::tokenarena::TokenArena; -use rustc_ast::tokenstream::{self, DelimSpan, TokenStream}; +use rustc_ast::tokenarena::{DelimitedBounds, DelimitedData, TokenArena}; +use rustc_ast::tokenstream::{DelimSpan, TokenStream}; use rustc_ast::{self as ast, DUMMY_NODE_ID, NodeId, Safety}; use rustc_ast_pretty::pprust; use rustc_attr_ir::diagnostic::Directive; @@ -823,9 +823,17 @@ pub fn compile_declarative_macro( if let Some(guar) = check_no_eof(sess, &p, "expected macro attr args") { return dummy_syn_ext(guar); } - let args = p.parse_token_tree(); - check_args_parens(sess, sym::attr, &args); - let args = parse_one_tt(args, RulePart::Pattern, sess, node_id, features, edition); + let tt = p.parse_token_tree(); + let args = tt.to_delimited_data(); + check_args_parens(sess, sym::attr, args); + let args = parse_one_tt( + tt.to_token_tree(p.arena()), + RulePart::Pattern, + sess, + node_id, + features, + edition, + ); check_emission(check_lhs(sess, features, node_id, &args)); if let Some(guar) = check_no_eof(sess, &p, "expected macro attr body") { return dummy_syn_ext(guar); @@ -845,9 +853,10 @@ pub fn compile_declarative_macro( if let Some(guar) = check_no_eof(sess, &p, "expected `()` after `derive`") { return dummy_syn_ext(guar); } - let args = p.parse_token_tree(); - check_args_parens(sess, sym::derive, &args); - let args_empty_result = check_args_empty(sess, &args); + let tt = p.parse_token_tree(); + let args = tt.to_delimited_data(); + check_args_parens(sess, sym::derive, args); + let args_empty_result = check_args_empty(sess, tt.to_delimited_bounds(), tt.span()); let args_not_empty = args_empty_result.is_err(); check_emission(args_empty_result); if let Some(guar) = check_no_eof(sess, &p, "expected macro derive body") { @@ -873,7 +882,7 @@ pub fn compile_declarative_macro( } (None, false) }; - let lhs_tt = p.parse_token_tree(); + let lhs_tt = p.parse_token_tree().to_token_tree(p.arena()); let lhs_tt = parse_one_tt(lhs_tt, RulePart::Pattern, sess, node_id, features, edition); check_emission(check_lhs(sess, features, node_id, &lhs_tt)); if let Err(e) = p.expect(exp!(FatArrow)) { @@ -882,7 +891,7 @@ pub fn compile_declarative_macro( if let Some(guar) = check_no_eof(sess, &p, "expected right-hand side of macro rule") { return dummy_syn_ext(guar); } - let rhs = p.parse_token_tree(); + let rhs = p.parse_token_tree().to_token_tree(p.arena()); let rhs = parse_one_tt(rhs, RulePart::Body, sess, node_id, features, edition); check_emission(check_rhs(sess, &rhs)); check_emission(check_meta_variables(&sess.psess, node_id, args.as_ref(), &lhs_tt, &rhs)); @@ -962,25 +971,32 @@ fn check_no_eof(sess: &Session, p: &Parser<'_>, msg: &'static str) -> Option) { // This does not handle the non-delimited case; that gets handled separately by `check_lhs`. - if let tokenstream::TokenTree::Delimited(dspan, _, delim, _) = args - && *delim != Delimiter::Parenthesis + if let Some(data) = args + && data.delimiter != Delimiter::Parenthesis { sess.dcx().emit_err(diagnostics::MacroArgsBadDelim { - span: dspan.entire(), - sugg: diagnostics::MacroArgsBadDelimSugg { open: dspan.open, close: dspan.close }, + span: data.span.entire(), + sugg: diagnostics::MacroArgsBadDelimSugg { + open: data.span.open, + close: data.span.close, + }, rule_kw, }); } } -fn check_args_empty(sess: &Session, args: &tokenstream::TokenTree) -> Result<(), ErrorGuaranteed> { +fn check_args_empty( + sess: &Session, + args: Option<&DelimitedBounds>, + span: Span, +) -> Result<(), ErrorGuaranteed> { match args { - tokenstream::TokenTree::Delimited(.., delimited) if delimited.is_empty() => Ok(()), + Some(bounds) if bounds.is_empty() => Ok(()), _ => { let msg = "`derive` rules do not accept arguments; `derive` must be followed by `()`"; - Err(sess.dcx().span_err(args.span(), msg)) + Err(sess.dcx().span_err(span, msg)) } } } diff --git a/compiler/rustc_parse/src/parser/cfg_select.rs b/compiler/rustc_parse/src/parser/cfg_select.rs index 3d89cabbbc655..cb8489fd03c86 100644 --- a/compiler/rustc_parse/src/parser/cfg_select.rs +++ b/compiler/rustc_parse/src/parser/cfg_select.rs @@ -1,3 +1,4 @@ +use rustc_ast::tokenarena::ArenaTokenTree; use rustc_ast::tokenstream::{TokenStream, TokenTree}; use rustc_ast::util::classify; use rustc_ast::{AttrKind, token}; @@ -20,12 +21,16 @@ impl<'a> Parser<'a> { if self.token == token::OpenBrace { // Strip the outer '{' and '}'. match self.parse_token_tree() { - TokenTree::Token(..) => unreachable!("because the current token is a '{{'"), - TokenTree::Delimited(.., tts) => { + ArenaTokenTree::Token(..) => unreachable!("because the current token is a '{{'"), + tree @ ArenaTokenTree::DelimitedStart(..) => { // Optionally end with a comma. let _ = self.eat(exp!(Comma)); - return Ok(tts); + return Ok(match tree.to_token_tree(&self.token_cursor.arena) { + TokenTree::Token(_, _) => unreachable!(), + TokenTree::Delimited(_, _, _, tts) => tts, + }); } + _ => unreachable!(), } } let attrs = AttrWrapper::empty(); // FIXME expressions with attributes can be supported here diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index e4a70d28225bc..4b40b40a6dbec 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -2603,7 +2603,11 @@ impl<'a> Parser<'a> { // Convert `MacParams MacBody` into `{ MacParams => MacBody }`. let bspan = body.span(); let arrow = TokenTree::token_alone(token::FatArrow, pspan.between(bspan)); // `=>` - let tokens = TokenStream::new(vec![params, arrow, body]); + let tokens = TokenStream::new(vec![ + params.to_token_tree(&self.token_cursor.arena), + arrow, + body.to_token_tree(&self.token_cursor.arena), + ]); let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi()); Box::new(DelimArgs { dspan, delim: Delimiter::Brace, tokens }) } else { diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 670a1c23922a0..46ea57215a849 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -244,6 +244,12 @@ pub struct Parser<'a> { pub fn_body_missing_semi_guar: Option = None, } +impl<'a> Parser<'a> { + pub fn arena(&self) -> &TokenArena { + &self.token_cursor.arena + } +} + // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with // nonterminals. Make sure it doesn't unintentionally get bigger. We only check a few arches // though, because `TokenTypeSet(u128)` alignment varies on others, changing the total size. @@ -1381,7 +1387,9 @@ impl<'a> Parser<'a> { || self.check(exp!(OpenBrace)); delimited.then(|| { - let TokenTree::Delimited(dspan, _, delim, tokens) = self.parse_token_tree() else { + let TokenTree::Delimited(dspan, _, delim, tokens) = + self.parse_token_tree().to_token_tree(&self.token_cursor.arena) + else { unreachable!() }; DelimArgs { dspan, delim, tokens } @@ -1389,13 +1397,12 @@ impl<'a> Parser<'a> { } /// Parses a single token tree from the input. - pub fn parse_token_tree(&mut self) -> TokenTree { + pub fn parse_token_tree(&mut self) -> ArenaTokenTree { if self.token.kind.open_delim().is_some() { // Clone the `TokenTree::Delimited` that we are currently // within. That's what we are going to return. let tree = self.token_cursor.clone_enclosing_delim(); - let tree = tree.to_token_tree(&self.token_cursor.arena); - debug_assert_matches!(tree, TokenTree::Delimited(..)); + debug_assert_matches!(tree, ArenaTokenTree::DelimitedStart(..)); // Advance the token cursor through the entire delimited // sequence. After getting the `OpenDelim` we are *within* the @@ -1431,7 +1438,7 @@ impl<'a> Parser<'a> { assert!(!self.token.kind.is_close_delim_or_eof()); let prev_spacing = self.token_spacing; self.bump(); - TokenTree::Token(self.prev_token, prev_spacing) + ArenaTokenTree::Token(self.prev_token, prev_spacing) } } @@ -1444,7 +1451,9 @@ impl<'a> Parser<'a> { result.push(self.parse_token_tree()); } } - TokenStream::new(result) + TokenStream::new( + result.into_iter().map(|tt| tt.to_token_tree(&self.token_cursor.arena)).collect(), + ) } /// Evaluates the closure with restrictions in place. diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 9f9545c194082..67f8fd0964b34 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -125,7 +125,9 @@ impl<'a> Parser<'a> { // we always capture tokens for any nonterminal that needs them. match kind { // Note that TT is treated differently to all the others. - NonterminalKind::TT => Ok(ParseNtResult::Tt(self.parse_token_tree())), + NonterminalKind::TT => Ok(ParseNtResult::Tt( + self.parse_token_tree().to_token_tree(&self.token_cursor.arena), + )), NonterminalKind::Item => match self .parse_item(ForceCollect::Yes, AllowConstBlockItems::Yes)? { From 4b166991dd23319f0f2fd14d47b0f6d5c1f21a1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 26 Aug 2026 14:30:31 +0200 Subject: [PATCH 6/8] Pass `&mut TokenArena` to `lex_token_trees` --- compiler/rustc_parse/src/lexer/mod.rs | 8 ++++---- compiler/rustc_parse/src/lib.rs | 23 ++++++++++++++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index 2738440ebafff..aa83c19a685ed 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -66,9 +66,10 @@ pub(crate) fn lex_token_trees<'psess, 'src>( psess: &'psess ParseSess, mut src: &'src str, mut start_pos: BytePos, + arena: &mut TokenArena, override_span: Option, strip_tokens: StripTokens, -) -> Result>> { +) -> Result<(), Vec>> { match strip_tokens { StripTokens::Shebang | StripTokens::ShebangAndFrontmatter => { if let Some(shebang_len) = rustc_lexer::strip_shebang(src) { @@ -97,8 +98,7 @@ pub(crate) fn lex_token_trees<'psess, 'src>( token: Token::dummy(), diag_info: TokenTreeDiagInfo::default(), }; - let mut arena = TokenArena::new(Vec::new()); - let res = lexer.lex_token_trees(&mut arena, /* is_delimited */ false); + let res = lexer.lex_token_trees(arena, /* is_delimited */ false); let mut unmatched_closing_delims: Vec<_> = make_errors_for_mismatched_closing_delims(&lexer.diag_info.unmatched_delims, psess); @@ -106,7 +106,7 @@ pub(crate) fn lex_token_trees<'psess, 'src>( match res { Ok(_) => { if unmatched_closing_delims.is_empty() { - Ok(arena) + Ok(()) } else { // Return error if there are unmatched delimiters or unclosed delimiters. Err(unmatched_closing_delims) diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 1f0416f8dc102..f5ee5fdb96df7 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -271,7 +271,16 @@ fn source_file_to_stream<'psess>( )); }); - lexer::lex_token_trees(psess, src.as_str(), source_file.start_pos, override_span, strip_tokens) + let mut arena = TokenArena::default(); + lexer::lex_token_trees( + psess, + src.as_str(), + source_file.start_pos, + &mut arena, + override_span, + strip_tokens, + )?; + Ok(arena) } /// Runs the given subparser `f` on the tokens of the given `attr`'s item. @@ -350,8 +359,16 @@ fn lex_token_trees_for_span( span: Span, ) -> Option> { let src = psess.source_map().span_to_snippet(span).ok()?; - let stream = match lexer::lex_token_trees(psess, &src, span.lo(), None, StripTokens::Nothing) { - Ok(arena) => arena.to_token_stream(), + let mut arena = TokenArena::default(); + let stream = match lexer::lex_token_trees( + psess, + &src, + span.lo(), + &mut arena, + None, + StripTokens::Nothing, + ) { + Ok(_) => arena.to_token_stream(), Err(errs) => { errs.into_iter().for_each(|err| err.cancel()); return None; From 05d1ef92abc13bdfa9f71783591ad32d1d746796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 26 Aug 2026 14:39:21 +0200 Subject: [PATCH 7/8] Migrate `fake_token_stream_for_file_mod` to `TokenArena` --- compiler/rustc_ast/src/attr/mod.rs | 23 ++++++ compiler/rustc_ast/src/tokenarena.rs | 53 ++++++++------ compiler/rustc_parse/src/lib.rs | 71 ++++++++----------- src/librustdoc/clean/render_macro_matchers.rs | 6 +- 4 files changed, 92 insertions(+), 61 deletions(-) diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 40a1b4bd32218..59d60c3268bf6 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -19,6 +19,7 @@ use crate::ast::{ use crate::token::{ self, CommentKind, Delimiter, DocFragmentKind, InvisibleOrigin, MetaVarKind, Token, }; +use crate::tokenarena::{ArenaTokenTree, TokenArena}; use crate::tokenstream::{ AttrTokenStream, AttrTokenTree, DelimSpacing, DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenStreamIter, TokenTree, @@ -308,6 +309,28 @@ impl Attribute { } } + pub fn push_token_trees(&self, arena: &mut TokenArena) { + match self.kind { + AttrKind::Normal(ref normal) => { + for token_tree in normal + .tokens + .as_ref() + .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}")) + .to_attr_token_stream() + .to_token_trees() + { + arena.push_token_tree(&token_tree); + } + } + // Empty tokens here ensures synthetic attributes are invisible to proc macros. + AttrKind::Synthetic(..) => {} + AttrKind::DocComment(comment_kind, data) => arena.push(ArenaTokenTree::token_alone( + token::DocComment(comment_kind, self.style, data), + self.span, + )), + } + } + pub fn deprecation_note(&self) -> Option { match &self.kind { AttrKind::Normal(normal) if normal.item.path == sym::deprecated => { diff --git a/compiler/rustc_ast/src/tokenarena.rs b/compiler/rustc_ast/src/tokenarena.rs index afbe9289cfc92..b17c35d67931f 100644 --- a/compiler/rustc_ast/src/tokenarena.rs +++ b/compiler/rustc_ast/src/tokenarena.rs @@ -2,7 +2,7 @@ use rustc_index::static_assert_size; use rustc_macros::{Decodable, Encodable, StableHash}; use rustc_span::Span; -use crate::token::{Delimiter, Token}; +use crate::token::{Delimiter, Token, TokenKind}; use crate::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; /// Part of a `TokenArena`. @@ -13,11 +13,16 @@ pub enum ArenaTokenTree { Token(Token, Spacing), /// A delimited sequence of token trees. DelimitedStart(DelimitedBounds, DelimitedData), - // TODO: get rid of this and represent it implicitly + // FIXME: get rid of this and represent it implicitly DelimitedEnd, } impl ArenaTokenTree { + /// Create a `TokenTree::Token` with alone spacing. + pub fn token_alone(kind: TokenKind, span: Span) -> ArenaTokenTree { + ArenaTokenTree::Token(Token::new(kind, span), Spacing::Alone) + } + /// Convert an arena token tree to the tree-shaped token tree. pub fn to_token_tree(&self, arena: &TokenArena) -> TokenTree { match self { @@ -64,14 +69,18 @@ pub struct TokenArena { } impl TokenArena { - pub fn new(tokens: Vec) -> Self { - Self { tokens } - } - pub fn push(&mut self, token: ArenaTokenTree) { self.tokens.push(token); } + pub fn pop(&mut self) -> Option { + let tree = self.tokens.pop(); + if let Some(tree) = &tree { + assert!(matches!(tree, ArenaTokenTree::Token(..))); + } + tree + } + /// Iter top-level token trees of a delimited token sequence. pub fn iter_delimited(&self, bounds: &DelimitedBounds) -> impl Iterator { let mut index = (bounds.start + 1) as usize; @@ -163,24 +172,28 @@ impl TokenArena { arena } - fn fill(&mut self, stream: &TokenStream) { - for item in stream.iter() { - match item { - TokenTree::Token(token, spacing) => { - self.tokens.push(ArenaTokenTree::Token(*token, *spacing)); - } - TokenTree::Delimited(span, spacing, delimiter, stream) => { - let start = self.start_delimited(); - self.fill(stream); - self.finish_delimited( - start, - DelimitedData { span: *span, spacing: *spacing, delimiter: *delimiter }, - ); - } + pub fn push_token_tree(&mut self, tt: &TokenTree) { + match tt { + TokenTree::Token(token, spacing) => { + self.tokens.push(ArenaTokenTree::Token(*token, *spacing)); + } + TokenTree::Delimited(span, spacing, delimiter, stream) => { + let start = self.start_delimited(); + self.fill(stream); + self.finish_delimited( + start, + DelimitedData { span: *span, spacing: *spacing, delimiter: *delimiter }, + ); } } } + fn fill(&mut self, stream: &TokenStream) { + for tt in stream.iter() { + self.push_token_tree(tt); + } + } + pub fn to_token_stream(&self) -> TokenStream { let mut tokens = vec![]; for tt in self.iter_top_level_trees() { diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index f5ee5fdb96df7..dcafa9fd111bc 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use rustc_ast as ast; use rustc_ast::token; -use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; +use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing}; use rustc_ast_pretty::pprust; use rustc_errors::{Diag, EmissionGuarantee, FatalError, PResult, pluralize}; pub use rustc_lexer::UNICODE_VERSION; @@ -29,7 +29,7 @@ pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments"); #[macro_use] pub mod parser; use parser::Parser; -use rustc_ast::tokenarena::TokenArena; +use rustc_ast::tokenarena::{ArenaTokenTree, DelimitedData, TokenArena}; use crate::lexer::StripTokens; @@ -303,8 +303,8 @@ pub fn fake_token_stream_for_item( item: &ast::Item, attr_to_exclude: Option<&ast::Attribute>, ) -> TokenArena { - if let Some(tokens) = fake_token_stream_for_file_mod(psess, item, attr_to_exclude) { - return TokenArena::from_stream(&tokens); + if let Some(arena) = fake_token_stream_for_file_mod(psess, item, attr_to_exclude) { + return arena; } let source = pprust::item_to_string(item); @@ -316,7 +316,7 @@ fn fake_token_stream_for_file_mod( psess: &ParseSess, item: &ast::Item, attr_to_exclude: Option<&ast::Attribute>, -) -> Option { +) -> Option { let ast::ItemKind::Mod(_, _, ast::ModKind::Loaded(_, ast::Inline::No { .. }, spans)) = &item.kind else { @@ -326,55 +326,46 @@ fn fake_token_stream_for_file_mod( let attr = attr_to_exclude.expect("file modules must have an attribute to exclude"); assert_eq!(attr.style, ast::AttrStyle::Inner); - let mut body_tts = Vec::new(); - body_tts.extend(lex_token_trees_for_span(psess, spans.inner_span.until(attr.span))?); - body_tts.extend(lex_token_trees_for_span( - psess, - attr.span.between(spans.inner_span.shrink_to_hi()), - )?); + let mut arena = TokenArena::default(); - let mut wrapper_tts = Vec::new(); for attr in item.attrs.iter().filter(|attr| attr.style == ast::AttrStyle::Outer) { - wrapper_tts.extend(attr.token_trees()); + attr.push_token_trees(&mut arena); } - wrapper_tts.extend(lex_token_trees_for_span(psess, item.span)?); - let Some(TokenTree::Token(semi, _)) = wrapper_tts.pop() else { + lex_token_trees_for_span(psess, item.span, &mut arena)?; + let Some(ArenaTokenTree::Token(semi, _)) = arena.pop() else { return None; }; if semi.kind != token::Semi { return None; } - wrapper_tts.push(TokenTree::Delimited( - DelimSpan::from_single(semi.span), - DelimSpacing::new(Spacing::Alone, Spacing::Alone), - token::Delimiter::Brace, - TokenStream::new(body_tts), - )); - - Some(TokenStream::new(wrapper_tts)) -} -fn lex_token_trees_for_span( - psess: &ParseSess, - span: Span, -) -> Option> { - let src = psess.source_map().span_to_snippet(span).ok()?; - let mut arena = TokenArena::default(); - let stream = match lexer::lex_token_trees( + let start = arena.start_delimited(); + lex_token_trees_for_span(psess, spans.inner_span.until(attr.span), &mut arena)?; + lex_token_trees_for_span( psess, - &src, - span.lo(), + attr.span.between(spans.inner_span.shrink_to_hi()), &mut arena, - None, - StripTokens::Nothing, - ) { - Ok(_) => arena.to_token_stream(), + )?; + arena.finish_delimited( + start, + DelimitedData { + span: DelimSpan::from_single(semi.span), + spacing: DelimSpacing::new(Spacing::Alone, Spacing::Alone), + delimiter: token::Delimiter::Brace, + }, + ); + Some(arena) +} + +fn lex_token_trees_for_span(psess: &ParseSess, span: Span, arena: &mut TokenArena) -> Option<()> { + let src = psess.source_map().span_to_snippet(span).ok()?; + match lexer::lex_token_trees(psess, &src, span.lo(), arena, None, StripTokens::Nothing) { + Ok(_) => Some(()), Err(errs) => { errs.into_iter().for_each(|err| err.cancel()); - return None; + None } - }; - Some((0..).map_while(move |index| stream.get(index).cloned())) + } } pub fn fake_token_stream_for_foreign_item( diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs index a69e3808bd7f7..70498dd3ef090 100644 --- a/src/librustdoc/clean/render_macro_matchers.rs +++ b/src/librustdoc/clean/render_macro_matchers.rs @@ -88,7 +88,11 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option, tt: &TokenTree) { From 46738d1bf0ffc117fe9fd101b86b88cf804ab22f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 27 Aug 2026 10:23:13 +0200 Subject: [PATCH 8/8] WIP --- compiler/rustc_ast/src/ast.rs | 7 +-- compiler/rustc_ast/src/attr/mod.rs | 16 +++--- compiler/rustc_ast/src/tokenarena.rs | 53 +++++++++++++++++++ compiler/rustc_ast/src/tokenstream.rs | 10 ++-- compiler/rustc_ast/src/visit.rs | 1 + compiler/rustc_ast_pretty/src/pprust/state.rs | 11 ++-- compiler/rustc_attr_ir/src/attr.rs | 3 +- .../rustc_attr_parsing/src/attributes/cfg.rs | 2 +- compiler/rustc_hir_pretty/src/lib.rs | 22 ++++---- compiler/rustc_parse/src/lib.rs | 4 +- compiler/rustc_parse/src/parser/item.rs | 11 ++-- compiler/rustc_parse/src/parser/mod.rs | 11 ++-- src/doc/reference | 2 +- 13 files changed, 104 insertions(+), 49 deletions(-) diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 45ea2dcd121ff..9718ee8489a3e 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -371,6 +371,7 @@ impl ParenthesizedArgs { } pub use crate::node_id::{CRATE_NODE_ID, DUMMY_NODE_ID, NodeId}; +use crate::tokenarena::TokenArenaStream; /// Modifiers on a trait bound like `[const]`, `?` and `!`. #[derive(Copy, Clone, PartialEq, Eq, Encodable, Decodable, Debug, Walkable)] @@ -2090,7 +2091,7 @@ impl AttrArgs { pub fn inner_tokens(&self) -> TokenStream { match self { AttrArgs::Empty => TokenStream::default(), - AttrArgs::Delimited(args) => args.tokens.clone(), + AttrArgs::Delimited(args) => todo!(), //args.tokens.clone(), AttrArgs::Eq { expr, .. } => TokenStream::from_ast(expr), } } @@ -2101,7 +2102,7 @@ impl AttrArgs { pub struct DelimArgs { pub dspan: DelimSpan, pub delim: Delimiter, // Note: `Delimiter::Invisible` never occurs - pub tokens: TokenStream, + pub tokens: TokenArenaStream, } impl DelimArgs { @@ -4455,7 +4456,7 @@ mod size_asserts { static_assert_size!(MetaItem, 80); static_assert_size!(MetaItemKind, 40); static_assert_size!(MetaItemLit, 40); - static_assert_size!(NormalAttr, 80); + static_assert_size!(NormalAttr, 104); static_assert_size!(Param, 40); static_assert_size!(Pat, 64); static_assert_size!(PatKind, 48); diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 59d60c3268bf6..9d7f18ccc0dc2 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -19,7 +19,7 @@ use crate::ast::{ use crate::token::{ self, CommentKind, Delimiter, DocFragmentKind, InvisibleOrigin, MetaVarKind, Token, }; -use crate::tokenarena::{ArenaTokenTree, TokenArena}; +use crate::tokenarena::{ArenaTokenTree, TokenArena, TokenArenaStream}; use crate::tokenstream::{ AttrTokenStream, AttrTokenTree, DelimSpacing, DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenStreamIter, TokenTree, @@ -368,7 +368,7 @@ impl AttrItem { pub fn meta_item_list(&self) -> Option> { match &self.args { AttrArgs::Delimited(args) if args.delim == Delimiter::Parenthesis => { - MetaItemKind::list_from_tokens(args.tokens.clone()) + MetaItemKind::list_from_tokens(todo!()) //args.tokens.clone()) } AttrArgs::Delimited(_) | AttrArgs::Eq { .. } | AttrArgs::Empty => None, } @@ -612,8 +612,9 @@ impl MetaItemKind { fn from_attr_args(args: &AttrArgs) -> Option { match args { AttrArgs::Empty => Some(MetaItemKind::Word), - AttrArgs::Delimited(DelimArgs { dspan: _, delim: Delimiter::Parenthesis, tokens }) => { - MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List) + AttrArgs::Delimited(DelimArgs { dspan: _, delim: Delimiter::Parenthesis, .. }) => { + // MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List) + todo!() } AttrArgs::Delimited(..) => None, AttrArgs::Eq { expr, .. } => match expr.kind { @@ -826,16 +827,13 @@ pub fn mk_attr_nested_word( inner: Symbol, span: Span, ) -> Attribute { - let inner_tokens = TokenStream::new(vec![TokenTree::Token( - Token::from_ast_ident(Ident::new(inner, span)), - Spacing::Alone, - )]); + let token = Token::from_ast_ident(Ident::new(inner, span)); let outer_ident = Ident::new(outer, span); let path = Path::from_ident(outer_ident); let attr_args = AttrArgs::Delimited(DelimArgs { dspan: DelimSpan::from_single(span), delim: Delimiter::Parenthesis, - tokens: inner_tokens, + tokens: TokenArenaStream::from_token(token, Spacing::Alone), }); let tokens = Some(mk_attr_tokens( diff --git a/compiler/rustc_ast/src/tokenarena.rs b/compiler/rustc_ast/src/tokenarena.rs index b17c35d67931f..87bb9a53e5ae9 100644 --- a/compiler/rustc_ast/src/tokenarena.rs +++ b/compiler/rustc_ast/src/tokenarena.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use rustc_index::static_assert_size; use rustc_macros::{Decodable, Encodable, StableHash}; use rustc_span::Span; @@ -61,6 +63,14 @@ impl ArenaTokenTree { } } +pub fn children_to_owned_stream( + arena: Arc, + bounds: DelimitedBounds, +) -> TokenArenaStream { + let trees: Vec = arena.iter_delimited(&bounds).collect(); + TokenArenaStream { arena, trees } +} + static_assert_size!(ArenaTokenTree, 36); #[derive(Debug, Default, PartialEq, Eq, Hash, Encodable, Decodable)] @@ -203,6 +213,49 @@ impl TokenArena { } } +#[derive(Clone, Encodable, Decodable, Debug, StableHash)] +pub enum TokenArenaView { + Inline(Arc), + View(TokenArenaStream), +} + +impl TokenArenaView { + pub fn arena_arc(&self) -> Arc { + match self { + TokenArenaView::Inline(a) => a.clone(), + TokenArenaView::View(arena) => arena.arena.clone(), + } + } +} + +#[derive(Clone, Encodable, Decodable, Debug, StableHash)] +pub struct TokenArenaStream { + #[stable_hash(ignore)] + arena: Arc, + trees: Arc>, +} + +impl TokenArenaStream { + pub fn new(arena: Arc, trees: Vec) -> Self { + Self { arena, trees: Arc::new(trees) } + } + + pub fn from_token(token: Token, spacing: Spacing) -> TokenArenaStream { + Self { + arena: Arc::new(Default::default()), + trees: Arc::new(vec![ArenaTokenTree::Token(token, spacing)]), + } + } + + pub fn is_empty(&self) -> bool { + self.trees.is_empty() + } + + pub fn as_view(&self) -> TokenArenaView { + TokenArenaView::View(self.clone()) + } +} + pub struct OpenDelimited { start: usize, } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 35900d5e3b198..f8397ace680c4 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -20,7 +20,9 @@ use thin_vec::ThinVec; use crate::ast::AttrStyle; use crate::ast_traits::HasTokens; use crate::token::{self, Delimiter, Token, TokenKind}; -use crate::tokenarena::{ArenaTokenTree, DelimitedBounds, DelimitedData, TokenArena}; +use crate::tokenarena::{ + ArenaTokenTree, DelimitedBounds, DelimitedData, TokenArena, TokenArenaView, +}; use crate::{AttrVec, Attribute}; #[cfg(test)] @@ -884,7 +886,7 @@ impl<'t> Iterator for TokenStreamIter<'t> { /// A `TokenArena` cursor that produces `Token`s. #[derive(Clone, Debug)] pub struct TokenCursor { - pub arena: Arc, + pub arena: TokenArenaView, /// Global index into the token arena. index: usize, /// The current delimited sequences that we are inside of. @@ -893,8 +895,8 @@ pub struct TokenCursor { impl TokenCursor { #[inline] - pub fn new(arena: TokenArena) -> Self { - TokenCursor { arena: Arc::new(arena), index: 0, stack: vec![] } + pub fn new(arena: TokenArenaView) -> Self { + TokenCursor { arena, index: 0, stack: vec![] } } /// Gets the next token and advances the cursor by one. diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs index 6b7ea072da61b..a9a5d1976b217 100644 --- a/compiler/rustc_ast/src/visit.rs +++ b/compiler/rustc_ast/src/visit.rs @@ -371,6 +371,7 @@ macro_rules! common_visitor_and_walkers { crate::token::LitKind, crate::tokenstream::LazyAttrTokenStream, crate::tokenstream::TokenStream, + crate::tokenarena::TokenArenaStream, Movability, Mutability, Pinnedness, diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 462ac4a317611..fe8c15679b1f1 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -709,13 +709,14 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere ast::Safety::Default | ast::Safety::Safe(_) => {} } match &item.args { - AttrArgs::Delimited(DelimArgs { dspan: _, delim, tokens }) => self.print_mac_common( + AttrArgs::Delimited(DelimArgs { dspan: _, delim, .. }) => self.print_mac_common( Some(MacHeader::Path(&item.path)), false, None, *delim, None, - tokens, + todo!(), + // tokens, true, span, ), @@ -926,7 +927,8 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere Some(*ident), macro_def.body.delim, None, - ¯o_def.body.tokens, + todo!(), + // ¯o_def.body.tokens, true, sp, ); @@ -1674,7 +1676,8 @@ impl<'a> State<'a> { None, m.args.delim, None, - &m.args.tokens, + // &m.args.tokens, + todo!(), true, m.span(), ); diff --git a/compiler/rustc_attr_ir/src/attr.rs b/compiler/rustc_attr_ir/src/attr.rs index 6068c11590a23..f06c06b5b3d43 100644 --- a/compiler/rustc_attr_ir/src/attr.rs +++ b/compiler/rustc_attr_ir/src/attr.rs @@ -154,7 +154,8 @@ impl AttributeExt for Attribute { match &self { Attribute::Unparsed(n) => match n.as_ref() { AttrItem { args: AttrArgs::Delimited(d), .. } => { - ast::MetaItemKind::list_from_tokens(d.tokens.clone()) + todo!() + // ast::MetaItemKind::list_from_tokens(d.tokens.clone()) } _ => None, }, diff --git a/compiler/rustc_attr_parsing/src/attributes/cfg.rs b/compiler/rustc_attr_parsing/src/attributes/cfg.rs index 8af43d059bfe6..b17ff908d44b6 100644 --- a/compiler/rustc_attr_parsing/src/attributes/cfg.rs +++ b/compiler/rustc_attr_parsing/src/attributes/cfg.rs @@ -313,7 +313,7 @@ pub fn parse_cfg_attr( match &cfg_attr.get_normal_item().args { ast::AttrArgs::Delimited(ast::DelimArgs { dspan, delim, tokens }) if !tokens.is_empty() => { check_cfg_attr_bad_delim(&sess.psess, *dspan, *delim); - match parse_in(&sess.psess, TokenArena::from_stream(tokens), "`cfg_attr` input", |p| { + match parse_in(&sess.psess, tokens.as_view(), "`cfg_attr` input", |p| { parse_cfg_attr_internal(p, sess, features, lint_node_id, cfg_attr) }) { Ok(r) => return Some(r), diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs index f2f485a30300a..67899c04cd486 100644 --- a/compiler/rustc_hir_pretty/src/lib.rs +++ b/compiler/rustc_hir_pretty/src/lib.rs @@ -146,17 +146,17 @@ impl<'a> State<'a> { }; match &item.args { - hir::AttrArgs::Delimited(DelimArgs { dspan: _, delim, tokens }) => self - .print_mac_common( - Some(MacHeader::Path(&path)), - false, - None, - *delim, - None, - &tokens, - true, - span, - ), + hir::AttrArgs::Delimited(DelimArgs { dspan: _, delim, .. }) => self.print_mac_common( + Some(MacHeader::Path(&path)), + false, + None, + *delim, + None, + // &tokens, + todo!(), + true, + span, + ), hir::AttrArgs::Empty => { PrintState::print_path(self, &path, false, 0); } diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index dcafa9fd111bc..d3ac9551dc63d 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -29,7 +29,7 @@ pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments"); #[macro_use] pub mod parser; use parser::Parser; -use rustc_ast::tokenarena::{ArenaTokenTree, DelimitedData, TokenArena}; +use rustc_ast::tokenarena::{ArenaTokenTree, DelimitedData, TokenArena, TokenArenaView}; use crate::lexer::StripTokens; @@ -286,7 +286,7 @@ fn source_file_to_stream<'psess>( /// Runs the given subparser `f` on the tokens of the given `attr`'s item. pub fn parse_in<'a, T>( psess: &'a ParseSess, - arena: TokenArena, + arena: TokenArenaView, name: &'static str, mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, ) -> PResult<'a, T> { diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 4b40b40a6dbec..78cefb2f04eca 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -5,7 +5,7 @@ use ast::token::IdentIsRaw; use rustc_ast as ast; use rustc_ast::ast::*; use rustc_ast::token::{self, Delimiter, MetaVarKind, TokenKind}; -use rustc_ast::tokenarena::ArenaTokenTree; +use rustc_ast::tokenarena::{ArenaTokenTree, TokenArenaStream}; use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_ast::util::case::Case; use rustc_ast_pretty::pprust; @@ -2602,12 +2602,9 @@ impl<'a> Parser<'a> { let body = self.parse_token_tree(); // `MacBody` // Convert `MacParams MacBody` into `{ MacParams => MacBody }`. let bspan = body.span(); - let arrow = TokenTree::token_alone(token::FatArrow, pspan.between(bspan)); // `=>` - let tokens = TokenStream::new(vec![ - params.to_token_tree(&self.token_cursor.arena), - arrow, - body.to_token_tree(&self.token_cursor.arena), - ]); + let arrow = ArenaTokenTree::token_alone(token::FatArrow, pspan.between(bspan)); // `=>` + let tokens = + TokenArenaStream::new(self.token_cursor.arena.clone(), vec![params, arrow, body]); let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi()); Box::new(DelimArgs { dspan, delim: Delimiter::Brace, tokens }) } else { diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 46ea57215a849..d2034afbaa133 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -29,7 +29,7 @@ pub use path::PathStyle; use rustc_ast::token::{ self, IdentIsRaw, InvisibleOrigin, MetaVarKind, NtExprKind, NtPatKind, Token, TokenKind, }; -use rustc_ast::tokenarena::{ArenaTokenTree, TokenArena}; +use rustc_ast::tokenarena::{ArenaTokenTree, TokenArena, TokenArenaView, children_to_owned_stream}; use rustc_ast::tokenstream::{ ParserRange, ParserReplacement, Spacing, TokenCursor, TokenStream, TokenTree, WithTokens, }; @@ -349,7 +349,7 @@ pub fn token_descr(token: &Token) -> String { impl<'a> Parser<'a> { pub fn new( psess: &'a ParseSess, - arena: TokenArena, + arena: TokenArenaView, subparser_name: Option<&'static str>, ) -> Self { let mut parser = Parser { @@ -1387,12 +1387,11 @@ impl<'a> Parser<'a> { || self.check(exp!(OpenBrace)); delimited.then(|| { - let TokenTree::Delimited(dspan, _, delim, tokens) = - self.parse_token_tree().to_token_tree(&self.token_cursor.arena) - else { + let ArenaTokenTree::DelimitedStart(bounds, data) = self.parse_token_tree() else { unreachable!() }; - DelimArgs { dspan, delim, tokens } + let tokens = children_to_owned_stream(self.token_cursor.arena.clone(), bounds); + DelimArgs { dspan: data.span, delim: data.delimiter, tokens } }) } diff --git a/src/doc/reference b/src/doc/reference index eda708334abad..3b38834b39f73 160000 --- a/src/doc/reference +++ b/src/doc/reference @@ -1 +1 @@ -Subproject commit eda708334abad285d63bc1f3558a51e8b790346e +Subproject commit 3b38834b39f732c64686f7c64aa29dcf3cd83ba5