[perf] Use a flat token stream representation when lexing

compiler/rustc_ast/src/ast.rs

@@ -371,6 +371,7 @@ impl ParenthesizedArgs {
 }
 
 pub use crate::node_id::{CRATE_NODE_ID, DUMMY_NODE_ID, NodeId};
+use crate::tokenarena::TokenArenaStream;
 
 /// Modifiers on a trait bound like `[const]`, `?` and `!`.
 #[derive(Copy, Clone, PartialEq, Eq, Encodable, Decodable, Debug, Walkable)]
@@ -2090,7 +2091,7 @@ impl AttrArgs {
     pub fn inner_tokens(&self) -> TokenStream {
         match self {
             AttrArgs::Empty => TokenStream::default(),
-            AttrArgs::Delimited(args) => args.tokens.clone(),
+            AttrArgs::Delimited(args) => todo!(), //args.tokens.clone(),
             AttrArgs::Eq { expr, .. } => TokenStream::from_ast(expr),
         }
     }
@@ -2101,7 +2102,7 @@ impl AttrArgs {
 pub struct DelimArgs {
     pub dspan: DelimSpan,
     pub delim: Delimiter, // Note: `Delimiter::Invisible` never occurs
-    pub tokens: TokenStream,
+    pub tokens: TokenArenaStream,
 }
 
 impl DelimArgs {
@@ -4455,7 +4456,7 @@ mod size_asserts {
     static_assert_size!(MetaItem, 80);
     static_assert_size!(MetaItemKind, 40);
     static_assert_size!(MetaItemLit, 40);
-    static_assert_size!(NormalAttr, 80);
+    static_assert_size!(NormalAttr, 104);
     static_assert_size!(Param, 40);
     static_assert_size!(Pat, 64);
     static_assert_size!(PatKind, 48);

compiler/rustc_ast/src/attr/mod.rs

@@ -19,6 +19,7 @@ use crate::ast::{
 use crate::token::{
     self, CommentKind, Delimiter, DocFragmentKind, InvisibleOrigin, MetaVarKind, Token,
 };
+use crate::tokenarena::{ArenaTokenTree, TokenArena, TokenArenaStream};
 use crate::tokenstream::{
     AttrTokenStream, AttrTokenTree, DelimSpacing, DelimSpan, LazyAttrTokenStream, Spacing,
     TokenStream, TokenStreamIter, TokenTree,
@@ -308,6 +309,28 @@ impl Attribute {
         }
     }
 
+    pub fn push_token_trees(&self, arena: &mut TokenArena) {
+        match self.kind {
+            AttrKind::Normal(ref normal) => {
+                for token_tree in normal
+                    .tokens
+                    .as_ref()
+                    .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
+                    .to_attr_token_stream()
+                    .to_token_trees()
+                {
+                    arena.push_token_tree(&token_tree);
+                }
+            }
+            // Empty tokens here ensures synthetic attributes are invisible to proc macros.
+            AttrKind::Synthetic(..) => {}
+            AttrKind::DocComment(comment_kind, data) => arena.push(ArenaTokenTree::token_alone(
+                token::DocComment(comment_kind, self.style, data),
+                self.span,
+            )),
+        }
+    }
+
     pub fn deprecation_note(&self) -> Option<Ident> {
         match &self.kind {
             AttrKind::Normal(normal) if normal.item.path == sym::deprecated => {
@@ -345,7 +368,7 @@ impl AttrItem {
     pub fn meta_item_list(&self) -> Option<ThinVec<MetaItemInner>> {
         match &self.args {
             AttrArgs::Delimited(args) if args.delim == Delimiter::Parenthesis => {
-                MetaItemKind::list_from_tokens(args.tokens.clone())
+                MetaItemKind::list_from_tokens(todo!()) //args.tokens.clone())
             }
             AttrArgs::Delimited(_) | AttrArgs::Eq { .. } | AttrArgs::Empty => None,
         }
@@ -589,8 +612,9 @@ impl MetaItemKind {
     fn from_attr_args(args: &AttrArgs) -> Option<MetaItemKind> {
         match args {
             AttrArgs::Empty => Some(MetaItemKind::Word),
-            AttrArgs::Delimited(DelimArgs { dspan: _, delim: Delimiter::Parenthesis, tokens }) => {
-                MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List)
+            AttrArgs::Delimited(DelimArgs { dspan: _, delim: Delimiter::Parenthesis, .. }) => {
+                // MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List)
+                todo!()
             }
             AttrArgs::Delimited(..) => None,
             AttrArgs::Eq { expr, .. } => match expr.kind {
@@ -803,16 +827,13 @@ pub fn mk_attr_nested_word(
     inner: Symbol,
     span: Span,
 ) -> Attribute {
-    let inner_tokens = TokenStream::new(vec![TokenTree::Token(
-        Token::from_ast_ident(Ident::new(inner, span)),
-        Spacing::Alone,
-    )]);
+    let token = Token::from_ast_ident(Ident::new(inner, span));
     let outer_ident = Ident::new(outer, span);
     let path = Path::from_ident(outer_ident);
     let attr_args = AttrArgs::Delimited(DelimArgs {
         dspan: DelimSpan::from_single(span),
         delim: Delimiter::Parenthesis,
-        tokens: inner_tokens,
+        tokens: TokenArenaStream::from_token(token, Spacing::Alone),
     });
 
     let tokens = Some(mk_attr_tokens(

compiler/rustc_ast/src/lib.rs

@@ -31,6 +31,7 @@ pub mod format;
 pub mod mut_visit;
 pub mod node_id;
 pub mod token;
+pub mod tokenarena;
 pub mod tokenstream;
 pub mod visit;