Return a plain Token for Prism.lex etc.

ext/prism/extension.c

@@ -846,14 +846,7 @@ parse_lex_token(pm_parser_t *parser, pm_token_t *token, void *data) {
     parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) data;
 
     VALUE value = pm_token_new(parser, token, parse_lex_data->encoding, parse_lex_data->source, parse_lex_data->freeze);
-    VALUE yields = rb_assoc_new(value, INT2FIX(pm_parser_lex_state(parser)));
-
-    if (parse_lex_data->freeze) {
-        rb_obj_freeze(value);
-        rb_obj_freeze(yields);
-    }
-
-    rb_ary_push(parse_lex_data->tokens, yields);
+    rb_ary_push(parse_lex_data->tokens, value);
 }
 
 /**
@@ -874,9 +867,7 @@ parse_lex_encoding_changed_callback(pm_parser_t *parser) {
     VALUE next_tokens = rb_ary_new();
 
     for (long index = 0; index < RARRAY_LEN(tokens); index++) {
-        VALUE yields = rb_ary_entry(tokens, index);
-        VALUE token = rb_ary_entry(yields, 0);
-
+        VALUE token = rb_ary_entry(tokens, index);
         VALUE value = rb_ivar_get(token, rb_intern("@value"));
         VALUE next_value = rb_str_dup(value);
 
@@ -887,18 +878,17 @@ parse_lex_encoding_changed_callback(pm_parser_t *parser) {
             parse_lex_data->source,
             rb_ivar_get(token, rb_intern("@type")),
             next_value,
-            rb_ivar_get(token, rb_intern("@location"))
+            rb_ivar_get(token, rb_intern("@location")),
+            rb_ivar_get(token, rb_intern("@state")),
         };
 
-        VALUE next_token = rb_class_new_instance(4, next_token_argv, rb_cPrismToken);
-        VALUE next_yields = rb_assoc_new(next_token, rb_ary_entry(yields, 1));
+        VALUE next_token = rb_class_new_instance(5, next_token_argv, rb_cPrismToken);
 
         if (parse_lex_data->freeze) {
             rb_obj_freeze(next_token);
-            rb_obj_freeze(next_yields);
         }
 
-        rb_ary_push(next_tokens, next_yields);
+        rb_ary_push(next_tokens, next_token);
     }
 
     rb_ary_replace(parse_lex_data->tokens, next_tokens);

lib/prism/lex_compat.rb

@@ -621,13 +621,13 @@ def result
       last_comment_token = nil #: lex_compat_token?
       last_comment_end = nil #: Integer?
 
-      result_value.each_with_index do |(prism_token, prism_state), index|
+      result_value.each_with_index do |prism_token, index|
         lineno = prism_token.location.start_line
         column = prism_token.location.start_column
 
         event = RIPPER.fetch(prism_token.type)
         value = prism_token.value
-        lex_state = Translation::Ripper::Lexer::State[prism_state]
+        lex_state = Translation::Ripper::Lexer::State[prism_token.instance_variable_get(:@state)]
 
         # A comment token does not include its terminating newline, but
         # ripper's comment value does, so the newline token that directly
@@ -701,7 +701,7 @@ def result
             # Ripper's lexed state. So here, if it's a regexp end token, we
             # output the state as the previous state, solely for the sake of
             # comparison.
-            previous_token = result_value[index - 1][0]
+            previous_token = result_value[index - 1]
             lex_state =
               if RIPPER.fetch(previous_token.type) == :on_embexpr_end
                 # If the previous token is embexpr_end, then we have to do even
@@ -714,24 +714,24 @@ def result
 
                 until counter == 0
                   current_index -= 1
-                  current_event = RIPPER.fetch(result_value[current_index][0].type)
+                  current_event = RIPPER.fetch(result_value[current_index].type)
                   counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0
                 end
 
-                Translation::Ripper::Lexer::State[result_value[current_index][1]]
+                Translation::Ripper::Lexer::State[result_value[current_index].instance_variable_get(:@state)]
               else
                 previous_state
               end
 
             [[lineno, column], event, value, lex_state]
           when :on_eof
             eof_token = prism_token
-            previous_token = result_value[index - 1][0]
+            previous_token = result_value[index - 1]
 
             # A newline that was folded back into a comment still marks the
             # comment boundary for the check below.
             comment_boundary = previous_token.type == :COMMENT ||
-              (index >= 2 && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous_token.type) && result_value[index - 2][0].type == :COMMENT && result_value[index - 2][0].location.end_offset == previous_token.location.start_offset)
+              (index >= 2 && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous_token.type) && result_value[index - 2].type == :COMMENT && result_value[index - 2].location.end_offset == previous_token.location.start_offset)
 
             # If we're at the end of the file and the previous token was a
             # comment and there is still whitespace after the comment, then

lib/prism/parse_result.rb

@@ -1053,11 +1053,11 @@ def errors_format
   # This is a result specific to the `lex` and `lex_file` methods.
   class LexResult < Result
     # The list of tokens that were parsed from the source code.
-    attr_reader :value #: Array[[Token, Integer]]
+    attr_reader :value #: Array[Token]
 
     # Create a new lex result object with the given values.
     #--
-    #: (Array[[Token, Integer]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    #: (Array[Token] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
     def initialize(value, comments, magic_comments, data_loc, errors, warnings, continuable, source)
       @value = value
       super(comments, magic_comments, data_loc, errors, warnings, continuable, source)
@@ -1075,11 +1075,11 @@ def deconstruct_keys(keys) # :nodoc:
   class ParseLexResult < Result
     # A tuple of the syntax tree and the list of tokens that were parsed from
     # the source code.
-    attr_reader :value #: [ProgramNode, Array[[Token, Integer]]]
+    attr_reader :value #: [ProgramNode, Array[Token]]
 
     # Create a new parse lex result object with the given values.
     #--
-    #: ([ProgramNode, Array[[Token, Integer]]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    #: ([ProgramNode, Array[Token]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
     def initialize(value, comments, magic_comments, data_loc, errors, warnings, continuable, source)
       @value = value
       super(comments, magic_comments, data_loc, errors, warnings, continuable, source)
@@ -1109,12 +1109,13 @@ class Token
 
     # Create a new token object with the given type, value, and location.
     #--
-    #: (Source source, Symbol type, String value, Location | Integer location) -> void
-    def initialize(source, type, value, location)
+    #: (Source source, Symbol type, String value, Location | Integer location, Integer state) -> void
+    def initialize(source, type, value, location, state)
       @source = source
       @type = type
       @value = value
       @location = location
+      @state = state
     end
 
     # Implement the hash pattern matching interface for Token.
@@ -1175,6 +1176,20 @@ def deep_freeze
       location.freeze
       freeze
     end
+
+    # Backwards compatibility for Prism.lex/Prism.lex_file/Prism.parse_lex
+    # when they returned a 2-element array.
+
+    #: (Integer index) -> Token
+    def [](index) # :nodoc:
+      return self if index == 0
+      raise ArgumentError, "Invalid index #{index}"
+    end
+
+    #: () -> Token
+    def first # :nodoc:
+      self
+    end
   end
 
   # This object is passed to the various Prism.* methods that accept the

lib/prism/translation/parser/lexer.rb

@@ -230,7 +230,7 @@ def to_a
           comment_newline_location = nil
 
           while index < length
-            token, _ = lexed[index]
+            token = lexed[index]
             index += 1
             next if TYPES_ALWAYS_SKIP.include?(token.type)
 
@@ -253,7 +253,7 @@ def to_a
             when :tCOMMENT
               if token.type == :EMBDOC_BEGIN
 
-                while !((next_token = lexed[index]&.first) && next_token.type == :EMBDOC_END) && (index < length - 1)
+                while !((next_token = lexed[index]) && next_token.type == :EMBDOC_END) && (index < length - 1)
                   value += next_token.value
                   index += 1
                 end
@@ -267,7 +267,7 @@ def to_a
                 location = range(token.location.start_offset, token.location.end_offset - 1) if value.chomp!
               end
             when :tNL
-              next_token, _ = lexed[index]
+              next_token = lexed[index]
               # Newlines after comments are emitted out of order.
               if next_token&.type == :COMMENT
                 comment_newline_location = location
@@ -300,8 +300,8 @@ def to_a
               location = range(token.location.start_offset, token.location.start_offset + percent_array_leading_whitespace(value))
               value = nil
             when :tSTRING_BEG
-              next_token, _ = lexed[index]
-              next_next_token, _ = lexed[index + 1]
+              next_token = lexed[index]
+              next_next_token = lexed[index + 1]
               basic_quotes = value == '"' || value == "'"
 
               if basic_quotes && next_token&.type == :STRING_END
@@ -369,7 +369,7 @@ def to_a
                 while token.type == :STRING_CONTENT
                   current_length += token.value.bytesize
                   # Heredoc interpolation can have multiple STRING_CONTENT nodes on the same line.
-                  prev_token, _ = lexed[index - 2] if index - 2 >= 0
+                  prev_token = lexed[index - 2] if index - 2 >= 0
                   is_first_token_on_line = prev_token && token.location.start_line != prev_token.location.start_line
                   # The parser gem only removes indentation when the heredoc is not nested
                   not_nested = heredoc_stack.size == 1
@@ -389,7 +389,7 @@ def to_a
                     tokens << [:tSTRING_CONTENT, [current_string, range(start_offset, start_offset + current_length)]]
                     break
                   end
-                  token, _ = lexed[index]
+                  token = lexed[index]
                   index += 1
                 end
               else
@@ -444,7 +444,7 @@ def to_a
               end
 
               if percent_array?(quote_stack.pop)
-                prev_token, _ = lexed[index - 2] if index - 2 >= 0
+                prev_token = lexed[index - 2] if index - 2 >= 0
                 empty = %i[PERCENT_LOWER_I PERCENT_LOWER_W PERCENT_UPPER_I PERCENT_UPPER_W].include?(prev_token&.type)
                 ends_with_whitespace = prev_token&.type == :WORDS_SEP
                 # parser always emits a space token after content in a percent array, even if no actual whitespace is present.
@@ -453,7 +453,7 @@ def to_a
                 end
               end
             when :tSYMBEG
-              if (next_token = lexed[index]&.first) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
+              if (next_token = lexed[index]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
                 next_location = token.location.join(next_token.location)
                 type = :tSYMBOL
                 value = next_token.value
@@ -470,7 +470,7 @@ def to_a
             when :tXSTRING_BEG
               quote_stack.push(value)
             when :tSYMBOLS_BEG, :tQSYMBOLS_BEG, :tWORDS_BEG, :tQWORDS_BEG
-              if (next_token = lexed[index]&.first) && next_token.type == :WORDS_SEP
+              if (next_token = lexed[index]) && next_token.type == :WORDS_SEP
                 index += 1
               end
 
@@ -550,9 +550,9 @@ def calculate_heredoc_whitespace(heredoc_token_index)
           previous_line = -1
           result = Float::MAX
 
-          while (next_token = lexed[next_token_index]&.first)
+          while (next_token = lexed[next_token_index])
             next_token_index += 1
-            next_next_token, _ = lexed[next_token_index]
+            next_next_token = lexed[next_token_index]
             first_token_on_line = next_token.location.start_column == 0
 
             # String content inside nested heredocs and interpolation is ignored