[pull] master from ruby:master

lib/prism/lex_compat.rb

@@ -621,13 +621,13 @@ def result
       last_comment_token = nil #: lex_compat_token?
       last_comment_end = nil #: Integer?
 
-      result_value.each_with_index do |(prism_token, prism_state), index|
+      result_value.each_with_index do |prism_token, index|
         lineno = prism_token.location.start_line
         column = prism_token.location.start_column
 
         event = RIPPER.fetch(prism_token.type)
         value = prism_token.value
-        lex_state = Translation::Ripper::Lexer::State[prism_state]
+        lex_state = Translation::Ripper::Lexer::State[prism_token._ripper_state]
 
         # A comment token does not include its terminating newline, but
         # ripper's comment value does, so the newline token that directly
@@ -701,7 +701,7 @@ def result
             # Ripper's lexed state. So here, if it's a regexp end token, we
             # output the state as the previous state, solely for the sake of
             # comparison.
-            previous_token = result_value[index - 1][0]
+            previous_token = result_value[index - 1]
             lex_state =
               if RIPPER.fetch(previous_token.type) == :on_embexpr_end
                 # If the previous token is embexpr_end, then we have to do even
@@ -714,24 +714,24 @@ def result
 
                 until counter == 0
                   current_index -= 1
-                  current_event = RIPPER.fetch(result_value[current_index][0].type)
+                  current_event = RIPPER.fetch(result_value[current_index].type)
                   counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0
                 end
 
-                Translation::Ripper::Lexer::State[result_value[current_index][1]]
+                Translation::Ripper::Lexer::State[result_value[current_index]._ripper_state]
               else
                 previous_state
               end
 
             [[lineno, column], event, value, lex_state]
           when :on_eof
             eof_token = prism_token
-            previous_token = result_value[index - 1][0]
+            previous_token = result_value[index - 1]
 
             # A newline that was folded back into a comment still marks the
             # comment boundary for the check below.
             comment_boundary = previous_token.type == :COMMENT ||
-              (index >= 2 && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous_token.type) && result_value[index - 2][0].type == :COMMENT && result_value[index - 2][0].location.end_offset == previous_token.location.start_offset)
+              (index >= 2 && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous_token.type) && result_value[index - 2].type == :COMMENT && result_value[index - 2].location.end_offset == previous_token.location.start_offset)
 
             # If we're at the end of the file and the previous token was a
             # comment and there is still whitespace after the comment, then

lib/prism/parse_result.rb

@@ -1053,11 +1053,11 @@ def errors_format
   # This is a result specific to the `lex` and `lex_file` methods.
   class LexResult < Result
     # The list of tokens that were parsed from the source code.
-    attr_reader :value #: Array[[Token, Integer]]
+    attr_reader :value #: Array[Token]
 
     # Create a new lex result object with the given values.
     #--
-    #: (Array[[Token, Integer]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    #: (Array[Token] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
     def initialize(value, comments, magic_comments, data_loc, errors, warnings, continuable, source)
       @value = value
       super(comments, magic_comments, data_loc, errors, warnings, continuable, source)
@@ -1075,11 +1075,11 @@ def deconstruct_keys(keys) # :nodoc:
   class ParseLexResult < Result
     # A tuple of the syntax tree and the list of tokens that were parsed from
     # the source code.
-    attr_reader :value #: [ProgramNode, Array[[Token, Integer]]]
+    attr_reader :value #: [ProgramNode, Array[Token]]
 
     # Create a new parse lex result object with the given values.
     #--
-    #: ([ProgramNode, Array[[Token, Integer]]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
+    #: ([ProgramNode, Array[Token]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void
     def initialize(value, comments, magic_comments, data_loc, errors, warnings, continuable, source)
       @value = value
       super(comments, magic_comments, data_loc, errors, warnings, continuable, source)
@@ -1109,12 +1109,13 @@ class Token
 
     # Create a new token object with the given type, value, and location.
     #--
-    #: (Source source, Symbol type, String value, Location | Integer location) -> void
-    def initialize(source, type, value, location)
+    #: (Source source, Symbol type, String value, Location | Integer location, Integer state) -> void
+    def initialize(source, type, value, location, state)
       @source = source
       @type = type
       @value = value
       @location = location
+      @state = state
     end
 
     # Implement the hash pattern matching interface for Token.
@@ -1175,6 +1176,27 @@ def deep_freeze
       location.freeze
       freeze
     end
+
+    # For internal use only.
+    #: () -> Integer
+    def _ripper_state # :nodoc:
+      @state
+    end
+
+    # Backwards compatibility for Prism.lex/Prism.lex_file/Prism.parse_lex
+    # when they returned a 2-element array.
+
+    #: (Integer index) -> Token | Integer
+    def [](index) # :nodoc:
+      return self if index == 0
+      return @state if index == 1
+      raise ArgumentError, "Invalid index #{index}"
+    end
+
+    #: () -> Token
+    def first # :nodoc:
+      self
+    end
   end
 
   # This object is passed to the various Prism.* methods that accept the

lib/prism/translation/parser/lexer.rb

@@ -230,7 +230,7 @@ def to_a
           comment_newline_location = nil
 
           while index < length
-            token, _ = lexed[index]
+            token = lexed[index]
             index += 1
             next if TYPES_ALWAYS_SKIP.include?(token.type)
 
@@ -253,7 +253,7 @@ def to_a
             when :tCOMMENT
               if token.type == :EMBDOC_BEGIN
 
-                while !((next_token = lexed[index]&.first) && next_token.type == :EMBDOC_END) && (index < length - 1)
+                while !((next_token = lexed[index]) && next_token.type == :EMBDOC_END) && (index < length - 1)
                   value += next_token.value
                   index += 1
                 end
@@ -267,7 +267,7 @@ def to_a
                 location = range(token.location.start_offset, token.location.end_offset - 1) if value.chomp!
               end
             when :tNL
-              next_token, _ = lexed[index]
+              next_token = lexed[index]
               # Newlines after comments are emitted out of order.
               if next_token&.type == :COMMENT
                 comment_newline_location = location
@@ -300,8 +300,8 @@ def to_a
               location = range(token.location.start_offset, token.location.start_offset + percent_array_leading_whitespace(value))
               value = nil
             when :tSTRING_BEG
-              next_token, _ = lexed[index]
-              next_next_token, _ = lexed[index + 1]
+              next_token = lexed[index]
+              next_next_token = lexed[index + 1]
               basic_quotes = value == '"' || value == "'"
 
               if basic_quotes && next_token&.type == :STRING_END
@@ -369,7 +369,7 @@ def to_a
                 while token.type == :STRING_CONTENT
                   current_length += token.value.bytesize
                   # Heredoc interpolation can have multiple STRING_CONTENT nodes on the same line.
-                  prev_token, _ = lexed[index - 2] if index - 2 >= 0
+                  prev_token = lexed[index - 2] if index - 2 >= 0
                   is_first_token_on_line = prev_token && token.location.start_line != prev_token.location.start_line
                   # The parser gem only removes indentation when the heredoc is not nested
                   not_nested = heredoc_stack.size == 1
@@ -389,7 +389,7 @@ def to_a
                     tokens << [:tSTRING_CONTENT, [current_string, range(start_offset, start_offset + current_length)]]
                     break
                   end
-                  token, _ = lexed[index]
+                  token = lexed[index]
                   index += 1
                 end
               else
@@ -444,7 +444,7 @@ def to_a
               end
 
               if percent_array?(quote_stack.pop)
-                prev_token, _ = lexed[index - 2] if index - 2 >= 0
+                prev_token = lexed[index - 2] if index - 2 >= 0
                 empty = %i[PERCENT_LOWER_I PERCENT_LOWER_W PERCENT_UPPER_I PERCENT_UPPER_W].include?(prev_token&.type)
                 ends_with_whitespace = prev_token&.type == :WORDS_SEP
                 # parser always emits a space token after content in a percent array, even if no actual whitespace is present.
@@ -453,7 +453,7 @@ def to_a
                 end
               end
             when :tSYMBEG
-              if (next_token = lexed[index]&.first) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
+              if (next_token = lexed[index]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END
                 next_location = token.location.join(next_token.location)
                 type = :tSYMBOL
                 value = next_token.value
@@ -470,7 +470,7 @@ def to_a
             when :tXSTRING_BEG
               quote_stack.push(value)
             when :tSYMBOLS_BEG, :tQSYMBOLS_BEG, :tWORDS_BEG, :tQWORDS_BEG
-              if (next_token = lexed[index]&.first) && next_token.type == :WORDS_SEP
+              if (next_token = lexed[index]) && next_token.type == :WORDS_SEP
                 index += 1
               end
 
@@ -550,9 +550,9 @@ def calculate_heredoc_whitespace(heredoc_token_index)
           previous_line = -1
           result = Float::MAX
 
-          while (next_token = lexed[next_token_index]&.first)
+          while (next_token = lexed[next_token_index])
             next_token_index += 1
-            next_next_token, _ = lexed[next_token_index]
+            next_next_token = lexed[next_token_index]
             first_token_on_line = next_token.location.start_column == 0
 
             # String content inside nested heredocs and interpolation is ignored

prism/extension.c

@@ -846,14 +846,7 @@ parse_lex_token(pm_parser_t *parser, pm_token_t *token, void *data) {
     parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) data;
 
     VALUE value = pm_token_new(parser, token, parse_lex_data->encoding, parse_lex_data->source, parse_lex_data->freeze);
-    VALUE yields = rb_assoc_new(value, INT2FIX(pm_parser_lex_state(parser)));
-
-    if (parse_lex_data->freeze) {
-        rb_obj_freeze(value);
-        rb_obj_freeze(yields);
-    }
-
-    rb_ary_push(parse_lex_data->tokens, yields);
+    rb_ary_push(parse_lex_data->tokens, value);
 }
 
 /**
@@ -874,9 +867,7 @@ parse_lex_encoding_changed_callback(pm_parser_t *parser) {
     VALUE next_tokens = rb_ary_new();
 
     for (long index = 0; index < RARRAY_LEN(tokens); index++) {
-        VALUE yields = rb_ary_entry(tokens, index);
-        VALUE token = rb_ary_entry(yields, 0);
-
+        VALUE token = rb_ary_entry(tokens, index);
         VALUE value = rb_ivar_get(token, rb_intern("@value"));
         VALUE next_value = rb_str_dup(value);
 
@@ -887,18 +878,17 @@ parse_lex_encoding_changed_callback(pm_parser_t *parser) {
             parse_lex_data->source,
             rb_ivar_get(token, rb_intern("@type")),
             next_value,
-            rb_ivar_get(token, rb_intern("@location"))
+            rb_ivar_get(token, rb_intern("@location")),
+            rb_ivar_get(token, rb_intern("@state")),
         };
 
-        VALUE next_token = rb_class_new_instance(4, next_token_argv, rb_cPrismToken);
-        VALUE next_yields = rb_assoc_new(next_token, rb_ary_entry(yields, 1));
+        VALUE next_token = rb_class_new_instance(5, next_token_argv, rb_cPrismToken);
 
         if (parse_lex_data->freeze) {
             rb_obj_freeze(next_token);
-            rb_obj_freeze(next_yields);
         }
 
-        rb_ary_push(next_tokens, next_yields);
+        rb_ary_push(next_tokens, next_token);
     }
 
     rb_ary_replace(parse_lex_data->tokens, next_tokens);

prism/templates/ext/prism/api_node.c.erb

@@ -34,8 +34,8 @@ pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *en
     VALUE slice = rb_enc_str_new((const char *) token->start, token->end - token->start, encoding);
     if (freeze) rb_obj_freeze(slice);
 
-    VALUE argv[] = { source, ID2SYM(type), slice, location };
-    VALUE value = rb_class_new_instance(4, argv, rb_cPrismToken);
+    VALUE argv[] = { source, ID2SYM(type), slice, location, INT2FIX(pm_parser_lex_state(parser)) };
+    VALUE value = rb_class_new_instance(5, argv, rb_cPrismToken);
     if (freeze) rb_obj_freeze(value);
 
     return value;

prism/templates/lib/prism/serialize.rb.erb

@@ -182,7 +182,7 @@ module Prism
                        loader.load_constant_pool(constant_pool)
       raise unless     loader.eof?
 
-      value = [node, tokens] #: [ProgramNode, Array[[Token, Integer]]]
+      value = [node, tokens] #: [ProgramNode, Array[Token]]
       result = ParseLexResult.new(value, comments, magic_comments, data_loc, errors, warnings, continuable, source)
 
       tokens.each do |token|
@@ -434,18 +434,18 @@ module Prism
         warnings
       end
 
-      #: () -> Array[[Token, Integer]]
+      #: () -> Array[Token]
       def load_tokens
-        tokens = [] #: Array[[Token, Integer]]
+        tokens = [] #: Array[Token]
 
         while (type = TOKEN_TYPES.fetch(load_varuint))
           location = load_location_object(false)
 
           lex_state = load_varuint
 
-          token = Token.new(@source, type, location.slice, location)
+          token = Token.new(@source, type, location.slice, location, lex_state)
 
-          tokens << [token, lex_state]
+          tokens << token
         end
 
         tokens

test/prism/encoding/string_encoding_test.rb

@@ -46,7 +46,7 @@ def test_utf_8_star
     end
 
     def test_first_lexed_token
-      encoding = Prism.lex("# encoding: ascii-8bit").value[0][0].value.encoding
+      encoding = Prism.lex("# encoding: ascii-8bit").value[0].value.encoding
       assert_equal Encoding::ASCII_8BIT, encoding
     end
 

test/prism/lex_test.rb

@@ -50,7 +50,7 @@ def test_parse_lex_file
     def test_lex_encoding
       tokens = Prism.lex('"わたし"', encoding: Encoding::Windows_31J).value
       tokens.each do |t|
-        assert_equal(Encoding::Windows_31J, t[0].value.encoding)
+        assert_equal(Encoding::Windows_31J, t.value.encoding)
       end
 
       # Shebangs must appear on the first line. For these cases, the encoding
@@ -61,7 +61,18 @@ def test_lex_encoding
         "わたし"
       RUBY
       tokens.each do |t|
-        assert_equal(Encoding::UTF_8, t[0].value.encoding)
+        assert_equal(Encoding::UTF_8, t.value.encoding)
+      end
+    end
+
+    def test_lex_legacy
+      tokens = Prism.lex("foo").value
+
+      tokens.each do |token, state|
+        assert_nil(state)
+        assert_equal(token, token[0])
+        assert(token[1])
+        assert_equal(token, token.first)
       end
     end
 
@@ -117,7 +128,7 @@ def test_lex_heredoc_unterminated
     end
 
     def token_types(code)
-      Prism.lex(code).value.map { |token, _state| token.type }
+      Prism.lex(code).value.map(&:type)
     end
   end
 end