From f3ed64d30c71ed5b2f93b619485b6e842a034152 Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:38:30 +0200 Subject: [PATCH] [ruby/prism] Return a plain Token for `Prism.lex` etc. Instead of a 2-element array. Nobody should care about the state, and nobody seems to. A code search on Github shows that it only gets discarded (like via `value.map(&:first)`, or similar things. But because our own ripper translator still needs it for compatibility, we have to keep it around in some way. I played around with `Prism.lex_internal` that keeps the same return value but that duplicates much logic like serialization/deserialization. This achieves the same thing and is much smaller in scope. https://github.com/ruby/prism/commit/32455d191c --- lib/prism/lex_compat.rb | 14 ++++----- lib/prism/parse_result.rb | 34 +++++++++++++++++---- lib/prism/translation/parser/lexer.rb | 24 +++++++-------- prism/extension.c | 22 ++++--------- prism/templates/ext/prism/api_node.c.erb | 4 +-- prism/templates/lib/prism/serialize.rb.erb | 10 +++--- test/prism/encoding/string_encoding_test.rb | 2 +- test/prism/lex_test.rb | 17 +++++++++-- 8 files changed, 75 insertions(+), 52 deletions(-) diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index a2ad69cd2982a6..deb9ab99d87a1f 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -621,13 +621,13 @@ def result last_comment_token = nil #: lex_compat_token? last_comment_end = nil #: Integer? - result_value.each_with_index do |(prism_token, prism_state), index| + result_value.each_with_index do |prism_token, index| lineno = prism_token.location.start_line column = prism_token.location.start_column event = RIPPER.fetch(prism_token.type) value = prism_token.value - lex_state = Translation::Ripper::Lexer::State[prism_state] + lex_state = Translation::Ripper::Lexer::State[prism_token._ripper_state] # A comment token does not include its terminating newline, but # ripper's comment value does, so the newline token that directly @@ -701,7 +701,7 @@ def result # Ripper's lexed state. So here, if it's a regexp end token, we # output the state as the previous state, solely for the sake of # comparison. - previous_token = result_value[index - 1][0] + previous_token = result_value[index - 1] lex_state = if RIPPER.fetch(previous_token.type) == :on_embexpr_end # If the previous token is embexpr_end, then we have to do even @@ -714,11 +714,11 @@ def result until counter == 0 current_index -= 1 - current_event = RIPPER.fetch(result_value[current_index][0].type) + current_event = RIPPER.fetch(result_value[current_index].type) counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0 end - Translation::Ripper::Lexer::State[result_value[current_index][1]] + Translation::Ripper::Lexer::State[result_value[current_index]._ripper_state] else previous_state end @@ -726,12 +726,12 @@ def result [[lineno, column], event, value, lex_state] when :on_eof eof_token = prism_token - previous_token = result_value[index - 1][0] + previous_token = result_value[index - 1] # A newline that was folded back into a comment still marks the # comment boundary for the check below. comment_boundary = previous_token.type == :COMMENT || - (index >= 2 && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous_token.type) && result_value[index - 2][0].type == :COMMENT && result_value[index - 2][0].location.end_offset == previous_token.location.start_offset) + (index >= 2 && %i[NEWLINE NEWLINE_TERMINATOR IGNORED_NEWLINE].include?(previous_token.type) && result_value[index - 2].type == :COMMENT && result_value[index - 2].location.end_offset == previous_token.location.start_offset) # If we're at the end of the file and the previous token was a # comment and there is still whitespace after the comment, then diff --git a/lib/prism/parse_result.rb b/lib/prism/parse_result.rb index 93d3c006b72b65..0d1d2fc84556f5 100644 --- a/lib/prism/parse_result.rb +++ b/lib/prism/parse_result.rb @@ -1053,11 +1053,11 @@ def errors_format # This is a result specific to the `lex` and `lex_file` methods. class LexResult < Result # The list of tokens that were parsed from the source code. - attr_reader :value #: Array[[Token, Integer]] + attr_reader :value #: Array[Token] # Create a new lex result object with the given values. #-- - #: (Array[[Token, Integer]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void + #: (Array[Token] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void def initialize(value, comments, magic_comments, data_loc, errors, warnings, continuable, source) @value = value super(comments, magic_comments, data_loc, errors, warnings, continuable, source) @@ -1075,11 +1075,11 @@ def deconstruct_keys(keys) # :nodoc: class ParseLexResult < Result # A tuple of the syntax tree and the list of tokens that were parsed from # the source code. - attr_reader :value #: [ProgramNode, Array[[Token, Integer]]] + attr_reader :value #: [ProgramNode, Array[Token]] # Create a new parse lex result object with the given values. #-- - #: ([ProgramNode, Array[[Token, Integer]]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void + #: ([ProgramNode, Array[Token]] value, Array[Comment] comments, Array[MagicComment] magic_comments, Location? data_loc, Array[ParseError] errors, Array[ParseWarning] warnings, bool continuable, Source source) -> void def initialize(value, comments, magic_comments, data_loc, errors, warnings, continuable, source) @value = value super(comments, magic_comments, data_loc, errors, warnings, continuable, source) @@ -1109,12 +1109,13 @@ class Token # Create a new token object with the given type, value, and location. #-- - #: (Source source, Symbol type, String value, Location | Integer location) -> void - def initialize(source, type, value, location) + #: (Source source, Symbol type, String value, Location | Integer location, Integer state) -> void + def initialize(source, type, value, location, state) @source = source @type = type @value = value @location = location + @state = state end # Implement the hash pattern matching interface for Token. @@ -1175,6 +1176,27 @@ def deep_freeze location.freeze freeze end + + # For internal use only. + #: () -> Integer + def _ripper_state # :nodoc: + @state + end + + # Backwards compatibility for Prism.lex/Prism.lex_file/Prism.parse_lex + # when they returned a 2-element array. + + #: (Integer index) -> Token | Integer + def [](index) # :nodoc: + return self if index == 0 + return @state if index == 1 + raise ArgumentError, "Invalid index #{index}" + end + + #: () -> Token + def first # :nodoc: + self + end end # This object is passed to the various Prism.* methods that accept the diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb index 34d7e637490937..a2f517298aae8b 100644 --- a/lib/prism/translation/parser/lexer.rb +++ b/lib/prism/translation/parser/lexer.rb @@ -230,7 +230,7 @@ def to_a comment_newline_location = nil while index < length - token, _ = lexed[index] + token = lexed[index] index += 1 next if TYPES_ALWAYS_SKIP.include?(token.type) @@ -253,7 +253,7 @@ def to_a when :tCOMMENT if token.type == :EMBDOC_BEGIN - while !((next_token = lexed[index]&.first) && next_token.type == :EMBDOC_END) && (index < length - 1) + while !((next_token = lexed[index]) && next_token.type == :EMBDOC_END) && (index < length - 1) value += next_token.value index += 1 end @@ -267,7 +267,7 @@ def to_a location = range(token.location.start_offset, token.location.end_offset - 1) if value.chomp! end when :tNL - next_token, _ = lexed[index] + next_token = lexed[index] # Newlines after comments are emitted out of order. if next_token&.type == :COMMENT comment_newline_location = location @@ -300,8 +300,8 @@ def to_a location = range(token.location.start_offset, token.location.start_offset + percent_array_leading_whitespace(value)) value = nil when :tSTRING_BEG - next_token, _ = lexed[index] - next_next_token, _ = lexed[index + 1] + next_token = lexed[index] + next_next_token = lexed[index + 1] basic_quotes = value == '"' || value == "'" if basic_quotes && next_token&.type == :STRING_END @@ -369,7 +369,7 @@ def to_a while token.type == :STRING_CONTENT current_length += token.value.bytesize # Heredoc interpolation can have multiple STRING_CONTENT nodes on the same line. - prev_token, _ = lexed[index - 2] if index - 2 >= 0 + prev_token = lexed[index - 2] if index - 2 >= 0 is_first_token_on_line = prev_token && token.location.start_line != prev_token.location.start_line # The parser gem only removes indentation when the heredoc is not nested not_nested = heredoc_stack.size == 1 @@ -389,7 +389,7 @@ def to_a tokens << [:tSTRING_CONTENT, [current_string, range(start_offset, start_offset + current_length)]] break end - token, _ = lexed[index] + token = lexed[index] index += 1 end else @@ -444,7 +444,7 @@ def to_a end if percent_array?(quote_stack.pop) - prev_token, _ = lexed[index - 2] if index - 2 >= 0 + prev_token = lexed[index - 2] if index - 2 >= 0 empty = %i[PERCENT_LOWER_I PERCENT_LOWER_W PERCENT_UPPER_I PERCENT_UPPER_W].include?(prev_token&.type) ends_with_whitespace = prev_token&.type == :WORDS_SEP # parser always emits a space token after content in a percent array, even if no actual whitespace is present. @@ -453,7 +453,7 @@ def to_a end end when :tSYMBEG - if (next_token = lexed[index]&.first) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END + if (next_token = lexed[index]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR && next_token.type != :STRING_END next_location = token.location.join(next_token.location) type = :tSYMBOL value = next_token.value @@ -470,7 +470,7 @@ def to_a when :tXSTRING_BEG quote_stack.push(value) when :tSYMBOLS_BEG, :tQSYMBOLS_BEG, :tWORDS_BEG, :tQWORDS_BEG - if (next_token = lexed[index]&.first) && next_token.type == :WORDS_SEP + if (next_token = lexed[index]) && next_token.type == :WORDS_SEP index += 1 end @@ -550,9 +550,9 @@ def calculate_heredoc_whitespace(heredoc_token_index) previous_line = -1 result = Float::MAX - while (next_token = lexed[next_token_index]&.first) + while (next_token = lexed[next_token_index]) next_token_index += 1 - next_next_token, _ = lexed[next_token_index] + next_next_token = lexed[next_token_index] first_token_on_line = next_token.location.start_column == 0 # String content inside nested heredocs and interpolation is ignored diff --git a/prism/extension.c b/prism/extension.c index e411a4972561c0..2e3992474adf8a 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -846,14 +846,7 @@ parse_lex_token(pm_parser_t *parser, pm_token_t *token, void *data) { parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) data; VALUE value = pm_token_new(parser, token, parse_lex_data->encoding, parse_lex_data->source, parse_lex_data->freeze); - VALUE yields = rb_assoc_new(value, INT2FIX(pm_parser_lex_state(parser))); - - if (parse_lex_data->freeze) { - rb_obj_freeze(value); - rb_obj_freeze(yields); - } - - rb_ary_push(parse_lex_data->tokens, yields); + rb_ary_push(parse_lex_data->tokens, value); } /** @@ -874,9 +867,7 @@ parse_lex_encoding_changed_callback(pm_parser_t *parser) { VALUE next_tokens = rb_ary_new(); for (long index = 0; index < RARRAY_LEN(tokens); index++) { - VALUE yields = rb_ary_entry(tokens, index); - VALUE token = rb_ary_entry(yields, 0); - + VALUE token = rb_ary_entry(tokens, index); VALUE value = rb_ivar_get(token, rb_intern("@value")); VALUE next_value = rb_str_dup(value); @@ -887,18 +878,17 @@ parse_lex_encoding_changed_callback(pm_parser_t *parser) { parse_lex_data->source, rb_ivar_get(token, rb_intern("@type")), next_value, - rb_ivar_get(token, rb_intern("@location")) + rb_ivar_get(token, rb_intern("@location")), + rb_ivar_get(token, rb_intern("@state")), }; - VALUE next_token = rb_class_new_instance(4, next_token_argv, rb_cPrismToken); - VALUE next_yields = rb_assoc_new(next_token, rb_ary_entry(yields, 1)); + VALUE next_token = rb_class_new_instance(5, next_token_argv, rb_cPrismToken); if (parse_lex_data->freeze) { rb_obj_freeze(next_token); - rb_obj_freeze(next_yields); } - rb_ary_push(next_tokens, next_yields); + rb_ary_push(next_tokens, next_token); } rb_ary_replace(parse_lex_data->tokens, next_tokens); diff --git a/prism/templates/ext/prism/api_node.c.erb b/prism/templates/ext/prism/api_node.c.erb index 41d7165930d38a..9ef799be1060be 100644 --- a/prism/templates/ext/prism/api_node.c.erb +++ b/prism/templates/ext/prism/api_node.c.erb @@ -34,8 +34,8 @@ pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *en VALUE slice = rb_enc_str_new((const char *) token->start, token->end - token->start, encoding); if (freeze) rb_obj_freeze(slice); - VALUE argv[] = { source, ID2SYM(type), slice, location }; - VALUE value = rb_class_new_instance(4, argv, rb_cPrismToken); + VALUE argv[] = { source, ID2SYM(type), slice, location, INT2FIX(pm_parser_lex_state(parser)) }; + VALUE value = rb_class_new_instance(5, argv, rb_cPrismToken); if (freeze) rb_obj_freeze(value); return value; diff --git a/prism/templates/lib/prism/serialize.rb.erb b/prism/templates/lib/prism/serialize.rb.erb index ec6a19ead014a0..e65acbfef81585 100644 --- a/prism/templates/lib/prism/serialize.rb.erb +++ b/prism/templates/lib/prism/serialize.rb.erb @@ -182,7 +182,7 @@ module Prism loader.load_constant_pool(constant_pool) raise unless loader.eof? - value = [node, tokens] #: [ProgramNode, Array[[Token, Integer]]] + value = [node, tokens] #: [ProgramNode, Array[Token]] result = ParseLexResult.new(value, comments, magic_comments, data_loc, errors, warnings, continuable, source) tokens.each do |token| @@ -434,18 +434,18 @@ module Prism warnings end - #: () -> Array[[Token, Integer]] + #: () -> Array[Token] def load_tokens - tokens = [] #: Array[[Token, Integer]] + tokens = [] #: Array[Token] while (type = TOKEN_TYPES.fetch(load_varuint)) location = load_location_object(false) lex_state = load_varuint - token = Token.new(@source, type, location.slice, location) + token = Token.new(@source, type, location.slice, location, lex_state) - tokens << [token, lex_state] + tokens << token end tokens diff --git a/test/prism/encoding/string_encoding_test.rb b/test/prism/encoding/string_encoding_test.rb index bc563b089384bd..db4d51f7bd61ec 100644 --- a/test/prism/encoding/string_encoding_test.rb +++ b/test/prism/encoding/string_encoding_test.rb @@ -46,7 +46,7 @@ def test_utf_8_star end def test_first_lexed_token - encoding = Prism.lex("# encoding: ascii-8bit").value[0][0].value.encoding + encoding = Prism.lex("# encoding: ascii-8bit").value[0].value.encoding assert_equal Encoding::ASCII_8BIT, encoding end diff --git a/test/prism/lex_test.rb b/test/prism/lex_test.rb index 1e06d52184b3c5..8aaf754d5dfa02 100644 --- a/test/prism/lex_test.rb +++ b/test/prism/lex_test.rb @@ -50,7 +50,7 @@ def test_parse_lex_file def test_lex_encoding tokens = Prism.lex('"わたし"', encoding: Encoding::Windows_31J).value tokens.each do |t| - assert_equal(Encoding::Windows_31J, t[0].value.encoding) + assert_equal(Encoding::Windows_31J, t.value.encoding) end # Shebangs must appear on the first line. For these cases, the encoding @@ -61,7 +61,18 @@ def test_lex_encoding "わたし" RUBY tokens.each do |t| - assert_equal(Encoding::UTF_8, t[0].value.encoding) + assert_equal(Encoding::UTF_8, t.value.encoding) + end + end + + def test_lex_legacy + tokens = Prism.lex("foo").value + + tokens.each do |token, state| + assert_nil(state) + assert_equal(token, token[0]) + assert(token[1]) + assert_equal(token, token.first) end end @@ -117,7 +128,7 @@ def test_lex_heredoc_unterminated end def token_types(code) - Prism.lex(code).value.map { |token, _state| token.type } + Prism.lex(code).value.map(&:type) end end end