From 554d384bcc6535465e69ef5e36dfa0544447336f Mon Sep 17 00:00:00 2001 From: Takumi Shotoku Date: Wed, 9 Sep 2026 21:15:57 +0900 Subject: [PATCH 1/5] Read source and config files as UTF-8 regardless of locale `File.read` without an encoding follows `Encoding.default_external`, which is US-ASCII when `LANG` is not set (e.g., ruby-lsp started from a GUI editor). With a `.rbs` file containing non-ASCII characters, `RBS::Parser` then raises `ArgumentError` (invalid byte sequence in US-ASCII). This is not a `RBS::ParsingError` and is not rescued, so one such file aborts `add_workspace` and `batch` entirely. `typeprof.conf.jsonc` has the same problem. Ruby files are mostly fine because Prism ignores the encoding of the source string. They still fail when `Encoding.default_internal` is set (e.g., `ruby -U`), since `File.read` then transcodes from US-ASCII. So this reads them as UTF-8 too. The rbs gem has read signature files as UTF-8 for the same reason since ruby/rbs#493. --- lib/typeprof/core/service.rb | 6 +++--- lib/typeprof/lsp/util.rb | 2 +- test/core/service_test.rb | 38 ++++++++++++++++++++++++++++++++++++ test/helper.rb | 12 ++++++++++++ test/lsp/util_test.rb | 19 ++++++++++++++++++ 5 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 test/lsp/util_test.rb diff --git a/lib/typeprof/core/service.rb b/lib/typeprof/core/service.rb index ed8077f9d..5adc6fbbd 100644 --- a/lib/typeprof/core/service.rb +++ b/lib/typeprof/core/service.rb @@ -55,7 +55,7 @@ def update_file(path, code) end def update_rb_file(path, code) - code = File.read(path) unless code + code = File.read(path, encoding: "UTF-8") unless code update_rb_ast(path, Prism.parse(code)) end @@ -122,7 +122,7 @@ def update_rb_ast(path, parse_result) def update_rbs_file(path, code) prev_decls = @rbs_text_nodes[path] - code = File.read(path) unless code + code = File.read(path, encoding: "UTF-8") unless code begin decls = AST.parse_rbs(path, code, @options[:position_encoding]) rescue RBS::ParsingError @@ -586,7 +586,7 @@ def batch(files, output) i += 1 end - res = update_file(file, File.read(file)) + res = update_file(file, File.read(file, encoding: "UTF-8")) if res true diff --git a/lib/typeprof/lsp/util.rb b/lib/typeprof/lsp/util.rb index 2cab8345c..f8f9f0bb8 100644 --- a/lib/typeprof/lsp/util.rb +++ b/lib/typeprof/lsp/util.rb @@ -1,6 +1,6 @@ module TypeProf::LSP def self.load_json_with_comments(path, **opts) - json = File.read(path) + json = File.read(path, encoding: "UTF-8") state = :normal last_comma_index = nil diff --git a/test/core/service_test.rb b/test/core/service_test.rb index 263b85b95..2fd7619c9 100644 --- a/test/core/service_test.rb +++ b/test/core/service_test.rb @@ -1,6 +1,7 @@ require_relative "../helper" require "stringio" require "tempfile" +require "tmpdir" module TypeProf::Core class ServiceTest < Test::Unit::TestCase @@ -21,5 +22,42 @@ def update_rb_file(*) assert_equal("# error: #{f.path}\n", output.string) end end + + def test_add_workspace_reads_non_ascii_files_as_utf8 + with_non_ascii_workspace do |rb_dir, rbs_dir| + with_default_external(Encoding::US_ASCII) do + service = TypeProf::Core::Service.new({}) + service.add_workspace(rb_dir, rbs_dir) + + assert_match(/def bar: -> String/, service.dump_declarations(File.join(rb_dir, "foo.rb"))) + end + end + end + + def test_batch_reads_non_ascii_files_as_utf8 + with_non_ascii_workspace do |rb_dir, rbs_dir| + with_default_external(Encoding::US_ASCII) do + service = TypeProf::Core::Service.new({}) + output = StringIO.new(+"") + files = [File.join(rbs_dir, "foo.rbs"), File.join(rb_dir, "foo.rb")] + service.batch(files, output) + assert_match(/def bar: -> String/, output.string) + end + end + end + + private + + def with_non_ascii_workspace + Dir.mktmpdir do |dir| + rb_dir = File.join(dir, "lib") + rbs_dir = File.join(dir, "sig") + Dir.mkdir(rb_dir) + Dir.mkdir(rbs_dir) + File.write(File.join(rbs_dir, "foo.rbs"), "# 日本語コメント\nclass Foo\n def bar: () -> String\nend\n", encoding: "UTF-8") + File.write(File.join(rb_dir, "foo.rb"), "# 日本語コメント\nclass Foo\n def bar = \"あ\"\nend\n", encoding: "UTF-8") + yield rb_dir, rbs_dir + end + end end end diff --git a/test/helper.rb b/test/helper.rb index 72f459c50..023f7ef06 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -5,3 +5,15 @@ require "test/unit" require_relative "../lib/typeprof" + +class Test::Unit::TestCase + # Encoding.default_external is US-ASCII when LANG is not set, which must + # not affect how TypeProf reads files + def with_default_external(encoding) + orig = Encoding.default_external + Encoding.default_external = encoding + yield + ensure + Encoding.default_external = orig + end +end diff --git a/test/lsp/util_test.rb b/test/lsp/util_test.rb new file mode 100644 index 000000000..099e5a5d9 --- /dev/null +++ b/test/lsp/util_test.rb @@ -0,0 +1,19 @@ +require_relative "../helper" +require "tempfile" + +module TypeProf::LSP + class UtilTest < Test::Unit::TestCase + def test_load_json_with_comments_reads_non_ascii_as_utf8 + Tempfile.create(["typeprof.conf", ".jsonc"]) do |f| + f.write("{\n // 日本語コメント\n \"typeprof_version\": \"experimental\",\n \"analysis_unit_dirs\": [\"lib\"],\n}\n") + f.flush + + conf = with_default_external(Encoding::US_ASCII) do + TypeProf::LSP.load_json_with_comments(f.path, symbolize_names: true) + end + + assert_equal({ typeprof_version: "experimental", analysis_unit_dirs: ["lib"] }, conf) + end + end + end +end From 51ef1923823b6cf3bb2bce2971e998aa8fb56c6c Mon Sep 17 00:00:00 2001 From: Takumi Shotoku Date: Mon, 28 Sep 2026 22:38:05 +0900 Subject: [PATCH 2/5] Support encodings other than UTF-8 Co-authored-by: Yusuke Endoh --- lib/typeprof/core/service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/typeprof/core/service.rb b/lib/typeprof/core/service.rb index 5adc6fbbd..a58461021 100644 --- a/lib/typeprof/core/service.rb +++ b/lib/typeprof/core/service.rb @@ -55,7 +55,7 @@ def update_file(path, code) end def update_rb_file(path, code) - code = File.read(path, encoding: "UTF-8") unless code + code = File.binread(path) unless code update_rb_ast(path, Prism.parse(code)) end From 4951edee2a513ca65bcadb6b8baf97388fb580c9 Mon Sep 17 00:00:00 2001 From: Takumi Shotoku Date: Mon, 28 Sep 2026 22:40:33 +0900 Subject: [PATCH 3/5] Treat an invalid `.rbs` as a parse failure Co-authored-by: Yusuke Endoh --- lib/typeprof/core/service.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/typeprof/core/service.rb b/lib/typeprof/core/service.rb index a58461021..f186bdadd 100644 --- a/lib/typeprof/core/service.rb +++ b/lib/typeprof/core/service.rb @@ -123,6 +123,7 @@ def update_rbs_file(path, code) prev_decls = @rbs_text_nodes[path] code = File.read(path, encoding: "UTF-8") unless code + return false unless code.valid_encoding? begin decls = AST.parse_rbs(path, code, @options[:position_encoding]) rescue RBS::ParsingError From 30de2f99e03d41c67a528c38c260900eb6e21d8e Mon Sep 17 00:00:00 2001 From: Takumi Shotoku Date: Mon, 28 Sep 2026 22:44:41 +0900 Subject: [PATCH 4/5] Use `update_rb_file` and `update_rbs_file` in their own way Co-authored-by: Yusuke Endoh --- lib/typeprof/core/service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/typeprof/core/service.rb b/lib/typeprof/core/service.rb index f186bdadd..8382cb76b 100644 --- a/lib/typeprof/core/service.rb +++ b/lib/typeprof/core/service.rb @@ -587,7 +587,7 @@ def batch(files, output) i += 1 end - res = update_file(file, File.read(file, encoding: "UTF-8")) + res = update_file(file, nil) if res true From f7e52810f40d12d15e913ef91adc13b6972ac720 Mon Sep 17 00:00:00 2001 From: Takumi Shotoku Date: Mon, 28 Sep 2026 22:46:00 +0900 Subject: [PATCH 5/5] Suppress warnings regarding Encoding.default_external Co-authored-by: Yusuke Endoh --- test/helper.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/helper.rb b/test/helper.rb index 023f7ef06..87a7eeae9 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -11,9 +11,13 @@ class Test::Unit::TestCase # not affect how TypeProf reads files def with_default_external(encoding) orig = Encoding.default_external + verbose, $VERBOSE = $VERBOSE, nil Encoding.default_external = encoding + $VERBOSE = verbose yield ensure + $VERBOSE = nil Encoding.default_external = orig + $VERBOSE = verbose end end