Read source and config files as UTF-8 regardless of locale

lib/typeprof/core/service.rb

@@ -55,7 +55,7 @@ def update_file(path, code)
     end
 
     def update_rb_file(path, code)
-      code = File.read(path) unless code
+      code = File.binread(path) unless code
       update_rb_ast(path, Prism.parse(code))
     end
 
@@ -122,7 +122,8 @@ def update_rb_ast(path, parse_result)
     def update_rbs_file(path, code)
       prev_decls = @rbs_text_nodes[path]
 
-      code = File.read(path) unless code
+      code = File.read(path, encoding: "UTF-8") unless code
+      return false unless code.valid_encoding?
       begin
         decls = AST.parse_rbs(path, code, @options[:position_encoding])
       rescue RBS::ParsingError
@@ -594,7 +595,7 @@ def batch(files, output)
           i += 1
         end
 
-        res = update_file(file, File.read(file))
+        res = update_file(file, nil)
 
         if res
           true

lib/typeprof/lsp/util.rb

@@ -1,6 +1,6 @@
 module TypeProf::LSP
   def self.load_json_with_comments(path, **opts)
-    json = File.read(path)
+    json = File.read(path, encoding: "UTF-8")
 
     state = :normal
     last_comma_index = nil

test/core/service_test.rb

@@ -1,6 +1,7 @@
 require_relative "../helper"
 require "stringio"
 require "tempfile"
+require "tmpdir"
 
 module TypeProf::Core
   class ServiceTest < Test::Unit::TestCase
@@ -21,5 +22,42 @@ def update_rb_file(*)
         assert_equal("# error: #{f.path}\n", output.string)
       end
     end
+
+    def test_add_workspace_reads_non_ascii_files_as_utf8
+      with_non_ascii_workspace do |rb_dir, rbs_dir|
+        with_default_external(Encoding::US_ASCII) do
+          service = TypeProf::Core::Service.new({})
+          service.add_workspace(rb_dir, rbs_dir)
+
+          assert_match(/def bar: -> String/, service.dump_declarations(File.join(rb_dir, "foo.rb")))
+        end
+      end
+    end
+
+    def test_batch_reads_non_ascii_files_as_utf8
+      with_non_ascii_workspace do |rb_dir, rbs_dir|
+        with_default_external(Encoding::US_ASCII) do
+          service = TypeProf::Core::Service.new({})
+          output = StringIO.new(+"")
+          files = [File.join(rbs_dir, "foo.rbs"), File.join(rb_dir, "foo.rb")]
+          service.batch(files, output)
+          assert_match(/def bar: -> String/, output.string)
+        end
+      end
+    end
+
+    private
+
+    def with_non_ascii_workspace
+      Dir.mktmpdir do |dir|
+        rb_dir = File.join(dir, "lib")
+        rbs_dir = File.join(dir, "sig")
+        Dir.mkdir(rb_dir)
+        Dir.mkdir(rbs_dir)
+        File.write(File.join(rbs_dir, "foo.rbs"), "# 日本語コメント\nclass Foo\n  def bar: () -> String\nend\n", encoding: "UTF-8")
+        File.write(File.join(rb_dir, "foo.rb"), "# 日本語コメント\nclass Foo\n  def bar = \"あ\"\nend\n", encoding: "UTF-8")
+        yield rb_dir, rbs_dir
+      end
+    end
   end
 end

test/helper.rb

@@ -5,3 +5,19 @@
 
 require "test/unit"
 require_relative "../lib/typeprof"
+
+class Test::Unit::TestCase
+  # Encoding.default_external is US-ASCII when LANG is not set, which must
+  # not affect how TypeProf reads files
+  def with_default_external(encoding)
+    orig = Encoding.default_external
+    verbose, $VERBOSE = $VERBOSE, nil
+    Encoding.default_external = encoding
+    $VERBOSE = verbose
+    yield
+  ensure
+    $VERBOSE = nil
+    Encoding.default_external = orig
+    $VERBOSE = verbose
+  end
+end

test/lsp/util_test.rb

@@ -0,0 +1,19 @@
+require_relative "../helper"
+require "tempfile"
+
+module TypeProf::LSP
+  class UtilTest < Test::Unit::TestCase
+    def test_load_json_with_comments_reads_non_ascii_as_utf8
+      Tempfile.create(["typeprof.conf", ".jsonc"]) do |f|
+        f.write("{\n  // 日本語コメント\n  \"typeprof_version\": \"experimental\",\n  \"analysis_unit_dirs\": [\"lib\"],\n}\n")
+        f.flush
+
+        conf = with_default_external(Encoding::US_ASCII) do
+          TypeProf::LSP.load_json_with_comments(f.path, symbolize_names: true)
+        end
+
+        assert_equal({ typeprof_version: "experimental", analysis_unit_dirs: ["lib"] }, conf)
+      end
+    end
+  end
+end