diff --git a/tsc/internal/scanner/regexp.go b/tsc/internal/scanner/regexp.go index 0e83c1c589094..fb76fe68b2c28 100644 --- a/tsc/internal/scanner/regexp.go +++ b/tsc/internal/scanner/regexp.go @@ -1028,7 +1028,12 @@ func (p *regExpParser) scanSourceCharacter() string { return stringutil.EncodeJSStringRune(low) } ch, size := utf8.DecodeRuneInString(p.text()[p.pos():]) - if ch == utf8.RuneError || size == 0 { + // utf8.RuneError is U+FFFD. A real U+FFFD decodes with size 3; only size 1 + // means the input byte was not valid UTF-8 (see utf8.DecodeRune). + if size == 0 { + return "" + } + if ch == utf8.RuneError && size == 1 { // Not a valid rune; consume one raw byte. p.incPos(1) return string(p.text()[p.pos()-1]) @@ -1047,9 +1052,9 @@ func (p *regExpParser) scanSourceCharacter() string { if size == 0 { return "" } - if ch == utf8.RuneError { + if ch == utf8.RuneError && size == 1 { // Invalid UTF-8; consume the byte to avoid infinite loops. - p.incPos(size) + p.incPos(1) return "" } p.incPos(size) diff --git a/tsc/internal/scanner/scanner_test.go b/tsc/internal/scanner/scanner_test.go index ac6f864943f70..a2e666214c561 100644 --- a/tsc/internal/scanner/scanner_test.go +++ b/tsc/internal/scanner/scanner_test.go @@ -3,9 +3,11 @@ package scanner import ( "strings" "testing" + "unicode/utf8" "github.com/microsoft/TypeScript/tsc/internal/ast" "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/diagnostics" "github.com/microsoft/TypeScript/tsc/internal/stringutil" "gotest.tools/v3/assert" ) @@ -125,3 +127,49 @@ func TestLookupKeywordMatchesKeywordMap(t *testing.T) { assert.Equal(t, GetIdentifierToken(text), ast.KindIdentifier, text) } } + +// https://github.com/microsoft/TypeScript/issues/65076 +// A literal U+FFFD in a regex character-class range must not be mistaken for +// utf8.RuneError (which is the same code point, but only signals invalid UTF-8 +// when DecodeRune returns size 1). +func TestRegExpLiteralReplacementCharacterInClassRange(t *testing.T) { + t.Parallel() + + fffd := string(utf8.RuneError) // U+FFFD, encoded as the 3-byte UTF-8 sequence + assert.Equal(t, len(fffd), 3) + + tests := []struct { + name string + text string + }{ + {name: "range ending in literal U+FFFD", text: `export const r = /[\uFFF9-` + fffd + `]/;`}, + {name: "range ending in literal U+FFFD with u flag", text: `export const r = /[\uFFF9-` + fffd + `]/u;`}, + {name: "escaped U+FFFD range end", text: `export const r = /[\uFFF9-\uFFFD]/;`}, + {name: "lone literal U+FFFD", text: `export const r = /` + fffd + `/;`}, + {name: "range starting with literal U+FFFD", text: `export const r = /[` + fffd + `-\uFFFF]/;`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + s := NewScanner() + var codes []int32 + s.SetOnError(func(d *diagnostics.Message, start, length int, args ...any) { + codes = append(codes, d.Code()) + }) + s.SetText(test.text) + for { + tok := s.Scan() + if tok == ast.KindSlashToken || tok == ast.KindSlashEqualsToken { + tok = s.ReScanSlashToken(true) + } + if tok == ast.KindEndOfFile { + break + } + } + for _, code := range codes { + assert.Assert(t, code != 1516 && code != 1517, "unexpected regex character-class diagnostic %d for %q (got %v)", code, test.text, codes) + } + }) + } +}