Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions tsc/internal/scanner/regexp.go
Original file line number Diff line number Diff line change
Expand Up @@ -1028,7 +1028,12 @@ func (p *regExpParser) scanSourceCharacter() string {
return stringutil.EncodeJSStringRune(low)
}
ch, size := utf8.DecodeRuneInString(p.text()[p.pos():])
if ch == utf8.RuneError || size == 0 {
// utf8.RuneError is U+FFFD. A real U+FFFD decodes with size 3; only size 1
// means the input byte was not valid UTF-8 (see utf8.DecodeRune).
if size == 0 {
return ""
}
if ch == utf8.RuneError && size == 1 {
// Not a valid rune; consume one raw byte.
p.incPos(1)
return string(p.text()[p.pos()-1])
Expand All @@ -1047,9 +1052,9 @@ func (p *regExpParser) scanSourceCharacter() string {
if size == 0 {
return ""
}
if ch == utf8.RuneError {
if ch == utf8.RuneError && size == 1 {
// Invalid UTF-8; consume the byte to avoid infinite loops.
p.incPos(size)
p.incPos(1)
return ""
}
p.incPos(size)
Expand Down
48 changes: 48 additions & 0 deletions tsc/internal/scanner/scanner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ package scanner
import (
"strings"
"testing"
"unicode/utf8"

"github.com/microsoft/TypeScript/tsc/internal/ast"
"github.com/microsoft/TypeScript/tsc/internal/core"
"github.com/microsoft/TypeScript/tsc/internal/diagnostics"
"github.com/microsoft/TypeScript/tsc/internal/stringutil"
"gotest.tools/v3/assert"
)
Expand Down Expand Up @@ -125,3 +127,49 @@ func TestLookupKeywordMatchesKeywordMap(t *testing.T) {
assert.Equal(t, GetIdentifierToken(text), ast.KindIdentifier, text)
}
}

// https://github.com/microsoft/TypeScript/issues/65076
// A literal U+FFFD in a regex character-class range must not be mistaken for
// utf8.RuneError (which is the same code point, but only signals invalid UTF-8
// when DecodeRune returns size 1).
func TestRegExpLiteralReplacementCharacterInClassRange(t *testing.T) {
t.Parallel()

fffd := string(utf8.RuneError) // U+FFFD, encoded as the 3-byte UTF-8 sequence
assert.Equal(t, len(fffd), 3)

tests := []struct {
name string
text string
}{
{name: "range ending in literal U+FFFD", text: `export const r = /[\uFFF9-` + fffd + `]/;`},
{name: "range ending in literal U+FFFD with u flag", text: `export const r = /[\uFFF9-` + fffd + `]/u;`},
{name: "escaped U+FFFD range end", text: `export const r = /[\uFFF9-\uFFFD]/;`},
{name: "lone literal U+FFFD", text: `export const r = /` + fffd + `/;`},
{name: "range starting with literal U+FFFD", text: `export const r = /[` + fffd + `-\uFFFF]/;`},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
s := NewScanner()
var codes []int32
s.SetOnError(func(d *diagnostics.Message, start, length int, args ...any) {
codes = append(codes, d.Code())
})
s.SetText(test.text)
for {
tok := s.Scan()
if tok == ast.KindSlashToken || tok == ast.KindSlashEqualsToken {
tok = s.ReScanSlashToken(true)
}
if tok == ast.KindEndOfFile {
break
}
}
for _, code := range codes {
assert.Assert(t, code != 1516 && code != 1517, "unexpected regex character-class diagnostic %d for %q (got %v)", code, test.text, codes)
}
})
}
}
Loading