From 3350dda3a7fdf106e183dc44aa1eba3c9c9dfdeb Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Mon, 26 Oct 2026 12:50:09 -0700 Subject: [PATCH 1/5] Add SIMD buffer traversal and matching for scanner-heavy operations --- .github/workflows/ci.yml | 6 ++ tsc/internal/ast/positionmap.go | 2 +- tsc/internal/ast/positionmap_scan_test.go | 72 ++++++++++++++++ tsc/internal/core/core.go | 39 +++++++-- tsc/internal/core/textscan_test.go | 88 ++++++++++++++++++++ tsc/internal/parser/parser_test.go | 18 +++- tsc/internal/printer/textscan_test.go | 62 ++++++++++++++ tsc/internal/printer/utilities.go | 11 +++ tsc/internal/stringutil/textscan.go | 41 +++++++++ tsc/internal/stringutil/textscan_scalar.go | 19 +++++ tsc/internal/stringutil/textscan_simd.go | 96 ++++++++++++++++++++++ tsc/internal/stringutil/textscan_test.go | 47 +++++++++++ 12 files changed, 490 insertions(+), 11 deletions(-) create mode 100644 tsc/internal/ast/positionmap_scan_test.go create mode 100644 tsc/internal/core/textscan_test.go create mode 100644 tsc/internal/printer/textscan_test.go create mode 100644 tsc/internal/stringutil/textscan.go create mode 100644 tsc/internal/stringutil/textscan_scalar.go create mode 100644 tsc/internal/stringutil/textscan_simd.go create mode 100644 tsc/internal/stringutil/textscan_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f61c5c914c24..3d2ded3b5c284 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,11 @@ jobs: os: linux name: 'noembed' noembed: true + - runner: ubuntu-latest + os: linux + name: 'simd' + simd: true + skip: ${{ github.event_name == 'merge_group' }} - runner: ubuntu-latest os: linux name: 'concurrent test programs' @@ -132,6 +137,7 @@ jobs: id-token: write contents: read env: + GOEXPERIMENT: ${{ (matrix.config.simd && 'simd') || '' }} TSGO_HEREBY_RACE: ${{ (matrix.config.race && 'true') || 'false' }} TSGO_HEREBY_NOEMBED: ${{ (matrix.config.noembed && 'true') || 'false' }} TSGO_HEREBY_CONCURRENT_TEST_PROGRAMS: ${{ (matrix.config.concurrent-test-programs && 'true') || 'false' }} diff --git a/tsc/internal/ast/positionmap.go b/tsc/internal/ast/positionmap.go index 4c2fde6b6842e..13a2a24fca6ca 100644 --- a/tsc/internal/ast/positionmap.go +++ b/tsc/internal/ast/positionmap.go @@ -43,7 +43,7 @@ func ComputePositionMap(text string) *PositionMap { for i := 0; i < len(text); { b := text[i] if b < utf8.RuneSelf { - i++ + i += stringutil.ASCIIPrefixLen(text[i:]) continue } r, size := stringutil.DecodeJSStringRune(text[i:]) diff --git a/tsc/internal/ast/positionmap_scan_test.go b/tsc/internal/ast/positionmap_scan_test.go new file mode 100644 index 0000000000000..6a4da8a3b46a6 --- /dev/null +++ b/tsc/internal/ast/positionmap_scan_test.go @@ -0,0 +1,72 @@ +package ast_test + +import ( + "strings" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/ast" + "github.com/microsoft/TypeScript/tsc/internal/stringutil" + "github.com/microsoft/TypeScript/tsc/internal/testutil/fixtures" +) + +func TestPositionMapScanBoundaries(t *testing.T) { + t.Parallel() + for prefix := range 130 { + for _, middle := range []string{ + "", "\u00E9", "\u65E5", "\U0001F980", "\xff", "\xe2\x80", + stringutil.EncodeJSStringRune(0xD800), "\u00E9a\U0001F980", + } { + text := strings.Repeat("a", prefix) + middle + strings.Repeat("z", 129) + pm := ast.ComputePositionMap(text) + type entry struct{ bytePos, delta int } + var entries []entry + delta := 0 + for i := 0; i < len(text); { + r, size := stringutil.DecodeJSStringRune(text[i:]) + units := 1 + if r >= 0x10000 { + units = 2 + } + if text[i] >= 0x80 { + delta += size - units + entries = append(entries, entry{i + size, delta}) + } + i += size + } + if pm.IsAsciiOnly() != (len(entries) == 0) { + t.Fatalf("IsAsciiOnly for %q", text) + } + for pos := 0; pos <= len(text); pos++ { + want16, want8 := pos, pos + for _, e := range entries { + if e.bytePos <= pos { + want16 = pos - e.delta + } + if e.bytePos-e.delta <= pos { + want8 = pos + e.delta + } + } + if got := pm.UTF8ToUTF16(pos); got != want16 { + t.Fatalf("UTF8ToUTF16(%d) = %d, want %d for %q", pos, got, want16, text) + } + if got := pm.UTF16ToUTF8(pos); got != want8 { + t.Fatalf("UTF16ToUTF8(%d) = %d, want %d for %q", pos, got, want8, text) + } + } + } + } +} + +func BenchmarkPositionMapScan(b *testing.B) { + for _, fixture := range fixtures.BenchFixtures { + b.Run(fixture.Name(), func(b *testing.B) { + fixture.SkipIfNotExist(b) + text := fixture.ReadFile(b) + b.SetBytes(int64(len(text))) + b.ReportAllocs() + for b.Loop() { + ast.ComputePositionMap(text) + } + }) + } +} diff --git a/tsc/internal/core/core.go b/tsc/internal/core/core.go index f5c296b9b5bb0..a28c0739bee7b 100644 --- a/tsc/internal/core/core.go +++ b/tsc/internal/core/core.go @@ -436,6 +436,10 @@ func ComputeECMALineStartsSeq(text string) iter.Seq[TextPos] { var pos TextPos var lineStart TextPos for pos < textLen { + pos += TextPos(stringutil.LineBreakPrefixLen(text[pos:])) + if pos == textLen { + break + } b := text[pos] if b < utf8.RuneSelf { pos++ @@ -484,17 +488,34 @@ type UTF16Offset int func UTF16Len(s string) UTF16Offset { // Fast path: scan for non-ASCII bytes. For ASCII-only strings, // each byte is one UTF-16 code unit, so we can return len(s) directly. - for i := range len(s) { - if s[i] >= utf8.RuneSelf { - // Found non-ASCII; count the ASCII prefix, then decode the rest. - n := UTF16Offset(i) - for _, r := range s[i:] { - n += UTF16Offset(utf16.RuneLen(r)) - } - return n + i := 0 + if len(s) < 16 { + for i < len(s) && s[i] < utf8.RuneSelf { + i++ + } + } else { + i = stringutil.ASCIIPrefixLen(s) + } + n := UTF16Offset(i) + // A non-ASCII start often means dense Unicode. Keep the compact rune loop + // rather than repeatedly dispatching SIMD for very short ASCII runs. + if i == 0 { + for _, r := range s { + n += UTF16Offset(utf16.RuneLen(r)) + } + return n + } + for i < len(s) { + r, size := utf8.DecodeRuneInString(s[i:]) + n += UTF16Offset(utf16.RuneLen(r)) + i += size + if i < len(s) && s[i] < utf8.RuneSelf { + size = stringutil.ASCIIPrefixLen(s[i:]) + n += UTF16Offset(size) + i += size } } - return UTF16Offset(len(s)) + return n } func Flatten[T any](array [][]T) []T { diff --git a/tsc/internal/core/textscan_test.go b/tsc/internal/core/textscan_test.go new file mode 100644 index 0000000000000..f400d83b92816 --- /dev/null +++ b/tsc/internal/core/textscan_test.go @@ -0,0 +1,88 @@ +package core_test + +import ( + "slices" + "strings" + "testing" + "unicode/utf16" + "unicode/utf8" + + "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/stringutil" + "github.com/microsoft/TypeScript/tsc/internal/testutil/filefixture" + "github.com/microsoft/TypeScript/tsc/internal/testutil/fixtures" + "gotest.tools/v3/assert" +) + +func TestTextScanBoundaries(t *testing.T) { + t.Parallel() + for prefix := range 130 { + for _, middle := range []string{ + "", "\n", "\r", "\r\n", "\n\r", "\u2028", "\u2029", "\u0085", + "\uFEFF", "\u00E9", "\U0001F980", "\xff", "\xe2", "\xe2\x80", + "\xe2\x80\n", "\xf0\x9f", "\xed\xa0\x80", + } { + for _, suffix := range []string{"", strings.Repeat("z", 129), "\r\nlast\n"} { + text := strings.Repeat("a", prefix) + middle + suffix + var length core.UTF16Offset + for _, r := range text { + length += core.UTF16Offset(utf16.RuneLen(r)) + } + assert.Equal(t, core.UTF16Len(text), length, "text: %q", text) + + want := []core.TextPos{0} + for i := 0; i < len(text); { + r, size := utf8.DecodeRuneInString(text[i:]) + i += size + if r == '\r' && i < len(text) && text[i] == '\n' { + i++ + } + if stringutil.IsLineBreak(r) { + want = append(want, core.TextPos(i)) + } + } + assert.DeepEqual(t, core.ComputeECMALineStarts(text), core.ECMALineStarts(want)) + var first []core.TextPos + for pos := range core.ComputeECMALineStartsSeq(text) { + first = append(first, pos) + if len(first) == 2 { + break + } + } + assert.DeepEqual(t, first, want[:min(2, len(want))]) + assert.DeepEqual(t, slices.Collect(core.ComputeECMALineStartsSeq(text)), want) + } + } + } +} + +func BenchmarkTextScan(b *testing.B) { + texts := []filefixture.Fixture{ + filefixture.FromString("short", "", "const x = 1;"), + filefixture.FromString("ASCII", "", strings.Repeat("const variable = someFunction(argument1, argument2);\n", 200)), + filefixture.FromString("mixed", "", strings.Repeat("const caf\u00E9 = \"h\u00E9llo \U0001F980\"; // a longer ASCII comment for scanning\n", 200)), + filefixture.FromString("Unicode", "", strings.Repeat("\u65E5\u672C\u8A9E\U0001F980\u2028", 400)), + filefixture.FromString("CRLF", "", strings.Repeat("const variable = someFunction(argument1, argument2);\r\n", 200)), + filefixture.FromString("longLine", "", strings.Repeat("a", 16384)), + } + for _, fixture := range append(texts, fixtures.BenchFixtures...) { + b.Run(fixture.Name(), func(b *testing.B) { + fixture.SkipIfNotExist(b) + text := fixture.ReadFile(b) + b.Run("UTF16Len", func(b *testing.B) { + b.SetBytes(int64(len(text))) + b.ReportAllocs() + for b.Loop() { + core.UTF16Len(text) + } + }) + b.Run("LineStarts", func(b *testing.B) { + b.SetBytes(int64(len(text))) + b.ReportAllocs() + for b.Loop() { + core.ComputeECMALineStarts(text) + } + }) + }) + } +} diff --git a/tsc/internal/parser/parser_test.go b/tsc/internal/parser/parser_test.go index ffa96713eb444..94a993be0cf0e 100644 --- a/tsc/internal/parser/parser_test.go +++ b/tsc/internal/parser/parser_test.go @@ -24,6 +24,16 @@ import ( ) func BenchmarkParse(b *testing.B) { + benchmarkParse(b, false) +} + +// BenchmarkParseWithPositionMaps includes the lazy maps used for diagnostics +// and JavaScript API positions; BenchmarkParse measures syntax parsing alone. +func BenchmarkParseWithPositionMaps(b *testing.B) { + benchmarkParse(b, true) +} + +func benchmarkParse(b *testing.B, withPositionMaps bool) { for _, f := range fixtures.BenchFixtures { b.Run(f.Name(), func(b *testing.B) { f.SkipIfNotExist(b) @@ -38,8 +48,14 @@ func BenchmarkParse(b *testing.B) { PathKey: path, } + b.SetBytes(int64(len(sourceText))) + b.ReportAllocs() for b.Loop() { - parser.ParseSourceFile(opts, sourceText, scriptKind) + file := parser.ParseSourceFile(opts, sourceText, scriptKind) + if withPositionMaps { + file.ECMALineMap() + file.GetPositionMap() + } } }) } diff --git a/tsc/internal/printer/textscan_test.go b/tsc/internal/printer/textscan_test.go new file mode 100644 index 0000000000000..cbcf6e4e3b159 --- /dev/null +++ b/tsc/internal/printer/textscan_test.go @@ -0,0 +1,62 @@ +package printer + +import ( + "strings" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/stringutil" + "github.com/microsoft/TypeScript/tsc/internal/testutil/filefixture" + "github.com/microsoft/TypeScript/tsc/internal/testutil/fixtures" + "gotest.tools/v3/assert" +) + +func TestEscapeScanBoundaries(t *testing.T) { + t.Parallel() + for prefix := range 130 { + for _, middle := range []string{ + "", "\"", "'", "`", "\\", "${", "$x", "\r\n", "\n", "\x001", "\x00a", + "\x1f", "\x7f", "\u0085", "\u2028", "\u2029", "\u00E9", "\U0001F980", + "\xff", "\xe2\x80", stringutil.EncodeJSStringRune(0xD800), + } { + for _, quote := range []QuoteChar{QuoteCharSingleQuote, QuoteCharDoubleQuote, QuoteCharBacktick} { + for _, flags := range []getLiteralTextFlags{ + getLiteralTextFlagsNone, getLiteralTextFlagsNeverAsciiEscape, + getLiteralTextFlagsJsxAttributeEscape | getLiteralTextFlagsNeverAsciiEscape, + } { + text := strings.Repeat("a", prefix) + middle + strings.Repeat("z", 129) + var got, want strings.Builder + escapeStringWorker(text, quote, flags, &got) + // One-byte ASCII runs cannot use a vector fast path. + for i := range prefix { + escapeStringWorker(text[i:i+1], quote, flags, &want) + } + escapeStringWorker(middle, quote, flags, &want) + for range 129 { + escapeStringWorker("z", quote, flags, &want) + } + assert.Equal(t, got.String(), want.String(), "text %q, quote %c, flags %d", text, quote, flags) + } + } + } + } +} + +func BenchmarkEscapeScan(b *testing.B) { + texts := []filefixture.Fixture{ + filefixture.FromString("short", "", "hello world"), + filefixture.FromString("ASCII", "", strings.Repeat("a longer plain ASCII string without any escaping ", 256)), + filefixture.FromString("sparse", "", strings.Repeat("a longer plain ASCII string with an escaped quote \" and caf\u00E9 \U0001F980 ", 200)), + filefixture.FromString("dense", "", strings.Repeat("\\\"\n\u2028", 1024)), + } + for _, fixture := range append(texts, fixtures.BenchFixtures...) { + b.Run(fixture.Name(), func(b *testing.B) { + fixture.SkipIfNotExist(b) + text := fixture.ReadFile(b) + b.SetBytes(int64(len(text))) + b.ReportAllocs() + for b.Loop() { + EscapeString(text, QuoteCharDoubleQuote) + } + }) + } +} diff --git a/tsc/internal/printer/utilities.go b/tsc/internal/printer/utilities.go index 096b4d27afe1f..eaa99df9ab958 100644 --- a/tsc/internal/printer/utilities.go +++ b/tsc/internal/printer/utilities.go @@ -77,7 +77,18 @@ func encodeUtf16EscapeSequence(b *strings.Builder, charCode rune) { func escapeStringWorker(s string, quoteChar QuoteChar, flags getLiteralTextFlags, b *strings.Builder) { pos := 0 i := 0 + // Other quote values cannot match ASCII bytes; avoid truncating them. + quote := byte(0) + if quoteChar >= 0 && quoteChar < utf8.RuneSelf { + quote = byte(quoteChar) + } for i < len(s) { + if ch := s[i]; ch >= 0x20 && ch < utf8.RuneSelf && ch != '\\' && ch != quote && (quote != '`' || ch != '$') { + i += stringutil.UnescapedASCIIPrefixLen(s[i:], quote) + if i == len(s) { + break + } + } ch, size := stringutil.DecodeJSStringRune(s[i:]) escape := false diff --git a/tsc/internal/stringutil/textscan.go b/tsc/internal/stringutil/textscan.go new file mode 100644 index 0000000000000..9cd59b9e1efe5 --- /dev/null +++ b/tsc/internal/stringutil/textscan.go @@ -0,0 +1,41 @@ +// Builds with GOEXPERIMENT=simd use portable, vector-size-agnostic SIMD for +// long text scans. Normal builds retain scalar implementations. +package stringutil + +// ASCIIPrefixLen returns the length in bytes of the leading ASCII portion of text. +func ASCIIPrefixLen(text string) int { + // Every supported vector is at least 16 bytes. Keep short strings out of + // the experimental SIMD runtime dispatch. + if len(text) < 16 { + return asciiPrefixLen(text) + } + return asciiPrefixLenLong(text) +} + +func asciiPrefixLen(text string) int { + for i := range len(text) { + if text[i] >= 0x80 { + return i + } + } + return len(text) +} + +func lineBreakPrefixLen(text string) int { + for i := range len(text) { + if b := text[i]; b == '\r' || b == '\n' || b == 0xe2 { + return i + } + } + return len(text) +} + +func unescapedASCIIPrefixLen(text string, quote byte) int { + for i := range len(text) { + b := text[i] + if b < 0x20 || b >= 0x80 || b == '\\' || b == quote || quote == '`' && b == '$' { + return i + } + } + return len(text) +} diff --git a/tsc/internal/stringutil/textscan_scalar.go b/tsc/internal/stringutil/textscan_scalar.go new file mode 100644 index 0000000000000..786b041efa762 --- /dev/null +++ b/tsc/internal/stringutil/textscan_scalar.go @@ -0,0 +1,19 @@ +//go:build !goexperiment.simd + +package stringutil + +func asciiPrefixLenLong(text string) int { + return asciiPrefixLen(text) +} + +// LineBreakPrefixLen returns the byte offset of the first possible ECMAScript +// line break (CR, LF, or the leading byte of U+2028/U+2029), or len(text). +func LineBreakPrefixLen(text string) int { + return lineBreakPrefixLen(text) +} + +// UnescapedASCIIPrefixLen returns the length of the ASCII prefix that needs no +// escaping. It conservatively stops at all controls, backslashes, and template dollars. +func UnescapedASCIIPrefixLen(text string, quote byte) int { + return unescapedASCIIPrefixLen(text, quote) +} diff --git a/tsc/internal/stringutil/textscan_simd.go b/tsc/internal/stringutil/textscan_simd.go new file mode 100644 index 0000000000000..6a1d34fe46a9f --- /dev/null +++ b/tsc/internal/stringutil/textscan_simd.go @@ -0,0 +1,96 @@ +//go:build goexperiment.simd + +package stringutil + +import ( + "simd" + "unsafe" +) + +func asciiPrefixLenLong(text string) int { + width := (simd.Uint8s{}).Len() + if len(text) < width || text[0] >= 0x80 { + return asciiPrefixLen(text) + } + // SIMD loads only read this view; converting to []byte would copy the input. + bytes := unsafe.Slice(unsafe.StringData(text), len(text)) + highBit := simd.BroadcastUint8s(0x80) + // Go 1.27 has no portable mask reduction. Store machine words and reduce + // them without assuming a vector width or the machine's byte order. + words := make([]uint64, (simd.Uint64s{}).Len()) + i := 0 + for len(text)-i >= width { + simd.LoadUint8s(bytes[i:]).And(highBit).ReshapeToUint64s().Store(words) + var combined uint64 + for _, word := range words { + combined |= word + } + if combined != 0 { + break + } + i += width + } + return i + asciiPrefixLen(text[i:]) +} + +// LineBreakPrefixLen returns the byte offset of the first possible ECMAScript +// line break (CR, LF, or the leading byte of U+2028/U+2029), or len(text). +func LineBreakPrefixLen(text string) int { + width := (simd.Uint8s{}).Len() + if len(text) < width || text[0] == '\r' || text[0] == '\n' || text[0] == 0xe2 { + return lineBreakPrefixLen(text) + } + bytes := unsafe.Slice(unsafe.StringData(text), len(text)) + cr := simd.BroadcastUint8s('\r') + lf := simd.BroadcastUint8s('\n') + separator := simd.BroadcastUint8s(0xe2) + words := make([]uint64, (simd.Uint64s{}).Len()) + i := 0 + for len(text)-i >= width { + v := simd.LoadUint8s(bytes[i:]) + v.Equal(cr).Or(v.Equal(lf)).Or(v.Equal(separator)).ToInt8s().ToBits().ReshapeToUint64s().Store(words) + var combined uint64 + for _, word := range words { + combined |= word + } + if combined != 0 { + break + } + i += width + } + return i + lineBreakPrefixLen(text[i:]) +} + +// UnescapedASCIIPrefixLen returns the length of the ASCII prefix that needs no +// escaping. It conservatively stops at all controls, backslashes, and template dollars. +func UnescapedASCIIPrefixLen(text string, quote byte) int { + width := (simd.Uint8s{}).Len() + if len(text) < width || unescapedASCIIPrefixLen(text[:1], quote) == 0 { + return unescapedASCIIPrefixLen(text, quote) + } + bytes := unsafe.Slice(unsafe.StringData(text), len(text)) + space := simd.BroadcastInt8s(0x20) + backslash := simd.BroadcastUint8s('\\') + quoteVector := simd.BroadcastUint8s(quote) + dollar := simd.BroadcastUint8s('$') + words := make([]uint64, (simd.Uint64s{}).Len()) + i := 0 + for len(text)-i >= width { + v := simd.LoadUint8s(bytes[i:]) + // Signed comparison includes both controls and non-ASCII bytes. + mask := v.BitsToInt8().Less(space).Or(v.Equal(backslash)).Or(v.Equal(quoteVector)) + if quote == '`' { + mask = mask.Or(v.Equal(dollar)) + } + mask.ToInt8s().ToBits().ReshapeToUint64s().Store(words) + var combined uint64 + for _, word := range words { + combined |= word + } + if combined != 0 { + break + } + i += width + } + return i + unescapedASCIIPrefixLen(text[i:], quote) +} diff --git a/tsc/internal/stringutil/textscan_test.go b/tsc/internal/stringutil/textscan_test.go new file mode 100644 index 0000000000000..7c7d3fcd1efbc --- /dev/null +++ b/tsc/internal/stringutil/textscan_test.go @@ -0,0 +1,47 @@ +package stringutil + +import ( + "strings" + "testing" +) + +func checkTextScan(t *testing.T, text string) { + t.Helper() + if got, want := ASCIIPrefixLen(text), asciiPrefixLen(text); got != want { + t.Fatalf("ASCIIPrefixLen(%q) = %d, want %d", text, got, want) + } + if got, want := LineBreakPrefixLen(text), lineBreakPrefixLen(text); got != want { + t.Fatalf("LineBreakPrefixLen(%q) = %d, want %d", text, got, want) + } + for _, quote := range []byte{0, '\'', '"', '`', 0xff} { + if got, want := UnescapedASCIIPrefixLen(text, quote), unescapedASCIIPrefixLen(text, quote); got != want { + t.Fatalf("UnescapedASCIIPrefixLen(%q, %q) = %d, want %d", text, quote, got, want) + } + } +} + +func TestTextScanBytes(t *testing.T) { + t.Parallel() + for prefix := range 130 { + padding := strings.Repeat("a", prefix) + checkTextScan(t, padding) + for b := range 256 { + text := padding + string(byte(b)) + strings.Repeat("z", 129) + checkTextScan(t, text) + checkTextScan(t, text[1:]) + checkTextScan(t, text[:prefix+1]) + } + } +} + +func FuzzTextScan(f *testing.F) { + for _, seed := range []string{ + "", "a", strings.Repeat("a", 1024), "\r\n\u2028\u2029", + "\xff\xe2\x80\xf0\x9f", "abc\"${\\\x00", "\u00E9\U0001F980", + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, text string) { + checkTextScan(t, text) + }) +} From fa2d8440039852b3f136a1eccb422c7ed4b84555 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Mon, 26 Oct 2026 20:46:02 -0700 Subject: [PATCH 2/5] Fix simd code doing silly non-simd traversals --- tsc/internal/core/core.go | 7 +- tsc/internal/core/linestarts_scalar.go | 13 +++ tsc/internal/core/linestarts_simd.go | 90 +++++++++++++++++++ tsc/internal/core/linestarts_simd_test.go | 20 +++++ tsc/internal/core/textscan_test.go | 18 ++++ tsc/internal/stringutil/textscan_simd.go | 44 +++++---- tsc/internal/stringutil/textscan_simd_test.go | 41 +++++++++ tsc/internal/stringutil/textscan_test.go | 36 ++++++++ 8 files changed, 251 insertions(+), 18 deletions(-) create mode 100644 tsc/internal/core/linestarts_scalar.go create mode 100644 tsc/internal/core/linestarts_simd.go create mode 100644 tsc/internal/core/linestarts_simd_test.go create mode 100644 tsc/internal/stringutil/textscan_simd_test.go diff --git a/tsc/internal/core/core.go b/tsc/internal/core/core.go index a28c0739bee7b..9a86278e74cb8 100644 --- a/tsc/internal/core/core.go +++ b/tsc/internal/core/core.go @@ -426,8 +426,11 @@ func Coalesce[T *U, U any](a T, b T) T { type ECMALineStarts []TextPos func ComputeECMALineStarts(text string) ECMALineStarts { - result := make([]TextPos, 0, strings.Count(text, "\n")+1) - return slices.AppendSeq(result, ComputeECMALineStartsSeq(text)) + if len(text) < 16 { + result := make([]TextPos, 0, strings.Count(text, "\n")+1) + return slices.AppendSeq(result, ComputeECMALineStartsSeq(text)) + } + return computeECMALineStarts(text) } func ComputeECMALineStartsSeq(text string) iter.Seq[TextPos] { diff --git a/tsc/internal/core/linestarts_scalar.go b/tsc/internal/core/linestarts_scalar.go new file mode 100644 index 0000000000000..f87d105c5f94a --- /dev/null +++ b/tsc/internal/core/linestarts_scalar.go @@ -0,0 +1,13 @@ +//go:build !goexperiment.simd + +package core + +import ( + "slices" + "strings" +) + +func computeECMALineStarts(text string) ECMALineStarts { + result := make([]TextPos, 0, strings.Count(text, "\n")+1) + return slices.AppendSeq(result, ComputeECMALineStartsSeq(text)) +} diff --git a/tsc/internal/core/linestarts_simd.go b/tsc/internal/core/linestarts_simd.go new file mode 100644 index 0000000000000..3e00ddc105417 --- /dev/null +++ b/tsc/internal/core/linestarts_simd.go @@ -0,0 +1,90 @@ +//go:build goexperiment.simd + +package core + +import ( + "math/bits" + "simd" + "strings" + "unsafe" +) + +func ecmaLineBreakEnd(text string, pos int) int { + switch text[pos] { + case '\r': + if pos+1 < len(text) && text[pos+1] == '\n' { + return pos + 2 + } + return pos + 1 + case '\n': + if pos == 0 || text[pos-1] != '\r' { + return pos + 1 + } + case 0xe2: + // U+2028 and U+2029 differ only in the low bit of their final byte. + if pos+2 < len(text) && text[pos+1] == 0x80 && text[pos+2]&0xfe == 0xa8 { + return pos + 3 + } + } + return 0 +} + +func lineBreakBits(word uint64) uint64 { + // Pack the low bit of each byte into the low eight bits. The multiply + // places source byte i at bit 56+i without overlapping partial products. + return ((word & 0x0101010101010101) * 0x0102040810204080) >> 56 +} + +func computeECMALineStarts(text string) ECMALineStarts { + result := make(ECMALineStarts, 1, strings.Count(text, "\n")+1) + length := int(TextPos(len(text))) + if length <= 0 { + return result + } + text = text[:length] + // Read-only loads avoid copying the source text into a byte slice. + bytes := unsafe.Slice(unsafe.StringData(text), len(text)) + width := (simd.Uint8s{}).Len() + cr := simd.BroadcastUint8s('\r') + lf := simd.BroadcastUint8s('\n') + separator := simd.BroadcastUint8s(0xe2) + mask := make([]uint64, (simd.Uint64s{}).Len()) + pos := 0 + for len(text)-pos >= width { + v := simd.LoadUint8s(bytes[pos:]) + v.Equal(cr).Or(v.Equal(lf)).Or(v.Equal(separator)).ToInt8s().ToBits().ReshapeToUint64s().Store(mask) + word := 0 + for ; len(mask)-word >= 2; word += 2 { + lo := mask[word] + hi := mask[word+1] + if lo|hi == 0 { + continue + } + matches := lineBreakBits(lo) | lineBreakBits(hi)<<8 + for matches != 0 { + index := pos + word*8 + bits.TrailingZeros64(matches) + matches &= matches - 1 + if end := ecmaLineBreakEnd(text, index); end != 0 { + result = append(result, TextPos(end)) + } + } + } + if word < len(mask) { + matches := lineBreakBits(mask[word]) + for matches != 0 { + index := pos + word*8 + bits.TrailingZeros64(matches) + matches &= matches - 1 + if end := ecmaLineBreakEnd(text, index); end != 0 { + result = append(result, TextPos(end)) + } + } + } + pos += width + } + for ; pos < len(text); pos++ { + if end := ecmaLineBreakEnd(text, pos); end != 0 { + result = append(result, TextPos(end)) + } + } + return result +} diff --git a/tsc/internal/core/linestarts_simd_test.go b/tsc/internal/core/linestarts_simd_test.go new file mode 100644 index 0000000000000..973ecee92c08f --- /dev/null +++ b/tsc/internal/core/linestarts_simd_test.go @@ -0,0 +1,20 @@ +//go:build goexperiment.simd + +package core + +import "testing" + +func TestLineBreakBits(t *testing.T) { + t.Parallel() + for mask := range 256 { + var bytes uint64 + for i := range 8 { + if mask&(1<= 0x80 { @@ -44,17 +64,13 @@ func LineBreakPrefixLen(text string) int { cr := simd.BroadcastUint8s('\r') lf := simd.BroadcastUint8s('\n') separator := simd.BroadcastUint8s(0xe2) - words := make([]uint64, (simd.Uint64s{}).Len()) + mask := make([]uint64, (simd.Uint64s{}).Len()) i := 0 for len(text)-i >= width { v := simd.LoadUint8s(bytes[i:]) - v.Equal(cr).Or(v.Equal(lf)).Or(v.Equal(separator)).ToInt8s().ToBits().ReshapeToUint64s().Store(words) - var combined uint64 - for _, word := range words { - combined |= word - } - if combined != 0 { - break + v.Equal(cr).Or(v.Equal(lf)).Or(v.Equal(separator)).ToInt8s().ToBits().ReshapeToUint64s().Store(mask) + if offset := firstNonzeroByte(mask); offset < width { + return i + offset } i += width } @@ -73,7 +89,7 @@ func UnescapedASCIIPrefixLen(text string, quote byte) int { backslash := simd.BroadcastUint8s('\\') quoteVector := simd.BroadcastUint8s(quote) dollar := simd.BroadcastUint8s('$') - words := make([]uint64, (simd.Uint64s{}).Len()) + matches := make([]uint64, (simd.Uint64s{}).Len()) i := 0 for len(text)-i >= width { v := simd.LoadUint8s(bytes[i:]) @@ -82,13 +98,9 @@ func UnescapedASCIIPrefixLen(text string, quote byte) int { if quote == '`' { mask = mask.Or(v.Equal(dollar)) } - mask.ToInt8s().ToBits().ReshapeToUint64s().Store(words) - var combined uint64 - for _, word := range words { - combined |= word - } - if combined != 0 { - break + mask.ToInt8s().ToBits().ReshapeToUint64s().Store(matches) + if offset := firstNonzeroByte(matches); offset < width { + return i + offset } i += width } diff --git a/tsc/internal/stringutil/textscan_simd_test.go b/tsc/internal/stringutil/textscan_simd_test.go new file mode 100644 index 0000000000000..d1fdc3aa2adc2 --- /dev/null +++ b/tsc/internal/stringutil/textscan_simd_test.go @@ -0,0 +1,41 @@ +//go:build goexperiment.simd + +package stringutil + +import ( + "simd" + "testing" +) + +func TestFirstNonzeroByte(t *testing.T) { + t.Parallel() + for _, length := range []int{16, 24, 32, 64, 128, 256} { + mask := make([]uint64, length/8) + if got := firstNonzeroByte(mask); got != length { + t.Fatalf("zero mask length %d: got %d", length, got) + } + for i := range length { + for _, value := range []byte{1, 0x80, 0xff} { + mask[i/8] = uint64(value) << ((i % 8) * 8) + if got := firstNonzeroByte(mask); got != i { + t.Fatalf("length %d, byte %d=%x: got %d", length, i, value, got) + } + } + mask[i/8] = 0 + } + } +} + +func TestSIMDMaskByteOrder(t *testing.T) { + t.Parallel() + bytes := make([]byte, (simd.Uint8s{}).Len()) + words := make([]uint64, (simd.Uint64s{}).Len()) + for i := range bytes { + bytes[i] = 0xff + simd.LoadUint8s(bytes).ReshapeToUint64s().Store(words) + if got := firstNonzeroByte(words); got != i { + t.Fatalf("byte %d: got %d", i, got) + } + bytes[i] = 0 + } +} diff --git a/tsc/internal/stringutil/textscan_test.go b/tsc/internal/stringutil/textscan_test.go index 7c7d3fcd1efbc..64f4ae06cbae2 100644 --- a/tsc/internal/stringutil/textscan_test.go +++ b/tsc/internal/stringutil/textscan_test.go @@ -1,10 +1,46 @@ package stringutil import ( + "strconv" "strings" "testing" ) +func TestTextScanMultipleCandidates(t *testing.T) { + t.Parallel() + for prefix := range 130 { + for gap := range 66 { + for _, middle := range []string{"\n", "\r\n", "\xe2", "\"", "\\", "$", "\x00", "\xff"} { + text := strings.Repeat("a", prefix) + middle + strings.Repeat("b", gap) + middle + strings.Repeat("c", 129) + checkTextScan(t, text) + } + } + } +} + +func BenchmarkTextScanPrefixes(b *testing.B) { + for _, length := range []int{8, 16, 31, 32, 63, 64, 127, 128, 1024} { + b.Run(strconv.Itoa(length), func(b *testing.B) { + b.Run("LineBreak", func(b *testing.B) { + text := strings.Repeat("a", length) + "\n" + strings.Repeat("z", 128) + b.SetBytes(int64(length + 1)) + b.ReportAllocs() + for b.Loop() { + LineBreakPrefixLen(text) + } + }) + b.Run("Escape", func(b *testing.B) { + text := strings.Repeat("a", length) + "\"" + strings.Repeat("z", 128) + b.SetBytes(int64(length + 1)) + b.ReportAllocs() + for b.Loop() { + UnescapedASCIIPrefixLen(text, '"') + } + }) + }) + } +} + func checkTextScan(t *testing.T, text string) { t.Helper() if got, want := ASCIIPrefixLen(text), asciiPrefixLen(text); got != want { From 5196925b0f14cc5c40d555a567f59bc4a64d8ee7 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Tue, 27 Oct 2026 14:49:10 -0700 Subject: [PATCH 3/5] Revision 3, faster still --- tsc/internal/core/core.go | 101 ++++++++----- tsc/internal/core/linestarts_simd.go | 43 ++---- tsc/internal/core/textscan_test.go | 85 +++++++++-- tsc/internal/printer/escaping.go | 90 +++++++++++ .../printer/escaping_reference_test.go | 86 +++++++++++ tsc/internal/printer/escaping_test.go | 39 +++++ tsc/internal/printer/textscan_test.go | 85 ++++++++++- tsc/internal/printer/utilities.go | 142 ++---------------- tsc/internal/stringutil/textscan_simd.go | 14 ++ 9 files changed, 462 insertions(+), 223 deletions(-) create mode 100644 tsc/internal/printer/escaping.go create mode 100644 tsc/internal/printer/escaping_reference_test.go create mode 100644 tsc/internal/printer/escaping_test.go diff --git a/tsc/internal/core/core.go b/tsc/internal/core/core.go index 9a86278e74cb8..42fe8a3982f35 100644 --- a/tsc/internal/core/core.go +++ b/tsc/internal/core/core.go @@ -427,8 +427,18 @@ type ECMALineStarts []TextPos func ComputeECMALineStarts(text string) ECMALineStarts { if len(text) < 16 { - result := make([]TextPos, 0, strings.Count(text, "\n")+1) - return slices.AppendSeq(result, ComputeECMALineStartsSeq(text)) + // Avoid iterator and vector dispatch overhead for short eager scans. + result := make(ECMALineStarts, 1, strings.Count(text, "\n")+1) + for pos := 0; pos < len(text); pos++ { + if b := text[pos]; b != '\r' && b != '\n' && b != 0xe2 { + continue + } + if end := ecmaLineBreakEnd(text, pos); end != 0 { + result = append(result, TextPos(end)) + pos = end - 1 + } + } + return result } return computeECMALineStarts(text) } @@ -443,36 +453,40 @@ func ComputeECMALineStartsSeq(text string) iter.Seq[TextPos] { if pos == textLen { break } - b := text[pos] - if b < utf8.RuneSelf { - pos++ - switch b { - case '\r': - if pos < textLen && text[pos] == '\n' { - pos++ - } - fallthrough - case '\n': - if !yield(lineStart) { - return - } - lineStart = pos + if end := ecmaLineBreakEnd(text, int(pos)); end != 0 { + if !yield(lineStart) { + return } + pos = TextPos(end) + lineStart = pos } else { - ch, size := utf8.DecodeRuneInString(text[pos:]) - pos += TextPos(size) - if stringutil.IsLineBreak(ch) { - if !yield(lineStart) { - return - } - lineStart = pos - } + pos++ } } yield(lineStart) } } +func ecmaLineBreakEnd(text string, pos int) int { + switch text[pos] { + case '\r': + if pos+1 < len(text) && text[pos+1] == '\n' { + return pos + 2 + } + return pos + 1 + case '\n': + if pos == 0 || text[pos-1] != '\r' { + return pos + 1 + } + case 0xe2: + // U+2028 and U+2029 differ only in the low bit of their final byte. + if pos+2 < len(text) && text[pos+1] == 0x80 && text[pos+2]&0xfe == 0xa8 { + return pos + 3 + } + } + return 0 +} + // PositionToLineAndByteOffset returns the 0-based line and byte offset from the // start of that line for the given byte position, using the provided line starts. // The byte offset is a raw UTF-8 byte offset from the line start, not a UTF-16 code unit count. @@ -489,25 +503,30 @@ type UTF16Offset int // UTF16Len returns the number of UTF-16 code units needed to // represent the given UTF-8 encoded string. func UTF16Len(s string) UTF16Offset { - // Fast path: scan for non-ASCII bytes. For ASCII-only strings, - // each byte is one UTF-16 code unit, so we can return len(s) directly. - i := 0 - if len(s) < 16 { - for i < len(s) && s[i] < utf8.RuneSelf { - i++ + // Amortize vector dispatch and keep the long scan's stack frame out of + // short strings, where the compact byte loop is faster. + if len(s) >= 64 { + return utf16LenLong(s) + } + for i := range len(s) { + if s[i] >= utf8.RuneSelf { + return UTF16Offset(i) + utf16LenRunes(s[i:]) } - } else { - i = stringutil.ASCIIPrefixLen(s) } - n := UTF16Offset(i) + return UTF16Offset(len(s)) +} + +func utf16LenLong(s string) UTF16Offset { + i := stringutil.ASCIIPrefixLen(s) + if i == len(s) { + return UTF16Offset(i) + } // A non-ASCII start often means dense Unicode. Keep the compact rune loop // rather than repeatedly dispatching SIMD for very short ASCII runs. if i == 0 { - for _, r := range s { - n += UTF16Offset(utf16.RuneLen(r)) - } - return n + return utf16LenRunes(s) } + n := UTF16Offset(i) for i < len(s) { r, size := utf8.DecodeRuneInString(s[i:]) n += UTF16Offset(utf16.RuneLen(r)) @@ -521,6 +540,14 @@ func UTF16Len(s string) UTF16Offset { return n } +func utf16LenRunes(s string) UTF16Offset { + var n UTF16Offset + for _, r := range s { + n += UTF16Offset(utf16.RuneLen(r)) + } + return n +} + func Flatten[T any](array [][]T) []T { var result []T for _, subArray := range array { diff --git a/tsc/internal/core/linestarts_simd.go b/tsc/internal/core/linestarts_simd.go index 3e00ddc105417..4984f820f5c94 100644 --- a/tsc/internal/core/linestarts_simd.go +++ b/tsc/internal/core/linestarts_simd.go @@ -9,26 +9,6 @@ import ( "unsafe" ) -func ecmaLineBreakEnd(text string, pos int) int { - switch text[pos] { - case '\r': - if pos+1 < len(text) && text[pos+1] == '\n' { - return pos + 2 - } - return pos + 1 - case '\n': - if pos == 0 || text[pos-1] != '\r' { - return pos + 1 - } - case 0xe2: - // U+2028 and U+2029 differ only in the low bit of their final byte. - if pos+2 < len(text) && text[pos+1] == 0x80 && text[pos+2]&0xfe == 0xa8 { - return pos + 3 - } - } - return 0 -} - func lineBreakBits(word uint64) uint64 { // Pack the low bit of each byte into the low eight bits. The multiply // places source byte i at bit 56+i without overlapping partial products. @@ -53,10 +33,12 @@ func computeECMALineStarts(text string) ECMALineStarts { for len(text)-pos >= width { v := simd.LoadUint8s(bytes[pos:]) v.Equal(cr).Or(v.Equal(lf)).Or(v.Equal(separator)).ToInt8s().ToBits().ReshapeToUint64s().Store(mask) - word := 0 - for ; len(mask)-word >= 2; word += 2 { + for word := 0; word < len(mask); word += 2 { lo := mask[word] - hi := mask[word+1] + var hi uint64 + if word+1 < len(mask) { + hi = mask[word+1] + } if lo|hi == 0 { continue } @@ -66,16 +48,10 @@ func computeECMALineStarts(text string) ECMALineStarts { matches &= matches - 1 if end := ecmaLineBreakEnd(text, index); end != 0 { result = append(result, TextPos(end)) - } - } - } - if word < len(mask) { - matches := lineBreakBits(mask[word]) - for matches != 0 { - index := pos + word*8 + bits.TrailingZeros64(matches) - matches &= matches - 1 - if end := ecmaLineBreakEnd(text, index); end != 0 { - result = append(result, TextPos(end)) + if end == index+2 { + // The next candidate is the LF of this CRLF pair. + matches &= matches - 1 + } } } } @@ -84,6 +60,7 @@ func computeECMALineStarts(text string) ECMALineStarts { for ; pos < len(text); pos++ { if end := ecmaLineBreakEnd(text, pos); end != 0 { result = append(result, TextPos(end)) + pos = end - 1 } } return result diff --git a/tsc/internal/core/textscan_test.go b/tsc/internal/core/textscan_test.go index 016aed3cb5a02..adc3e3abab90d 100644 --- a/tsc/internal/core/textscan_test.go +++ b/tsc/internal/core/textscan_test.go @@ -1,7 +1,9 @@ package core_test import ( + "runtime" "slices" + "strconv" "strings" "testing" "unicode/utf16" @@ -21,9 +23,11 @@ func TestTextScanBoundaries(t *testing.T) { "", "\n", "\r", "\r\n", "\n\r", "\u2028", "\u2029", "\u0085", "\uFEFF", "\u00E9", "\U0001F980", "\xff", "\xe2", "\xe2\x80", "\xe2\x80\n", "\xf0\x9f", "\xed\xa0\x80", + "\u2027", "\u202A", "\u20AC", "\xe2\x80\xff", "\xe2\r\n", "\r\u2028\n", "\n\n\r\r\n\r\n\u2028\u2029", strings.Repeat("x\r\n", 33), strings.Repeat("\u2028\u2029", 33), + strings.Repeat("\r\n\r\r\n\n\u2028\u2029", 33), } { for _, suffix := range []string{"", strings.Repeat("z", 129), "\r\nlast\n"} { text := strings.Repeat("a", prefix) + middle + suffix @@ -33,17 +37,7 @@ func TestTextScanBoundaries(t *testing.T) { } assert.Equal(t, core.UTF16Len(text), length, "text: %q", text) - want := []core.TextPos{0} - for i := 0; i < len(text); { - r, size := utf8.DecodeRuneInString(text[i:]) - i += size - if r == '\r' && i < len(text) && text[i] == '\n' { - i++ - } - if stringutil.IsLineBreak(r) { - want = append(want, core.TextPos(i)) - } - } + want := referenceECMALineStarts(text) assert.DeepEqual(t, core.ComputeECMALineStarts(text), core.ECMALineStarts(want)) var first []core.TextPos for pos := range core.ComputeECMALineStartsSeq(text) { @@ -59,6 +53,37 @@ func TestTextScanBoundaries(t *testing.T) { } } +func referenceECMALineStarts(text string) []core.TextPos { + result := []core.TextPos{0} + for pos := 0; pos < len(text); { + r, size := utf8.DecodeRuneInString(text[pos:]) + pos += size + if r == '\r' && pos < len(text) && text[pos] == '\n' { + pos++ + } + if stringutil.IsLineBreak(r) { + result = append(result, core.TextPos(pos)) + } + } + return result +} + +func TestUTF16LenShort(t *testing.T) { + t.Parallel() + for padding := range 130 { + for _, suffix := range []string{"", "\u00e9", "\u65e5\u672c\u8a9e", "\U0001f600", "\xff", "\xe2\x80", "\xed\xa0\x80"} { + text := strings.Repeat("a", padding) + suffix + for end := range min(len(text), 129) + 1 { + var want core.UTF16Offset + for _, r := range text[:end] { + want += core.UTF16Offset(utf16.RuneLen(r)) + } + assert.Equal(t, core.UTF16Len(text[:end]), want, "text: %q", text[:end]) + } + } + } +} + func FuzzECMALineStarts(f *testing.F) { for _, text := range []string{ "", "\r\n", "\n\r\r\n", "\u2028\u2029", @@ -68,14 +93,17 @@ func FuzzECMALineStarts(f *testing.F) { f.Add(text) } f.Fuzz(func(t *testing.T, text string) { - want := core.ECMALineStarts(slices.Collect(core.ComputeECMALineStartsSeq(text))) - assert.DeepEqual(t, core.ComputeECMALineStarts(text), want) + want := referenceECMALineStarts(text) + assert.DeepEqual(t, core.ComputeECMALineStarts(text), core.ECMALineStarts(want)) + assert.DeepEqual(t, slices.Collect(core.ComputeECMALineStartsSeq(text)), want) }) } func BenchmarkTextScan(b *testing.B) { texts := []filefixture.Fixture{ filefixture.FromString("short", "", "const x = 1;"), + filefixture.FromString("shortUnicode", "", "caf\u00e9 \U0001f600"), + filefixture.FromString("shortInvalid", "", "a\xf0\x9f"), filefixture.FromString("ASCII", "", strings.Repeat("const variable = someFunction(argument1, argument2);\n", 200)), filefixture.FromString("mixed", "", strings.Repeat("const caf\u00E9 = \"h\u00E9llo \U0001F980\"; // a longer ASCII comment for scanning\n", 200)), filefixture.FromString("Unicode", "", strings.Repeat("\u65E5\u672C\u8A9E\U0001F980\u2028", 400)), @@ -83,6 +111,10 @@ func BenchmarkTextScan(b *testing.B) { filefixture.FromString("longLine", "", strings.Repeat("a", 16384)), filefixture.FromString("shortLines", "", strings.Repeat("x\r\n", 4096)), } + + for _, length := range []int{1, 4, 8, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128} { + texts = append(texts, filefixture.FromString("ASCII"+strconv.Itoa(length), "", strings.Repeat("a", length))) + } for _, fixture := range append(texts, fixtures.BenchFixtures...) { b.Run(fixture.Name(), func(b *testing.B) { fixture.SkipIfNotExist(b) @@ -104,3 +136,30 @@ func BenchmarkTextScan(b *testing.B) { }) } } + +func BenchmarkECMALineStartsSeq(b *testing.B) { + texts := []filefixture.Fixture{ + filefixture.FromString("short", "", "const x = 1;"), + filefixture.FromString("ASCII1", "", "a"), + filefixture.FromString("ASCII15", "", strings.Repeat("a", 15)), + filefixture.FromString("shortUnicode", "", "caf\u00e9 \U0001f600"), + filefixture.FromString("shortInvalid", "", "a\xf0\x9f"), + filefixture.FromString("shortLines", "", strings.Repeat("x\r\n", 4096)), + filefixture.FromString("Unicode", "", strings.Repeat("\u65E5\u672C\u8A9E\U0001F980\u2028", 400)), + } + for _, fixture := range append(texts, fixtures.BenchFixtures...) { + b.Run(fixture.Name(), func(b *testing.B) { + fixture.SkipIfNotExist(b) + text := fixture.ReadFile(b) + b.SetBytes(int64(len(text))) + b.ReportAllocs() + var last core.TextPos + for b.Loop() { + for pos := range core.ComputeECMALineStartsSeq(text) { + last = pos + } + } + runtime.KeepAlive(last) + }) + } +} diff --git a/tsc/internal/printer/escaping.go b/tsc/internal/printer/escaping.go new file mode 100644 index 0000000000000..48051c0b6c9f9 --- /dev/null +++ b/tsc/internal/printer/escaping.go @@ -0,0 +1,90 @@ +package printer + +import ( + "strings" + "unicode/utf8" + + "github.com/microsoft/TypeScript/tsc/internal/stringutil" +) + +var asciiEscapeCodes = [128]byte{ + 0: '0', '\b': 'b', '\t': 't', '\n': 'n', '\v': 'v', '\f': 'f', '\r': 'r', + '\\': '\\', '"': '"', '\'': '\'', '`': '`', '$': '$', +} + +func appendEscapedString(s string, quoteChar QuoteChar, flags getLiteralTextFlags, b *strings.Builder) bool { + quote := byte(0) + if quoteChar >= 0 && quoteChar < utf8.RuneSelf { + quote = byte(quoteChar) + } + backtick := quoteChar == QuoteCharBacktick + asciiOnly := flags&getLiteralTextFlagsNeverAsciiEscape == 0 + jsx := flags&getLiteralTextFlagsJsxAttributeEscape != 0 + pos, pending := 0, 0 + changed := false + for pos < len(s) { + if ch := s[pos]; ch >= 0x20 && ch < utf8.RuneSelf && ch != '\\' && ch != quote && (!backtick || ch != '$') { + pos += stringutil.UnescapedASCIIPrefixLen(s[pos:], quote) + if pos == len(s) { + break + } + } + ch, size := rune(s[pos]), 1 + if ch < utf8.RuneSelf { + if ch == '\\' && jsx || ch == '\n' && backtick || ch == '$' && (!backtick || pos+1 == len(s) || s[pos+1] != '{') { + pos++ + continue + } + } else { + ch, size = stringutil.DecodeJSStringRune(s[pos:]) + if !asciiOnly && ch != rune(quoteChar) && ch != 0x85 && ch != 0x2028 && ch != 0x2029 && + !stringutil.IsSurrogate(ch) && !(ch == utf8.RuneError && size == 1) { + pos += size + continue + } + } + if !changed { + if b.Cap() == 0 { + b.Grow(len(s) + 2) + } else if b.Cap()-b.Len() < len(s) { + b.Grow(len(s)) + } + changed = true + } + if pending < pos { + b.WriteString(s[pending:pos]) + } + if jsx { + if ch == 0 { + b.WriteString("�") + } else if match, ok := jsxEscapedCharsMap[ch]; ok { + b.WriteString(match) + } else { + encodeJsxCharacterEntity(b, ch) + } + } else { + switch { + case ch == '\r' && pos+1 < len(s) && s[pos+1] == '\n': + size = 2 + b.WriteString(`\r\n`) + case ch == 0 && pos+1 < len(s) && s[pos+1] >= '0' && s[pos+1] <= '9': + b.WriteString(`\x00`) + case ch < 128 && asciiEscapeCodes[ch] != 0: + bytes := [2]byte{'\\', asciiEscapeCodes[ch]} + b.Write(bytes[:]) + case ch > 0xffff: + ch -= 0x10000 + encodeUtf16EscapeSequence(b, ch>>10+0xD800) + encodeUtf16EscapeSequence(b, ch&0x3ff+0xDC00) + default: + encodeUtf16EscapeSequence(b, ch) + } + } + pending = pos + size + pos += size + } + if changed && pending < len(s) { + b.WriteString(s[pending:]) + } + return changed +} diff --git a/tsc/internal/printer/escaping_reference_test.go b/tsc/internal/printer/escaping_reference_test.go new file mode 100644 index 0000000000000..b7bdedadc0ef8 --- /dev/null +++ b/tsc/internal/printer/escaping_reference_test.go @@ -0,0 +1,86 @@ +package printer + +import ( + "fmt" + "strings" + "unicode/utf8" + + "github.com/microsoft/TypeScript/tsc/internal/stringutil" +) + +var referenceEscapedChars = map[rune]string{ + '\t': `\t`, '\v': `\v`, '\f': `\f`, '\b': `\b`, '\r': `\r`, '\n': `\n`, + '\\': `\\`, '"': `\"`, '\'': `\'`, '`': "\\`", '$': `\$`, + '\u2028': `\u2028`, '\u2029': `\u2029`, '\u0085': `\u0085`, +} + +// Keep the original rune-by-rune emitter independent of production scanning, +// escape tables, and hexadecimal formatting. +func escapeStringWorkerReference(s string, quoteChar QuoteChar, flags getLiteralTextFlags, b *strings.Builder) { + pending := 0 + for pos := 0; pos < len(s); { + ch, size := stringutil.DecodeJSStringRune(s[pos:]) + escape := ch >= 0xD800 && ch <= 0xDFFF || ch == utf8.RuneError && size == 1 + switch ch { + case '\\': + if flags&getLiteralTextFlagsJsxAttributeEscape == 0 { + escape = true + } + case '$': + if quoteChar == QuoteCharBacktick && pos+1 < len(s) && s[pos+1] == '{' { + escape = true + } + case rune(quoteChar), '\u2028', '\u2029', '\u0085', '\r': + escape = true + case '\n': + if quoteChar != QuoteCharBacktick { + escape = true + } + default: + if ch <= '\u001f' || flags&getLiteralTextFlagsNeverAsciiEscape == 0 && ch > '\u007f' { + escape = true + } + } + if escape { + b.WriteString(s[pending:pos]) + if flags&getLiteralTextFlagsJsxAttributeEscape != 0 { + switch ch { + case 0: + b.WriteString("�") + case '"': + b.WriteString(""") + case '\'': + b.WriteString("'") + default: + b.WriteString(fmt.Sprintf("&#x%X;", ch)) + } + } else { + switch { + case ch == '\r' && quoteChar == QuoteCharBacktick && pos+1 < len(s) && s[pos+1] == '\n': + size++ + b.WriteString(`\r\n`) + case ch > 0xffff: + value := ch - 0x10000 + b.WriteString(fmt.Sprintf(`\u%04X\u%04X`, value>>10+0xD800, value&0x3ff+0xDC00)) + case ch >= 0xD800 && ch <= 0xDFFF: + b.WriteString(fmt.Sprintf(`\u%04X`, ch)) + case ch == 0: + if pos+1 < len(s) && stringutil.IsDigit(rune(s[pos+1])) { + b.WriteString(`\x00`) + } else { + b.WriteString(`\0`) + } + default: + if match, ok := referenceEscapedChars[ch]; ok { + b.WriteString(match) + } else { + b.WriteString(fmt.Sprintf(`\u%04X`, ch)) + } + } + } + pending = pos + size + } + pos += size + } + b.WriteString(s[pending:]) +} diff --git a/tsc/internal/printer/escaping_test.go b/tsc/internal/printer/escaping_test.go new file mode 100644 index 0000000000000..41d5786cac1b7 --- /dev/null +++ b/tsc/internal/printer/escaping_test.go @@ -0,0 +1,39 @@ +package printer + +import ( + "fmt" + "strings" + "testing" +) + +func TestEscapingAllocations(t *testing.T) { //nolint:paralleltest // AllocsPerRun changes GOMAXPROCS. + for _, text := range []string{ + "", "hello world", "caf\u00E9 \U0001F600", + strings.Repeat("plain ASCII text ", 128), + strings.Repeat("caf\u00E9 \U0001F600 ", 128), + } { + if allocations := testing.AllocsPerRun(25, func() { + EscapeString(text, QuoteCharDoubleQuote) + }); allocations != 0 { + t.Fatalf("unchanged text allocated %v times", allocations) + } + } + + text := strings.Repeat("abc\n", 4) + if allocations := testing.AllocsPerRun(25, func() { + EscapeString(text, QuoteCharDoubleQuote) + }); allocations != 1 { + t.Fatalf("small escaped text allocated %v times, want one output buffer", allocations) + } +} + +func TestEncodeUTF16Escape(t *testing.T) { + t.Parallel() + for ch := rune(0); ch <= 0xffff; ch++ { + var b strings.Builder + encodeUtf16EscapeSequence(&b, ch) + if got, want := b.String(), fmt.Sprintf(`\u%04X`, ch); got != want { + t.Fatalf("escape %U: got %q, want %q", ch, got, want) + } + } +} diff --git a/tsc/internal/printer/textscan_test.go b/tsc/internal/printer/textscan_test.go index cbcf6e4e3b159..0be2ab60d17b6 100644 --- a/tsc/internal/printer/textscan_test.go +++ b/tsc/internal/printer/textscan_test.go @@ -21,19 +21,13 @@ func TestEscapeScanBoundaries(t *testing.T) { for _, quote := range []QuoteChar{QuoteCharSingleQuote, QuoteCharDoubleQuote, QuoteCharBacktick} { for _, flags := range []getLiteralTextFlags{ getLiteralTextFlagsNone, getLiteralTextFlagsNeverAsciiEscape, + getLiteralTextFlagsJsxAttributeEscape, getLiteralTextFlagsJsxAttributeEscape | getLiteralTextFlagsNeverAsciiEscape, } { text := strings.Repeat("a", prefix) + middle + strings.Repeat("z", 129) var got, want strings.Builder escapeStringWorker(text, quote, flags, &got) - // One-byte ASCII runs cannot use a vector fast path. - for i := range prefix { - escapeStringWorker(text[i:i+1], quote, flags, &want) - } - escapeStringWorker(middle, quote, flags, &want) - for range 129 { - escapeStringWorker("z", quote, flags, &want) - } + escapeStringWorkerReference(text, quote, flags, &want) assert.Equal(t, got.String(), want.String(), "text %q, quote %c, flags %d", text, quote, flags) } } @@ -41,12 +35,70 @@ func TestEscapeScanBoundaries(t *testing.T) { } } +func TestEscapeASCIIAtVectorBoundaries(t *testing.T) { + t.Parallel() + for prefix := range 66 { + for character := range 256 { + for _, following := range []string{"", "1", "{", "\n"} { + middle := string(byte(character)) + following + for _, quote := range []QuoteChar{QuoteCharSingleQuote, QuoteCharDoubleQuote, QuoteCharBacktick} { + var expected strings.Builder + escapeStringWorkerReference(middle, quote, getLiteralTextFlagsNeverAsciiEscape, &expected) + text := strings.Repeat("a", prefix) + middle + strings.Repeat("z", 129) + want := strings.Repeat("a", prefix) + expected.String() + strings.Repeat("z", 129) + assert.Equal(t, EscapeString(text, quote), want) + var appended strings.Builder + appended.Grow(len(text) + 2) + appended.WriteString("prefix:") + escapeStringWorker(text, quote, getLiteralTextFlagsNeverAsciiEscape, &appended) + assert.Equal(t, appended.String(), "prefix:"+want) + } + } + } + } +} + +func FuzzEscaping(f *testing.F) { + for _, text := range []string{ + "", strings.Repeat("a", 129), "\x001\x00x\r\n\n${x}`\\\"'", + "\u0085\u2028\u2029\u00E9\U0001F600\uFFFD", "\xff\xe2\x80\xed\xa0\x80", + strings.Repeat("a", 63) + "\r\n\u2028" + strings.Repeat("z", 129), + } { + for _, quote := range []QuoteChar{QuoteCharDoubleQuote, QuoteCharSingleQuote, QuoteCharBacktick, '$', '\u2028', -1, 0, '\n', '\\', '\x7f', '\U0001f600'} { + for _, flags := range []getLiteralTextFlags{0, getLiteralTextFlagsNeverAsciiEscape, getLiteralTextFlagsJsxAttributeEscape, getLiteralTextFlagsJsxAttributeEscape | getLiteralTextFlagsNeverAsciiEscape} { + f.Add(text, int32(quote), uint8(flags), uint8(3)) + } + } + } + f.Fuzz(func(t *testing.T, text string, quote int32, flags uint8, capacity uint8) { + var got, want strings.Builder + if capacity&1 != 0 { + got.Grow(len(text) + 2) + want.Grow(len(text) + 2) + } + if capacity&2 != 0 { + got.WriteString("prefix:") + want.WriteString("prefix:") + } + escapeStringWorker(text, QuoteChar(quote), getLiteralTextFlags(flags), &got) + escapeStringWorkerReference(text, QuoteChar(quote), getLiteralTextFlags(flags), &want) + assert.Equal(t, got.String(), want.String()) + var reference strings.Builder + escapeStringWorkerReference(text, QuoteChar(quote), getLiteralTextFlagsNeverAsciiEscape, &reference) + assert.Equal(t, EscapeString(text, QuoteChar(quote)), reference.String()) + }) +} + func BenchmarkEscapeScan(b *testing.B) { texts := []filefixture.Fixture{ filefixture.FromString("short", "", "hello world"), filefixture.FromString("ASCII", "", strings.Repeat("a longer plain ASCII string without any escaping ", 256)), filefixture.FromString("sparse", "", strings.Repeat("a longer plain ASCII string with an escaped quote \" and caf\u00E9 \U0001F980 ", 200)), filefixture.FromString("dense", "", strings.Repeat("\\\"\n\u2028", 1024)), + filefixture.FromString("rust-ascii-lines", "", strings.Repeat("const value = 123; // source text\r\n", 2048)), + filefixture.FromString("rust-long-ascii", "", strings.Repeat("identifierWithSeveralLetters_0123456789 ", 2048)), + filefixture.FromString("rust-sparse-unicode", "", strings.Repeat(strings.Repeat("source text ", 32)+"\u00E9\U0001F600\u2028", 256)), + filefixture.FromString("rust-dense-unicode", "", strings.Repeat("\u00E9\u20AC\U0001F600\r\n\u2029", 4096)), } for _, fixture := range append(texts, fixtures.BenchFixtures...) { b.Run(fixture.Name(), func(b *testing.B) { @@ -60,3 +112,20 @@ func BenchmarkEscapeScan(b *testing.B) { }) } } + +func BenchmarkEscapePreallocated(b *testing.B) { + for _, fixture := range fixtures.BenchFixtures { + b.Run(fixture.Name(), func(b *testing.B) { + fixture.SkipIfNotExist(b) + text := fixture.ReadFile(b) + size := len(EscapeString(text, QuoteCharDoubleQuote)) + b.SetBytes(int64(len(text))) + b.ReportAllocs() + for b.Loop() { + var output strings.Builder + output.Grow(size) + escapeStringWorker(text, QuoteCharDoubleQuote, getLiteralTextFlagsNeverAsciiEscape, &output) + } + }) + } +} diff --git a/tsc/internal/printer/utilities.go b/tsc/internal/printer/utilities.go index eaa99df9ab958..8b60cce877a4f 100644 --- a/tsc/internal/printer/utilities.go +++ b/tsc/internal/printer/utilities.go @@ -38,23 +38,6 @@ var jsxEscapedCharsMap = map[rune]string{ '\'': "'", } -var escapedCharsMap = map[rune]string{ - '\t': `\t`, - '\v': `\v`, - '\f': `\f`, - '\b': `\b`, - '\r': `\r`, - '\n': `\n`, - '\\': `\\`, - '"': `\"`, - '\'': `\'`, - '`': "\\`", - '$': `\$`, // when quoteChar == '`' - '\u2028': `\u2028`, // lineSeparator - '\u2029': `\u2029`, // paragraphSeparator - '\u0085': `\u0085`, // nextLine -} - func encodeJsxCharacterEntity(b *strings.Builder, charCode rune) { hexCharCode := strings.ToUpper(strconv.FormatUint(uint64(charCode), 16)) b.WriteString("&#x") @@ -63,133 +46,28 @@ func encodeJsxCharacterEntity(b *strings.Builder, charCode rune) { } func encodeUtf16EscapeSequence(b *strings.Builder, charCode rune) { - hexCharCode := strings.ToUpper(strconv.FormatUint(uint64(charCode), 16)) - b.WriteString(`\u`) - for i := len(hexCharCode); i < 4; i++ { - b.WriteByte('0') - } - b.WriteString(hexCharCode) + const hex = "0123456789ABCDEF" + bytes := [6]byte{'\\', 'u', hex[charCode>>12&15], hex[charCode>>8&15], hex[charCode>>4&15], hex[charCode&15]} + b.Write(bytes[:]) } // Based heavily on the abstract 'Quote'/'QuoteJSONString' operation from ECMA-262 (24.3.2.2), // but augmented for a few select characters (e.g. lineSeparator, paragraphSeparator, nextLine) // Note that this doesn't actually wrap the input in double quotes. func escapeStringWorker(s string, quoteChar QuoteChar, flags getLiteralTextFlags, b *strings.Builder) { - pos := 0 - i := 0 - // Other quote values cannot match ASCII bytes; avoid truncating them. - quote := byte(0) - if quoteChar >= 0 && quoteChar < utf8.RuneSelf { - quote = byte(quoteChar) - } - for i < len(s) { - if ch := s[i]; ch >= 0x20 && ch < utf8.RuneSelf && ch != '\\' && ch != quote && (quote != '`' || ch != '$') { - i += stringutil.UnescapedASCIIPrefixLen(s[i:], quote) - if i == len(s) { - break - } - } - ch, size := stringutil.DecodeJSStringRune(s[i:]) - - escape := false - if ch >= 0xD800 && ch <= 0xDFFF { - escape = true - } else if ch == utf8.RuneError && size == 1 { - // A stray byte that is not valid UTF-8 (for example, a fragment of a - // surrogate sentinel left behind by code that sliced the string by - // byte). Escape it as the Unicode replacement character so the output - // is always well-formed rather than containing raw invalid bytes. - escape = true - } - - // This consists of the first 19 unprintable ASCII characters, canonical escapes, lineSeparator, - // paragraphSeparator, and nextLine. The latter three are just desirable to suppress new lines in - // the language service. These characters should be escaped when printing, and if any characters are added, - // `escapedCharsMap` and/or `jsxEscapedCharsMap` must be updated. Note that this *does not* include the 'delete' - // character. There is no reason for this other than that JSON.stringify does not handle it either. - switch ch { - case '\\': - if flags&getLiteralTextFlagsJsxAttributeEscape == 0 { - escape = true - } - case '$': - if quoteChar == QuoteCharBacktick && i+1 < len(s) && s[i+1] == '{' { - escape = true - } - case rune(quoteChar), '\u2028', '\u2029', '\u0085', '\r': - escape = true - case '\n': - if quoteChar != QuoteCharBacktick { - // Template strings preserve simple LF newlines, still encode CRLF (or CR). - escape = true - } - default: - if ch <= '\u001f' || flags&getLiteralTextFlagsNeverAsciiEscape == 0 && ch > '\u007f' { - escape = true - } - } - - if escape { - if pos < i { - // Write string up to this point - b.WriteString(s[pos:i]) - } - - switch { - case flags&getLiteralTextFlagsJsxAttributeEscape != 0: - if ch == 0 { - b.WriteString("�") - } else if match, ok := jsxEscapedCharsMap[ch]; ok { - b.WriteString(match) - } else { - encodeJsxCharacterEntity(b, ch) - } - - default: - if ch == '\r' && quoteChar == QuoteCharBacktick && i+1 < len(s) && s[i+1] == '\n' { - // Template strings preserve simple LF newlines, but still must escape CRLF. Left alone, the - // above cases for `\r` and `\n` would inadvertently escape CRLF as two independent characters. - size++ - b.WriteString(`\r\n`) - } else if ch > 0xffff { - // encode as surrogate pair - ch -= 0x10000 - encodeUtf16EscapeSequence(b, (ch&0b11111111110000000000>>10)+0xD800) - encodeUtf16EscapeSequence(b, (ch&0b00000000001111111111)+0xDC00) - } else if ch >= 0xD800 && ch <= 0xDFFF { - encodeUtf16EscapeSequence(b, ch) - } else if ch == 0 { - if i+1 < len(s) && stringutil.IsDigit(rune(s[i+1])) { - // If the null character is followed by digits, print as a hex escape to prevent the result from - // parsing as an octal (which is forbidden in strict mode) - b.WriteString(`\x00`) - } else { - // Otherwise, keep printing a literal \0 for the null character - b.WriteString(`\0`) - } - } else { - if match, ok := escapedCharsMap[ch]; ok { - b.WriteString(match) - } else { - encodeUtf16EscapeSequence(b, ch) - } - } - } - pos = i + size - } - - i += size + if len(s) == 0 { + return } - - if pos < i { - b.WriteString(s[pos:]) + if !appendEscapedString(s, quoteChar, flags, b) { + b.WriteString(s) } } func EscapeString(s string, quoteChar QuoteChar) string { var b strings.Builder - b.Grow(len(s) + 2) - escapeStringWorker(s, quoteChar, getLiteralTextFlagsNeverAsciiEscape, &b) + if !appendEscapedString(s, quoteChar, getLiteralTextFlagsNeverAsciiEscape, &b) { + return s + } return b.String() } diff --git a/tsc/internal/stringutil/textscan_simd.go b/tsc/internal/stringutil/textscan_simd.go index 5e40e62e5c9cb..418cee8b48cb6 100644 --- a/tsc/internal/stringutil/textscan_simd.go +++ b/tsc/internal/stringutil/textscan_simd.go @@ -56,6 +56,13 @@ func asciiPrefixLenLong(text string) int { // LineBreakPrefixLen returns the byte offset of the first possible ECMAScript // line break (CR, LF, or the leading byte of U+2028/U+2029), or len(text). func LineBreakPrefixLen(text string) int { + if len(text) < 16 { + return lineBreakPrefixLen(text) + } + return lineBreakPrefixLenLong(text) +} + +func lineBreakPrefixLenLong(text string) int { width := (simd.Uint8s{}).Len() if len(text) < width || text[0] == '\r' || text[0] == '\n' || text[0] == 0xe2 { return lineBreakPrefixLen(text) @@ -80,6 +87,13 @@ func LineBreakPrefixLen(text string) int { // UnescapedASCIIPrefixLen returns the length of the ASCII prefix that needs no // escaping. It conservatively stops at all controls, backslashes, and template dollars. func UnescapedASCIIPrefixLen(text string, quote byte) int { + if len(text) < 16 { + return unescapedASCIIPrefixLen(text, quote) + } + return unescapedASCIIPrefixLenLong(text, quote) +} + +func unescapedASCIIPrefixLenLong(text string, quote byte) int { width := (simd.Uint8s{}).Len() if len(text) < width || unescapedASCIIPrefixLen(text[:1], quote) == 0 { return unescapedASCIIPrefixLen(text, quote) From a5cc90c56c10d288d9e915f692af7a2b368b38b2 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Tue, 27 Oct 2026 15:24:38 -0700 Subject: [PATCH 4/5] Default simd experiment to on --- .github/workflows/ci.yml | 9 +++++--- Herebyfile.mjs | 3 +++ tools/scripts/gen/generatedFile.test.mts | 27 ++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d2ded3b5c284..066e444cfef79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,9 @@ on: permissions: contents: read +env: + GOEXPERIMENT: simd + defaults: run: shell: bash @@ -118,8 +121,8 @@ jobs: noembed: true - runner: ubuntu-latest os: linux - name: 'simd' - simd: true + name: 'nosimd' + nosimd: true skip: ${{ github.event_name == 'merge_group' }} - runner: ubuntu-latest os: linux @@ -137,7 +140,7 @@ jobs: id-token: write contents: read env: - GOEXPERIMENT: ${{ (matrix.config.simd && 'simd') || '' }} + GOEXPERIMENT: ${{ (matrix.config.nosimd && 'nosimd') || 'simd' }} TSGO_HEREBY_RACE: ${{ (matrix.config.race && 'true') || 'false' }} TSGO_HEREBY_NOEMBED: ${{ (matrix.config.noembed && 'true') || 'false' }} TSGO_HEREBY_CONCURRENT_TEST_PROGRAMS: ${{ (matrix.config.concurrent-test-programs && 'true') || 'false' }} diff --git a/Herebyfile.mjs b/Herebyfile.mjs index bf3010588e80a..45a361076442b 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -21,6 +21,9 @@ import { run, } from "./tools/scripts/gen/utils.mts"; +// Set GOEXPERIMENT=nosimd to opt out of SIMD for any task. +process.env.GOEXPERIMENT ||= "simd"; + enableFileFingerprintCache(); if (process.platform === "win32") { diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts index eaf918e759fc3..6311df09f102b 100644 --- a/tools/scripts/gen/generatedFile.test.mts +++ b/tools/scripts/gen/generatedFile.test.mts @@ -21,6 +21,33 @@ import { run, } from "./utils.mts"; +test("Hereby defaults to SIMD without overriding explicit experiments", () => { + const fileName = path.resolve(import.meta.dirname, "../../../Herebyfile.mjs"); + const source = ts.createSourceFile(fileName, fs.readFileSync(fileName, "utf8"), ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); + const assignment = source.statements.find(statement => + ts.isExpressionStatement(statement) + && ts.isBinaryExpression(statement.expression) + && statement.expression.left.getText(source) === "process.env.GOEXPERIMENT" + ); + assert.ok(assignment, "Hereby must set the default experiment before running tasks"); + for ( + const { initial, expected } of [ + { initial: undefined, expected: "simd" }, + { initial: "", expected: "simd" }, + { initial: "nosimd", expected: "nosimd" }, + { initial: "simd", expected: "simd" }, + { initial: "jsonv2", expected: "jsonv2" }, + { initial: "simd,nosimd", expected: "simd,nosimd" }, + ] + ) { + const environment: NodeJS.ProcessEnv = { GODEBUG: "tracebackancestors=10" }; + if (initial !== undefined) environment.GOEXPERIMENT = initial; + runInNewContext(assignment.getText(source), { process: { env: environment } }); + assert.equal(environment.GOEXPERIMENT, expected); + assert.equal(environment.GODEBUG, "tracebackancestors=10"); + } +}); + test("validate generates before building and selects the generation scope", async () => { const fileName = path.resolve(import.meta.dirname, "../../../Herebyfile.mjs"); const source = ts.createSourceFile(fileName, fs.readFileSync(fileName, "utf8"), ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); From 4f086082851758a5ea4c35fa163f5e8e7946efc2 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Wed, 28 Oct 2026 14:32:57 -0700 Subject: [PATCH 5/5] Improve formatter perf --- tsc/internal/format/README.md | 22 +++ tsc/internal/format/benchmark_test.go | 130 ++++++++++++++++++ tsc/internal/format/context.go | 22 ++- tsc/internal/format/context_test.go | 34 +++++ tsc/internal/format/format_test.go | 34 +++++ tsc/internal/format/rulecontext.go | 4 +- tsc/internal/format/scanner.go | 7 +- tsc/internal/format/scanner_test.go | 51 +++++++ tsc/internal/format/span.go | 34 ++--- .../fourslash/tests/formatBulkTrivia_test.go | 25 ++++ tsc/internal/scanner/lines_test.go | 39 ++++++ tsc/internal/scanner/scanner.go | 49 ++++--- tsc/internal/scanner/scanner_test.go | 42 ++++++ tsc/internal/stringutil/textscan.go | 18 +++ tsc/internal/stringutil/textscan_scalar.go | 4 + tsc/internal/stringutil/textscan_simd.go | 22 +++ tsc/internal/stringutil/textscan_test.go | 34 +++++ 17 files changed, 530 insertions(+), 41 deletions(-) create mode 100644 tsc/internal/format/benchmark_test.go create mode 100644 tsc/internal/format/context_test.go create mode 100644 tsc/internal/format/scanner_test.go create mode 100644 tsc/internal/fourslash/tests/formatBulkTrivia_test.go create mode 100644 tsc/internal/scanner/lines_test.go diff --git a/tsc/internal/format/README.md b/tsc/internal/format/README.md index d1086eaa0b485..e2397375807cf 100644 --- a/tsc/internal/format/README.md +++ b/tsc/internal/format/README.md @@ -24,3 +24,25 @@ There are a lot of rules, which you can find in [rules.ts](./rules.ts) each one ### Where is this used? The formatter is used mainly from any language service operation that inserts or modifies code. The formatter is not exported publicly, and so all usage can only come through the language server. + +## Benchmarks + +Run the formatter benchmarks with: + +```sh +GOEXPERIMENT=simd go -C ./tsc test ./internal/format -run '^$' -bench '^BenchmarkFormat(Document|Editor)$' -benchmem -count=5 +``` + +In PowerShell, set `$env:GOEXPERIMENT = "simd"` before running the Go command. +Use `GOEXPERIMENT=nosimd` for the scalar comparison. + +`BenchmarkFormatDocument` measures edit generation on an already parsed source file, +both in its original form and after formatting. It covers the shared parser fixtures +(compiler source, declarations, JavaScript, and TSX), a small editor document, long +comments and templates, long whitespace runs, Unicode with CRLF, and minified JavaScript. +`BenchmarkFormatEditor` measures selection, semicolon, and enter requests. +Parsing, rule initialization, and edit application are outside the timed loops. +The existing `BenchmarkFormat` also measures edit application and printer performance. + +For CPU profiles, select a case with `-bench`, add `-cpuprofile` and a longer +`-benchtime`, and inspect the result with `go tool pprof`. diff --git a/tsc/internal/format/benchmark_test.go b/tsc/internal/format/benchmark_test.go new file mode 100644 index 0000000000000..7f4a4c05f8969 --- /dev/null +++ b/tsc/internal/format/benchmark_test.go @@ -0,0 +1,130 @@ +package format_test + +import ( + "slices" + "strings" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/ast" + "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/format" + "github.com/microsoft/TypeScript/tsc/internal/ls/lsutil" + "github.com/microsoft/TypeScript/tsc/internal/parser" + "github.com/microsoft/TypeScript/tsc/internal/testutil/filefixture" + "github.com/microsoft/TypeScript/tsc/internal/testutil/fixtures" + "github.com/microsoft/TypeScript/tsc/internal/tspath" + "gotest.tools/v3/assert" +) + +const editorText = `import { readFile } from "node:fs/promises"; + +export async function readValues(path:string,convert:(text:string)=>T){ +// Ignore empty lines, but preserve quoted text. +const text=await readFile(path,"utf8"); +return text.split(/\r?\n/).filter(line=>line.length>0).map(convert); +} + +export class Cache{ +private values=new Map(); +get(key:string){return this.values.get(key);} +set(key:string,value:T){this.values.set(key,value);} +} +` + +func BenchmarkFormatDocument(b *testing.B) { + cases := append([]filefixture.Fixture{}, fixtures.BenchFixtures...) + cases = append(cases, + filefixture.FromString("editor.ts", "editor.ts", editorText), + filefixture.FromString("comments.ts", "comments.ts", strings.Repeat( + "/**\n * "+strings.Repeat("Documentation with ordinary ASCII words. ", 8)+"\n */\n"+ + "export function value(input:string){ // Keep the trailing comment.\n"+ + "return `"+strings.Repeat("template text ", 32)+"${input}`;\n}\n", 64)), + filefixture.FromString("whitespace.ts", "whitespace.ts", strings.Repeat( + "export function value(input:number){\n"+ + strings.Repeat(" ", 128)+"const result=input+1;"+strings.Repeat(" ", 128)+"\n"+ + strings.Repeat(" ", 128)+"return result;"+strings.Repeat(" ", 128)+"\n}\n", 128)), + filefixture.FromString("unicode-crlf.ts", "unicode-crlf.ts", strings.Repeat( + "// Unicode text: \u03bb \u4e2d \U0001f600\r\n"+ + "export function value(input:string){\r\n"+ + "\t\u00a0const text=\"\u03bb \u4e2d \U0001f600\";\r\n"+ + "\treturn input+text;\t \r\n}\r\n", 128)), + filefixture.FromString("minified.js", "minified.js", strings.Repeat( + "function value(input){if(input){return{value:input+1,valid:true};}return null;}", 256)), + ) + for _, fixture := range cases { + b.Run(fixture.Name(), func(b *testing.B) { + fixture.SkipIfNotExist(b) + fileName := tspath.ToRootedFilePath(fixture.Path(), "/") + parse := func(text string) *ast.SourceFile { + return parser.ParseSourceFile(ast.SourceFileParseOptions{ + FileName: fileName, + PathKey: tspath.CaseSensitive.PathKey(tspath.RootedPath(fileName)), + }, text, core.GetScriptKindFromFileName(fileName)) + } + options := lsutil.GetDefaultFormatCodeSettings() + options.TrimTrailingWhitespace = core.TSTrue + ctx := format.WithFormatCodeSettings(b.Context(), options, "\n") + text := fixture.ReadFile(b) + file := parse(text) + assert.Equal(b, len(file.Diagnostics()), 0) + formatted := applyBulkEdits(text, format.FormatDocument(ctx, file)) + for _, input := range []struct { + name string + text string + file *ast.SourceFile + }{ + {"original", text, file}, + {"formatted", formatted, parse(formatted)}, + } { + b.Run(input.name, func(b *testing.B) { + expected := format.FormatDocument(ctx, input.file) + if input.name == "formatted" { + assert.Assert(b, applyBulkEdits(input.text, expected) == input.text) + } + b.SetBytes(int64(len(input.text))) + b.ReportAllocs() + var edits []core.TextChange + for b.Loop() { + edits = format.FormatDocument(ctx, input.file) + } + assert.Assert(b, slices.Equal(edits, expected)) + }) + } + }) + } +} + +func BenchmarkFormatEditor(b *testing.B) { + options := lsutil.GetDefaultFormatCodeSettings() + options.TrimTrailingWhitespace = core.TSTrue + ctx := format.WithFormatCodeSettings(b.Context(), options, "\n") + file := parser.ParseSourceFile(ast.SourceFileParseOptions{ + FileName: "/editor.ts", + PathKey: "/editor.ts", + }, editorText, core.ScriptKindTS) + for _, operation := range []struct { + name string + run func() []core.TextChange + }{ + {"selection", func() []core.TextChange { + return format.FormatSelection(ctx, file, strings.Index(editorText, "const text"), strings.Index(editorText, "\n}\n")) + }}, + {"semicolon", func() []core.TextChange { + return format.FormatOnSemicolon(ctx, file, strings.Index(editorText, "utf8\");")+len("utf8\");")) + }}, + {"enter", func() []core.TextChange { + return format.FormatOnEnter(ctx, file, strings.Index(editorText, "return text")) + }}, + } { + b.Run(operation.name, func(b *testing.B) { + expected := operation.run() + assert.Assert(b, len(expected) > 0) + b.ReportAllocs() + var edits []core.TextChange + for b.Loop() { + edits = operation.run() + } + assert.Assert(b, slices.Equal(edits, expected)) + }) + } +} diff --git a/tsc/internal/format/context.go b/tsc/internal/format/context.go index 3833413b26109..e2680623b7077 100644 --- a/tsc/internal/format/context.go +++ b/tsc/internal/format/context.go @@ -21,6 +21,9 @@ type FormattingContext struct { contextNodeBlockIsOnOneLine core.Tristate nextNodeBlockIsOnOneLine core.Tristate + lineStarts []core.TextPos + lastLine int + SourceFile *ast.SourceFile FormattingRequestKind FormatRequestKind Options lsutil.FormatCodeSettings @@ -31,6 +34,7 @@ func NewFormattingContext(file *ast.SourceFile, kind FormatRequestKind, options SourceFile: file, FormattingRequestKind: kind, Options: options, + lineStarts: scanner.GetECMALineStarts(file), } return res } @@ -60,12 +64,28 @@ func (this *FormattingContext) UpdateContext(cur TextRangeWithKind, curParent *a } func (this *FormattingContext) rangeIsOnOneLine(node core.TextRange) core.Tristate { - if rangeIsOnOneLine(node, this.SourceFile) { + if this.getLineOfPosition(node.Pos()) == this.getLineOfPosition(node.End()) { return core.TSTrue } return core.TSFalse } +func (this *FormattingContext) getLineOfPosition(pos int) int { + line := this.lastLine + if line >= 0 && pos >= int(this.lineStarts[line]) && + (line+1 == len(this.lineStarts) || pos < int(this.lineStarts[line+1])) { + return line + } + line = scanner.ComputeLineOfPosition(this.lineStarts, pos) + this.lastLine = line + return line +} + +func (this *FormattingContext) getLineAndByteOffsetOfPosition(pos int) (int, int) { + line := this.getLineOfPosition(pos) + return line, pos - int(this.lineStarts[line]) +} + func (this *FormattingContext) nodeIsOnOneLine(node *ast.Node) core.Tristate { return this.rangeIsOnOneLine(withTokenStart(node, this.SourceFile)) } diff --git a/tsc/internal/format/context_test.go b/tsc/internal/format/context_test.go new file mode 100644 index 0000000000000..0e7bf80566274 --- /dev/null +++ b/tsc/internal/format/context_test.go @@ -0,0 +1,34 @@ +package format + +import ( + "strings" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/ast" + "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/ls/lsutil" + "github.com/microsoft/TypeScript/tsc/internal/parser" + "github.com/microsoft/TypeScript/tsc/internal/scanner" + "gotest.tools/v3/assert" +) + +func TestFormattingLinePositions(t *testing.T) { + t.Parallel() + for _, text := range []string{"", "value", strings.Repeat("first\r\n\u4e2d\u2028next\nlast\u2029", 32)} { + file := parser.ParseSourceFile(ast.SourceFileParseOptions{FileName: "/test.ts", PathKey: "/test.ts"}, text, core.ScriptKindTS) + context := NewFormattingContext(file, FormatRequestKindFormatDocument, lsutil.GetDefaultFormatCodeSettings()) + for _, reverse := range []bool{false, true, false} { + for i := range len(text) + 1 { + pos := i + if reverse { + pos = len(text) - i + } + start := scanner.GetECMALineOfPosition(file, pos) + for _, end := range []int{pos, min(pos+1, len(text)), len(text)} { + want := start == scanner.GetECMALineOfPosition(file, end) + assert.Equal(t, context.rangeIsOnOneLine(core.NewTextRange(pos, end)).IsTrue(), want) + } + } + } + } +} diff --git a/tsc/internal/format/format_test.go b/tsc/internal/format/format_test.go index 494624d57b661..4fcc0f9f7e187 100644 --- a/tsc/internal/format/format_test.go +++ b/tsc/internal/format/format_test.go @@ -54,3 +54,37 @@ func TestFormatNoTrailingSpace(t *testing.T) { }) } } + +func TestFormatBulkTrivia(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + body string + newLine string + }{ + {"long indentation", strings.Repeat(" ", 129) + "return 1;" + strings.Repeat(" ", 129), "\n"}, + {"mixed indentation", strings.Repeat(" \t", 33) + "return 1;", "\r\n"}, + {"Unicode indentation", strings.Repeat(" ", 32) + "\u00a0\u2003return 1;", "\n"}, + {"long comment", "/* " + strings.Repeat("comment ", 64) + "*/" + "\nreturn 1;", "\n"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + options := lsutil.GetDefaultFormatCodeSettings() + options.NewLineCharacter = test.newLine + options.TrimTrailingWhitespace = core.TSTrue + ctx := format.WithFormatCodeSettings(t.Context(), options, test.newLine) + text := "function f(){" + test.newLine + test.body + test.newLine + "}" + test.newLine + expectedBody := " return 1;" + if test.name == "long comment" { + expectedBody = " /* " + strings.Repeat("comment ", 64) + "*/\n return 1;" + } + expected := "function f() {" + test.newLine + expectedBody + test.newLine + "}" + test.newLine + parse := func(text string) *ast.SourceFile { + return parser.ParseSourceFile(ast.SourceFileParseOptions{FileName: "/test.ts", PathKey: "/test.ts"}, text, core.ScriptKindTS) + } + formatted := applyBulkEdits(text, format.FormatDocument(ctx, parse(text))) + assert.Equal(t, formatted, expected) + assert.Equal(t, len(format.FormatDocument(ctx, parse(formatted))), 0) + }) + } +} diff --git a/tsc/internal/format/rulecontext.go b/tsc/internal/format/rulecontext.go index 175f43752f396..79ee249fa6b49 100644 --- a/tsc/internal/format/rulecontext.go +++ b/tsc/internal/format/rulecontext.go @@ -569,8 +569,8 @@ func isSemicolonDeletionContext(context *FormattingContext) bool { nextTokenStart = scanner.GetTokenPosOfNode(nextRealToken, context.SourceFile, false) } - startLine := scanner.GetECMALineOfPosition(context.SourceFile, context.currentTokenSpan.Loc.Pos()) - endLine := scanner.GetECMALineOfPosition(context.SourceFile, nextTokenStart) + startLine := context.getLineOfPosition(context.currentTokenSpan.Loc.Pos()) + endLine := context.getLineOfPosition(nextTokenStart) if startLine == endLine { return nextTokenKind == ast.KindCloseBraceToken || nextTokenKind == ast.KindEndOfFile } diff --git a/tsc/internal/format/scanner.go b/tsc/internal/format/scanner.go index 4eeb2e5770ac6..a639d4ffaba8f 100644 --- a/tsc/internal/format/scanner.go +++ b/tsc/internal/format/scanner.go @@ -1,8 +1,6 @@ package format import ( - "slices" - "github.com/microsoft/TypeScript/tsc/internal/ast" "github.com/microsoft/TypeScript/tsc/internal/core" "github.com/microsoft/TypeScript/tsc/internal/debug" @@ -266,10 +264,11 @@ func (s *formattingScanner) readTokenInfo(n *ast.Node) tokenInfo { } s.hasLastTokenInfo = true + // Trivia buffers are replaced rather than reused, so saved token infos can share them. s.lastTokenInfo = tokenInfo{ - leadingTrivia: slices.Clone(s.leadingTrivia), + leadingTrivia: s.leadingTrivia, token: token, - trailingTrivia: slices.Clone(s.trailingTrivia), + trailingTrivia: s.trailingTrivia, } s.lastTokenInfo = fixTokenKind(s.lastTokenInfo, n) diff --git a/tsc/internal/format/scanner_test.go b/tsc/internal/format/scanner_test.go new file mode 100644 index 0000000000000..cc4ed7a0bd057 --- /dev/null +++ b/tsc/internal/format/scanner_test.go @@ -0,0 +1,51 @@ +package format + +import ( + "slices" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/ast" + "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/scanner" + "gotest.tools/v3/assert" +) + +func TestFormattingScannerTriviaOwnership(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + text string + first ast.Kind + rescan ast.Kind + }{ + {"ordinary", " left /* first */ +\t right // last\n third", ast.KindIdentifier, ast.KindIdentifier}, + {"greater than", " >> /* first */ next // last\n third", ast.KindGreaterThanToken, ast.KindGreaterThanGreaterThanToken}, + {"regular expression", " /ab+/g /* first */ next // last\n third", ast.KindSlashToken, ast.KindRegularExpressionLiteral}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + scan := scanner.NewScanner() + scan.SetSkipTrivia(false) + scan.SetText(test.text) + s := formattingScanner{s: scan, endPos: len(test.text), wasNewLine: true} + s.advance() + first := s.readTokenInfo(&ast.Node{Kind: test.first}) + firstLeading := slices.Clone(first.leadingTrivia) + firstTrailing := slices.Clone(first.trailingTrivia) + rescanned := s.readTokenInfo(&ast.Node{Kind: test.rescan}) + rescannedLeading := slices.Clone(rescanned.leadingTrivia) + rescannedTrailing := slices.Clone(rescanned.trailingTrivia) + for s.isOnToken() { + s.advance() + if s.isOnToken() { + s.readTokenInfo(&ast.Node{Kind: scan.Token()}) + } + } + s.skipToStartOf(&core.TextRange{}) + assert.Assert(t, slices.Equal(first.leadingTrivia, firstLeading)) + assert.Assert(t, slices.Equal(first.trailingTrivia, firstTrailing)) + assert.Assert(t, slices.Equal(rescanned.leadingTrivia, rescannedLeading)) + assert.Assert(t, slices.Equal(rescanned.trailingTrivia, rescannedTrailing)) + }) + } +} diff --git a/tsc/internal/format/span.go b/tsc/internal/format/span.go index df9d11119b2cb..f625fa2573c1b 100644 --- a/tsc/internal/format/span.go +++ b/tsc/internal/format/span.go @@ -248,10 +248,10 @@ func (w *formatSpanWorker) execute(s *formattingScanner) []core.TextChange { w.formattingScanner.advance() if w.formattingScanner.isOnToken() { - startLine := scanner.GetECMALineOfPosition(w.sourceFile, withTokenStart(w.enclosingNode, w.sourceFile).Pos()) + startLine := w.formattingContext.getLineOfPosition(withTokenStart(w.enclosingNode, w.sourceFile).Pos()) undecoratedStartLine := startLine if ast.HasDecorators(w.enclosingNode) { - undecoratedStartLine = scanner.GetECMALineOfPosition(w.sourceFile, getNonDecoratorTokenPosOfNode(w.enclosingNode, w.sourceFile)) + undecoratedStartLine = w.formattingContext.getLineOfPosition(getNonDecoratorTokenPosOfNode(w.enclosingNode, w.sourceFile)) } w.processNode(w.enclosingNode, w.enclosingNode, startLine, undecoratedStartLine, w.initialIndentation, w.delta) @@ -268,7 +268,7 @@ func (w *formatSpanWorker) execute(s *formattingScanner) []core.TextChange { } w.indentTriviaItems(remainingTrivia, indentation, true, func(item TextRangeWithKind) { - startLine, startChar := scanner.GetECMALineAndByteOffsetOfPosition(w.sourceFile, item.Loc.Pos()) + startLine, startChar := w.formattingContext.getLineAndByteOffsetOfPosition(item.Loc.Pos()) w.processRange(item, startLine, startChar, w.enclosingNode, w.enclosingNode, nil) w.insertIndentation(item.Loc.Pos(), indentation, false) }) @@ -314,7 +314,7 @@ func (w *formatSpanWorker) execute(s *formattingScanner) []core.TextChange { if parent == nil { parent = w.previousParent } - line := scanner.GetECMALineOfPosition(w.sourceFile, tokenInfo.Loc.Pos()) + line := w.formattingContext.getLineOfPosition(tokenInfo.Loc.Pos()) w.processPair( tokenInfo, line, @@ -351,11 +351,11 @@ func (w *formatSpanWorker) processChildNode( return inheritedIndentation } childStartPos := scanner.GetTokenPosOfNode(child, w.sourceFile, false) - childStartLine := scanner.GetECMALineOfPosition(w.sourceFile, childStartPos) + childStartLine := w.formattingContext.getLineOfPosition(childStartPos) undecoratedChildStartLine := childStartLine if ast.HasDecorators(child) { - undecoratedChildStartLine = scanner.GetECMALineOfPosition(w.sourceFile, getNonDecoratorTokenPosOfNode(child, w.sourceFile)) + undecoratedChildStartLine = w.formattingContext.getLineOfPosition(getNonDecoratorTokenPosOfNode(child, w.sourceFile)) } isErrorMemberListElement := child.Flags&ast.NodeFlagsThisNodeHasError != 0 && isMemberListElement(parent, child) @@ -472,7 +472,7 @@ func (w *formatSpanWorker) processChildNodes( break } else if tokenInfo.token.Kind == listStartToken { // consume list start token - startLine = scanner.GetECMALineOfPosition(w.sourceFile, tokenInfo.token.Loc.Pos()) + startLine = w.formattingContext.getLineOfPosition(tokenInfo.token.Loc.Pos()) w.consumeTokenAndAdvanceScanner(tokenInfo, parent, parentDynamicIndentation, parent, false) @@ -597,7 +597,7 @@ func (w *formatSpanWorker) tryComputeIndentationForListItem(startPos int, endPos return inheritedIndentation } } else { - startLine := scanner.GetECMALineOfPosition(w.sourceFile, startPos) + startLine := w.formattingContext.getLineOfPosition(startPos) column := w.getCurrentIndentationAtPosition(startPos) if startLine != parentStartLine || startPos == column { // Use the base indent size if it is greater than @@ -765,7 +765,7 @@ func (w *formatSpanWorker) processRange(r TextRangeWithKind, rangeStartLine int, if !rangeHasError { if w.previousRange == NewTextRangeWithKind(0, 0, 0) { // trim whitespaces starting from the beginning of the span up to the current line - originalStartLine := scanner.GetECMALineOfPosition(w.sourceFile, w.originalRange.Pos()) + originalStartLine := w.formattingContext.getLineOfPosition(w.originalRange.Pos()) w.trimTrailingWhitespacesForLines(originalStartLine, rangeStartLine, NewTextRangeWithKind(0, 0, 0)) } else { lineAction = w.processPair(r, rangeStartLine, parent, w.previousRange, w.previousRangeStartLine, w.previousParent, contextNode, dynamicIndentation) @@ -783,7 +783,7 @@ func (w *formatSpanWorker) processRange(r TextRangeWithKind, rangeStartLine int, func (w *formatSpanWorker) processTrivia(trivia []TextRangeWithKind, parent *ast.Node, contextNode *ast.Node, dynamicIndentation *dynamicIndenter) { for _, triviaItem := range trivia { if isComment(triviaItem.Kind) && triviaItem.Loc.ContainedBy(w.originalRange) { - triviaItemStartLine, triviaItemStartCharacter := scanner.GetECMALineAndByteOffsetOfPosition(w.sourceFile, triviaItem.Loc.Pos()) + triviaItemStartLine, triviaItemStartCharacter := w.formattingContext.getLineAndByteOffsetOfPosition(triviaItem.Loc.Pos()) w.processRange(triviaItem, triviaItemStartLine, triviaItemStartCharacter, parent, contextNode, dynamicIndentation) } } @@ -815,8 +815,8 @@ func (w *formatSpanWorker) trimTrailingWhitespacesForRemainingRange(trivias []Te } func (w *formatSpanWorker) trimTrailingWitespacesForPositions(startPos int, endPos int, previousRange TextRangeWithKind) { - startLine := scanner.GetECMALineOfPosition(w.sourceFile, startPos) - endLine := scanner.GetECMALineOfPosition(w.sourceFile, endPos) + startLine := w.formattingContext.getLineOfPosition(startPos) + endLine := w.formattingContext.getLineOfPosition(endPos) w.trimTrailingWhitespacesForLines(startLine, endLine+1, previousRange) } @@ -882,7 +882,7 @@ func (w *formatSpanWorker) insertIndentation(pos int, indentation int, lineAdded // insert indentation string at the very beginning of the token w.recordReplace(pos, 0, indentationString) } else { - tokenStartLine, tokenStartCharacter := scanner.GetECMALineAndByteOffsetOfPosition(w.sourceFile, pos) + tokenStartLine, tokenStartCharacter := w.formattingContext.getLineAndByteOffsetOfPosition(pos) startLinePosition := int(scanner.GetECMALineStarts(w.sourceFile)[tokenStartLine]) if indentation != w.characterToColumn(startLinePosition, tokenStartCharacter) || w.indentationIsDifferent(indentationString, startLinePosition) { w.recordReplace(startLinePosition, tokenStartCharacter, indentationString) @@ -936,8 +936,8 @@ func (w *formatSpanWorker) indentTriviaItems(trivia []TextRangeWithKind, comment func (w *formatSpanWorker) indentMultilineComment(commentRange core.TextRange, indentation int, firstLineIsIndented bool, indentFinalLine bool) { // split comment in lines - startLine := scanner.GetECMALineOfPosition(w.sourceFile, commentRange.Pos()) - endLine := scanner.GetECMALineOfPosition(w.sourceFile, commentRange.End()) + startLine := w.formattingContext.getLineOfPosition(commentRange.Pos()) + endLine := w.formattingContext.getLineOfPosition(commentRange.End()) if startLine == endLine { if !firstLineIsIndented { @@ -1051,7 +1051,7 @@ func (w *formatSpanWorker) consumeTokenAndAdvanceScanner(currentTokenInfo tokenI lineAction := LineActionNone isTokenInRange := currentTokenInfo.token.Loc.ContainedBy(w.originalRange) - tokenStartLine, tokenStartChar := scanner.GetECMALineAndByteOffsetOfPosition(w.sourceFile, currentTokenInfo.token.Loc.Pos()) + tokenStartLine, tokenStartChar := w.formattingContext.getLineAndByteOffsetOfPosition(currentTokenInfo.token.Loc.Pos()) if isTokenInRange { rangeHasError := w.rangeContainsError(currentTokenInfo.token.Loc) @@ -1063,7 +1063,7 @@ func (w *formatSpanWorker) consumeTokenAndAdvanceScanner(currentTokenInfo tokenI if lineAction == LineActionNone { // indent token only if end line of previous range does not match start line of the token if savePreviousRange != NewTextRangeWithKind(0, 0, 0) { - prevEndLine := scanner.GetECMALineOfPosition(w.sourceFile, savePreviousRange.Loc.End()) + prevEndLine := w.formattingContext.getLineOfPosition(savePreviousRange.Loc.End()) indentToken = lastTriviaWasNewLine && tokenStartLine != prevEndLine } else { // When there's no previous range (first token), TS sets prevEndLine to undefined. diff --git a/tsc/internal/fourslash/tests/formatBulkTrivia_test.go b/tsc/internal/fourslash/tests/formatBulkTrivia_test.go new file mode 100644 index 0000000000000..452d8dc0e0e53 --- /dev/null +++ b/tsc/internal/fourslash/tests/formatBulkTrivia_test.go @@ -0,0 +1,25 @@ +package fourslash_test + +import ( + "strings" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/fourslash" + "github.com/microsoft/TypeScript/tsc/internal/testutil" +) + +func TestFormatBulkTrivia(t *testing.T) { + t.Parallel() + defer testutil.RecoverAndFail(t, "Panic on fourslash test") + comment := "/* " + strings.Repeat("Documentation ", 32) + "\u4e2d \U0001f600 */" + content := "function f(){\n" + + strings.Repeat(" ", 129) + comment + "\n" + + strings.Repeat(" ", 129) + "return \"\u4e2d \U0001f600\";\n}" + expected := "function f() {\n " + comment + "\n return \"\u4e2d \U0001f600\";\n}" + f, done := fourslash.NewFourslash(t, nil /*capabilities*/, content) + defer done() + f.FormatDocument(t, "") + f.VerifyCurrentFileContent(t, expected) + f.FormatDocument(t, "") + f.VerifyCurrentFileContent(t, expected) +} diff --git a/tsc/internal/scanner/lines_test.go b/tsc/internal/scanner/lines_test.go new file mode 100644 index 0000000000000..834605baf021c --- /dev/null +++ b/tsc/internal/scanner/lines_test.go @@ -0,0 +1,39 @@ +package scanner_test + +import ( + "strings" + "testing" + "unicode/utf8" + + "github.com/microsoft/TypeScript/tsc/internal/ast" + "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/parser" + "github.com/microsoft/TypeScript/tsc/internal/scanner" + "github.com/microsoft/TypeScript/tsc/internal/stringutil" + "gotest.tools/v3/assert" +) + +func TestGetECMAEndLinePosition(t *testing.T) { + t.Parallel() + for _, text := range []string{ + "", "value", "\n", "\r", "\r\n", "\u2028", "\u2029", + "\r\n\n\r\u2028\u2029", + "value\r\nnext\rthird\nfourth\u2028fifth\u2029last", + "\u4e2d\U0001f600\u2028\u03bb\r\n", + "\xff\n\xe2\x80\r\n", + strings.Repeat(" ", 1024) + "\r\nlast", + } { + file := parser.ParseSourceFile(ast.SourceFileParseOptions{FileName: "/test.ts", PathKey: "/test.ts"}, text, core.ScriptKindTS) + for line, start := range scanner.GetECMALineStarts(file) { + pos := int(start) + for { + ch, size := utf8.DecodeRuneInString(text[pos:]) + if size == 0 || stringutil.IsLineBreak(ch) { + break + } + pos += size + } + assert.Equal(t, scanner.GetECMAEndLinePosition(file, line), pos-1, "text %q, line %d", text, line) + } + } +} diff --git a/tsc/internal/scanner/scanner.go b/tsc/internal/scanner/scanner.go index 785fe50a4e7df..b3b74f7eb75a4 100644 --- a/tsc/internal/scanner/scanner.go +++ b/tsc/internal/scanner/scanner.go @@ -488,6 +488,7 @@ func (s *Scanner) Scan() ast.Kind { if s.skipTrivia { continue } + s.pos += stringutil.ASCIISingleLineWhitespacePrefixLen(s.text[s.pos:s.end]) for { ch, size := s.charAndSize() if !stringutil.IsWhiteSpaceSingleLine(ch) { @@ -627,9 +628,7 @@ func (s *Scanner) Scan() ast.Kind { s.pos += 2 for { - s.scanASCIIWhile(func(b byte) bool { - return b != '\n' && b != '\r' - }) + s.pos += stringutil.LineBreakPrefixLen(s.text[s.pos:s.end]) ch1, size := s.charAndSize() if size == 0 || stringutil.IsLineBreak(ch1) { break @@ -2323,7 +2322,10 @@ func SkipTriviaEx(text string, pos int, options *SkipTriviaOptions) int { if pos >= textLen { return pos } - ch, size := utf8.DecodeRuneInString(text[pos:]) + ch, size := rune(text[pos]), 1 + if ch >= utf8.RuneSelf { + ch, size = utf8.DecodeRuneInString(text[pos:]) + } switch ch { case '\r': if pos+1 < textLen && text[pos+1] == '\n' { @@ -2339,6 +2341,7 @@ func SkipTriviaEx(text string, pos int, options *SkipTriviaOptions) int { continue case '\t', '\v', '\f', ' ': pos++ + pos += stringutil.ASCIISingleLineWhitespacePrefixLen(text[pos:]) continue case '/': if options.StopAtComments { @@ -2348,6 +2351,10 @@ func SkipTriviaEx(text string, pos int, options *SkipTriviaOptions) int { if text[pos+1] == '/' { pos += 2 for pos < textLen { + pos += stringutil.LineBreakPrefixLen(text[pos:]) + if pos == textLen { + break + } ch, size := utf8.DecodeRuneInString(text[pos:]) if stringutil.IsLineBreak(ch) { break @@ -2359,13 +2366,10 @@ func SkipTriviaEx(text string, pos int, options *SkipTriviaOptions) int { } if text[pos+1] == '*' { pos += 2 - for pos < textLen { - if text[pos] == '*' && (pos+1 < textLen) && text[pos+1] == '/' { - pos += 2 - break - } - _, size := utf8.DecodeRuneInString(text[pos:]) - pos += size + if end := strings.Index(text[pos:], "*/"); end >= 0 { + pos += end + 2 + } else { + pos = textLen } canConsumeStar = false continue @@ -2701,14 +2705,25 @@ func GetECMALineAndByteOffsetOfPosition(sourceFile ast.SourceFileLike, pos int) } func GetECMAEndLinePosition(sourceFile *ast.SourceFile, line int) int { - pos := int(GetECMALineStarts(sourceFile)[line]) - for { - ch, size := utf8.DecodeRuneInString(sourceFile.Text()[pos:]) - if size == 0 || stringutil.IsLineBreak(ch) { - return pos - 1 + lineStarts := GetECMALineStarts(sourceFile)[line:] + text := sourceFile.Text() + if len(lineStarts) == 1 { + return len(text) - 1 + } + end := int(lineStarts[1]) + switch text[end-1] { + case '\n': + end-- + if end > 0 && text[end-1] == '\r' { + end-- } - pos += size + case '\r': + end-- + default: + // The remaining ECMAScript line separators, U+2028 and U+2029, are three bytes. + end -= 3 } + return end - 1 } // GetECMAPositionOfLineAndUTF16Character converts a 0-based line number and UTF-16 diff --git a/tsc/internal/scanner/scanner_test.go b/tsc/internal/scanner/scanner_test.go index 8f970f2af8467..5bee357727734 100644 --- a/tsc/internal/scanner/scanner_test.go +++ b/tsc/internal/scanner/scanner_test.go @@ -1,6 +1,7 @@ package scanner import ( + "strconv" "strings" "testing" @@ -103,3 +104,44 @@ func TestScanSourceKeyword(t *testing.T) { assert.Equal(t, s.Scan(), ast.KindIdentifier) assert.Equal(t, s.TokenValue(), "sourceValue") } + +func TestBulkTrivia(t *testing.T) { + t.Parallel() + for i, prefix := range []string{ + "", + strings.Repeat(" \t\v\f", 40), + strings.Repeat(" ", 129) + "\u00a0\u2003\ufeff", + "// " + strings.Repeat("comment ", 40) + "\r\n", + "// " + strings.Repeat("\u4e2d", 40) + "\u2028", + "// \u20ac is not a line break\u2029", + "/* " + strings.Repeat("\u4e2d * / ", 40) + "*/", + "/* first */ /* second */\n\t", + "\r\n" + strings.Repeat(" ", 65), + } { + t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Parallel() + text := prefix + "value" + assert.Equal(t, SkipTrivia(text, 0), len(prefix)) + s := NewScanner() + s.SetText(text) + assert.Equal(t, s.Scan(), ast.KindIdentifier) + assert.Equal(t, s.TokenStart(), len(prefix)) + s.ResetTokenState(0) + s.SetSkipTrivia(false) + end := 0 + for s.Scan() != ast.KindIdentifier { + assert.Assert(t, ast.IsTrivia(s.Token())) + assert.Equal(t, s.TokenFullStart(), end) + end = s.TokenEnd() + } + assert.Equal(t, s.TokenFullStart(), end) + assert.Equal(t, s.TokenStart(), len(prefix)) + }) + } + for _, text := range []string{"// unterminated", "/* unterminated", "/* \xff", "/* trailing *"} { + assert.Equal(t, SkipTrivia(text, 0), len(text)) + } + assert.Equal(t, SkipTriviaEx(" \t/* comment */value", 0, &SkipTriviaOptions{StopAtComments: true}), 2) + assert.Equal(t, SkipTriviaEx(" \t\r\n value", 0, &SkipTriviaOptions{StopAfterLineBreak: true}), 4) + assert.Equal(t, SkipTriviaEx("\n \t* value", 0, &SkipTriviaOptions{InJSDoc: true}), 5) +} diff --git a/tsc/internal/stringutil/textscan.go b/tsc/internal/stringutil/textscan.go index 9cd59b9e1efe5..d5721be78a222 100644 --- a/tsc/internal/stringutil/textscan.go +++ b/tsc/internal/stringutil/textscan.go @@ -21,6 +21,24 @@ func asciiPrefixLen(text string) int { return len(text) } +// ASCIISingleLineWhitespacePrefixLen returns the length of the leading run of +// ASCII whitespace other than line breaks. Non-ASCII whitespace is left to the caller. +func ASCIISingleLineWhitespacePrefixLen(text string) int { + if len(text) < 16 || text[0] != ' ' && text[0] != '\t' && text[0] != '\v' && text[0] != '\f' { + return asciiSingleLineWhitespacePrefixLen(text) + } + return asciiSingleLineWhitespacePrefixLenLong(text) +} + +func asciiSingleLineWhitespacePrefixLen(text string) int { + for i := range len(text) { + if b := text[i]; b != ' ' && b != '\t' && b != '\v' && b != '\f' { + return i + } + } + return len(text) +} + func lineBreakPrefixLen(text string) int { for i := range len(text) { if b := text[i]; b == '\r' || b == '\n' || b == 0xe2 { diff --git a/tsc/internal/stringutil/textscan_scalar.go b/tsc/internal/stringutil/textscan_scalar.go index 786b041efa762..f9917e207bcd4 100644 --- a/tsc/internal/stringutil/textscan_scalar.go +++ b/tsc/internal/stringutil/textscan_scalar.go @@ -6,6 +6,10 @@ func asciiPrefixLenLong(text string) int { return asciiPrefixLen(text) } +func asciiSingleLineWhitespacePrefixLenLong(text string) int { + return asciiSingleLineWhitespacePrefixLen(text) +} + // LineBreakPrefixLen returns the byte offset of the first possible ECMAScript // line break (CR, LF, or the leading byte of U+2028/U+2029), or len(text). func LineBreakPrefixLen(text string) int { diff --git a/tsc/internal/stringutil/textscan_simd.go b/tsc/internal/stringutil/textscan_simd.go index 418cee8b48cb6..7663d505ef671 100644 --- a/tsc/internal/stringutil/textscan_simd.go +++ b/tsc/internal/stringutil/textscan_simd.go @@ -53,6 +53,28 @@ func asciiPrefixLenLong(text string) int { return i + asciiPrefixLen(text[i:]) } +func asciiSingleLineWhitespacePrefixLenLong(text string) int { + width := (simd.Uint8s{}).Len() + bytes := unsafe.Slice(unsafe.StringData(text), len(text)) + space := simd.BroadcastUint8s(' ') + tab := simd.BroadcastUint8s('\t') + verticalTab := simd.BroadcastUint8s('\v') + formFeed := simd.BroadcastUint8s('\f') + allBits := simd.BroadcastUint8s(0xff) + words := make([]uint64, (simd.Uint64s{}).Len()) + i := 0 + for len(text)-i >= width { + v := simd.LoadUint8s(bytes[i:]) + v.Equal(space).Or(v.Equal(tab)).Or(v.Equal(verticalTab)).Or(v.Equal(formFeed)). + ToInt8s().ToBits().Xor(allBits).ReshapeToUint64s().Store(words) + if offset := firstNonzeroByte(words); offset < width { + return i + offset + } + i += width + } + return i + asciiSingleLineWhitespacePrefixLen(text[i:]) +} + // LineBreakPrefixLen returns the byte offset of the first possible ECMAScript // line break (CR, LF, or the leading byte of U+2028/U+2029), or len(text). func LineBreakPrefixLen(text string) int { diff --git a/tsc/internal/stringutil/textscan_test.go b/tsc/internal/stringutil/textscan_test.go index 64f4ae06cbae2..70dbd4fbe5c73 100644 --- a/tsc/internal/stringutil/textscan_test.go +++ b/tsc/internal/stringutil/textscan_test.go @@ -21,6 +21,14 @@ func TestTextScanMultipleCandidates(t *testing.T) { func BenchmarkTextScanPrefixes(b *testing.B) { for _, length := range []int{8, 16, 31, 32, 63, 64, 127, 128, 1024} { b.Run(strconv.Itoa(length), func(b *testing.B) { + b.Run("Whitespace", func(b *testing.B) { + text := strings.Repeat(" ", length) + "x" + strings.Repeat("z", 128) + b.SetBytes(int64(length)) + b.ReportAllocs() + for b.Loop() { + ASCIISingleLineWhitespacePrefixLen(text) + } + }) b.Run("LineBreak", func(b *testing.B) { text := strings.Repeat("a", length) + "\n" + strings.Repeat("z", 128) b.SetBytes(int64(length + 1)) @@ -49,6 +57,9 @@ func checkTextScan(t *testing.T, text string) { if got, want := LineBreakPrefixLen(text), lineBreakPrefixLen(text); got != want { t.Fatalf("LineBreakPrefixLen(%q) = %d, want %d", text, got, want) } + if got, want := ASCIISingleLineWhitespacePrefixLen(text), asciiSingleLineWhitespacePrefixLen(text); got != want { + t.Fatalf("ASCIISingleLineWhitespacePrefixLen(%q) = %d, want %d", text, got, want) + } for _, quote := range []byte{0, '\'', '"', '`', 0xff} { if got, want := UnescapedASCIIPrefixLen(text, quote), unescapedASCIIPrefixLen(text, quote); got != want { t.Fatalf("UnescapedASCIIPrefixLen(%q, %q) = %d, want %d", text, quote, got, want) @@ -56,6 +67,29 @@ func checkTextScan(t *testing.T, text string) { } } +func TestWhitespacePrefixBytes(t *testing.T) { + t.Parallel() + for prefix := range 130 { + whitespace := strings.Repeat(" \t\v\f", prefix) + for offset := range 4 { + padding := whitespace[min(offset, len(whitespace)):] + for value := range 256 { + text := padding + string(byte(value)) + "x" + want := len(padding) + if value == ' ' || value == '\t' || value == '\v' || value == '\f' { + want++ + } + if got := ASCIISingleLineWhitespacePrefixLen(text); got != want { + t.Fatalf("prefix %d, offset %d, byte %x: got %d, want %d", prefix, offset, value, got, want) + } + } + if got := ASCIISingleLineWhitespacePrefixLen(padding); got != len(padding) { + t.Fatalf("prefix %d, offset %d: got %d, want %d", prefix, offset, got, len(padding)) + } + } + } +} + func TestTextScanBytes(t *testing.T) { t.Parallel() for prefix := range 130 {