Formatter perf improvements

.github/workflows/ci.yml

@@ -11,6 +11,9 @@ on:
 permissions:
   contents: read
 
+env:
+  GOEXPERIMENT: simd
+
 defaults:
   run:
     shell: bash
@@ -116,6 +119,11 @@ jobs:
             os: linux
             name: 'noembed'
             noembed: true
+          - runner: ubuntu-latest
+            os: linux
+            name: 'nosimd'
+            nosimd: true
+            skip: ${{ github.event_name == 'merge_group' }}
           - runner: ubuntu-latest
             os: linux
             name: 'concurrent test programs'
@@ -132,6 +140,7 @@ jobs:
       id-token: write
       contents: read
     env:
+      GOEXPERIMENT: ${{ (matrix.config.nosimd && 'nosimd') || 'simd' }}
       TSGO_HEREBY_RACE: ${{ (matrix.config.race && 'true') || 'false' }}
       TSGO_HEREBY_NOEMBED: ${{ (matrix.config.noembed && 'true') || 'false' }}
       TSGO_HEREBY_CONCURRENT_TEST_PROGRAMS: ${{ (matrix.config.concurrent-test-programs && 'true') || 'false' }}

Herebyfile.mjs

@@ -34,6 +34,9 @@ import { outputs as goASTOutputs } from "./tools/scripts/tsc/generate-go-ast.ts"
 import { outputs as tsASTOutputs } from "./tools/scripts/tsc/generate-ts-ast.ts";
 import generateUnicodeData, { outputs as unicodeOutputs } from "./tsc/internal/stringutil/_scripts/generate-unicode-data.mts";
 
+// Set GOEXPERIMENT=nosimd to opt out of SIMD for any task.
+process.env.GOEXPERIMENT ||= "simd";
+
 enableFileFingerprintCache();
 
 if (process.platform === "win32") {

tools/scripts/gen/generatedFile.test.mts

@@ -5,6 +5,8 @@ import { syncBuiltinESMExports } from "node:module";
 import os from "node:os";
 import path from "node:path";
 import { test } from "node:test";
+import { runInNewContext } from "node:vm";
+import ts from "typescript";
 import {
     defaultCacheDirectory,
     GeneratedFile,
@@ -16,6 +18,33 @@ import {
     withFileFingerprintWrites,
 } from "./utils.mts";
 
+test("Hereby defaults to SIMD without overriding explicit experiments", () => {
+    const fileName = path.resolve(import.meta.dirname, "../../../Herebyfile.mjs");
+    const source = ts.createSourceFile(fileName, fs.readFileSync(fileName, "utf8"), ts.ScriptTarget.Latest, true, ts.ScriptKind.JS);
+    const assignment = source.statements.find(statement =>
+        ts.isExpressionStatement(statement)
+        && ts.isBinaryExpression(statement.expression)
+        && statement.expression.left.getText(source) === "process.env.GOEXPERIMENT"
+    );
+    assert.ok(assignment, "Hereby must set the default experiment before running tasks");
+    for (
+        const { initial, expected } of [
+            { initial: undefined, expected: "simd" },
+            { initial: "", expected: "simd" },
+            { initial: "nosimd", expected: "nosimd" },
+            { initial: "simd", expected: "simd" },
+            { initial: "jsonv2", expected: "jsonv2" },
+            { initial: "simd,nosimd", expected: "simd,nosimd" },
+        ]
+    ) {
+        const environment: NodeJS.ProcessEnv = { GODEBUG: "tracebackancestors=10" };
+        if (initial !== undefined) environment.GOEXPERIMENT = initial;
+        runInNewContext(assignment.getText(source), { process: { env: environment } });
+        assert.equal(environment.GOEXPERIMENT, expected);
+        assert.equal(environment.GODEBUG, "tracebackancestors=10");
+    }
+});
+
 test("unchanged generators produce no output", async context => {
     const generators = [
         ["Go AST", (await import("../tsc/generate-go-ast.ts")).default],

tsc/internal/ast/positionmap.go

@@ -43,7 +43,7 @@ func ComputePositionMap(text string) *PositionMap {
 	for i := 0; i < len(text); {
 		b := text[i]
 		if b < utf8.RuneSelf {
-			i++
+			i += stringutil.ASCIIPrefixLen(text[i:])
 			continue
 		}
 		r, size := stringutil.DecodeJSStringRune(text[i:])

tsc/internal/ast/positionmap_scan_test.go

@@ -0,0 +1,72 @@
+package ast_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/microsoft/TypeScript/tsc/internal/ast"
+	"github.com/microsoft/TypeScript/tsc/internal/stringutil"
+	"github.com/microsoft/TypeScript/tsc/internal/testutil/fixtures"
+)
+
+func TestPositionMapScanBoundaries(t *testing.T) {
+	t.Parallel()
+	for prefix := range 130 {
+		for _, middle := range []string{
+			"", "\u00E9", "\u65E5", "\U0001F980", "\xff", "\xe2\x80",
+			stringutil.EncodeJSStringRune(0xD800), "\u00E9a\U0001F980",
+		} {
+			text := strings.Repeat("a", prefix) + middle + strings.Repeat("z", 129)
+			pm := ast.ComputePositionMap(text)
+			type entry struct{ bytePos, delta int }
+			var entries []entry
+			delta := 0
+			for i := 0; i < len(text); {
+				r, size := stringutil.DecodeJSStringRune(text[i:])
+				units := 1
+				if r >= 0x10000 {
+					units = 2
+				}
+				if text[i] >= 0x80 {
+					delta += size - units
+					entries = append(entries, entry{i + size, delta})
+				}
+				i += size
+			}
+			if pm.IsAsciiOnly() != (len(entries) == 0) {
+				t.Fatalf("IsAsciiOnly for %q", text)
+			}
+			for pos := 0; pos <= len(text); pos++ {
+				want16, want8 := pos, pos
+				for _, e := range entries {
+					if e.bytePos <= pos {
+						want16 = pos - e.delta
+					}
+					if e.bytePos-e.delta <= pos {
+						want8 = pos + e.delta
+					}
+				}
+				if got := pm.UTF8ToUTF16(pos); got != want16 {
+					t.Fatalf("UTF8ToUTF16(%d) = %d, want %d for %q", pos, got, want16, text)
+				}
+				if got := pm.UTF16ToUTF8(pos); got != want8 {
+					t.Fatalf("UTF16ToUTF8(%d) = %d, want %d for %q", pos, got, want8, text)
+				}
+			}
+		}
+	}
+}
+
+func BenchmarkPositionMapScan(b *testing.B) {
+	for _, fixture := range fixtures.BenchFixtures {
+		b.Run(fixture.Name(), func(b *testing.B) {
+			fixture.SkipIfNotExist(b)
+			text := fixture.ReadFile(b)
+			b.SetBytes(int64(len(text)))
+			b.ReportAllocs()
+			for b.Loop() {
+				ast.ComputePositionMap(text)
+			}
+		})
+	}
+}

tsc/internal/core/core.go

@@ -426,8 +426,21 @@ func Coalesce[T *U, U any](a T, b T) T {
 type ECMALineStarts []TextPos
 
 func ComputeECMALineStarts(text string) ECMALineStarts {
-	result := make([]TextPos, 0, strings.Count(text, "\n")+1)
-	return slices.AppendSeq(result, ComputeECMALineStartsSeq(text))
+	if len(text) < 16 {
+		// Avoid iterator and vector dispatch overhead for short eager scans.
+		result := make(ECMALineStarts, 1, strings.Count(text, "\n")+1)
+		for pos := 0; pos < len(text); pos++ {
+			if b := text[pos]; b != '\r' && b != '\n' && b != 0xe2 {
+				continue
+			}
+			if end := ecmaLineBreakEnd(text, pos); end != 0 {
+				result = append(result, TextPos(end))
+				pos = end - 1
+			}
+		}
+		return result
+	}
+	return computeECMALineStarts(text)
 }
 
 func ComputeECMALineStartsSeq(text string) iter.Seq[TextPos] {
@@ -436,36 +449,44 @@ func ComputeECMALineStartsSeq(text string) iter.Seq[TextPos] {
 		var pos TextPos
 		var lineStart TextPos
 		for pos < textLen {
-			b := text[pos]
-			if b < utf8.RuneSelf {
-				pos++
-				switch b {
-				case '\r':
-					if pos < textLen && text[pos] == '\n' {
-						pos++
-					}
-					fallthrough
-				case '\n':
-					if !yield(lineStart) {
-						return
-					}
-					lineStart = pos
+			pos += TextPos(stringutil.LineBreakPrefixLen(text[pos:]))
+			if pos == textLen {
+				break
+			}
+			if end := ecmaLineBreakEnd(text, int(pos)); end != 0 {
+				if !yield(lineStart) {
+					return
 				}
+				pos = TextPos(end)
+				lineStart = pos
 			} else {
-				ch, size := utf8.DecodeRuneInString(text[pos:])
-				pos += TextPos(size)
-				if stringutil.IsLineBreak(ch) {
-					if !yield(lineStart) {
-						return
-					}
-					lineStart = pos
-				}
+				pos++
 			}
 		}
 		yield(lineStart)
 	}
 }
 
+func ecmaLineBreakEnd(text string, pos int) int {
+	switch text[pos] {
+	case '\r':
+		if pos+1 < len(text) && text[pos+1] == '\n' {
+			return pos + 2
+		}
+		return pos + 1
+	case '\n':
+		if pos == 0 || text[pos-1] != '\r' {
+			return pos + 1
+		}
+	case 0xe2:
+		// U+2028 and U+2029 differ only in the low bit of their final byte.
+		if pos+2 < len(text) && text[pos+1] == 0x80 && text[pos+2]&0xfe == 0xa8 {
+			return pos + 3
+		}
+	}
+	return 0
+}
+
 // PositionToLineAndByteOffset returns the 0-based line and byte offset from the
 // start of that line for the given byte position, using the provided line starts.
 // The byte offset is a raw UTF-8 byte offset from the line start, not a UTF-16 code unit count.
@@ -482,21 +503,51 @@ type UTF16Offset int
 // UTF16Len returns the number of UTF-16 code units needed to
 // represent the given UTF-8 encoded string.
 func UTF16Len(s string) UTF16Offset {
-	// Fast path: scan for non-ASCII bytes. For ASCII-only strings,
-	// each byte is one UTF-16 code unit, so we can return len(s) directly.
+	// Amortize vector dispatch and keep the long scan's stack frame out of
+	// short strings, where the compact byte loop is faster.
+	if len(s) >= 64 {
+		return utf16LenLong(s)
+	}
 	for i := range len(s) {
 		if s[i] >= utf8.RuneSelf {
-			// Found non-ASCII; count the ASCII prefix, then decode the rest.
-			n := UTF16Offset(i)
-			for _, r := range s[i:] {
-				n += UTF16Offset(utf16.RuneLen(r))
-			}
-			return n
+			return UTF16Offset(i) + utf16LenRunes(s[i:])
 		}
 	}
 	return UTF16Offset(len(s))
 }
 
+func utf16LenLong(s string) UTF16Offset {
+	i := stringutil.ASCIIPrefixLen(s)
+	if i == len(s) {
+		return UTF16Offset(i)
+	}
+	// A non-ASCII start often means dense Unicode. Keep the compact rune loop
+	// rather than repeatedly dispatching SIMD for very short ASCII runs.
+	if i == 0 {
+		return utf16LenRunes(s)
+	}
+	n := UTF16Offset(i)
+	for i < len(s) {
+		r, size := utf8.DecodeRuneInString(s[i:])
+		n += UTF16Offset(utf16.RuneLen(r))
+		i += size
+		if i < len(s) && s[i] < utf8.RuneSelf {
+			size = stringutil.ASCIIPrefixLen(s[i:])
+			n += UTF16Offset(size)
+			i += size
+		}
+	}
+	return n
+}
+
+func utf16LenRunes(s string) UTF16Offset {
+	var n UTF16Offset
+	for _, r := range s {
+		n += UTF16Offset(utf16.RuneLen(r))
+	}
+	return n
+}
+
 func Flatten[T any](array [][]T) []T {
 	var result []T
 	for _, subArray := range array {

tsc/internal/core/linestarts_scalar.go

@@ -0,0 +1,13 @@
+//go:build !goexperiment.simd
+
+package core
+
+import (
+	"slices"
+	"strings"
+)
+
+func computeECMALineStarts(text string) ECMALineStarts {
+	result := make([]TextPos, 0, strings.Count(text, "\n")+1)
+	return slices.AppendSeq(result, ComputeECMALineStartsSeq(text))
+}

tsc/internal/core/linestarts_simd.go

@@ -0,0 +1,67 @@
+//go:build goexperiment.simd
+
+package core
+
+import (
+	"math/bits"
+	"simd"
+	"strings"
+	"unsafe"
+)
+
+func lineBreakBits(word uint64) uint64 {
+	// Pack the low bit of each byte into the low eight bits. The multiply
+	// places source byte i at bit 56+i without overlapping partial products.
+	return ((word & 0x0101010101010101) * 0x0102040810204080) >> 56
+}
+
+func computeECMALineStarts(text string) ECMALineStarts {
+	result := make(ECMALineStarts, 1, strings.Count(text, "\n")+1)
+	length := int(TextPos(len(text)))
+	if length <= 0 {
+		return result
+	}
+	text = text[:length]
+	// Read-only loads avoid copying the source text into a byte slice.
+	bytes := unsafe.Slice(unsafe.StringData(text), len(text))
+	width := (simd.Uint8s{}).Len()
+	cr := simd.BroadcastUint8s('\r')
+	lf := simd.BroadcastUint8s('\n')
+	separator := simd.BroadcastUint8s(0xe2)
+	mask := make([]uint64, (simd.Uint64s{}).Len())
+	pos := 0
+	for len(text)-pos >= width {
+		v := simd.LoadUint8s(bytes[pos:])
+		v.Equal(cr).Or(v.Equal(lf)).Or(v.Equal(separator)).ToInt8s().ToBits().ReshapeToUint64s().Store(mask)
+		for word := 0; word < len(mask); word += 2 {
+			lo := mask[word]
+			var hi uint64
+			if word+1 < len(mask) {
+				hi = mask[word+1]
+			}
+			if lo|hi == 0 {
+				continue
+			}
+			matches := lineBreakBits(lo) | lineBreakBits(hi)<<8
+			for matches != 0 {
+				index := pos + word*8 + bits.TrailingZeros64(matches)
+				matches &= matches - 1
+				if end := ecmaLineBreakEnd(text, index); end != 0 {
+					result = append(result, TextPos(end))
+					if end == index+2 {
+						// The next candidate is the LF of this CRLF pair.
+						matches &= matches - 1
+					}
+				}
+			}
+		}
+		pos += width
+	}
+	for ; pos < len(text); pos++ {
+		if end := ecmaLineBreakEnd(text, pos); end != 0 {
+			result = append(result, TextPos(end))
+			pos = end - 1
+		}
+	}
+	return result
+}