diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f61c5c914c24..066e444cfef79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,9 @@ on: permissions: contents: read +env: + GOEXPERIMENT: simd + defaults: run: shell: bash @@ -116,6 +119,11 @@ jobs: os: linux name: 'noembed' noembed: true + - runner: ubuntu-latest + os: linux + name: 'nosimd' + nosimd: true + skip: ${{ github.event_name == 'merge_group' }} - runner: ubuntu-latest os: linux name: 'concurrent test programs' @@ -132,6 +140,7 @@ jobs: id-token: write contents: read env: + GOEXPERIMENT: ${{ (matrix.config.nosimd && 'nosimd') || 'simd' }} TSGO_HEREBY_RACE: ${{ (matrix.config.race && 'true') || 'false' }} TSGO_HEREBY_NOEMBED: ${{ (matrix.config.noembed && 'true') || 'false' }} TSGO_HEREBY_CONCURRENT_TEST_PROGRAMS: ${{ (matrix.config.concurrent-test-programs && 'true') || 'false' }} diff --git a/Herebyfile.mjs b/Herebyfile.mjs index 3ecad38769fd5..7eaf0f02d45a4 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -34,6 +34,9 @@ import { outputs as goASTOutputs } from "./tools/scripts/tsc/generate-go-ast.ts" import { outputs as tsASTOutputs } from "./tools/scripts/tsc/generate-ts-ast.ts"; import generateUnicodeData, { outputs as unicodeOutputs } from "./tsc/internal/stringutil/_scripts/generate-unicode-data.mts"; +// Set GOEXPERIMENT=nosimd to opt out of SIMD for any task. +process.env.GOEXPERIMENT ||= "simd"; + enableFileFingerprintCache(); if (process.platform === "win32") { diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts index bd85f8b03b536..d27948d5454d0 100644 --- a/tools/scripts/gen/generatedFile.test.mts +++ b/tools/scripts/gen/generatedFile.test.mts @@ -5,6 +5,8 @@ import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; +import { runInNewContext } from "node:vm"; +import ts from "typescript"; import { defaultCacheDirectory, GeneratedFile, @@ -16,6 +18,33 @@ import { withFileFingerprintWrites, } from "./utils.mts"; +test("Hereby defaults to SIMD without overriding explicit experiments", () => { + const fileName = path.resolve(import.meta.dirname, "../../../Herebyfile.mjs"); + const source = ts.createSourceFile(fileName, fs.readFileSync(fileName, "utf8"), ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); + const assignment = source.statements.find(statement => + ts.isExpressionStatement(statement) + && ts.isBinaryExpression(statement.expression) + && statement.expression.left.getText(source) === "process.env.GOEXPERIMENT" + ); + assert.ok(assignment, "Hereby must set the default experiment before running tasks"); + for ( + const { initial, expected } of [ + { initial: undefined, expected: "simd" }, + { initial: "", expected: "simd" }, + { initial: "nosimd", expected: "nosimd" }, + { initial: "simd", expected: "simd" }, + { initial: "jsonv2", expected: "jsonv2" }, + { initial: "simd,nosimd", expected: "simd,nosimd" }, + ] + ) { + const environment: NodeJS.ProcessEnv = { GODEBUG: "tracebackancestors=10" }; + if (initial !== undefined) environment.GOEXPERIMENT = initial; + runInNewContext(assignment.getText(source), { process: { env: environment } }); + assert.equal(environment.GOEXPERIMENT, expected); + assert.equal(environment.GODEBUG, "tracebackancestors=10"); + } +}); + test("unchanged generators produce no output", async context => { const generators = [ ["Go AST", (await import("../tsc/generate-go-ast.ts")).default], diff --git a/tsc/internal/ast/positionmap.go b/tsc/internal/ast/positionmap.go index 4c2fde6b6842e..13a2a24fca6ca 100644 --- a/tsc/internal/ast/positionmap.go +++ b/tsc/internal/ast/positionmap.go @@ -43,7 +43,7 @@ func ComputePositionMap(text string) *PositionMap { for i := 0; i < len(text); { b := text[i] if b < utf8.RuneSelf { - i++ + i += stringutil.ASCIIPrefixLen(text[i:]) continue } r, size := stringutil.DecodeJSStringRune(text[i:]) diff --git a/tsc/internal/ast/positionmap_scan_test.go b/tsc/internal/ast/positionmap_scan_test.go new file mode 100644 index 0000000000000..6a4da8a3b46a6 --- /dev/null +++ b/tsc/internal/ast/positionmap_scan_test.go @@ -0,0 +1,72 @@ +package ast_test + +import ( + "strings" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/ast" + "github.com/microsoft/TypeScript/tsc/internal/stringutil" + "github.com/microsoft/TypeScript/tsc/internal/testutil/fixtures" +) + +func TestPositionMapScanBoundaries(t *testing.T) { + t.Parallel() + for prefix := range 130 { + for _, middle := range []string{ + "", "\u00E9", "\u65E5", "\U0001F980", "\xff", "\xe2\x80", + stringutil.EncodeJSStringRune(0xD800), "\u00E9a\U0001F980", + } { + text := strings.Repeat("a", prefix) + middle + strings.Repeat("z", 129) + pm := ast.ComputePositionMap(text) + type entry struct{ bytePos, delta int } + var entries []entry + delta := 0 + for i := 0; i < len(text); { + r, size := stringutil.DecodeJSStringRune(text[i:]) + units := 1 + if r >= 0x10000 { + units = 2 + } + if text[i] >= 0x80 { + delta += size - units + entries = append(entries, entry{i + size, delta}) + } + i += size + } + if pm.IsAsciiOnly() != (len(entries) == 0) { + t.Fatalf("IsAsciiOnly for %q", text) + } + for pos := 0; pos <= len(text); pos++ { + want16, want8 := pos, pos + for _, e := range entries { + if e.bytePos <= pos { + want16 = pos - e.delta + } + if e.bytePos-e.delta <= pos { + want8 = pos + e.delta + } + } + if got := pm.UTF8ToUTF16(pos); got != want16 { + t.Fatalf("UTF8ToUTF16(%d) = %d, want %d for %q", pos, got, want16, text) + } + if got := pm.UTF16ToUTF8(pos); got != want8 { + t.Fatalf("UTF16ToUTF8(%d) = %d, want %d for %q", pos, got, want8, text) + } + } + } + } +} + +func BenchmarkPositionMapScan(b *testing.B) { + for _, fixture := range fixtures.BenchFixtures { + b.Run(fixture.Name(), func(b *testing.B) { + fixture.SkipIfNotExist(b) + text := fixture.ReadFile(b) + b.SetBytes(int64(len(text))) + b.ReportAllocs() + for b.Loop() { + ast.ComputePositionMap(text) + } + }) + } +} diff --git a/tsc/internal/core/core.go b/tsc/internal/core/core.go index f5c296b9b5bb0..42fe8a3982f35 100644 --- a/tsc/internal/core/core.go +++ b/tsc/internal/core/core.go @@ -426,8 +426,21 @@ func Coalesce[T *U, U any](a T, b T) T { type ECMALineStarts []TextPos func ComputeECMALineStarts(text string) ECMALineStarts { - result := make([]TextPos, 0, strings.Count(text, "\n")+1) - return slices.AppendSeq(result, ComputeECMALineStartsSeq(text)) + if len(text) < 16 { + // Avoid iterator and vector dispatch overhead for short eager scans. + result := make(ECMALineStarts, 1, strings.Count(text, "\n")+1) + for pos := 0; pos < len(text); pos++ { + if b := text[pos]; b != '\r' && b != '\n' && b != 0xe2 { + continue + } + if end := ecmaLineBreakEnd(text, pos); end != 0 { + result = append(result, TextPos(end)) + pos = end - 1 + } + } + return result + } + return computeECMALineStarts(text) } func ComputeECMALineStartsSeq(text string) iter.Seq[TextPos] { @@ -436,36 +449,44 @@ func ComputeECMALineStartsSeq(text string) iter.Seq[TextPos] { var pos TextPos var lineStart TextPos for pos < textLen { - b := text[pos] - if b < utf8.RuneSelf { - pos++ - switch b { - case '\r': - if pos < textLen && text[pos] == '\n' { - pos++ - } - fallthrough - case '\n': - if !yield(lineStart) { - return - } - lineStart = pos + pos += TextPos(stringutil.LineBreakPrefixLen(text[pos:])) + if pos == textLen { + break + } + if end := ecmaLineBreakEnd(text, int(pos)); end != 0 { + if !yield(lineStart) { + return } + pos = TextPos(end) + lineStart = pos } else { - ch, size := utf8.DecodeRuneInString(text[pos:]) - pos += TextPos(size) - if stringutil.IsLineBreak(ch) { - if !yield(lineStart) { - return - } - lineStart = pos - } + pos++ } } yield(lineStart) } } +func ecmaLineBreakEnd(text string, pos int) int { + switch text[pos] { + case '\r': + if pos+1 < len(text) && text[pos+1] == '\n' { + return pos + 2 + } + return pos + 1 + case '\n': + if pos == 0 || text[pos-1] != '\r' { + return pos + 1 + } + case 0xe2: + // U+2028 and U+2029 differ only in the low bit of their final byte. + if pos+2 < len(text) && text[pos+1] == 0x80 && text[pos+2]&0xfe == 0xa8 { + return pos + 3 + } + } + return 0 +} + // PositionToLineAndByteOffset returns the 0-based line and byte offset from the // start of that line for the given byte position, using the provided line starts. // The byte offset is a raw UTF-8 byte offset from the line start, not a UTF-16 code unit count. @@ -482,21 +503,51 @@ type UTF16Offset int // UTF16Len returns the number of UTF-16 code units needed to // represent the given UTF-8 encoded string. func UTF16Len(s string) UTF16Offset { - // Fast path: scan for non-ASCII bytes. For ASCII-only strings, - // each byte is one UTF-16 code unit, so we can return len(s) directly. + // Amortize vector dispatch and keep the long scan's stack frame out of + // short strings, where the compact byte loop is faster. + if len(s) >= 64 { + return utf16LenLong(s) + } for i := range len(s) { if s[i] >= utf8.RuneSelf { - // Found non-ASCII; count the ASCII prefix, then decode the rest. - n := UTF16Offset(i) - for _, r := range s[i:] { - n += UTF16Offset(utf16.RuneLen(r)) - } - return n + return UTF16Offset(i) + utf16LenRunes(s[i:]) } } return UTF16Offset(len(s)) } +func utf16LenLong(s string) UTF16Offset { + i := stringutil.ASCIIPrefixLen(s) + if i == len(s) { + return UTF16Offset(i) + } + // A non-ASCII start often means dense Unicode. Keep the compact rune loop + // rather than repeatedly dispatching SIMD for very short ASCII runs. + if i == 0 { + return utf16LenRunes(s) + } + n := UTF16Offset(i) + for i < len(s) { + r, size := utf8.DecodeRuneInString(s[i:]) + n += UTF16Offset(utf16.RuneLen(r)) + i += size + if i < len(s) && s[i] < utf8.RuneSelf { + size = stringutil.ASCIIPrefixLen(s[i:]) + n += UTF16Offset(size) + i += size + } + } + return n +} + +func utf16LenRunes(s string) UTF16Offset { + var n UTF16Offset + for _, r := range s { + n += UTF16Offset(utf16.RuneLen(r)) + } + return n +} + func Flatten[T any](array [][]T) []T { var result []T for _, subArray := range array { diff --git a/tsc/internal/core/linestarts_scalar.go b/tsc/internal/core/linestarts_scalar.go new file mode 100644 index 0000000000000..f87d105c5f94a --- /dev/null +++ b/tsc/internal/core/linestarts_scalar.go @@ -0,0 +1,13 @@ +//go:build !goexperiment.simd + +package core + +import ( + "slices" + "strings" +) + +func computeECMALineStarts(text string) ECMALineStarts { + result := make([]TextPos, 0, strings.Count(text, "\n")+1) + return slices.AppendSeq(result, ComputeECMALineStartsSeq(text)) +} diff --git a/tsc/internal/core/linestarts_simd.go b/tsc/internal/core/linestarts_simd.go new file mode 100644 index 0000000000000..4984f820f5c94 --- /dev/null +++ b/tsc/internal/core/linestarts_simd.go @@ -0,0 +1,67 @@ +//go:build goexperiment.simd + +package core + +import ( + "math/bits" + "simd" + "strings" + "unsafe" +) + +func lineBreakBits(word uint64) uint64 { + // Pack the low bit of each byte into the low eight bits. The multiply + // places source byte i at bit 56+i without overlapping partial products. + return ((word & 0x0101010101010101) * 0x0102040810204080) >> 56 +} + +func computeECMALineStarts(text string) ECMALineStarts { + result := make(ECMALineStarts, 1, strings.Count(text, "\n")+1) + length := int(TextPos(len(text))) + if length <= 0 { + return result + } + text = text[:length] + // Read-only loads avoid copying the source text into a byte slice. + bytes := unsafe.Slice(unsafe.StringData(text), len(text)) + width := (simd.Uint8s{}).Len() + cr := simd.BroadcastUint8s('\r') + lf := simd.BroadcastUint8s('\n') + separator := simd.BroadcastUint8s(0xe2) + mask := make([]uint64, (simd.Uint64s{}).Len()) + pos := 0 + for len(text)-pos >= width { + v := simd.LoadUint8s(bytes[pos:]) + v.Equal(cr).Or(v.Equal(lf)).Or(v.Equal(separator)).ToInt8s().ToBits().ReshapeToUint64s().Store(mask) + for word := 0; word < len(mask); word += 2 { + lo := mask[word] + var hi uint64 + if word+1 < len(mask) { + hi = mask[word+1] + } + if lo|hi == 0 { + continue + } + matches := lineBreakBits(lo) | lineBreakBits(hi)<<8 + for matches != 0 { + index := pos + word*8 + bits.TrailingZeros64(matches) + matches &= matches - 1 + if end := ecmaLineBreakEnd(text, index); end != 0 { + result = append(result, TextPos(end)) + if end == index+2 { + // The next candidate is the LF of this CRLF pair. + matches &= matches - 1 + } + } + } + } + pos += width + } + for ; pos < len(text); pos++ { + if end := ecmaLineBreakEnd(text, pos); end != 0 { + result = append(result, TextPos(end)) + pos = end - 1 + } + } + return result +} diff --git a/tsc/internal/core/linestarts_simd_test.go b/tsc/internal/core/linestarts_simd_test.go new file mode 100644 index 0000000000000..973ecee92c08f --- /dev/null +++ b/tsc/internal/core/linestarts_simd_test.go @@ -0,0 +1,20 @@ +//go:build goexperiment.simd + +package core + +import "testing" + +func TestLineBreakBits(t *testing.T) { + t.Parallel() + for mask := range 256 { + var bytes uint64 + for i := range 8 { + if mask&(1<(path:string,convert:(text:string)=>T){ +// Ignore empty lines, but preserve quoted text. +const text=await readFile(path,"utf8"); +return text.split(/\r?\n/).filter(line=>line.length>0).map(convert); +} + +export class Cache{ +private values=new Map(); +get(key:string){return this.values.get(key);} +set(key:string,value:T){this.values.set(key,value);} +} +` + +func BenchmarkFormatDocument(b *testing.B) { + cases := append([]filefixture.Fixture{}, fixtures.BenchFixtures...) + cases = append(cases, + filefixture.FromString("editor.ts", "editor.ts", editorText), + filefixture.FromString("comments.ts", "comments.ts", strings.Repeat( + "/**\n * "+strings.Repeat("Documentation with ordinary ASCII words. ", 8)+"\n */\n"+ + "export function value(input:string){ // Keep the trailing comment.\n"+ + "return `"+strings.Repeat("template text ", 32)+"${input}`;\n}\n", 64)), + filefixture.FromString("whitespace.ts", "whitespace.ts", strings.Repeat( + "export function value(input:number){\n"+ + strings.Repeat(" ", 128)+"const result=input+1;"+strings.Repeat(" ", 128)+"\n"+ + strings.Repeat(" ", 128)+"return result;"+strings.Repeat(" ", 128)+"\n}\n", 128)), + filefixture.FromString("unicode-crlf.ts", "unicode-crlf.ts", strings.Repeat( + "// Unicode text: \u03bb \u4e2d \U0001f600\r\n"+ + "export function value(input:string){\r\n"+ + "\t\u00a0const text=\"\u03bb \u4e2d \U0001f600\";\r\n"+ + "\treturn input+text;\t \r\n}\r\n", 128)), + filefixture.FromString("minified.js", "minified.js", strings.Repeat( + "function value(input){if(input){return{value:input+1,valid:true};}return null;}", 256)), + ) + for _, fixture := range cases { + b.Run(fixture.Name(), func(b *testing.B) { + fixture.SkipIfNotExist(b) + fileName := tspath.ToRootedFilePath(fixture.Path(), "/") + parse := func(text string) *ast.SourceFile { + return parser.ParseSourceFile(ast.SourceFileParseOptions{ + FileName: fileName, + PathKey: tspath.CaseSensitive.PathKey(tspath.RootedPath(fileName)), + }, text, core.GetScriptKindFromFileName(fileName)) + } + options := lsutil.GetDefaultFormatCodeSettings() + options.TrimTrailingWhitespace = core.TSTrue + ctx := format.WithFormatCodeSettings(b.Context(), options, "\n") + text := fixture.ReadFile(b) + file := parse(text) + assert.Equal(b, len(file.Diagnostics()), 0) + formatted := applyBulkEdits(text, format.FormatDocument(ctx, file)) + for _, input := range []struct { + name string + text string + file *ast.SourceFile + }{ + {"original", text, file}, + {"formatted", formatted, parse(formatted)}, + } { + b.Run(input.name, func(b *testing.B) { + expected := format.FormatDocument(ctx, input.file) + if input.name == "formatted" { + assert.Assert(b, applyBulkEdits(input.text, expected) == input.text) + } + b.SetBytes(int64(len(input.text))) + b.ReportAllocs() + var edits []core.TextChange + for b.Loop() { + edits = format.FormatDocument(ctx, input.file) + } + assert.Assert(b, slices.Equal(edits, expected)) + }) + } + }) + } +} + +func BenchmarkFormatEditor(b *testing.B) { + options := lsutil.GetDefaultFormatCodeSettings() + options.TrimTrailingWhitespace = core.TSTrue + ctx := format.WithFormatCodeSettings(b.Context(), options, "\n") + file := parser.ParseSourceFile(ast.SourceFileParseOptions{ + FileName: "/editor.ts", + PathKey: "/editor.ts", + }, editorText, core.ScriptKindTS) + for _, operation := range []struct { + name string + run func() []core.TextChange + }{ + {"selection", func() []core.TextChange { + return format.FormatSelection(ctx, file, strings.Index(editorText, "const text"), strings.Index(editorText, "\n}\n")) + }}, + {"semicolon", func() []core.TextChange { + return format.FormatOnSemicolon(ctx, file, strings.Index(editorText, "utf8\");")+len("utf8\");")) + }}, + {"enter", func() []core.TextChange { + return format.FormatOnEnter(ctx, file, strings.Index(editorText, "return text")) + }}, + } { + b.Run(operation.name, func(b *testing.B) { + expected := operation.run() + assert.Assert(b, len(expected) > 0) + b.ReportAllocs() + var edits []core.TextChange + for b.Loop() { + edits = operation.run() + } + assert.Assert(b, slices.Equal(edits, expected)) + }) + } +} diff --git a/tsc/internal/format/context.go b/tsc/internal/format/context.go index 3833413b26109..e2680623b7077 100644 --- a/tsc/internal/format/context.go +++ b/tsc/internal/format/context.go @@ -21,6 +21,9 @@ type FormattingContext struct { contextNodeBlockIsOnOneLine core.Tristate nextNodeBlockIsOnOneLine core.Tristate + lineStarts []core.TextPos + lastLine int + SourceFile *ast.SourceFile FormattingRequestKind FormatRequestKind Options lsutil.FormatCodeSettings @@ -31,6 +34,7 @@ func NewFormattingContext(file *ast.SourceFile, kind FormatRequestKind, options SourceFile: file, FormattingRequestKind: kind, Options: options, + lineStarts: scanner.GetECMALineStarts(file), } return res } @@ -60,12 +64,28 @@ func (this *FormattingContext) UpdateContext(cur TextRangeWithKind, curParent *a } func (this *FormattingContext) rangeIsOnOneLine(node core.TextRange) core.Tristate { - if rangeIsOnOneLine(node, this.SourceFile) { + if this.getLineOfPosition(node.Pos()) == this.getLineOfPosition(node.End()) { return core.TSTrue } return core.TSFalse } +func (this *FormattingContext) getLineOfPosition(pos int) int { + line := this.lastLine + if line >= 0 && pos >= int(this.lineStarts[line]) && + (line+1 == len(this.lineStarts) || pos < int(this.lineStarts[line+1])) { + return line + } + line = scanner.ComputeLineOfPosition(this.lineStarts, pos) + this.lastLine = line + return line +} + +func (this *FormattingContext) getLineAndByteOffsetOfPosition(pos int) (int, int) { + line := this.getLineOfPosition(pos) + return line, pos - int(this.lineStarts[line]) +} + func (this *FormattingContext) nodeIsOnOneLine(node *ast.Node) core.Tristate { return this.rangeIsOnOneLine(withTokenStart(node, this.SourceFile)) } diff --git a/tsc/internal/format/context_test.go b/tsc/internal/format/context_test.go new file mode 100644 index 0000000000000..0e7bf80566274 --- /dev/null +++ b/tsc/internal/format/context_test.go @@ -0,0 +1,34 @@ +package format + +import ( + "strings" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/ast" + "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/ls/lsutil" + "github.com/microsoft/TypeScript/tsc/internal/parser" + "github.com/microsoft/TypeScript/tsc/internal/scanner" + "gotest.tools/v3/assert" +) + +func TestFormattingLinePositions(t *testing.T) { + t.Parallel() + for _, text := range []string{"", "value", strings.Repeat("first\r\n\u4e2d\u2028next\nlast\u2029", 32)} { + file := parser.ParseSourceFile(ast.SourceFileParseOptions{FileName: "/test.ts", PathKey: "/test.ts"}, text, core.ScriptKindTS) + context := NewFormattingContext(file, FormatRequestKindFormatDocument, lsutil.GetDefaultFormatCodeSettings()) + for _, reverse := range []bool{false, true, false} { + for i := range len(text) + 1 { + pos := i + if reverse { + pos = len(text) - i + } + start := scanner.GetECMALineOfPosition(file, pos) + for _, end := range []int{pos, min(pos+1, len(text)), len(text)} { + want := start == scanner.GetECMALineOfPosition(file, end) + assert.Equal(t, context.rangeIsOnOneLine(core.NewTextRange(pos, end)).IsTrue(), want) + } + } + } + } +} diff --git a/tsc/internal/format/format_test.go b/tsc/internal/format/format_test.go index 494624d57b661..4fcc0f9f7e187 100644 --- a/tsc/internal/format/format_test.go +++ b/tsc/internal/format/format_test.go @@ -54,3 +54,37 @@ func TestFormatNoTrailingSpace(t *testing.T) { }) } } + +func TestFormatBulkTrivia(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + body string + newLine string + }{ + {"long indentation", strings.Repeat(" ", 129) + "return 1;" + strings.Repeat(" ", 129), "\n"}, + {"mixed indentation", strings.Repeat(" \t", 33) + "return 1;", "\r\n"}, + {"Unicode indentation", strings.Repeat(" ", 32) + "\u00a0\u2003return 1;", "\n"}, + {"long comment", "/* " + strings.Repeat("comment ", 64) + "*/" + "\nreturn 1;", "\n"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + options := lsutil.GetDefaultFormatCodeSettings() + options.NewLineCharacter = test.newLine + options.TrimTrailingWhitespace = core.TSTrue + ctx := format.WithFormatCodeSettings(t.Context(), options, test.newLine) + text := "function f(){" + test.newLine + test.body + test.newLine + "}" + test.newLine + expectedBody := " return 1;" + if test.name == "long comment" { + expectedBody = " /* " + strings.Repeat("comment ", 64) + "*/\n return 1;" + } + expected := "function f() {" + test.newLine + expectedBody + test.newLine + "}" + test.newLine + parse := func(text string) *ast.SourceFile { + return parser.ParseSourceFile(ast.SourceFileParseOptions{FileName: "/test.ts", PathKey: "/test.ts"}, text, core.ScriptKindTS) + } + formatted := applyBulkEdits(text, format.FormatDocument(ctx, parse(text))) + assert.Equal(t, formatted, expected) + assert.Equal(t, len(format.FormatDocument(ctx, parse(formatted))), 0) + }) + } +} diff --git a/tsc/internal/format/rulecontext.go b/tsc/internal/format/rulecontext.go index 175f43752f396..79ee249fa6b49 100644 --- a/tsc/internal/format/rulecontext.go +++ b/tsc/internal/format/rulecontext.go @@ -569,8 +569,8 @@ func isSemicolonDeletionContext(context *FormattingContext) bool { nextTokenStart = scanner.GetTokenPosOfNode(nextRealToken, context.SourceFile, false) } - startLine := scanner.GetECMALineOfPosition(context.SourceFile, context.currentTokenSpan.Loc.Pos()) - endLine := scanner.GetECMALineOfPosition(context.SourceFile, nextTokenStart) + startLine := context.getLineOfPosition(context.currentTokenSpan.Loc.Pos()) + endLine := context.getLineOfPosition(nextTokenStart) if startLine == endLine { return nextTokenKind == ast.KindCloseBraceToken || nextTokenKind == ast.KindEndOfFile } diff --git a/tsc/internal/format/scanner.go b/tsc/internal/format/scanner.go index 4eeb2e5770ac6..a639d4ffaba8f 100644 --- a/tsc/internal/format/scanner.go +++ b/tsc/internal/format/scanner.go @@ -1,8 +1,6 @@ package format import ( - "slices" - "github.com/microsoft/TypeScript/tsc/internal/ast" "github.com/microsoft/TypeScript/tsc/internal/core" "github.com/microsoft/TypeScript/tsc/internal/debug" @@ -266,10 +264,11 @@ func (s *formattingScanner) readTokenInfo(n *ast.Node) tokenInfo { } s.hasLastTokenInfo = true + // Trivia buffers are replaced rather than reused, so saved token infos can share them. s.lastTokenInfo = tokenInfo{ - leadingTrivia: slices.Clone(s.leadingTrivia), + leadingTrivia: s.leadingTrivia, token: token, - trailingTrivia: slices.Clone(s.trailingTrivia), + trailingTrivia: s.trailingTrivia, } s.lastTokenInfo = fixTokenKind(s.lastTokenInfo, n) diff --git a/tsc/internal/format/scanner_test.go b/tsc/internal/format/scanner_test.go new file mode 100644 index 0000000000000..cc4ed7a0bd057 --- /dev/null +++ b/tsc/internal/format/scanner_test.go @@ -0,0 +1,51 @@ +package format + +import ( + "slices" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/ast" + "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/scanner" + "gotest.tools/v3/assert" +) + +func TestFormattingScannerTriviaOwnership(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + text string + first ast.Kind + rescan ast.Kind + }{ + {"ordinary", " left /* first */ +\t right // last\n third", ast.KindIdentifier, ast.KindIdentifier}, + {"greater than", " >> /* first */ next // last\n third", ast.KindGreaterThanToken, ast.KindGreaterThanGreaterThanToken}, + {"regular expression", " /ab+/g /* first */ next // last\n third", ast.KindSlashToken, ast.KindRegularExpressionLiteral}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + scan := scanner.NewScanner() + scan.SetSkipTrivia(false) + scan.SetText(test.text) + s := formattingScanner{s: scan, endPos: len(test.text), wasNewLine: true} + s.advance() + first := s.readTokenInfo(&ast.Node{Kind: test.first}) + firstLeading := slices.Clone(first.leadingTrivia) + firstTrailing := slices.Clone(first.trailingTrivia) + rescanned := s.readTokenInfo(&ast.Node{Kind: test.rescan}) + rescannedLeading := slices.Clone(rescanned.leadingTrivia) + rescannedTrailing := slices.Clone(rescanned.trailingTrivia) + for s.isOnToken() { + s.advance() + if s.isOnToken() { + s.readTokenInfo(&ast.Node{Kind: scan.Token()}) + } + } + s.skipToStartOf(&core.TextRange{}) + assert.Assert(t, slices.Equal(first.leadingTrivia, firstLeading)) + assert.Assert(t, slices.Equal(first.trailingTrivia, firstTrailing)) + assert.Assert(t, slices.Equal(rescanned.leadingTrivia, rescannedLeading)) + assert.Assert(t, slices.Equal(rescanned.trailingTrivia, rescannedTrailing)) + }) + } +} diff --git a/tsc/internal/format/span.go b/tsc/internal/format/span.go index df9d11119b2cb..f625fa2573c1b 100644 --- a/tsc/internal/format/span.go +++ b/tsc/internal/format/span.go @@ -248,10 +248,10 @@ func (w *formatSpanWorker) execute(s *formattingScanner) []core.TextChange { w.formattingScanner.advance() if w.formattingScanner.isOnToken() { - startLine := scanner.GetECMALineOfPosition(w.sourceFile, withTokenStart(w.enclosingNode, w.sourceFile).Pos()) + startLine := w.formattingContext.getLineOfPosition(withTokenStart(w.enclosingNode, w.sourceFile).Pos()) undecoratedStartLine := startLine if ast.HasDecorators(w.enclosingNode) { - undecoratedStartLine = scanner.GetECMALineOfPosition(w.sourceFile, getNonDecoratorTokenPosOfNode(w.enclosingNode, w.sourceFile)) + undecoratedStartLine = w.formattingContext.getLineOfPosition(getNonDecoratorTokenPosOfNode(w.enclosingNode, w.sourceFile)) } w.processNode(w.enclosingNode, w.enclosingNode, startLine, undecoratedStartLine, w.initialIndentation, w.delta) @@ -268,7 +268,7 @@ func (w *formatSpanWorker) execute(s *formattingScanner) []core.TextChange { } w.indentTriviaItems(remainingTrivia, indentation, true, func(item TextRangeWithKind) { - startLine, startChar := scanner.GetECMALineAndByteOffsetOfPosition(w.sourceFile, item.Loc.Pos()) + startLine, startChar := w.formattingContext.getLineAndByteOffsetOfPosition(item.Loc.Pos()) w.processRange(item, startLine, startChar, w.enclosingNode, w.enclosingNode, nil) w.insertIndentation(item.Loc.Pos(), indentation, false) }) @@ -314,7 +314,7 @@ func (w *formatSpanWorker) execute(s *formattingScanner) []core.TextChange { if parent == nil { parent = w.previousParent } - line := scanner.GetECMALineOfPosition(w.sourceFile, tokenInfo.Loc.Pos()) + line := w.formattingContext.getLineOfPosition(tokenInfo.Loc.Pos()) w.processPair( tokenInfo, line, @@ -351,11 +351,11 @@ func (w *formatSpanWorker) processChildNode( return inheritedIndentation } childStartPos := scanner.GetTokenPosOfNode(child, w.sourceFile, false) - childStartLine := scanner.GetECMALineOfPosition(w.sourceFile, childStartPos) + childStartLine := w.formattingContext.getLineOfPosition(childStartPos) undecoratedChildStartLine := childStartLine if ast.HasDecorators(child) { - undecoratedChildStartLine = scanner.GetECMALineOfPosition(w.sourceFile, getNonDecoratorTokenPosOfNode(child, w.sourceFile)) + undecoratedChildStartLine = w.formattingContext.getLineOfPosition(getNonDecoratorTokenPosOfNode(child, w.sourceFile)) } isErrorMemberListElement := child.Flags&ast.NodeFlagsThisNodeHasError != 0 && isMemberListElement(parent, child) @@ -472,7 +472,7 @@ func (w *formatSpanWorker) processChildNodes( break } else if tokenInfo.token.Kind == listStartToken { // consume list start token - startLine = scanner.GetECMALineOfPosition(w.sourceFile, tokenInfo.token.Loc.Pos()) + startLine = w.formattingContext.getLineOfPosition(tokenInfo.token.Loc.Pos()) w.consumeTokenAndAdvanceScanner(tokenInfo, parent, parentDynamicIndentation, parent, false) @@ -597,7 +597,7 @@ func (w *formatSpanWorker) tryComputeIndentationForListItem(startPos int, endPos return inheritedIndentation } } else { - startLine := scanner.GetECMALineOfPosition(w.sourceFile, startPos) + startLine := w.formattingContext.getLineOfPosition(startPos) column := w.getCurrentIndentationAtPosition(startPos) if startLine != parentStartLine || startPos == column { // Use the base indent size if it is greater than @@ -765,7 +765,7 @@ func (w *formatSpanWorker) processRange(r TextRangeWithKind, rangeStartLine int, if !rangeHasError { if w.previousRange == NewTextRangeWithKind(0, 0, 0) { // trim whitespaces starting from the beginning of the span up to the current line - originalStartLine := scanner.GetECMALineOfPosition(w.sourceFile, w.originalRange.Pos()) + originalStartLine := w.formattingContext.getLineOfPosition(w.originalRange.Pos()) w.trimTrailingWhitespacesForLines(originalStartLine, rangeStartLine, NewTextRangeWithKind(0, 0, 0)) } else { lineAction = w.processPair(r, rangeStartLine, parent, w.previousRange, w.previousRangeStartLine, w.previousParent, contextNode, dynamicIndentation) @@ -783,7 +783,7 @@ func (w *formatSpanWorker) processRange(r TextRangeWithKind, rangeStartLine int, func (w *formatSpanWorker) processTrivia(trivia []TextRangeWithKind, parent *ast.Node, contextNode *ast.Node, dynamicIndentation *dynamicIndenter) { for _, triviaItem := range trivia { if isComment(triviaItem.Kind) && triviaItem.Loc.ContainedBy(w.originalRange) { - triviaItemStartLine, triviaItemStartCharacter := scanner.GetECMALineAndByteOffsetOfPosition(w.sourceFile, triviaItem.Loc.Pos()) + triviaItemStartLine, triviaItemStartCharacter := w.formattingContext.getLineAndByteOffsetOfPosition(triviaItem.Loc.Pos()) w.processRange(triviaItem, triviaItemStartLine, triviaItemStartCharacter, parent, contextNode, dynamicIndentation) } } @@ -815,8 +815,8 @@ func (w *formatSpanWorker) trimTrailingWhitespacesForRemainingRange(trivias []Te } func (w *formatSpanWorker) trimTrailingWitespacesForPositions(startPos int, endPos int, previousRange TextRangeWithKind) { - startLine := scanner.GetECMALineOfPosition(w.sourceFile, startPos) - endLine := scanner.GetECMALineOfPosition(w.sourceFile, endPos) + startLine := w.formattingContext.getLineOfPosition(startPos) + endLine := w.formattingContext.getLineOfPosition(endPos) w.trimTrailingWhitespacesForLines(startLine, endLine+1, previousRange) } @@ -882,7 +882,7 @@ func (w *formatSpanWorker) insertIndentation(pos int, indentation int, lineAdded // insert indentation string at the very beginning of the token w.recordReplace(pos, 0, indentationString) } else { - tokenStartLine, tokenStartCharacter := scanner.GetECMALineAndByteOffsetOfPosition(w.sourceFile, pos) + tokenStartLine, tokenStartCharacter := w.formattingContext.getLineAndByteOffsetOfPosition(pos) startLinePosition := int(scanner.GetECMALineStarts(w.sourceFile)[tokenStartLine]) if indentation != w.characterToColumn(startLinePosition, tokenStartCharacter) || w.indentationIsDifferent(indentationString, startLinePosition) { w.recordReplace(startLinePosition, tokenStartCharacter, indentationString) @@ -936,8 +936,8 @@ func (w *formatSpanWorker) indentTriviaItems(trivia []TextRangeWithKind, comment func (w *formatSpanWorker) indentMultilineComment(commentRange core.TextRange, indentation int, firstLineIsIndented bool, indentFinalLine bool) { // split comment in lines - startLine := scanner.GetECMALineOfPosition(w.sourceFile, commentRange.Pos()) - endLine := scanner.GetECMALineOfPosition(w.sourceFile, commentRange.End()) + startLine := w.formattingContext.getLineOfPosition(commentRange.Pos()) + endLine := w.formattingContext.getLineOfPosition(commentRange.End()) if startLine == endLine { if !firstLineIsIndented { @@ -1051,7 +1051,7 @@ func (w *formatSpanWorker) consumeTokenAndAdvanceScanner(currentTokenInfo tokenI lineAction := LineActionNone isTokenInRange := currentTokenInfo.token.Loc.ContainedBy(w.originalRange) - tokenStartLine, tokenStartChar := scanner.GetECMALineAndByteOffsetOfPosition(w.sourceFile, currentTokenInfo.token.Loc.Pos()) + tokenStartLine, tokenStartChar := w.formattingContext.getLineAndByteOffsetOfPosition(currentTokenInfo.token.Loc.Pos()) if isTokenInRange { rangeHasError := w.rangeContainsError(currentTokenInfo.token.Loc) @@ -1063,7 +1063,7 @@ func (w *formatSpanWorker) consumeTokenAndAdvanceScanner(currentTokenInfo tokenI if lineAction == LineActionNone { // indent token only if end line of previous range does not match start line of the token if savePreviousRange != NewTextRangeWithKind(0, 0, 0) { - prevEndLine := scanner.GetECMALineOfPosition(w.sourceFile, savePreviousRange.Loc.End()) + prevEndLine := w.formattingContext.getLineOfPosition(savePreviousRange.Loc.End()) indentToken = lastTriviaWasNewLine && tokenStartLine != prevEndLine } else { // When there's no previous range (first token), TS sets prevEndLine to undefined. diff --git a/tsc/internal/fourslash/tests/formatBulkTrivia_test.go b/tsc/internal/fourslash/tests/formatBulkTrivia_test.go new file mode 100644 index 0000000000000..452d8dc0e0e53 --- /dev/null +++ b/tsc/internal/fourslash/tests/formatBulkTrivia_test.go @@ -0,0 +1,25 @@ +package fourslash_test + +import ( + "strings" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/fourslash" + "github.com/microsoft/TypeScript/tsc/internal/testutil" +) + +func TestFormatBulkTrivia(t *testing.T) { + t.Parallel() + defer testutil.RecoverAndFail(t, "Panic on fourslash test") + comment := "/* " + strings.Repeat("Documentation ", 32) + "\u4e2d \U0001f600 */" + content := "function f(){\n" + + strings.Repeat(" ", 129) + comment + "\n" + + strings.Repeat(" ", 129) + "return \"\u4e2d \U0001f600\";\n}" + expected := "function f() {\n " + comment + "\n return \"\u4e2d \U0001f600\";\n}" + f, done := fourslash.NewFourslash(t, nil /*capabilities*/, content) + defer done() + f.FormatDocument(t, "") + f.VerifyCurrentFileContent(t, expected) + f.FormatDocument(t, "") + f.VerifyCurrentFileContent(t, expected) +} diff --git a/tsc/internal/parser/parser_test.go b/tsc/internal/parser/parser_test.go index ffa96713eb444..94a993be0cf0e 100644 --- a/tsc/internal/parser/parser_test.go +++ b/tsc/internal/parser/parser_test.go @@ -24,6 +24,16 @@ import ( ) func BenchmarkParse(b *testing.B) { + benchmarkParse(b, false) +} + +// BenchmarkParseWithPositionMaps includes the lazy maps used for diagnostics +// and JavaScript API positions; BenchmarkParse measures syntax parsing alone. +func BenchmarkParseWithPositionMaps(b *testing.B) { + benchmarkParse(b, true) +} + +func benchmarkParse(b *testing.B, withPositionMaps bool) { for _, f := range fixtures.BenchFixtures { b.Run(f.Name(), func(b *testing.B) { f.SkipIfNotExist(b) @@ -38,8 +48,14 @@ func BenchmarkParse(b *testing.B) { PathKey: path, } + b.SetBytes(int64(len(sourceText))) + b.ReportAllocs() for b.Loop() { - parser.ParseSourceFile(opts, sourceText, scriptKind) + file := parser.ParseSourceFile(opts, sourceText, scriptKind) + if withPositionMaps { + file.ECMALineMap() + file.GetPositionMap() + } } }) } diff --git a/tsc/internal/printer/escaping.go b/tsc/internal/printer/escaping.go new file mode 100644 index 0000000000000..48051c0b6c9f9 --- /dev/null +++ b/tsc/internal/printer/escaping.go @@ -0,0 +1,90 @@ +package printer + +import ( + "strings" + "unicode/utf8" + + "github.com/microsoft/TypeScript/tsc/internal/stringutil" +) + +var asciiEscapeCodes = [128]byte{ + 0: '0', '\b': 'b', '\t': 't', '\n': 'n', '\v': 'v', '\f': 'f', '\r': 'r', + '\\': '\\', '"': '"', '\'': '\'', '`': '`', '$': '$', +} + +func appendEscapedString(s string, quoteChar QuoteChar, flags getLiteralTextFlags, b *strings.Builder) bool { + quote := byte(0) + if quoteChar >= 0 && quoteChar < utf8.RuneSelf { + quote = byte(quoteChar) + } + backtick := quoteChar == QuoteCharBacktick + asciiOnly := flags&getLiteralTextFlagsNeverAsciiEscape == 0 + jsx := flags&getLiteralTextFlagsJsxAttributeEscape != 0 + pos, pending := 0, 0 + changed := false + for pos < len(s) { + if ch := s[pos]; ch >= 0x20 && ch < utf8.RuneSelf && ch != '\\' && ch != quote && (!backtick || ch != '$') { + pos += stringutil.UnescapedASCIIPrefixLen(s[pos:], quote) + if pos == len(s) { + break + } + } + ch, size := rune(s[pos]), 1 + if ch < utf8.RuneSelf { + if ch == '\\' && jsx || ch == '\n' && backtick || ch == '$' && (!backtick || pos+1 == len(s) || s[pos+1] != '{') { + pos++ + continue + } + } else { + ch, size = stringutil.DecodeJSStringRune(s[pos:]) + if !asciiOnly && ch != rune(quoteChar) && ch != 0x85 && ch != 0x2028 && ch != 0x2029 && + !stringutil.IsSurrogate(ch) && !(ch == utf8.RuneError && size == 1) { + pos += size + continue + } + } + if !changed { + if b.Cap() == 0 { + b.Grow(len(s) + 2) + } else if b.Cap()-b.Len() < len(s) { + b.Grow(len(s)) + } + changed = true + } + if pending < pos { + b.WriteString(s[pending:pos]) + } + if jsx { + if ch == 0 { + b.WriteString("�") + } else if match, ok := jsxEscapedCharsMap[ch]; ok { + b.WriteString(match) + } else { + encodeJsxCharacterEntity(b, ch) + } + } else { + switch { + case ch == '\r' && pos+1 < len(s) && s[pos+1] == '\n': + size = 2 + b.WriteString(`\r\n`) + case ch == 0 && pos+1 < len(s) && s[pos+1] >= '0' && s[pos+1] <= '9': + b.WriteString(`\x00`) + case ch < 128 && asciiEscapeCodes[ch] != 0: + bytes := [2]byte{'\\', asciiEscapeCodes[ch]} + b.Write(bytes[:]) + case ch > 0xffff: + ch -= 0x10000 + encodeUtf16EscapeSequence(b, ch>>10+0xD800) + encodeUtf16EscapeSequence(b, ch&0x3ff+0xDC00) + default: + encodeUtf16EscapeSequence(b, ch) + } + } + pending = pos + size + pos += size + } + if changed && pending < len(s) { + b.WriteString(s[pending:]) + } + return changed +} diff --git a/tsc/internal/printer/escaping_reference_test.go b/tsc/internal/printer/escaping_reference_test.go new file mode 100644 index 0000000000000..b7bdedadc0ef8 --- /dev/null +++ b/tsc/internal/printer/escaping_reference_test.go @@ -0,0 +1,86 @@ +package printer + +import ( + "fmt" + "strings" + "unicode/utf8" + + "github.com/microsoft/TypeScript/tsc/internal/stringutil" +) + +var referenceEscapedChars = map[rune]string{ + '\t': `\t`, '\v': `\v`, '\f': `\f`, '\b': `\b`, '\r': `\r`, '\n': `\n`, + '\\': `\\`, '"': `\"`, '\'': `\'`, '`': "\\`", '$': `\$`, + '\u2028': `\u2028`, '\u2029': `\u2029`, '\u0085': `\u0085`, +} + +// Keep the original rune-by-rune emitter independent of production scanning, +// escape tables, and hexadecimal formatting. +func escapeStringWorkerReference(s string, quoteChar QuoteChar, flags getLiteralTextFlags, b *strings.Builder) { + pending := 0 + for pos := 0; pos < len(s); { + ch, size := stringutil.DecodeJSStringRune(s[pos:]) + escape := ch >= 0xD800 && ch <= 0xDFFF || ch == utf8.RuneError && size == 1 + switch ch { + case '\\': + if flags&getLiteralTextFlagsJsxAttributeEscape == 0 { + escape = true + } + case '$': + if quoteChar == QuoteCharBacktick && pos+1 < len(s) && s[pos+1] == '{' { + escape = true + } + case rune(quoteChar), '\u2028', '\u2029', '\u0085', '\r': + escape = true + case '\n': + if quoteChar != QuoteCharBacktick { + escape = true + } + default: + if ch <= '\u001f' || flags&getLiteralTextFlagsNeverAsciiEscape == 0 && ch > '\u007f' { + escape = true + } + } + if escape { + b.WriteString(s[pending:pos]) + if flags&getLiteralTextFlagsJsxAttributeEscape != 0 { + switch ch { + case 0: + b.WriteString("�") + case '"': + b.WriteString(""") + case '\'': + b.WriteString("'") + default: + b.WriteString(fmt.Sprintf("&#x%X;", ch)) + } + } else { + switch { + case ch == '\r' && quoteChar == QuoteCharBacktick && pos+1 < len(s) && s[pos+1] == '\n': + size++ + b.WriteString(`\r\n`) + case ch > 0xffff: + value := ch - 0x10000 + b.WriteString(fmt.Sprintf(`\u%04X\u%04X`, value>>10+0xD800, value&0x3ff+0xDC00)) + case ch >= 0xD800 && ch <= 0xDFFF: + b.WriteString(fmt.Sprintf(`\u%04X`, ch)) + case ch == 0: + if pos+1 < len(s) && stringutil.IsDigit(rune(s[pos+1])) { + b.WriteString(`\x00`) + } else { + b.WriteString(`\0`) + } + default: + if match, ok := referenceEscapedChars[ch]; ok { + b.WriteString(match) + } else { + b.WriteString(fmt.Sprintf(`\u%04X`, ch)) + } + } + } + pending = pos + size + } + pos += size + } + b.WriteString(s[pending:]) +} diff --git a/tsc/internal/printer/escaping_test.go b/tsc/internal/printer/escaping_test.go new file mode 100644 index 0000000000000..41d5786cac1b7 --- /dev/null +++ b/tsc/internal/printer/escaping_test.go @@ -0,0 +1,39 @@ +package printer + +import ( + "fmt" + "strings" + "testing" +) + +func TestEscapingAllocations(t *testing.T) { //nolint:paralleltest // AllocsPerRun changes GOMAXPROCS. + for _, text := range []string{ + "", "hello world", "caf\u00E9 \U0001F600", + strings.Repeat("plain ASCII text ", 128), + strings.Repeat("caf\u00E9 \U0001F600 ", 128), + } { + if allocations := testing.AllocsPerRun(25, func() { + EscapeString(text, QuoteCharDoubleQuote) + }); allocations != 0 { + t.Fatalf("unchanged text allocated %v times", allocations) + } + } + + text := strings.Repeat("abc\n", 4) + if allocations := testing.AllocsPerRun(25, func() { + EscapeString(text, QuoteCharDoubleQuote) + }); allocations != 1 { + t.Fatalf("small escaped text allocated %v times, want one output buffer", allocations) + } +} + +func TestEncodeUTF16Escape(t *testing.T) { + t.Parallel() + for ch := rune(0); ch <= 0xffff; ch++ { + var b strings.Builder + encodeUtf16EscapeSequence(&b, ch) + if got, want := b.String(), fmt.Sprintf(`\u%04X`, ch); got != want { + t.Fatalf("escape %U: got %q, want %q", ch, got, want) + } + } +} diff --git a/tsc/internal/printer/textscan_test.go b/tsc/internal/printer/textscan_test.go new file mode 100644 index 0000000000000..0be2ab60d17b6 --- /dev/null +++ b/tsc/internal/printer/textscan_test.go @@ -0,0 +1,131 @@ +package printer + +import ( + "strings" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/stringutil" + "github.com/microsoft/TypeScript/tsc/internal/testutil/filefixture" + "github.com/microsoft/TypeScript/tsc/internal/testutil/fixtures" + "gotest.tools/v3/assert" +) + +func TestEscapeScanBoundaries(t *testing.T) { + t.Parallel() + for prefix := range 130 { + for _, middle := range []string{ + "", "\"", "'", "`", "\\", "${", "$x", "\r\n", "\n", "\x001", "\x00a", + "\x1f", "\x7f", "\u0085", "\u2028", "\u2029", "\u00E9", "\U0001F980", + "\xff", "\xe2\x80", stringutil.EncodeJSStringRune(0xD800), + } { + for _, quote := range []QuoteChar{QuoteCharSingleQuote, QuoteCharDoubleQuote, QuoteCharBacktick} { + for _, flags := range []getLiteralTextFlags{ + getLiteralTextFlagsNone, getLiteralTextFlagsNeverAsciiEscape, + getLiteralTextFlagsJsxAttributeEscape, + getLiteralTextFlagsJsxAttributeEscape | getLiteralTextFlagsNeverAsciiEscape, + } { + text := strings.Repeat("a", prefix) + middle + strings.Repeat("z", 129) + var got, want strings.Builder + escapeStringWorker(text, quote, flags, &got) + escapeStringWorkerReference(text, quote, flags, &want) + assert.Equal(t, got.String(), want.String(), "text %q, quote %c, flags %d", text, quote, flags) + } + } + } + } +} + +func TestEscapeASCIIAtVectorBoundaries(t *testing.T) { + t.Parallel() + for prefix := range 66 { + for character := range 256 { + for _, following := range []string{"", "1", "{", "\n"} { + middle := string(byte(character)) + following + for _, quote := range []QuoteChar{QuoteCharSingleQuote, QuoteCharDoubleQuote, QuoteCharBacktick} { + var expected strings.Builder + escapeStringWorkerReference(middle, quote, getLiteralTextFlagsNeverAsciiEscape, &expected) + text := strings.Repeat("a", prefix) + middle + strings.Repeat("z", 129) + want := strings.Repeat("a", prefix) + expected.String() + strings.Repeat("z", 129) + assert.Equal(t, EscapeString(text, quote), want) + var appended strings.Builder + appended.Grow(len(text) + 2) + appended.WriteString("prefix:") + escapeStringWorker(text, quote, getLiteralTextFlagsNeverAsciiEscape, &appended) + assert.Equal(t, appended.String(), "prefix:"+want) + } + } + } + } +} + +func FuzzEscaping(f *testing.F) { + for _, text := range []string{ + "", strings.Repeat("a", 129), "\x001\x00x\r\n\n${x}`\\\"'", + "\u0085\u2028\u2029\u00E9\U0001F600\uFFFD", "\xff\xe2\x80\xed\xa0\x80", + strings.Repeat("a", 63) + "\r\n\u2028" + strings.Repeat("z", 129), + } { + for _, quote := range []QuoteChar{QuoteCharDoubleQuote, QuoteCharSingleQuote, QuoteCharBacktick, '$', '\u2028', -1, 0, '\n', '\\', '\x7f', '\U0001f600'} { + for _, flags := range []getLiteralTextFlags{0, getLiteralTextFlagsNeverAsciiEscape, getLiteralTextFlagsJsxAttributeEscape, getLiteralTextFlagsJsxAttributeEscape | getLiteralTextFlagsNeverAsciiEscape} { + f.Add(text, int32(quote), uint8(flags), uint8(3)) + } + } + } + f.Fuzz(func(t *testing.T, text string, quote int32, flags uint8, capacity uint8) { + var got, want strings.Builder + if capacity&1 != 0 { + got.Grow(len(text) + 2) + want.Grow(len(text) + 2) + } + if capacity&2 != 0 { + got.WriteString("prefix:") + want.WriteString("prefix:") + } + escapeStringWorker(text, QuoteChar(quote), getLiteralTextFlags(flags), &got) + escapeStringWorkerReference(text, QuoteChar(quote), getLiteralTextFlags(flags), &want) + assert.Equal(t, got.String(), want.String()) + var reference strings.Builder + escapeStringWorkerReference(text, QuoteChar(quote), getLiteralTextFlagsNeverAsciiEscape, &reference) + assert.Equal(t, EscapeString(text, QuoteChar(quote)), reference.String()) + }) +} + +func BenchmarkEscapeScan(b *testing.B) { + texts := []filefixture.Fixture{ + filefixture.FromString("short", "", "hello world"), + filefixture.FromString("ASCII", "", strings.Repeat("a longer plain ASCII string without any escaping ", 256)), + filefixture.FromString("sparse", "", strings.Repeat("a longer plain ASCII string with an escaped quote \" and caf\u00E9 \U0001F980 ", 200)), + filefixture.FromString("dense", "", strings.Repeat("\\\"\n\u2028", 1024)), + filefixture.FromString("rust-ascii-lines", "", strings.Repeat("const value = 123; // source text\r\n", 2048)), + filefixture.FromString("rust-long-ascii", "", strings.Repeat("identifierWithSeveralLetters_0123456789 ", 2048)), + filefixture.FromString("rust-sparse-unicode", "", strings.Repeat(strings.Repeat("source text ", 32)+"\u00E9\U0001F600\u2028", 256)), + filefixture.FromString("rust-dense-unicode", "", strings.Repeat("\u00E9\u20AC\U0001F600\r\n\u2029", 4096)), + } + for _, fixture := range append(texts, fixtures.BenchFixtures...) { + b.Run(fixture.Name(), func(b *testing.B) { + fixture.SkipIfNotExist(b) + text := fixture.ReadFile(b) + b.SetBytes(int64(len(text))) + b.ReportAllocs() + for b.Loop() { + EscapeString(text, QuoteCharDoubleQuote) + } + }) + } +} + +func BenchmarkEscapePreallocated(b *testing.B) { + for _, fixture := range fixtures.BenchFixtures { + b.Run(fixture.Name(), func(b *testing.B) { + fixture.SkipIfNotExist(b) + text := fixture.ReadFile(b) + size := len(EscapeString(text, QuoteCharDoubleQuote)) + b.SetBytes(int64(len(text))) + b.ReportAllocs() + for b.Loop() { + var output strings.Builder + output.Grow(size) + escapeStringWorker(text, QuoteCharDoubleQuote, getLiteralTextFlagsNeverAsciiEscape, &output) + } + }) + } +} diff --git a/tsc/internal/printer/utilities.go b/tsc/internal/printer/utilities.go index 096b4d27afe1f..8b60cce877a4f 100644 --- a/tsc/internal/printer/utilities.go +++ b/tsc/internal/printer/utilities.go @@ -38,23 +38,6 @@ var jsxEscapedCharsMap = map[rune]string{ '\'': "'", } -var escapedCharsMap = map[rune]string{ - '\t': `\t`, - '\v': `\v`, - '\f': `\f`, - '\b': `\b`, - '\r': `\r`, - '\n': `\n`, - '\\': `\\`, - '"': `\"`, - '\'': `\'`, - '`': "\\`", - '$': `\$`, // when quoteChar == '`' - '\u2028': `\u2028`, // lineSeparator - '\u2029': `\u2029`, // paragraphSeparator - '\u0085': `\u0085`, // nextLine -} - func encodeJsxCharacterEntity(b *strings.Builder, charCode rune) { hexCharCode := strings.ToUpper(strconv.FormatUint(uint64(charCode), 16)) b.WriteString("&#x") @@ -63,122 +46,28 @@ func encodeJsxCharacterEntity(b *strings.Builder, charCode rune) { } func encodeUtf16EscapeSequence(b *strings.Builder, charCode rune) { - hexCharCode := strings.ToUpper(strconv.FormatUint(uint64(charCode), 16)) - b.WriteString(`\u`) - for i := len(hexCharCode); i < 4; i++ { - b.WriteByte('0') - } - b.WriteString(hexCharCode) + const hex = "0123456789ABCDEF" + bytes := [6]byte{'\\', 'u', hex[charCode>>12&15], hex[charCode>>8&15], hex[charCode>>4&15], hex[charCode&15]} + b.Write(bytes[:]) } // Based heavily on the abstract 'Quote'/'QuoteJSONString' operation from ECMA-262 (24.3.2.2), // but augmented for a few select characters (e.g. lineSeparator, paragraphSeparator, nextLine) // Note that this doesn't actually wrap the input in double quotes. func escapeStringWorker(s string, quoteChar QuoteChar, flags getLiteralTextFlags, b *strings.Builder) { - pos := 0 - i := 0 - for i < len(s) { - ch, size := stringutil.DecodeJSStringRune(s[i:]) - - escape := false - if ch >= 0xD800 && ch <= 0xDFFF { - escape = true - } else if ch == utf8.RuneError && size == 1 { - // A stray byte that is not valid UTF-8 (for example, a fragment of a - // surrogate sentinel left behind by code that sliced the string by - // byte). Escape it as the Unicode replacement character so the output - // is always well-formed rather than containing raw invalid bytes. - escape = true - } - - // This consists of the first 19 unprintable ASCII characters, canonical escapes, lineSeparator, - // paragraphSeparator, and nextLine. The latter three are just desirable to suppress new lines in - // the language service. These characters should be escaped when printing, and if any characters are added, - // `escapedCharsMap` and/or `jsxEscapedCharsMap` must be updated. Note that this *does not* include the 'delete' - // character. There is no reason for this other than that JSON.stringify does not handle it either. - switch ch { - case '\\': - if flags&getLiteralTextFlagsJsxAttributeEscape == 0 { - escape = true - } - case '$': - if quoteChar == QuoteCharBacktick && i+1 < len(s) && s[i+1] == '{' { - escape = true - } - case rune(quoteChar), '\u2028', '\u2029', '\u0085', '\r': - escape = true - case '\n': - if quoteChar != QuoteCharBacktick { - // Template strings preserve simple LF newlines, still encode CRLF (or CR). - escape = true - } - default: - if ch <= '\u001f' || flags&getLiteralTextFlagsNeverAsciiEscape == 0 && ch > '\u007f' { - escape = true - } - } - - if escape { - if pos < i { - // Write string up to this point - b.WriteString(s[pos:i]) - } - - switch { - case flags&getLiteralTextFlagsJsxAttributeEscape != 0: - if ch == 0 { - b.WriteString("�") - } else if match, ok := jsxEscapedCharsMap[ch]; ok { - b.WriteString(match) - } else { - encodeJsxCharacterEntity(b, ch) - } - - default: - if ch == '\r' && quoteChar == QuoteCharBacktick && i+1 < len(s) && s[i+1] == '\n' { - // Template strings preserve simple LF newlines, but still must escape CRLF. Left alone, the - // above cases for `\r` and `\n` would inadvertently escape CRLF as two independent characters. - size++ - b.WriteString(`\r\n`) - } else if ch > 0xffff { - // encode as surrogate pair - ch -= 0x10000 - encodeUtf16EscapeSequence(b, (ch&0b11111111110000000000>>10)+0xD800) - encodeUtf16EscapeSequence(b, (ch&0b00000000001111111111)+0xDC00) - } else if ch >= 0xD800 && ch <= 0xDFFF { - encodeUtf16EscapeSequence(b, ch) - } else if ch == 0 { - if i+1 < len(s) && stringutil.IsDigit(rune(s[i+1])) { - // If the null character is followed by digits, print as a hex escape to prevent the result from - // parsing as an octal (which is forbidden in strict mode) - b.WriteString(`\x00`) - } else { - // Otherwise, keep printing a literal \0 for the null character - b.WriteString(`\0`) - } - } else { - if match, ok := escapedCharsMap[ch]; ok { - b.WriteString(match) - } else { - encodeUtf16EscapeSequence(b, ch) - } - } - } - pos = i + size - } - - i += size + if len(s) == 0 { + return } - - if pos < i { - b.WriteString(s[pos:]) + if !appendEscapedString(s, quoteChar, flags, b) { + b.WriteString(s) } } func EscapeString(s string, quoteChar QuoteChar) string { var b strings.Builder - b.Grow(len(s) + 2) - escapeStringWorker(s, quoteChar, getLiteralTextFlagsNeverAsciiEscape, &b) + if !appendEscapedString(s, quoteChar, getLiteralTextFlagsNeverAsciiEscape, &b) { + return s + } return b.String() } diff --git a/tsc/internal/scanner/lines_test.go b/tsc/internal/scanner/lines_test.go new file mode 100644 index 0000000000000..834605baf021c --- /dev/null +++ b/tsc/internal/scanner/lines_test.go @@ -0,0 +1,39 @@ +package scanner_test + +import ( + "strings" + "testing" + "unicode/utf8" + + "github.com/microsoft/TypeScript/tsc/internal/ast" + "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/parser" + "github.com/microsoft/TypeScript/tsc/internal/scanner" + "github.com/microsoft/TypeScript/tsc/internal/stringutil" + "gotest.tools/v3/assert" +) + +func TestGetECMAEndLinePosition(t *testing.T) { + t.Parallel() + for _, text := range []string{ + "", "value", "\n", "\r", "\r\n", "\u2028", "\u2029", + "\r\n\n\r\u2028\u2029", + "value\r\nnext\rthird\nfourth\u2028fifth\u2029last", + "\u4e2d\U0001f600\u2028\u03bb\r\n", + "\xff\n\xe2\x80\r\n", + strings.Repeat(" ", 1024) + "\r\nlast", + } { + file := parser.ParseSourceFile(ast.SourceFileParseOptions{FileName: "/test.ts", PathKey: "/test.ts"}, text, core.ScriptKindTS) + for line, start := range scanner.GetECMALineStarts(file) { + pos := int(start) + for { + ch, size := utf8.DecodeRuneInString(text[pos:]) + if size == 0 || stringutil.IsLineBreak(ch) { + break + } + pos += size + } + assert.Equal(t, scanner.GetECMAEndLinePosition(file, line), pos-1, "text %q, line %d", text, line) + } + } +} diff --git a/tsc/internal/scanner/scanner.go b/tsc/internal/scanner/scanner.go index 785fe50a4e7df..b3b74f7eb75a4 100644 --- a/tsc/internal/scanner/scanner.go +++ b/tsc/internal/scanner/scanner.go @@ -488,6 +488,7 @@ func (s *Scanner) Scan() ast.Kind { if s.skipTrivia { continue } + s.pos += stringutil.ASCIISingleLineWhitespacePrefixLen(s.text[s.pos:s.end]) for { ch, size := s.charAndSize() if !stringutil.IsWhiteSpaceSingleLine(ch) { @@ -627,9 +628,7 @@ func (s *Scanner) Scan() ast.Kind { s.pos += 2 for { - s.scanASCIIWhile(func(b byte) bool { - return b != '\n' && b != '\r' - }) + s.pos += stringutil.LineBreakPrefixLen(s.text[s.pos:s.end]) ch1, size := s.charAndSize() if size == 0 || stringutil.IsLineBreak(ch1) { break @@ -2323,7 +2322,10 @@ func SkipTriviaEx(text string, pos int, options *SkipTriviaOptions) int { if pos >= textLen { return pos } - ch, size := utf8.DecodeRuneInString(text[pos:]) + ch, size := rune(text[pos]), 1 + if ch >= utf8.RuneSelf { + ch, size = utf8.DecodeRuneInString(text[pos:]) + } switch ch { case '\r': if pos+1 < textLen && text[pos+1] == '\n' { @@ -2339,6 +2341,7 @@ func SkipTriviaEx(text string, pos int, options *SkipTriviaOptions) int { continue case '\t', '\v', '\f', ' ': pos++ + pos += stringutil.ASCIISingleLineWhitespacePrefixLen(text[pos:]) continue case '/': if options.StopAtComments { @@ -2348,6 +2351,10 @@ func SkipTriviaEx(text string, pos int, options *SkipTriviaOptions) int { if text[pos+1] == '/' { pos += 2 for pos < textLen { + pos += stringutil.LineBreakPrefixLen(text[pos:]) + if pos == textLen { + break + } ch, size := utf8.DecodeRuneInString(text[pos:]) if stringutil.IsLineBreak(ch) { break @@ -2359,13 +2366,10 @@ func SkipTriviaEx(text string, pos int, options *SkipTriviaOptions) int { } if text[pos+1] == '*' { pos += 2 - for pos < textLen { - if text[pos] == '*' && (pos+1 < textLen) && text[pos+1] == '/' { - pos += 2 - break - } - _, size := utf8.DecodeRuneInString(text[pos:]) - pos += size + if end := strings.Index(text[pos:], "*/"); end >= 0 { + pos += end + 2 + } else { + pos = textLen } canConsumeStar = false continue @@ -2701,14 +2705,25 @@ func GetECMALineAndByteOffsetOfPosition(sourceFile ast.SourceFileLike, pos int) } func GetECMAEndLinePosition(sourceFile *ast.SourceFile, line int) int { - pos := int(GetECMALineStarts(sourceFile)[line]) - for { - ch, size := utf8.DecodeRuneInString(sourceFile.Text()[pos:]) - if size == 0 || stringutil.IsLineBreak(ch) { - return pos - 1 + lineStarts := GetECMALineStarts(sourceFile)[line:] + text := sourceFile.Text() + if len(lineStarts) == 1 { + return len(text) - 1 + } + end := int(lineStarts[1]) + switch text[end-1] { + case '\n': + end-- + if end > 0 && text[end-1] == '\r' { + end-- } - pos += size + case '\r': + end-- + default: + // The remaining ECMAScript line separators, U+2028 and U+2029, are three bytes. + end -= 3 } + return end - 1 } // GetECMAPositionOfLineAndUTF16Character converts a 0-based line number and UTF-16 diff --git a/tsc/internal/scanner/scanner_test.go b/tsc/internal/scanner/scanner_test.go index 8f970f2af8467..5bee357727734 100644 --- a/tsc/internal/scanner/scanner_test.go +++ b/tsc/internal/scanner/scanner_test.go @@ -1,6 +1,7 @@ package scanner import ( + "strconv" "strings" "testing" @@ -103,3 +104,44 @@ func TestScanSourceKeyword(t *testing.T) { assert.Equal(t, s.Scan(), ast.KindIdentifier) assert.Equal(t, s.TokenValue(), "sourceValue") } + +func TestBulkTrivia(t *testing.T) { + t.Parallel() + for i, prefix := range []string{ + "", + strings.Repeat(" \t\v\f", 40), + strings.Repeat(" ", 129) + "\u00a0\u2003\ufeff", + "// " + strings.Repeat("comment ", 40) + "\r\n", + "// " + strings.Repeat("\u4e2d", 40) + "\u2028", + "// \u20ac is not a line break\u2029", + "/* " + strings.Repeat("\u4e2d * / ", 40) + "*/", + "/* first */ /* second */\n\t", + "\r\n" + strings.Repeat(" ", 65), + } { + t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Parallel() + text := prefix + "value" + assert.Equal(t, SkipTrivia(text, 0), len(prefix)) + s := NewScanner() + s.SetText(text) + assert.Equal(t, s.Scan(), ast.KindIdentifier) + assert.Equal(t, s.TokenStart(), len(prefix)) + s.ResetTokenState(0) + s.SetSkipTrivia(false) + end := 0 + for s.Scan() != ast.KindIdentifier { + assert.Assert(t, ast.IsTrivia(s.Token())) + assert.Equal(t, s.TokenFullStart(), end) + end = s.TokenEnd() + } + assert.Equal(t, s.TokenFullStart(), end) + assert.Equal(t, s.TokenStart(), len(prefix)) + }) + } + for _, text := range []string{"// unterminated", "/* unterminated", "/* \xff", "/* trailing *"} { + assert.Equal(t, SkipTrivia(text, 0), len(text)) + } + assert.Equal(t, SkipTriviaEx(" \t/* comment */value", 0, &SkipTriviaOptions{StopAtComments: true}), 2) + assert.Equal(t, SkipTriviaEx(" \t\r\n value", 0, &SkipTriviaOptions{StopAfterLineBreak: true}), 4) + assert.Equal(t, SkipTriviaEx("\n \t* value", 0, &SkipTriviaOptions{InJSDoc: true}), 5) +} diff --git a/tsc/internal/stringutil/textscan.go b/tsc/internal/stringutil/textscan.go new file mode 100644 index 0000000000000..d5721be78a222 --- /dev/null +++ b/tsc/internal/stringutil/textscan.go @@ -0,0 +1,59 @@ +// Builds with GOEXPERIMENT=simd use portable, vector-size-agnostic SIMD for +// long text scans. Normal builds retain scalar implementations. +package stringutil + +// ASCIIPrefixLen returns the length in bytes of the leading ASCII portion of text. +func ASCIIPrefixLen(text string) int { + // Every supported vector is at least 16 bytes. Keep short strings out of + // the experimental SIMD runtime dispatch. + if len(text) < 16 { + return asciiPrefixLen(text) + } + return asciiPrefixLenLong(text) +} + +func asciiPrefixLen(text string) int { + for i := range len(text) { + if text[i] >= 0x80 { + return i + } + } + return len(text) +} + +// ASCIISingleLineWhitespacePrefixLen returns the length of the leading run of +// ASCII whitespace other than line breaks. Non-ASCII whitespace is left to the caller. +func ASCIISingleLineWhitespacePrefixLen(text string) int { + if len(text) < 16 || text[0] != ' ' && text[0] != '\t' && text[0] != '\v' && text[0] != '\f' { + return asciiSingleLineWhitespacePrefixLen(text) + } + return asciiSingleLineWhitespacePrefixLenLong(text) +} + +func asciiSingleLineWhitespacePrefixLen(text string) int { + for i := range len(text) { + if b := text[i]; b != ' ' && b != '\t' && b != '\v' && b != '\f' { + return i + } + } + return len(text) +} + +func lineBreakPrefixLen(text string) int { + for i := range len(text) { + if b := text[i]; b == '\r' || b == '\n' || b == 0xe2 { + return i + } + } + return len(text) +} + +func unescapedASCIIPrefixLen(text string, quote byte) int { + for i := range len(text) { + b := text[i] + if b < 0x20 || b >= 0x80 || b == '\\' || b == quote || quote == '`' && b == '$' { + return i + } + } + return len(text) +} diff --git a/tsc/internal/stringutil/textscan_scalar.go b/tsc/internal/stringutil/textscan_scalar.go new file mode 100644 index 0000000000000..f9917e207bcd4 --- /dev/null +++ b/tsc/internal/stringutil/textscan_scalar.go @@ -0,0 +1,23 @@ +//go:build !goexperiment.simd + +package stringutil + +func asciiPrefixLenLong(text string) int { + return asciiPrefixLen(text) +} + +func asciiSingleLineWhitespacePrefixLenLong(text string) int { + return asciiSingleLineWhitespacePrefixLen(text) +} + +// LineBreakPrefixLen returns the byte offset of the first possible ECMAScript +// line break (CR, LF, or the leading byte of U+2028/U+2029), or len(text). +func LineBreakPrefixLen(text string) int { + return lineBreakPrefixLen(text) +} + +// UnescapedASCIIPrefixLen returns the length of the ASCII prefix that needs no +// escaping. It conservatively stops at all controls, backslashes, and template dollars. +func UnescapedASCIIPrefixLen(text string, quote byte) int { + return unescapedASCIIPrefixLen(text, quote) +} diff --git a/tsc/internal/stringutil/textscan_simd.go b/tsc/internal/stringutil/textscan_simd.go new file mode 100644 index 0000000000000..7663d505ef671 --- /dev/null +++ b/tsc/internal/stringutil/textscan_simd.go @@ -0,0 +1,144 @@ +//go:build goexperiment.simd + +package stringutil + +import ( + "math/bits" + "simd" + "unsafe" +) + +// Keep this small enough to inline: it is called once per vector. +// ReshapeToUint64s places each group's first byte in the low bits. +// A full SIMD mask has at least two uint64 lanes. +func firstNonzeroByte(mask []uint64) int { + lo, hi := mask[0], mask[1] + if lo != 0 { + return bits.TrailingZeros64(lo) / 8 + } + if hi != 0 { + return 8 + bits.TrailingZeros64(hi)/8 + } + for i := 2; i < len(mask); i++ { + if word := mask[i]; word != 0 { + return i*8 + bits.TrailingZeros64(word)/8 + } + } + return len(mask) * 8 +} + +func asciiPrefixLenLong(text string) int { + width := (simd.Uint8s{}).Len() + if len(text) < width || text[0] >= 0x80 { + return asciiPrefixLen(text) + } + // SIMD loads only read this view; converting to []byte would copy the input. + bytes := unsafe.Slice(unsafe.StringData(text), len(text)) + highBit := simd.BroadcastUint8s(0x80) + // Go 1.27 has no portable mask reduction. Store machine words and reduce + // them without assuming a vector width or the machine's byte order. + words := make([]uint64, (simd.Uint64s{}).Len()) + i := 0 + for len(text)-i >= width { + simd.LoadUint8s(bytes[i:]).And(highBit).ReshapeToUint64s().Store(words) + var combined uint64 + for _, word := range words { + combined |= word + } + if combined != 0 { + break + } + i += width + } + return i + asciiPrefixLen(text[i:]) +} + +func asciiSingleLineWhitespacePrefixLenLong(text string) int { + width := (simd.Uint8s{}).Len() + bytes := unsafe.Slice(unsafe.StringData(text), len(text)) + space := simd.BroadcastUint8s(' ') + tab := simd.BroadcastUint8s('\t') + verticalTab := simd.BroadcastUint8s('\v') + formFeed := simd.BroadcastUint8s('\f') + allBits := simd.BroadcastUint8s(0xff) + words := make([]uint64, (simd.Uint64s{}).Len()) + i := 0 + for len(text)-i >= width { + v := simd.LoadUint8s(bytes[i:]) + v.Equal(space).Or(v.Equal(tab)).Or(v.Equal(verticalTab)).Or(v.Equal(formFeed)). + ToInt8s().ToBits().Xor(allBits).ReshapeToUint64s().Store(words) + if offset := firstNonzeroByte(words); offset < width { + return i + offset + } + i += width + } + return i + asciiSingleLineWhitespacePrefixLen(text[i:]) +} + +// LineBreakPrefixLen returns the byte offset of the first possible ECMAScript +// line break (CR, LF, or the leading byte of U+2028/U+2029), or len(text). +func LineBreakPrefixLen(text string) int { + if len(text) < 16 { + return lineBreakPrefixLen(text) + } + return lineBreakPrefixLenLong(text) +} + +func lineBreakPrefixLenLong(text string) int { + width := (simd.Uint8s{}).Len() + if len(text) < width || text[0] == '\r' || text[0] == '\n' || text[0] == 0xe2 { + return lineBreakPrefixLen(text) + } + bytes := unsafe.Slice(unsafe.StringData(text), len(text)) + cr := simd.BroadcastUint8s('\r') + lf := simd.BroadcastUint8s('\n') + separator := simd.BroadcastUint8s(0xe2) + mask := make([]uint64, (simd.Uint64s{}).Len()) + i := 0 + for len(text)-i >= width { + v := simd.LoadUint8s(bytes[i:]) + v.Equal(cr).Or(v.Equal(lf)).Or(v.Equal(separator)).ToInt8s().ToBits().ReshapeToUint64s().Store(mask) + if offset := firstNonzeroByte(mask); offset < width { + return i + offset + } + i += width + } + return i + lineBreakPrefixLen(text[i:]) +} + +// UnescapedASCIIPrefixLen returns the length of the ASCII prefix that needs no +// escaping. It conservatively stops at all controls, backslashes, and template dollars. +func UnescapedASCIIPrefixLen(text string, quote byte) int { + if len(text) < 16 { + return unescapedASCIIPrefixLen(text, quote) + } + return unescapedASCIIPrefixLenLong(text, quote) +} + +func unescapedASCIIPrefixLenLong(text string, quote byte) int { + width := (simd.Uint8s{}).Len() + if len(text) < width || unescapedASCIIPrefixLen(text[:1], quote) == 0 { + return unescapedASCIIPrefixLen(text, quote) + } + bytes := unsafe.Slice(unsafe.StringData(text), len(text)) + space := simd.BroadcastInt8s(0x20) + backslash := simd.BroadcastUint8s('\\') + quoteVector := simd.BroadcastUint8s(quote) + dollar := simd.BroadcastUint8s('$') + matches := make([]uint64, (simd.Uint64s{}).Len()) + i := 0 + for len(text)-i >= width { + v := simd.LoadUint8s(bytes[i:]) + // Signed comparison includes both controls and non-ASCII bytes. + mask := v.BitsToInt8().Less(space).Or(v.Equal(backslash)).Or(v.Equal(quoteVector)) + if quote == '`' { + mask = mask.Or(v.Equal(dollar)) + } + mask.ToInt8s().ToBits().ReshapeToUint64s().Store(matches) + if offset := firstNonzeroByte(matches); offset < width { + return i + offset + } + i += width + } + return i + unescapedASCIIPrefixLen(text[i:], quote) +} diff --git a/tsc/internal/stringutil/textscan_simd_test.go b/tsc/internal/stringutil/textscan_simd_test.go new file mode 100644 index 0000000000000..d1fdc3aa2adc2 --- /dev/null +++ b/tsc/internal/stringutil/textscan_simd_test.go @@ -0,0 +1,41 @@ +//go:build goexperiment.simd + +package stringutil + +import ( + "simd" + "testing" +) + +func TestFirstNonzeroByte(t *testing.T) { + t.Parallel() + for _, length := range []int{16, 24, 32, 64, 128, 256} { + mask := make([]uint64, length/8) + if got := firstNonzeroByte(mask); got != length { + t.Fatalf("zero mask length %d: got %d", length, got) + } + for i := range length { + for _, value := range []byte{1, 0x80, 0xff} { + mask[i/8] = uint64(value) << ((i % 8) * 8) + if got := firstNonzeroByte(mask); got != i { + t.Fatalf("length %d, byte %d=%x: got %d", length, i, value, got) + } + } + mask[i/8] = 0 + } + } +} + +func TestSIMDMaskByteOrder(t *testing.T) { + t.Parallel() + bytes := make([]byte, (simd.Uint8s{}).Len()) + words := make([]uint64, (simd.Uint64s{}).Len()) + for i := range bytes { + bytes[i] = 0xff + simd.LoadUint8s(bytes).ReshapeToUint64s().Store(words) + if got := firstNonzeroByte(words); got != i { + t.Fatalf("byte %d: got %d", i, got) + } + bytes[i] = 0 + } +} diff --git a/tsc/internal/stringutil/textscan_test.go b/tsc/internal/stringutil/textscan_test.go new file mode 100644 index 0000000000000..70dbd4fbe5c73 --- /dev/null +++ b/tsc/internal/stringutil/textscan_test.go @@ -0,0 +1,117 @@ +package stringutil + +import ( + "strconv" + "strings" + "testing" +) + +func TestTextScanMultipleCandidates(t *testing.T) { + t.Parallel() + for prefix := range 130 { + for gap := range 66 { + for _, middle := range []string{"\n", "\r\n", "\xe2", "\"", "\\", "$", "\x00", "\xff"} { + text := strings.Repeat("a", prefix) + middle + strings.Repeat("b", gap) + middle + strings.Repeat("c", 129) + checkTextScan(t, text) + } + } + } +} + +func BenchmarkTextScanPrefixes(b *testing.B) { + for _, length := range []int{8, 16, 31, 32, 63, 64, 127, 128, 1024} { + b.Run(strconv.Itoa(length), func(b *testing.B) { + b.Run("Whitespace", func(b *testing.B) { + text := strings.Repeat(" ", length) + "x" + strings.Repeat("z", 128) + b.SetBytes(int64(length)) + b.ReportAllocs() + for b.Loop() { + ASCIISingleLineWhitespacePrefixLen(text) + } + }) + b.Run("LineBreak", func(b *testing.B) { + text := strings.Repeat("a", length) + "\n" + strings.Repeat("z", 128) + b.SetBytes(int64(length + 1)) + b.ReportAllocs() + for b.Loop() { + LineBreakPrefixLen(text) + } + }) + b.Run("Escape", func(b *testing.B) { + text := strings.Repeat("a", length) + "\"" + strings.Repeat("z", 128) + b.SetBytes(int64(length + 1)) + b.ReportAllocs() + for b.Loop() { + UnescapedASCIIPrefixLen(text, '"') + } + }) + }) + } +} + +func checkTextScan(t *testing.T, text string) { + t.Helper() + if got, want := ASCIIPrefixLen(text), asciiPrefixLen(text); got != want { + t.Fatalf("ASCIIPrefixLen(%q) = %d, want %d", text, got, want) + } + if got, want := LineBreakPrefixLen(text), lineBreakPrefixLen(text); got != want { + t.Fatalf("LineBreakPrefixLen(%q) = %d, want %d", text, got, want) + } + if got, want := ASCIISingleLineWhitespacePrefixLen(text), asciiSingleLineWhitespacePrefixLen(text); got != want { + t.Fatalf("ASCIISingleLineWhitespacePrefixLen(%q) = %d, want %d", text, got, want) + } + for _, quote := range []byte{0, '\'', '"', '`', 0xff} { + if got, want := UnescapedASCIIPrefixLen(text, quote), unescapedASCIIPrefixLen(text, quote); got != want { + t.Fatalf("UnescapedASCIIPrefixLen(%q, %q) = %d, want %d", text, quote, got, want) + } + } +} + +func TestWhitespacePrefixBytes(t *testing.T) { + t.Parallel() + for prefix := range 130 { + whitespace := strings.Repeat(" \t\v\f", prefix) + for offset := range 4 { + padding := whitespace[min(offset, len(whitespace)):] + for value := range 256 { + text := padding + string(byte(value)) + "x" + want := len(padding) + if value == ' ' || value == '\t' || value == '\v' || value == '\f' { + want++ + } + if got := ASCIISingleLineWhitespacePrefixLen(text); got != want { + t.Fatalf("prefix %d, offset %d, byte %x: got %d, want %d", prefix, offset, value, got, want) + } + } + if got := ASCIISingleLineWhitespacePrefixLen(padding); got != len(padding) { + t.Fatalf("prefix %d, offset %d: got %d, want %d", prefix, offset, got, len(padding)) + } + } + } +} + +func TestTextScanBytes(t *testing.T) { + t.Parallel() + for prefix := range 130 { + padding := strings.Repeat("a", prefix) + checkTextScan(t, padding) + for b := range 256 { + text := padding + string(byte(b)) + strings.Repeat("z", 129) + checkTextScan(t, text) + checkTextScan(t, text[1:]) + checkTextScan(t, text[:prefix+1]) + } + } +} + +func FuzzTextScan(f *testing.F) { + for _, seed := range []string{ + "", "a", strings.Repeat("a", 1024), "\r\n\u2028\u2029", + "\xff\xe2\x80\xf0\x9f", "abc\"${\\\x00", "\u00E9\U0001F980", + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, text string) { + checkTextScan(t, text) + }) +}