feat(symbols): upload Java/Kotlin sources with R8 mappings (--include-sources)

cmd/symbols/android_sources.go

@@ -0,0 +1,131 @@
+package symbols
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/launchdarkly/ldcli/internal/symbols/srcbundle"
+)
+
+// androidSourceBundleName is the object name of the source bundle uploaded beside
+// an R8 mapping, so a build's mapping and its sources share one key prefix:
+// _sym/android/id/<symbolsID>/mapping.txt and .../sources.srcbundle.
+const androidSourceBundleName = "sources.srcbundle"
+
+// androidSourceExtensions are the JVM source types R8 stack frames can point at.
+var androidSourceExtensions = map[string]bool{
+	".java": true,
+	".kt":   true,
+}
+
+// androidSkipDirs are trees that never hold hand-written sources worth shipping:
+// build output (including R8's own generated code), caches, and JS deps.
+var androidSkipDirs = map[string]bool{
+	"build":        true,
+	".gradle":      true,
+	".git":         true,
+	".idea":        true,
+	"node_modules": true,
+}
+
+// buildAndroidSourceBundle scans root for .java/.kt files and packs them keyed by
+// their package-relative path (e.g. com/example/MainActivity.kt), which is what
+// the backend reconstructs from a retraced frame's fully-qualified class name.
+//
+// The key comes from each file's own `package` declaration rather than its
+// directory layout, because Kotlin does not require the two to agree. A file with
+// no package declaration is keyed by its bare name (the JVM default package).
+// Returns nil when nothing was found, so the caller can skip the upload.
+func buildAndroidSourceBundle(root string) ([]byte, int, error) {
+	info, err := os.Stat(root)
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to read source path %s: %w", root, err)
+	}
+	if !info.IsDir() {
+		return nil, 0, fmt.Errorf("source path %s is not a directory", root)
+	}
+
+	b := &srcbundle.Builder{}
+	total := 0
+	err = filepath.WalkDir(root, func(p string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		if d.IsDir() {
+			if androidSkipDirs[d.Name()] {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if !androidSourceExtensions[strings.ToLower(filepath.Ext(d.Name()))] {
+			return nil
+		}
+		data, err := os.ReadFile(p)
+		if err != nil {
+			return nil // unreadable: skip, same as Apple
+		}
+		if len(data) > maxSourceFileBytes || total+len(data) > maxSourceBundleBytes {
+			return nil
+		}
+		total += len(data)
+		b.Add(androidSourceKey(data, d.Name()), data)
+		return nil
+	})
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to scan sources in %s: %w", root, err)
+	}
+	if b.Len() == 0 {
+		return nil, 0, nil
+	}
+
+	var buf bytes.Buffer
+	if err := b.Encode(&buf); err != nil {
+		return nil, 0, fmt.Errorf("failed to encode source bundle: %w", err)
+	}
+	return buf.Bytes(), b.Len(), nil
+}
+
+// androidSourceKey is the bundle key for one source file: its package as a path,
+// plus the file name (com/example/MainActivity.kt).
+func androidSourceKey(data []byte, fileName string) string {
+	pkg := javaPackageOf(data)
+	if pkg == "" {
+		return fileName
+	}
+	return path.Join(strings.ReplaceAll(pkg, ".", "/"), fileName)
+}
+
+// javaPackageOf extracts the package name from Java or Kotlin source, or "" when
+// there is none. It reads the leading declarations only: the package statement
+// must precede any type declaration in both languages, so scanning stops at the
+// first import (Kotlin allows a semicolon-less package line, hence the trim).
+func javaPackageOf(data []byte) string {
+	scanner := bufio.NewScanner(bytes.NewReader(data))
+	scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "//") || strings.HasPrefix(line, "/*") ||
+			strings.HasPrefix(line, "*") || strings.HasPrefix(line, "@") {
+			continue
+		}
+		if rest, ok := strings.CutPrefix(line, "package "); ok {
+			pkg := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(rest), ";"))
+			// Guard against a stray trailing comment ("package a.b; // note").
+			if i := strings.IndexAny(pkg, " \t/"); i >= 0 {
+				pkg = pkg[:i]
+			}
+			return strings.TrimSuffix(pkg, ";")
+		}
+		if strings.HasPrefix(line, "import ") {
+			return "" // past the package slot
+		}
+		return "" // a real declaration: this file has no package
+	}
+	return ""
+}

cmd/symbols/android_sources_test.go

@@ -0,0 +1,135 @@
+package symbols
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/launchdarkly/ldcli/internal/symbols/srcbundle"
+)
+
+func writeFile(t *testing.T, dir, rel, content string) string {
+	t.Helper()
+	full := filepath.Join(dir, rel)
+	require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
+	require.NoError(t, os.WriteFile(full, []byte(content), 0o644))
+	return full
+}
+
+func TestJavaPackageOf(t *testing.T) {
+	cases := []struct {
+		name string
+		src  string
+		want string
+	}{
+		{"java", "package com.example.app;\n\nclass Foo {}\n", "com.example.app"},
+		{"kotlin no semicolon", "package com.example.app\n\nclass Foo\n", "com.example.app"},
+		{"after license header", "/*\n * (c) 2026\n */\npackage com.example;\n", "com.example"},
+		{"after line comment", "// generated\npackage com.example\n", "com.example"},
+		{"after file annotation", "@file:JvmName(\"Utils\")\npackage com.example\n", "com.example"},
+		{"trailing comment", "package com.example; // note\n", "com.example"},
+		{"default package", "class Foo {}\n", ""},
+		{"import first means none", "import java.util.List;\nclass Foo {}\n", ""},
+		{"empty", "", ""},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			assert.Equal(t, c.want, javaPackageOf([]byte(c.src)))
+		})
+	}
+}
+
+// The bundle key is the package-relative path, which is what the backend
+// reconstructs from a retraced frame's fully-qualified class name.
+func TestAndroidSourceKey(t *testing.T) {
+	assert.Equal(t, "com/example/app/MainActivity.kt",
+		androidSourceKey([]byte("package com.example.app\n"), "MainActivity.kt"))
+	assert.Equal(t, "com/example/Foo.java",
+		androidSourceKey([]byte("package com.example;\n"), "Foo.java"))
+	// No package declaration: the JVM default package, keyed by bare name.
+	assert.Equal(t, "Foo.java", androidSourceKey([]byte("class Foo {}\n"), "Foo.java"))
+}
+
+// The key comes from the package declaration, not the directory layout, because
+// Kotlin does not require the two to agree.
+func TestBuildAndroidSourceBundleKeysByPackageNotLayout(t *testing.T) {
+	dir := t.TempDir()
+	writeFile(t, dir, "app/src/main/kotlin/MainActivity.kt", "package com.example.app\n\nclass MainActivity\n")
+	writeFile(t, dir, "app/src/main/java/com/example/util/Helper.java", "package com.example.util;\n\nclass Helper {}\n")
+
+	data, count, err := buildAndroidSourceBundle(dir)
+	require.NoError(t, err)
+	require.NotNil(t, data)
+	assert.Equal(t, 2, count)
+
+	bundle, err := srcbundle.Open(data)
+	require.NoError(t, err)
+
+	// Declared package wins over the flat directory it happened to live in.
+	_, content, _, ok := bundle.Window("com/example/app/MainActivity.kt", 3, 1)
+	require.True(t, ok)
+	assert.Equal(t, "class MainActivity\n", content)
+
+	_, ok = bundle.File("com/example/util/Helper.java")
+	assert.True(t, ok)
+
+	_, ok = bundle.File("app/src/main/kotlin/MainActivity.kt")
+	assert.False(t, ok, "on-disk path must not be a key")
+}
+
+func TestBuildAndroidSourceBundleSkipsBuildDirs(t *testing.T) {
+	dir := t.TempDir()
+	writeFile(t, dir, "app/src/main/java/com/example/Kept.java", "package com.example;\nclass Kept {}\n")
+	writeFile(t, dir, "app/build/generated/com/example/Generated.java", "package com.example;\nclass Generated {}\n")
+	writeFile(t, dir, "node_modules/pkg/Vendored.java", "package vendored;\nclass Vendored {}\n")
+	writeFile(t, dir, "app/src/main/res/values/strings.xml", "<resources/>\n")
+
+	data, count, err := buildAndroidSourceBundle(dir)
+	require.NoError(t, err)
+	require.NotNil(t, data)
+	assert.Equal(t, 1, count, "only the hand-written source is bundled")
+
+	bundle, err := srcbundle.Open(data)
+	require.NoError(t, err)
+	_, ok := bundle.File("com/example/Kept.java")
+	assert.True(t, ok)
+	_, ok = bundle.File("com/example/Generated.java")
+	assert.False(t, ok, "build output must be skipped")
+	_, ok = bundle.File("vendored/Vendored.java")
+	assert.False(t, ok, "node_modules must be skipped")
+}
+
+func TestBuildAndroidSourceBundleSkipsOversizeFile(t *testing.T) {
+	dir := t.TempDir()
+	writeFile(t, dir, "com/example/Small.java", "package com.example;\nclass Small {}\n")
+	big := make([]byte, maxSourceFileBytes+1)
+	copy(big, []byte("package com.example;\n"))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "Big.java"), big, 0o644))
+
+	data, count, err := buildAndroidSourceBundle(dir)
+	require.NoError(t, err)
+	require.NotNil(t, data)
+	assert.Equal(t, 1, count)
+}
+
+func TestBuildAndroidSourceBundleNoSources(t *testing.T) {
+	dir := t.TempDir()
+	writeFile(t, dir, "README.md", "# nothing to see\n")
+
+	data, count, err := buildAndroidSourceBundle(dir)
+	require.NoError(t, err)
+	assert.Nil(t, data, "no sources yields no bundle rather than an error")
+	assert.Zero(t, count)
+}
+
+func TestBuildAndroidSourceBundleRejectsBadPath(t *testing.T) {
+	_, _, err := buildAndroidSourceBundle(filepath.Join(t.TempDir(), "missing"))
+	assert.Error(t, err)
+
+	file := writeFile(t, t.TempDir(), "a.java", "package a;\n")
+	_, _, err = buildAndroidSourceBundle(file)
+	assert.Error(t, err, "a file is not a source root")
+}

cmd/symbols/apple_sources.go

@@ -0,0 +1,83 @@
+package symbols
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/launchdarkly/ldcli/internal/symbols/apple"
+	"github.com/launchdarkly/ldcli/internal/symbols/srcbundle"
+)
+
+// appleSourceExt is appended to the build UUID to form the source-bundle object
+// name, so an image's map and its sources sit side by side under the same key:
+// _sym/apple/id/<UUID>.dsymmap and _sym/apple/id/<UUID>.srcbundle.
+const appleSourceExt = ".srcbundle"
+
+const (
+	// maxSourceFileBytes skips individual files too large to be worth shipping
+	// (generated code, vendored blobs). Only a handful of lines around a frame is
+	// ever rendered, so a huge file adds size for no benefit.
+	maxSourceFileBytes = 2 << 20 // 2 MiB
+
+	// maxSourceBundleBytes caps one image's uncompressed source total, so an
+	// accidental `--include-sources` over a huge workspace can't produce an
+	// unbounded upload.
+	maxSourceBundleBytes = 256 << 20 // 256 MiB
+)
+
+// appleSourceExtensions are the file types the UI can render as source context.
+// Restricting to these keeps unrelated DWARF-referenced paths (module maps,
+// generated interfaces) out of the bundle.
+var appleSourceExtensions = map[string]bool{
+	".swift": true,
+	".m":     true,
+	".mm":    true,
+	".h":     true,
+	".hpp":   true,
+	".hh":    true,
+	".c":     true,
+	".cc":    true,
+	".cpp":   true,
+	".cxx":   true,
+}
+
+// buildAppleSourceBundle packs the source files referenced by one architecture's
+// DWARF into a .srcbundle. Files that aren't readable on this machine (SDK,
+// system, or third-party code built elsewhere) are skipped — the backend simply
+// renders those frames without source context. Returns nil when nothing local
+// was found, so the caller can skip the upload entirely.
+func buildAppleSourceBundle(a apple.Arch) ([]byte, int, error) {
+	b := &srcbundle.Builder{}
+	total := 0
+	for key, path := range a.Sources {
+		if !appleSourceExtensions[strings.ToLower(filepath.Ext(key))] {
+			continue
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			continue // not on this machine: expected for SDK/system sources
+		}
+		if len(data) > maxSourceFileBytes || total+len(data) > maxSourceBundleBytes {
+			continue
+		}
+		total += len(data)
+		b.Add(key, data)
+	}
+	if b.Len() == 0 {
+		return nil, 0, nil
+	}
+
+	var buf bytes.Buffer
+	if err := b.Encode(&buf); err != nil {
+		return nil, 0, fmt.Errorf("failed to encode source bundle for %s: %w", a.UUID, err)
+	}
+	return buf.Bytes(), b.Len(), nil
+}
+
+// appleSourceKey is the storage key for an image's source bundle.
+func appleSourceKey(uuid string) string {
+	return fmt.Sprintf("%s/%s%s", appleSymbolsIDPrefix, uuid, appleSourceExt)
+}