diff --git a/cmd/symbols/apple_sources.go b/cmd/symbols/apple_sources.go new file mode 100644 index 00000000..3ccab488 --- /dev/null +++ b/cmd/symbols/apple_sources.go @@ -0,0 +1,83 @@ +package symbols + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/launchdarkly/ldcli/internal/symbols/apple" + "github.com/launchdarkly/ldcli/internal/symbols/srcbundle" +) + +// appleSourceExt is appended to the build UUID to form the source-bundle object +// name, so an image's map and its sources sit side by side under the same key: +// _sym/apple/id/.dsymmap and _sym/apple/id/.srcbundle. +const appleSourceExt = ".srcbundle" + +const ( + // maxSourceFileBytes skips individual files too large to be worth shipping + // (generated code, vendored blobs). Only a handful of lines around a frame is + // ever rendered, so a huge file adds size for no benefit. + maxSourceFileBytes = 2 << 20 // 2 MiB + + // maxSourceBundleBytes caps one image's uncompressed source total, so an + // accidental `--include-sources` over a huge workspace can't produce an + // unbounded upload. + maxSourceBundleBytes = 256 << 20 // 256 MiB +) + +// appleSourceExtensions are the file types the UI can render as source context. +// Restricting to these keeps unrelated DWARF-referenced paths (module maps, +// generated interfaces) out of the bundle. +var appleSourceExtensions = map[string]bool{ + ".swift": true, + ".m": true, + ".mm": true, + ".h": true, + ".hpp": true, + ".hh": true, + ".c": true, + ".cc": true, + ".cpp": true, + ".cxx": true, +} + +// buildAppleSourceBundle packs the source files referenced by one architecture's +// DWARF into a .srcbundle. Files that aren't readable on this machine (SDK, +// system, or third-party code built elsewhere) are skipped — the backend simply +// renders those frames without source context. Returns nil when nothing local +// was found, so the caller can skip the upload entirely. +func buildAppleSourceBundle(a apple.Arch) ([]byte, int, error) { + b := &srcbundle.Builder{} + total := 0 + for key, path := range a.Sources { + if !appleSourceExtensions[strings.ToLower(filepath.Ext(key))] { + continue + } + data, err := os.ReadFile(path) + if err != nil { + continue // not on this machine: expected for SDK/system sources + } + if len(data) > maxSourceFileBytes || total+len(data) > maxSourceBundleBytes { + continue + } + total += len(data) + b.Add(key, data) + } + if b.Len() == 0 { + return nil, 0, nil + } + + var buf bytes.Buffer + if err := b.Encode(&buf); err != nil { + return nil, 0, fmt.Errorf("failed to encode source bundle for %s: %w", a.UUID, err) + } + return buf.Bytes(), b.Len(), nil +} + +// appleSourceKey is the storage key for an image's source bundle. +func appleSourceKey(uuid string) string { + return fmt.Sprintf("%s/%s%s", appleSymbolsIDPrefix, uuid, appleSourceExt) +} diff --git a/cmd/symbols/apple_sources_test.go b/cmd/symbols/apple_sources_test.go new file mode 100644 index 00000000..689d38ff --- /dev/null +++ b/cmd/symbols/apple_sources_test.go @@ -0,0 +1,135 @@ +package symbols + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/launchdarkly/ldcli/internal/symbols/apple" + "github.com/launchdarkly/ldcli/internal/symbols/srcbundle" +) + +func TestAppleSourceKey(t *testing.T) { + key := appleSourceKey("A5121984B70C3CA0BCC22FB671D75A20") + assert.Equal(t, "_sym/apple/id/A5121984B70C3CA0BCC22FB671D75A20.srcbundle", key) + // The source bundle sits beside the map under the same UUID. + assert.Equal(t, strings.TrimSuffix(appleKey("A5121984B70C3CA0BCC22FB671D75A20"), appleSymbolExt), + strings.TrimSuffix(key, appleSourceExt)) +} + +// buildAppleSourceBundle packs readable sources keyed exactly as the .dsymmap +// spells them, so a resolved frame's FileName is the lookup key. +func TestBuildAppleSourceBundle(t *testing.T) { + dir := t.TempDir() + swiftPath := filepath.Join(dir, "Cart.swift") + require.NoError(t, os.WriteFile(swiftPath, []byte("import Foundation\nlet total = 1\n"), 0o644)) + + a := apple.Arch{ + UUID: "A5121984B70C3CA0BCC22FB671D75A20", + Sources: map[string]string{"/build/Sources/Cart.swift": swiftPath}, + } + data, nFiles, err := buildAppleSourceBundle(a) + require.NoError(t, err) + require.NotNil(t, data) + assert.Equal(t, 1, nFiles) + + bundle, err := srcbundle.Open(data) + require.NoError(t, err) + // Keyed by the DWARF path, not the local path it was read from. + _, content, _, ok := bundle.Window("/build/Sources/Cart.swift", 2, 2) + require.True(t, ok) + assert.Equal(t, "let total = 1\n", content) + + _, ok = bundle.File(swiftPath) + assert.False(t, ok, "local path must not be a key") +} + +func TestBuildAppleSourceBundleSkipsUnreadableAndNonSource(t *testing.T) { + dir := t.TempDir() + txtPath := filepath.Join(dir, "notes.txt") + require.NoError(t, os.WriteFile(txtPath, []byte("not source\n"), 0o644)) + + a := apple.Arch{ + UUID: "A5121984B70C3CA0BCC22FB671D75A20", + Sources: map[string]string{ + "/build/notes.txt": txtPath, // wrong extension + "/build/Gone.swift": filepath.Join(dir, "Gone.swift"), // not on this machine + "/SDK/Vendor.swift": filepath.Join(dir, "nope.swift"), // not on this machine + "/build/Modules/module.map": filepath.Join(dir, "module.map"), // wrong extension + }, + } + data, nFiles, err := buildAppleSourceBundle(a) + require.NoError(t, err) + assert.Nil(t, data, "nothing bundleable yields no bundle") + assert.Zero(t, nFiles) +} + +func TestBuildAppleSourceBundleSkipsOversizeFile(t *testing.T) { + dir := t.TempDir() + big := filepath.Join(dir, "Generated.swift") + require.NoError(t, os.WriteFile(big, make([]byte, maxSourceFileBytes+1), 0o644)) + small := filepath.Join(dir, "Small.swift") + require.NoError(t, os.WriteFile(small, []byte("let a = 1\n"), 0o644)) + + a := apple.Arch{ + UUID: "A5121984B70C3CA0BCC22FB671D75A20", + Sources: map[string]string{ + "/build/Generated.swift": big, + "/build/Small.swift": small, + }, + } + data, nFiles, err := buildAppleSourceBundle(a) + require.NoError(t, err) + require.NotNil(t, data) + assert.Equal(t, 1, nFiles, "oversize file is skipped") + + bundle, err := srcbundle.Open(data) + require.NoError(t, err) + _, ok := bundle.File("/build/Small.swift") + assert.True(t, ok) + _, ok = bundle.File("/build/Generated.swift") + assert.False(t, ok) +} + +// The fixture dSYM's sources aren't checked out on this machine, so +// --include-sources must degrade to "map only" instead of failing. +func TestBuildAppleMapsWithSourcesDoesNotFail(t *testing.T) { + image := filepath.Join(fixtureDSYM, "Contents", "Resources", "DWARF", "symbolsdemo") + maps, err := buildAppleMaps([]string{image}, true) + require.NoError(t, err) + require.NotEmpty(t, maps) + + for _, m := range maps { + if m.Kind == "sources" { + assert.Equal(t, appleSourceKey(m.UUID), m.Key) + _, err := srcbundle.Open(m.Data) + require.NoError(t, err, "uploaded source bundle must decode") + } else { + assert.Equal(t, appleKey(m.UUID), m.Key) + } + } +} + +// The DWARF walk must record the source paths the map references. +func TestBuildFromMachOCollectsSources(t *testing.T) { + image := filepath.Join(fixtureDSYM, "Contents", "Resources", "DWARF", "symbolsdemo") + arches, err := apple.BuildFromMachO(image) + require.NoError(t, err) + require.NotEmpty(t, arches) + + found := false + for _, a := range arches { + if len(a.Sources) > 0 { + found = true + for key, abs := range a.Sources { + assert.NotEmpty(t, key) + assert.True(t, filepath.IsAbs(abs), "%s should resolve to an absolute path", key) + } + } + } + assert.True(t, found, "fixture DWARF should reference at least one source file") +} diff --git a/cmd/symbols/apple_upload.go b/cmd/symbols/apple_upload.go index caf9d162..a2ac5a22 100644 --- a/cmd/symbols/apple_upload.go +++ b/cmd/symbols/apple_upload.go @@ -22,17 +22,31 @@ const appleSymbolsIDPrefix = "_sym/apple/id" // in lookup (maps are keyed by UUID); the backend appends the same extension. const appleSymbolExt = ".dsymmap" -// appleSymbolMap is one architecture's compiled .dsymmap ready to upload. +// appleSymbolMap is one compiled artifact ready to upload: an architecture's +// .dsymmap symbol map, or (with --include-sources) its .srcbundle sources. type appleSymbolMap struct { Key string UUID string Arch string Data []byte + // Kind labels the artifact in progress output ("" for a symbol map). + Kind string +} + +// label describes the artifact for upload logs. +func (m appleSymbolMap) label() string { + if m.Kind == "" { + return fmt.Sprintf("%s (%s)", m.UUID, m.Arch) + } + return fmt.Sprintf("%s (%s, %s)", m.UUID, m.Arch, m.Kind) } // uploadAppleDSYMs discovers .dSYM bundles under path, compiles each contained // architecture to a .dsymmap symbol map, and uploads one object per build UUID. -func uploadAppleDSYMs(apiKey, projectID, path, backendURL string) error { +// When includeSources is set, each image's referenced source files are packed +// into a .srcbundle and uploaded alongside its map so the UI can show source +// context around native frames. +func uploadAppleDSYMs(apiKey, projectID, path, backendURL string, includeSources bool) error { images, err := findDSYMImages(path) if err != nil { return fmt.Errorf("failed to find dSYM files: %w", err) @@ -41,7 +55,7 @@ func uploadAppleDSYMs(apiKey, projectID, path, backendURL string) error { return fmt.Errorf("no .dSYM bundles found in %s, is this the correct path?", path) } - maps, err := buildAppleMaps(images) + maps, err := buildAppleMaps(images, includeSources) if err != nil { return err } @@ -65,7 +79,7 @@ func uploadAppleDSYMs(apiKey, projectID, path, backendURL string) error { } for i, m := range maps { - if err := uploadBytes(m.Data, uploadURLs[i], fmt.Sprintf("%s (%s)", m.UUID, m.Arch)); err != nil { + if err := uploadBytes(m.Data, uploadURLs[i], m.label()); err != nil { return fmt.Errorf("failed to upload symbol map for %s: %w", m.UUID, err) } } @@ -76,7 +90,9 @@ func uploadAppleDSYMs(apiKey, projectID, path, backendURL string) error { // buildAppleMaps compiles every architecture of every dSYM image into a .dsymmap, // deduplicating by UUID (a universal binary and its per-arch slices can repeat). -func buildAppleMaps(images []string) ([]appleSymbolMap, error) { +// With includeSources it also emits a .srcbundle per image, keyed by the same +// UUID. +func buildAppleMaps(images []string, includeSources bool) ([]appleSymbolMap, error) { var maps []appleSymbolMap seen := make(map[string]bool) @@ -103,6 +119,29 @@ func buildAppleMaps(images []string) ([]appleSymbolMap, error) { Data: buf.Bytes(), }) fmt.Printf("Built symbol map for %s (%s, %d bytes)\n", a.UUID, arch, buf.Len()) + + if !includeSources { + continue + } + srcData, nFiles, err := buildAppleSourceBundle(a) + if err != nil { + return nil, err + } + if srcData == nil { + // None of the DWARF-referenced sources are readable here (e.g. + // uploading a dSYM archived on another machine). Not fatal: the map + // alone still symbolicates, just without source context. + fmt.Printf("No local sources found for %s (%s); skipping source bundle\n", a.UUID, arch) + continue + } + maps = append(maps, appleSymbolMap{ + Key: appleSourceKey(a.UUID), + UUID: a.UUID, + Arch: arch, + Data: srcData, + Kind: "sources", + }) + fmt.Printf("Built source bundle for %s (%s, %d files, %d bytes)\n", a.UUID, arch, nFiles, len(srcData)) } } return maps, nil diff --git a/cmd/symbols/apple_upload_test.go b/cmd/symbols/apple_upload_test.go index 9bf6c3fd..e734016b 100644 --- a/cmd/symbols/apple_upload_test.go +++ b/cmd/symbols/apple_upload_test.go @@ -57,7 +57,7 @@ func TestFindDSYMImages_Missing(t *testing.T) { // the real fixture and confirms the produced bytes are valid, keyed .dsymmap maps. func TestBuildAppleMaps(t *testing.T) { image := filepath.Join(fixtureDSYM, "Contents", "Resources", "DWARF", "symbolsdemo") - maps, err := buildAppleMaps([]string{image}) + maps, err := buildAppleMaps([]string{image}, false) require.NoError(t, err) require.Len(t, maps, 2, "universal fixture yields arm64 + x86_64 maps") @@ -81,7 +81,7 @@ func TestBuildAppleMaps(t *testing.T) { // produce duplicate uploads. func TestBuildAppleMaps_DedupesUUID(t *testing.T) { image := filepath.Join(fixtureDSYM, "Contents", "Resources", "DWARF", "symbolsdemo") - maps, err := buildAppleMaps([]string{image, image}) + maps, err := buildAppleMaps([]string{image, image}, false) require.NoError(t, err) assert.Len(t, maps, 2, "duplicate images must be deduplicated by UUID") } diff --git a/cmd/symbols/flutter_upload.go b/cmd/symbols/flutter_upload.go new file mode 100644 index 00000000..10cdb318 --- /dev/null +++ b/cmd/symbols/flutter_upload.go @@ -0,0 +1,225 @@ +package symbols + +import ( + "bytes" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/launchdarkly/ldcli/internal/symbols/flutter" +) + +const ( + // flutterSymbolsIDPrefix is the Id-lane storage segment for Flutter/Dart + // symbol maps. Each map is keyed by the Dart snapshot build id (symbols_id): + // _sym/flutter/id//app.dartmap. The backend derives the same key + // from the symbols_id an obfuscated crash reports. + flutterSymbolsIDPrefix = "_sym/flutter/id" + + // flutterSymbolExt is the object extension for a compiled Flutter symbol map + // (the shared dsymmap binary format under a Flutter-specific name). + flutterSymbolExt = ".dartmap" + + // flutterSymbolMapName is the Id-lane object name. A symbols_id is unique per + // (build, arch), so it fully identifies one map and no platform token is + // needed in the Id-lane key. + flutterSymbolMapName = "app" + flutterSymbolExt + + // flutterSymbolFileSuffix is the Flutter debug-symbols file `ldcli` discovers + // for --type flutter (e.g. app.android-arm64.symbols). + flutterSymbolFileSuffix = ".symbols" +) + +// flutterUpload is one .dartmap object to store at one key. A map is uploaded to +// the Id lane always, and to the Version lane too when --app-version is given +// (same bytes, two keys). +type flutterUpload struct { + Data []byte + Key string + Label string +} + +// uploadFlutterSymbols discovers app.*.symbols files under path, compiles each +// to a .dartmap, and uploads it to the Id lane (and the Version lane when +// appVersion is set). +func uploadFlutterSymbols(apiKey, projectID, path, appVersion, backendURL string) error { + uploads, err := buildFlutterMaps(path, appVersion) + if err != nil { + return err + } + + keys := make([]string, len(uploads)) + for i, u := range uploads { + keys[i] = u.Key + } + + uploadURLs, err := getSymbolUploadUrls(apiKey, projectID, keys, backendURL) + if err != nil { + return fmt.Errorf("failed to get upload URLs: %w", err) + } + // One URL per requested key, in order; a short list would misalign the pairing. + if len(uploadURLs) != len(uploads) { + return fmt.Errorf("expected %d upload URLs but received %d", len(uploads), len(uploadURLs)) + } + + for i, u := range uploads { + if err := uploadBytes(u.Data, uploadURLs[i], u.Label); err != nil { + return fmt.Errorf("failed to upload symbol map %s: %w", u.Label, err) + } + } + + fmt.Println("Successfully uploaded all symbols") + return nil +} + +// buildFlutterMaps compiles every discovered app.*.symbols to a .dartmap and +// returns the objects to store, deduplicating by symbols_id (the same build can +// be discovered more than once). Each map yields an Id-lane upload, plus a +// Version-lane upload when appVersion and a platform token are both available. +func buildFlutterMaps(path, appVersion string) ([]flutterUpload, error) { + files, err := findFlutterSymbolFiles(path) + if err != nil { + return nil, fmt.Errorf("failed to find Flutter symbol files: %w", err) + } + if len(files) == 0 { + return nil, fmt.Errorf("no Flutter symbol files found in %s (looked for app.*.symbols). Build with `flutter build ... --obfuscate --split-debug-info=`", path) + } + + var uploads []flutterUpload + seenID := make(map[string]bool) + seenVersionKey := make(map[string]bool) + var noBuildID []string + var missingAppVersion, missingPlatform int + for _, file := range files { + img, err := flutter.BuildFromELF(file) + if err != nil { + return nil, fmt.Errorf("failed to process %s: %w", file, err) + } + + var buf bytes.Buffer + if err := img.Builder.Encode(&buf); err != nil { + return nil, fmt.Errorf("failed to encode symbol map for %s: %w", file, err) + } + data := buf.Bytes() + + // iOS/macOS .symbols files carry no Dart build id (see readBuildID), so the + // Id lane can't be keyed from the file. Fall back to the Version lane, which + // the backend also tries for Flutter crashes (keyed by app version + + // platform). This requires --app-version and a platform token. + if img.SymbolsID == "" { + if appVersion == "" || img.Platform == "" { + noBuildID = append(noBuildID, file) + if appVersion == "" { + missingAppVersion++ + fmt.Printf("Skipping %s: no build id in file (e.g. iOS .symbols). Re-run with --app-version to upload it to the Version lane.\n", filepath.Base(file)) + } else { + missingPlatform++ + fmt.Printf("Skipping %s: no build id in file and no platform token could be parsed from the filename (expected app..symbols), so the Version lane can't be keyed.\n", filepath.Base(file)) + } + continue + } + vKey := flutterVersionKey(appVersion, img.Platform) + if seenVersionKey[vKey] { + continue + } + seenVersionKey[vKey] = true + uploads = append(uploads, flutterUpload{ + Data: data, + Key: vKey, + Label: fmt.Sprintf("%s (Version Lane, no build id)", img.Platform), + }) + fmt.Printf("Built symbol map for %s (Version lane only, no build id, %d bytes)\n", img.Platform, len(data)) + continue + } + + if seenID[img.SymbolsID] { + continue + } + seenID[img.SymbolsID] = true + + uploads = append(uploads, flutterUpload{ + Data: data, + Key: flutterIDKey(img.SymbolsID), + Label: fmt.Sprintf("%s (%s, Id Lane)", img.SymbolsID, img.Platform), + }) + if appVersion != "" && img.Platform != "" { + vKey := flutterVersionKey(appVersion, img.Platform) + if !seenVersionKey[vKey] { + seenVersionKey[vKey] = true + uploads = append(uploads, flutterUpload{ + Data: data, + Key: vKey, + Label: fmt.Sprintf("%s (%s, Version Lane)", img.SymbolsID, img.Platform), + }) + } + } + fmt.Printf("Built symbol map for %s (%s, %d bytes)\n", img.SymbolsID, img.Platform, len(data)) + } + + if len(uploads) == 0 { + if len(noBuildID) > 0 { + switch { + case missingAppVersion > 0 && missingPlatform > 0: + return nil, fmt.Errorf("found %d Flutter symbol file(s) with no build id (e.g. iOS .symbols) that could not be uploaded: %d were missing --app-version, and %d had no platform token parsable from the filename (expected app..symbols). Re-run with --app-version and ensure filenames include a platform token to use the Version lane", len(noBuildID), missingAppVersion, missingPlatform) + case missingPlatform > 0: + return nil, fmt.Errorf("found %d Flutter symbol file(s) with no build id (e.g. iOS .symbols) and no platform token parsable from the filename (expected app..symbols), so the Version lane could not be keyed and none could be uploaded", len(noBuildID)) + default: + return nil, fmt.Errorf("found %d Flutter symbol file(s) with no build id (e.g. iOS .symbols) and no --app-version was given, so none could be uploaded. Re-run with --app-version to use the Version lane", len(noBuildID)) + } + } + return nil, fmt.Errorf("no Flutter symbol maps could be built from %s", path) + } + return uploads, nil +} + +// flutterIDKey is the Id-lane storage key for a symbols_id: +// _sym/flutter/id//app.dartmap. +func flutterIDKey(symbolsID string) string { + return fmt.Sprintf("%s/%s/%s", flutterSymbolsIDPrefix, symbolsID, flutterSymbolMapName) +} + +// flutterVersionKey is the Version-lane storage key for a platform token: +// /app..dartmap. The platform token (e.g. "android-arm64") +// disambiguates the per-arch maps that share one app version, and matches the +// "-" the backend builds from the crash header. +func flutterVersionKey(version, platform string) string { + return fmt.Sprintf("%s/app.%s%s", version, platform, flutterSymbolExt) +} + +// findFlutterSymbolFiles resolves path to the app.*.symbols files to compile. +// path may be a single .symbols file or a directory tree (e.g. the +// --split-debug-info output folder). +func findFlutterSymbolFiles(path string) ([]string, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if !info.IsDir() { + if !isFlutterSymbolFile(path) { + return nil, fmt.Errorf("file %s is not a Flutter symbol file (expected app.*.symbols)", path) + } + return []string{path}, nil + } + + var out []string + err = filepath.WalkDir(path, func(p string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !d.IsDir() && isFlutterSymbolFile(p) { + out = append(out, p) + } + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +func isFlutterSymbolFile(name string) bool { + base := filepath.Base(name) + return strings.HasPrefix(base, "app.") && strings.HasSuffix(base, flutterSymbolFileSuffix) +} diff --git a/cmd/symbols/generate.go b/cmd/symbols/generate.go index 544af7dd..32b68a6b 100644 --- a/cmd/symbols/generate.go +++ b/cmd/symbols/generate.go @@ -62,7 +62,7 @@ func generateRunE() func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { symbolType := canonicalizeSymbolType(viper.GetString(typeFlag)) if !isSupportedType(symbolType) { - return fmt.Errorf("unsupported --type %q; supported types: %s, %s, %s", viper.GetString(typeFlag), typeReactNative, typeAndroid, typeAppleDSYM) + return fmt.Errorf("unsupported --type %q; supported types: %s, %s, %s, %s", viper.GetString(typeFlag), typeReactNative, typeAndroid, typeAppleDSYM, typeFlutter) } path := viper.GetString(pathFlag) @@ -76,7 +76,13 @@ func generateRunE() func(cmd *cobra.Command, args []string) error { // Apple dSYMs are compiled into per-arch .dsymmap symbol maps keyed by build // UUID, ignoring the version/symbols-id lanes. if symbolType == typeAppleDSYM { - return generateAppleDSYMs(path, outputDir) + return generateAppleDSYMs(path, outputDir, viper.GetBool(includeSourcesFlag)) + } + + // Flutter symbols compile to .dartmap maps keyed by build id (Id Lane), + // plus a Version-lane copy when --app-version is set. + if symbolType == typeFlutter { + return generateFlutterSymbols(path, viper.GetString(appVersionFlag), outputDir) } return generateSymbolFiles(symbolType, path, outputDir) @@ -86,7 +92,7 @@ func generateRunE() func(cmd *cobra.Command, args []string) error { // generateAppleDSYMs compiles the discovered dSYM images to .dsymmap symbol maps // and writes one file per build UUID under outputDir, using the same storage // key (_sym/apple/id/.dsymmap) that `symbols upload` would use. -func generateAppleDSYMs(path, outputDir string) error { +func generateAppleDSYMs(path, outputDir string, includeSources bool) error { images, err := findDSYMImages(path) if err != nil { return fmt.Errorf("failed to find dSYM files: %w", err) @@ -95,7 +101,7 @@ func generateAppleDSYMs(path, outputDir string) error { return fmt.Errorf("no .dSYM bundles found in %s, is this the correct path?", path) } - maps, err := buildAppleMaps(images) + maps, err := buildAppleMaps(images, includeSources) if err != nil { return err } @@ -113,6 +119,23 @@ func generateAppleDSYMs(path, outputDir string) error { return nil } +// generateFlutterSymbols compiles the discovered app.*.symbols to .dartmap +// symbol maps and writes them under outputDir using the same storage keys +// `symbols upload` would use (Id lane, plus Version lane when appVersion is set). +func generateFlutterSymbols(path, appVersion, outputDir string) error { + uploads, err := buildFlutterMaps(path, appVersion) + if err != nil { + return err + } + for _, u := range uploads { + if err := writeSymbolFile(outputDir, u.Key, u.Data); err != nil { + return fmt.Errorf("failed to write symbol map %s: %w", u.Label, err) + } + } + fmt.Printf("Successfully generated %d symbol file(s) in %s\n", len(uploads), outputDir) + return nil +} + // generateSymbolFiles discovers React Native or Android artifacts and copies // each one to outputDir under the same storage key `symbols upload` would use, // so the generated folder matches what the backend expects. @@ -166,7 +189,7 @@ func writeSymbolFile(outputDir, key string, data []byte) error { } func initGenerateFlags(cmd *cobra.Command) { - cmd.Flags().String(typeFlag, "", fmt.Sprintf("The symbol type to generate (supported: %s, %s, %s; %s also accepts ios/ipados/tvos/watchos/visionos/macos/apple/dsym)", typeReactNative, typeAndroid, typeAppleDSYM, typeAppleDSYM)) + cmd.Flags().String(typeFlag, "", fmt.Sprintf("The symbol type to generate (supported: %s, %s, %s, %s; %s also accepts ios/ipados/tvos/watchos/visionos/macos/apple/dsym; %s also accepts dart)", typeReactNative, typeAndroid, typeAppleDSYM, typeFlutter, typeAppleDSYM, typeFlutter)) _ = cmd.MarkFlagRequired(typeFlag) _ = cmd.Flags().SetAnnotation(typeFlag, "required", []string{"true"}) _ = viper.BindPFlag(typeFlag, cmd.Flags().Lookup(typeFlag)) @@ -180,6 +203,9 @@ func initGenerateFlags(cmd *cobra.Command) { cmd.Flags().String(appVersionFlag, "", "The current version of your deploy") _ = viper.BindPFlag(appVersionFlag, cmd.Flags().Lookup(appVersionFlag)) + cmd.Flags().Bool(includeSourcesFlag, false, fmt.Sprintf("Also generate a source bundle from the files referenced by the debug info, for source context on native frames (%s only)", typeAppleDSYM)) + _ = viper.BindPFlag(includeSourcesFlag, cmd.Flags().Lookup(includeSourcesFlag)) + cmd.Flags().String(symbolsIdFlag, "", "The symbols id (launchdarkly.symbols_id.htlhash) to key files by (Symbols Id Lane). If omitted, a *.symbolsid sidecar next to the bundle is used when present") _ = viper.BindPFlag(symbolsIdFlag, cmd.Flags().Lookup(symbolsIdFlag)) diff --git a/cmd/symbols/upload.go b/cmd/symbols/upload.go index 0250df71..55dcfa50 100644 --- a/cmd/symbols/upload.go +++ b/cmd/symbols/upload.go @@ -32,6 +32,12 @@ const ( basePathFlag = "base-path" backendUrlFlag = "backend-url" + // includeSourcesFlag opts into uploading the source files a dSYM's DWARF + // references (as a .srcbundle beside the .dsymmap) so the errors page can show + // source context around native frames. Off by default: it ships your source + // to LaunchDarkly. + includeSourcesFlag = "include-sources" + defaultPath = "." defaultBackendUrl = "https://pri.observability.app.launchdarkly.com" @@ -67,6 +73,11 @@ const ( // the canonical value; see symbolTypeAliases for accepted synonyms. typeAppleDSYM = "apple-dsym" + // typeFlutter compiles Flutter/Dart AOT debug symbols (app..symbols) + // into per-build .dartmap symbol maps (keyed by the Dart snapshot build id, + // surfaced as symbols_id) for obfuscated Dart crash symbolication. + typeFlutter = "flutter" + // getSymbolUrlsQuery uses the dedicated `get_symbol_upload_urls_ld` query // (separate from `sourcemaps upload`) so symbol uploads travel over the // symbol endpoint, which accepts larger, multi-segment uploads. @@ -136,7 +147,7 @@ func runE(client resources.Client) func(cmd *cobra.Command, args []string) error return func(cmd *cobra.Command, args []string) error { symbolType := canonicalizeSymbolType(viper.GetString(typeFlag)) if !isSupportedType(symbolType) { - return fmt.Errorf("unsupported --type %q; supported types: %s, %s, %s", viper.GetString(typeFlag), typeReactNative, typeAndroid, typeAppleDSYM) + return fmt.Errorf("unsupported --type %q; supported types: %s, %s, %s, %s", viper.GetString(typeFlag), typeReactNative, typeAndroid, typeAppleDSYM, typeFlutter) } projectKey := viper.GetString(cliflags.ProjectFlag) @@ -182,7 +193,15 @@ func runE(client resources.Client) func(cmd *cobra.Command, args []string) error // symbol maps keyed by build UUID, ignoring the version/symbols-id lanes. if symbolType == typeAppleDSYM { fmt.Printf("Starting to upload %s symbols from %s\n", symbolType, path) - return uploadAppleDSYMs(viper.GetString(cliflags.AccessTokenFlag), projectResult.ID, path, backendUrl) + return uploadAppleDSYMs(viper.GetString(cliflags.AccessTokenFlag), projectResult.ID, path, backendUrl, viper.GetBool(includeSourcesFlag)) + } + + // Flutter/Dart symbols take a dedicated path too: each app..symbols + // is compiled to a .dartmap keyed by its build id (Id Lane), plus a + // Version-lane copy when --app-version is set. + if symbolType == typeFlutter { + fmt.Printf("Starting to upload %s symbols from %s\n", symbolType, path) + return uploadFlutterSymbols(viper.GetString(cliflags.AccessTokenFlag), projectResult.ID, path, appVersion, backendUrl) } symbolsIDPrefix := symbolsIDPrefixForType(symbolType) @@ -255,6 +274,8 @@ var symbolTypeAliases = map[string]string{ "visionos": typeAppleDSYM, "macos": typeAppleDSYM, "osx": typeAppleDSYM, + "flutter": typeFlutter, + "dart": typeFlutter, } // canonicalizeSymbolType resolves a user-supplied --type to its canonical value. @@ -270,7 +291,7 @@ func canonicalizeSymbolType(symbolType string) string { } func isSupportedType(symbolType string) bool { - return symbolType == typeReactNative || symbolType == typeAndroid || symbolType == typeAppleDSYM + return symbolType == typeReactNative || symbolType == typeAndroid || symbolType == typeAppleDSYM || symbolType == typeFlutter } // symbolsIDPrefixForType picks the Symbols Id Lane storage segment for the symbol @@ -501,7 +522,7 @@ func uploadFile(filePath, uploadUrl, name string) error { } func initFlags(cmd *cobra.Command) { - cmd.Flags().String(typeFlag, "", fmt.Sprintf("The symbol type to upload (supported: %s, %s, %s; %s also accepts ios/ipados/tvos/watchos/visionos/macos/apple/dsym)", typeReactNative, typeAndroid, typeAppleDSYM, typeAppleDSYM)) + cmd.Flags().String(typeFlag, "", fmt.Sprintf("The symbol type to upload (supported: %s, %s, %s, %s; %s also accepts ios/ipados/tvos/watchos/visionos/macos/apple/dsym; %s also accepts dart)", typeReactNative, typeAndroid, typeAppleDSYM, typeFlutter, typeAppleDSYM, typeFlutter)) _ = cmd.MarkFlagRequired(typeFlag) _ = cmd.Flags().SetAnnotation(typeFlag, "required", []string{"true"}) _ = viper.BindPFlag(typeFlag, cmd.Flags().Lookup(typeFlag)) @@ -525,4 +546,7 @@ func initFlags(cmd *cobra.Command) { cmd.Flags().String(backendUrlFlag, defaultBackendUrl, "An optional backend url for self-hosted deployments") _ = viper.BindPFlag(backendUrlFlag, cmd.Flags().Lookup(backendUrlFlag)) + + cmd.Flags().Bool(includeSourcesFlag, false, fmt.Sprintf("Also upload the source files referenced by the debug info so the errors page can show source context around native frames (%s only). Your source is stored in LaunchDarkly", typeAppleDSYM)) + _ = viper.BindPFlag(includeSourcesFlag, cmd.Flags().Lookup(includeSourcesFlag)) } diff --git a/cmd/symbols/upload_test.go b/cmd/symbols/upload_test.go index 9fc2b2ed..117565ad 100644 --- a/cmd/symbols/upload_test.go +++ b/cmd/symbols/upload_test.go @@ -248,7 +248,7 @@ func TestSymbolsIDForArtifact(t *testing.T) { } func TestUnsupportedType(t *testing.T) { - viper.Set(typeFlag, "flutter") + viper.Set(typeFlag, "totally-unknown") defer viper.Set(typeFlag, "") client := resources.NewClient("") @@ -261,7 +261,8 @@ func TestIsSupportedType(t *testing.T) { assert.True(t, isSupportedType(typeReactNative)) assert.True(t, isSupportedType(typeAndroid)) assert.True(t, isSupportedType(typeAppleDSYM)) - assert.False(t, isSupportedType("flutter")) + assert.True(t, isSupportedType(typeFlutter)) + assert.False(t, isSupportedType("totally-unknown")) assert.False(t, isSupportedType("")) } @@ -271,6 +272,11 @@ func TestCanonicalizeSymbolType(t *testing.T) { assert.Equal(t, typeAppleDSYM, canonicalizeSymbolType(alias), alias) } + // Flutter synonyms resolve to the flutter type. + assert.Equal(t, typeFlutter, canonicalizeSymbolType("flutter")) + assert.Equal(t, typeFlutter, canonicalizeSymbolType("dart")) + assert.Equal(t, typeFlutter, canonicalizeSymbolType("Flutter")) + // Case-insensitive and whitespace-tolerant. assert.Equal(t, typeAppleDSYM, canonicalizeSymbolType("iOS")) assert.Equal(t, typeAppleDSYM, canonicalizeSymbolType(" Apple-DSYM ")) @@ -279,11 +285,28 @@ func TestCanonicalizeSymbolType(t *testing.T) { // Canonical values pass through; unknown values are lower-cased for rejection. assert.Equal(t, typeReactNative, canonicalizeSymbolType(typeReactNative)) assert.Equal(t, typeAndroid, canonicalizeSymbolType(typeAndroid)) - assert.Equal(t, "flutter", canonicalizeSymbolType("Flutter")) - assert.False(t, isSupportedType(canonicalizeSymbolType("flutter"))) + assert.Equal(t, "totally-unknown", canonicalizeSymbolType("Totally-Unknown")) + assert.False(t, isSupportedType(canonicalizeSymbolType("totally-unknown"))) // Every alias must map to a supported canonical type. for alias, canonical := range symbolTypeAliases { assert.True(t, isSupportedType(canonical), alias) } } + +func TestFlutterKeys(t *testing.T) { + symbolsID := "0f8a1b2c3d4e5f60718293a4b5c6d7e8" + // Id lane: keyed by symbols_id (arch-unique), constant object name. + assert.Equal(t, "_sym/flutter/id/"+symbolsID+"/app.dartmap", flutterIDKey(symbolsID)) + // Version lane: keyed by version, with the platform token disambiguating arches. + assert.Equal(t, "1.2.3/app.android-arm64.dartmap", flutterVersionKey("1.2.3", "android-arm64")) + assert.Equal(t, "1.2.3/app.ios-arm64.dartmap", flutterVersionKey("1.2.3", "ios-arm64")) +} + +func TestIsFlutterSymbolFile(t *testing.T) { + assert.True(t, isFlutterSymbolFile("app.android-arm64.symbols")) + assert.True(t, isFlutterSymbolFile("build/symbols/app.ios-arm64.symbols")) + assert.False(t, isFlutterSymbolFile("app.android-arm64.dartmap")) + assert.False(t, isFlutterSymbolFile("mapping.txt")) + assert.False(t, isFlutterSymbolFile("other.symbols")) +} diff --git a/internal/symbols/apple/dsym.go b/internal/symbols/apple/dsym.go index bc06c768..470f0944 100644 --- a/internal/symbols/apple/dsym.go +++ b/internal/symbols/apple/dsym.go @@ -17,6 +17,7 @@ package apple import ( "encoding/hex" "os" + "path/filepath" "strings" "github.com/blacktop/go-dwarf" @@ -36,6 +37,12 @@ type Arch struct { CPUType uint32 CPUSubtype uint32 Builder *dsymmap.Builder + // Sources maps each source path referenced by this image's DWARF — keyed by + // the exact string stored in the .dsymmap, so a resolved frame's FileName is + // the lookup key — to its absolute path on this machine. It is only used to + // build the optional .srcbundle (`--include-sources`); files that aren't + // present locally (SDK/system code) simply fail to read and are skipped. + Sources map[string]string } // BuildFromMachO opens a dSYM's DWARF Mach-O image at path — thin or fat — @@ -98,7 +105,8 @@ func buildArch(f *macho.File) (Arch, error) { CPUType: uint32(f.CPU), CPUSubtype: uint32(f.SubCPU), } - if err := populate(d, textVM, b); err != nil { + sources := make(map[string]string) + if err := populate(d, textVM, b, sources); err != nil { return Arch{}, err } @@ -107,6 +115,7 @@ func buildArch(f *macho.File) (Arch, error) { CPUType: uint32(f.CPU), CPUSubtype: uint32(f.SubCPU), Builder: b, + Sources: sources, }, nil } @@ -119,11 +128,15 @@ type scope struct { inlineDepth uint32 } -func populate(d *dwarf.Data, textVM uint64, b *dsymmap.Builder) error { +// populate walks the DWARF DIE tree into b. It also records every source path it +// encounters into sources (keyed exactly as stored in the map) so the caller can +// optionally bundle those files; pass nil to skip that. +func populate(d *dwarf.Data, textVM uint64, b *dsymmap.Builder, sources map[string]string) error { r := d.Reader() var stack []scope var funcs []*dsymmap.Function var curFiles []*dwarf.LineFile + var curCompDir string top := func() scope { if len(stack) == 0 { @@ -153,7 +166,8 @@ func populate(d *dwarf.Data, textVM uint64, b *dsymmap.Builder) error { switch ent.Tag { case dwarf.TagCompileUnit: curFiles = filesForEntry(d, ent) - addLines(d, ent, textVM, b) + curCompDir, _ = ent.Val(dwarf.AttrCompDir).(string) + addLines(d, ent, textVM, b, sources, curCompDir) case dwarf.TagSubprogram: if fn := makeFunction(d, ent, textVM); fn != nil { @@ -165,7 +179,13 @@ func populate(d *dwarf.Data, textVM uint64, b *dsymmap.Builder) error { case dwarf.TagInlinedSubroutine: depth := top().inlineDepth + 1 if fn := top().fn; fn != nil { - fn.Inlines = append(fn.Inlines, makeInlines(d, ent, textVM, depth, curFiles)...) + inlines := makeInlines(d, ent, textVM, depth, curFiles) + for i := range inlines { + // Call-site files are rendered for outer inline frames, so they + // need to be bundled too. + recordSource(sources, inlines[i].CallFile, curCompDir) + } + fn.Inlines = append(fn.Inlines, inlines...) } push.inlineDepth = depth } @@ -263,7 +283,7 @@ func callSite(ent *dwarf.Entry, files []*dwarf.LineFile) (string, uint32) { return file, line } -func addLines(d *dwarf.Data, cu *dwarf.Entry, textVM uint64, b *dsymmap.Builder) { +func addLines(d *dwarf.Data, cu *dwarf.Entry, textVM uint64, b *dsymmap.Builder, sources map[string]string, compDir string) { lr, err := d.LineReader(cu) if err != nil || lr == nil { return @@ -287,10 +307,28 @@ func addLines(d *dwarf.Data, cu *dwarf.Entry, textVM uint64, b *dsymmap.Builder) if le.File != nil { file = le.File.Name } + recordSource(sources, file, compDir) b.Lines = append(b.Lines, dsymmap.LineRow{Addr: rel, File: file, Line: uint32(le.Line)}) } } +// recordSource notes that file (as spelled in the map) is referenced by this +// image, resolving where to read it from on this machine. DWARF paths are +// usually absolute; relative ones are resolved against the CU's DW_AT_comp_dir. +func recordSource(sources map[string]string, file, compDir string) { + if sources == nil || file == "" { + return + } + if _, ok := sources[file]; ok { + return + } + abs := file + if !filepath.IsAbs(abs) && compDir != "" { + abs = filepath.Join(compDir, abs) + } + sources[file] = abs +} + func filesForEntry(d *dwarf.Data, cu *dwarf.Entry) []*dwarf.LineFile { files, err := d.FilesForEntry(cu) if err != nil { diff --git a/internal/symbols/flutter/elf.go b/internal/symbols/flutter/elf.go new file mode 100644 index 00000000..2173c8d6 --- /dev/null +++ b/internal/symbols/flutter/elf.go @@ -0,0 +1,378 @@ +// Package flutter converts a Flutter/Dart AOT debug-symbols file +// (app..symbols — an ELF containing DWARF) into the compact, +// mmap-friendly .dartmap symbol map the backend consumes. It is the ldcli-side +// (encode) half of the Flutter symbolication pipeline. +// +// A release Flutter build produced with `--obfuscate --split-debug-info=` +// strips Dart names and emits address-based crash frames plus a header carrying +// a build id. The matching per-arch .symbols ELF carries the same build id (as a +// GNU build-id note) and the DWARF needed to resolve those addresses back to +// Dart function + file:line, expanding inlined calls. +// +// This uses only the Go standard library (debug/elf + debug/dwarf) so ldcli +// cross-compiles cleanly, and reuses the dsymmap codec shared with Apple. Dart +// symbol names in the .symbols DWARF are already human-readable, so — unlike the +// Apple path — no demangling is applied. +package flutter + +import ( + "debug/dwarf" + "debug/elf" + "encoding/binary" + "encoding/hex" + "path/filepath" + "strings" + + e "github.com/pkg/errors" + + "github.com/launchdarkly/ldcli/internal/symbols/dsymmap" +) + +// ntGNUBuildID is the ELF note type for a GNU build id (NT_GNU_BUILD_ID). +const ntGNUBuildID = 3 + +// Image is one .symbols file's recovered symbol map. +type Image struct { + // SymbolsID is the Dart snapshot build id as lowercase hex — the Id-lane + // lookup key (the same value the VM prints in every obfuscated crash header + // and the backend recovers from it). Named symbols_id everywhere; it is not + // exposed under the Dart "build id" term. + SymbolsID string + // Platform is the "-" token (e.g. "android-arm64") parsed from the + // app..symbols filename — used for the Version-lane object name. + Platform string + Builder *dsymmap.Builder +} + +// BuildFromELF opens a Flutter app..symbols ELF at path and returns +// its symbol map: the build id (symbols_id), platform token, and a dsymmap +// Builder the caller encodes to a .dartmap. +func BuildFromELF(path string) (Image, error) { + f, err := elf.Open(path) + if err != nil { + return Image{}, e.Wrapf(err, "opening %s", path) + } + defer f.Close() + + symbolsID, err := readBuildID(f) + if err != nil { + return Image{}, e.Wrapf(err, "reading build id from %s", path) + } + + d, err := f.DWARF() + if err != nil { + return Image{}, e.Wrapf(err, "reading DWARF from %s (was it built with --split-debug-info?)", path) + } + + // Dart AOT addresses in the crash `virt` column are already snapshot-relative + // and match the DWARF vaddr, so no rebasing is needed (TextVMAddr = 0). + b := &dsymmap.Builder{} + if err := populate(d, b); err != nil { + return Image{}, err + } + + return Image{ + SymbolsID: symbolsID, + Platform: platformFromFilename(path), + Builder: b, + }, nil +} + +// platformFromFilename extracts the "-" token from an +// app..symbols filename (e.g. "app.android-arm64.symbols" -> +// "android-arm64"). It returns "" when the name doesn't match, so the caller can +// fall back or surface a clear error; the Id lane does not depend on it. +func platformFromFilename(path string) string { + base := filepath.Base(path) + base = strings.TrimSuffix(base, ".symbols") + base = strings.TrimPrefix(base, "app.") + if base == "" || strings.Contains(base, ".") { + return "" + } + return base +} + +// readBuildID returns the GNU build id of an ELF as lowercase hex, scanning both +// note sections and PT_NOTE segments (Dart emits it as a .note.gnu.build-id on +// ELF targets such as Android/Linux). iOS/macOS .symbols files are ELF but carry +// no build-id note (the snapshot ships in a Mach-O App.framework), so this +// returns "" (not an error) there; the caller then falls back to the Version +// lane, which keys by app version + platform instead of the build id. +func readBuildID(f *elf.File) (string, error) { + for _, sec := range f.Sections { + if sec.Type != elf.SHT_NOTE { + continue + } + data, err := sec.Data() + if err != nil { + continue + } + if id := buildIDFromNotes(data, f.ByteOrder); id != "" { + return id, nil + } + } + for _, prog := range f.Progs { + if prog.Type != elf.PT_NOTE { + continue + } + data := make([]byte, prog.Filesz) + if _, err := prog.ReadAt(data, 0); err != nil { + continue + } + if id := buildIDFromNotes(data, f.ByteOrder); id != "" { + return id, nil + } + } + // No build-id note (e.g. iOS/macOS .symbols): not an error — the caller uses + // the Version lane instead. + return "", nil +} + +// buildIDFromNotes walks a note section/segment (a sequence of ELF notes) and +// returns the descriptor of the first NT_GNU_BUILD_ID note as lowercase hex, or +// "" if there is none. Each note is: namesz(4) descsz(4) type(4), then the name +// and descriptor, each padded to a 4-byte boundary. +func buildIDFromNotes(data []byte, bo binary.ByteOrder) string { + for len(data) >= 12 { + namesz := bo.Uint32(data[0:4]) + descsz := bo.Uint32(data[4:8]) + ntype := bo.Uint32(data[8:12]) + nameEnd := 12 + int(namesz) + descStart := align4(nameEnd) + descEnd := descStart + int(descsz) + if descEnd > len(data) || nameEnd > len(data) { + return "" + } + if ntype == ntGNUBuildID && descsz > 0 { + return hex.EncodeToString(data[descStart:descEnd]) + } + // The descriptor's padding may run past the end when the note is the last + // one and the section isn't 4-byte aligned; there is nothing left to read. + next := align4(descEnd) + if next > len(data) { + return "" + } + // Each note record is at least 12 bytes (header), so this always advances. + data = data[next:] + } + return "" +} + +func align4(n int) int { return (n + 3) &^ 3 } + +// scope is one level of the DWARF DIE tree during the depth-first walk (see the +// Apple builder for the shared shape). fn is the nearest enclosing concrete +// function; inlineDepth counts inlined_subroutine ancestors. +type scope struct { + fn *dsymmap.Function + inlineDepth uint32 +} + +// populate walks the DWARF DIE tree into b: physical functions, their line +// tables, and inlined-call chains. Mirrors apple.populate but on the standard +// library's debug/dwarf types. +func populate(d *dwarf.Data, b *dsymmap.Builder) error { + r := d.Reader() + var stack []scope + var funcs []*dsymmap.Function + var curFiles []*dwarf.LineFile + + top := func() scope { + if len(stack) == 0 { + return scope{} + } + return stack[len(stack)-1] + } + + for { + ent, err := r.Next() + if err != nil { + return e.Wrap(err, "walking DWARF") + } + if ent == nil { + break + } + // A Tag==0 entry terminates the current sibling list: pop one scope. + if ent.Tag == 0 { + if len(stack) > 0 { + stack = stack[:len(stack)-1] + } + continue + } + + push := top() + + switch ent.Tag { + case dwarf.TagCompileUnit: + curFiles = filesForCU(d, ent) + addLines(d, ent, b) + + case dwarf.TagSubprogram: + if fn := makeFunction(d, ent); fn != nil { + funcs = append(funcs, fn) + push.fn = fn + } + push.inlineDepth = 0 + + case dwarf.TagInlinedSubroutine: + depth := top().inlineDepth + 1 + if fn := top().fn; fn != nil { + fn.Inlines = append(fn.Inlines, makeInlines(d, ent, depth, curFiles)...) + } + push.inlineDepth = depth + } + + if ent.Children { + stack = append(stack, push) + } + } + + for _, fp := range funcs { + b.Funcs = append(b.Funcs, *fp) + } + return nil +} + +func makeFunction(d *dwarf.Data, ent *dwarf.Entry) *dsymmap.Function { + lo, hi, ok := spanOf(d, ent) + if !ok { + return nil + } + return &dsymmap.Function{Start: lo, End: hi, Name: entryName(d, ent)} +} + +func makeInlines(d *dwarf.Data, ent *dwarf.Entry, depth uint32, files []*dwarf.LineFile) []dsymmap.Inline { + ranges, err := d.Ranges(ent) + if err != nil || len(ranges) == 0 { + return nil + } + name := entryName(d, ent) + callFile, callLine := callSite(ent, files) + + out := make([]dsymmap.Inline, 0, len(ranges)) + for _, rng := range ranges { + if rng[1] <= rng[0] { + continue + } + out = append(out, dsymmap.Inline{ + Start: rng[0], + End: rng[1], + Name: name, + CallFile: callFile, + CallLine: callLine, + Depth: depth, + }) + } + return out +} + +// spanOf returns the [low,high) covering all of an entry's PC ranges. +func spanOf(d *dwarf.Data, ent *dwarf.Entry) (uint64, uint64, bool) { + ranges, err := d.Ranges(ent) + if err != nil || len(ranges) == 0 { + return 0, 0, false + } + lo, hi := ranges[0][0], ranges[0][1] + for _, rng := range ranges[1:] { + if rng[0] < lo { + lo = rng[0] + } + if rng[1] > hi { + hi = rng[1] + } + } + if hi <= lo { + return 0, 0, false + } + return lo, hi, true +} + +func callSite(ent *dwarf.Entry, files []*dwarf.LineFile) (string, uint32) { + var line uint32 + if v, ok := ent.Val(dwarf.AttrCallLine).(int64); ok { + line = uint32(v) + } + file := "" + if v, ok := ent.Val(dwarf.AttrCallFile).(int64); ok { + if idx := int(v); idx >= 0 && idx < len(files) && files[idx] != nil { + file = files[idx].Name + } + } + return file, line +} + +func addLines(d *dwarf.Data, cu *dwarf.Entry, b *dsymmap.Builder) { + lr, err := d.LineReader(cu) + if err != nil || lr == nil { + return + } + var le dwarf.LineEntry + for { + if err := lr.Next(&le); err != nil { + break // io.EOF or malformed table: stop, keep what we have + } + if le.EndSequence { + // Mark the end of a code range so a lookup in the following gap + // resolves to no source location rather than the previous row. + b.Lines = append(b.Lines, dsymmap.LineRow{Addr: le.Address}) + continue + } + file := "" + if le.File != nil { + file = le.File.Name + } + b.Lines = append(b.Lines, dsymmap.LineRow{Addr: le.Address, File: file, Line: uint32(le.Line)}) + } +} + +func filesForCU(d *dwarf.Data, cu *dwarf.Entry) []*dwarf.LineFile { + lr, err := d.LineReader(cu) + if err != nil || lr == nil { + return nil + } + return lr.Files() +} + +// entryName resolves a subprogram/inlined_subroutine name, following +// abstract_origin/specification references. Dart DWARF names are already +// readable, so no demangling is applied. +func entryName(d *dwarf.Data, ent *dwarf.Entry) string { + if name, _ := ent.Val(dwarf.AttrName).(string); name != "" { + return name + } + if off, ok := refOffset(ent); ok { + if name := resolveName(d, off, 0); name != "" { + return name + } + } + return "" +} + +func refOffset(ent *dwarf.Entry) (dwarf.Offset, bool) { + if off, ok := ent.Val(dwarf.AttrAbstractOrigin).(dwarf.Offset); ok { + return off, true + } + if off, ok := ent.Val(dwarf.AttrSpecification).(dwarf.Offset); ok { + return off, true + } + return 0, false +} + +func resolveName(d *dwarf.Data, off dwarf.Offset, depth int) string { + if depth > 4 { + return "" + } + r := d.Reader() + r.Seek(off) + ent, err := r.Next() + if err != nil || ent == nil { + return "" + } + if name, _ := ent.Val(dwarf.AttrName).(string); name != "" { + return name + } + if next, ok := refOffset(ent); ok { + return resolveName(d, next, depth+1) + } + return "" +} diff --git a/internal/symbols/flutter/elf_test.go b/internal/symbols/flutter/elf_test.go new file mode 100644 index 00000000..87fac629 --- /dev/null +++ b/internal/symbols/flutter/elf_test.go @@ -0,0 +1,66 @@ +package flutter + +import ( + "encoding/binary" + "testing" + + "github.com/stretchr/testify/assert" +) + +// note builds a single ELF note record: namesz descsz type, then the padded +// name and descriptor, matching what buildIDFromNotes parses. +func note(bo binary.ByteOrder, name string, ntype uint32, desc []byte) []byte { + nameB := append([]byte(name), 0) // NUL-terminated + var b []byte + hdr := make([]byte, 12) + bo.PutUint32(hdr[0:], uint32(len(nameB))) + bo.PutUint32(hdr[4:], uint32(len(desc))) + bo.PutUint32(hdr[8:], ntype) + b = append(b, hdr...) + b = append(b, nameB...) + for len(b)%4 != 0 { + b = append(b, 0) + } + b = append(b, desc...) + for len(b)%4 != 0 { + b = append(b, 0) + } + return b +} + +func TestBuildIDFromNotes(t *testing.T) { + bo := binary.LittleEndian + id := []byte{0x0f, 0x8a, 0x1b, 0x2c, 0x3d, 0x4e, 0x5f, 0x60} + + // A GNU build-id note on its own. + data := note(bo, "GNU", ntGNUBuildID, id) + assert.Equal(t, "0f8a1b2c3d4e5f60", buildIDFromNotes(data, bo)) + + // Preceded by an unrelated note (e.g. NT_GNU_ABI_TAG=1): must skip and find it. + data = append(note(bo, "GNU", 1, []byte{0, 0, 0, 0, 1, 2, 3, 4}), note(bo, "GNU", ntGNUBuildID, id)...) + assert.Equal(t, "0f8a1b2c3d4e5f60", buildIDFromNotes(data, bo)) + + // No build-id note present. + assert.Equal(t, "", buildIDFromNotes(note(bo, "GNU", 1, []byte{1, 2, 3, 4}), bo)) + + // Truncated data must not panic and must return "". + assert.Equal(t, "", buildIDFromNotes([]byte{1, 2, 3}, bo)) + + // A skipped note whose descriptor ends at the very end of the data but whose + // padding would run past it must not panic. + unaligned := note(bo, "GNU", 1, []byte{1, 2, 3}) + assert.Equal(t, "", buildIDFromNotes(unaligned[:len(unaligned)-1], bo)) + + // Big-endian is honored. + be := binary.BigEndian + assert.Equal(t, "0f8a1b2c3d4e5f60", buildIDFromNotes(note(be, "GNU", ntGNUBuildID, id), be)) +} + +func TestPlatformFromFilename(t *testing.T) { + assert.Equal(t, "android-arm64", platformFromFilename("app.android-arm64.symbols")) + assert.Equal(t, "ios-arm64", platformFromFilename("build/symbols/app.ios-arm64.symbols")) + assert.Equal(t, "android-x64", platformFromFilename("app.android-x64.symbols")) + // Names that don't match the app..symbols shape yield "". + assert.Equal(t, "", platformFromFilename("app..symbols")) + assert.Equal(t, "", platformFromFilename("mapping.txt")) +} diff --git a/internal/symbols/srcbundle/srcbundle.go b/internal/symbols/srcbundle/srcbundle.go new file mode 100644 index 00000000..9258237f --- /dev/null +++ b/internal/symbols/srcbundle/srcbundle.go @@ -0,0 +1,366 @@ +// Package srcbundle implements the Source Bundle (".srcbundle") format: a +// compact, binary-searchable archive of the source files referenced by a dSYM's +// DWARF, used to render source context (the lines around a frame) for native +// stack frames. +// +// It is produced by `ldcli symbols upload --type apple-dsym --include-sources` +// and consumed by the backend Apple enhancer, which hydrates +// lineContent/linesBefore/linesAfter after a frame resolves to file:line. The two +// sides live in separate repos and intentionally duplicate this layout rather +// than share a module; this file is byte-for-byte identical to +// backend/stacktraces/srcbundle/srcbundle.go, and the `Magic`+`Version` header +// guards against drift. +// +// # Keying +// +// Files are keyed by the *exact* path string the sibling .dsymmap stores for the +// same build (DWARF's file name). Both artifacts are produced from one DWARF +// pass, so a frame resolved through the map looks its source up here with no +// path normalization. +// +// # Design goals +// +// - Fetch little: each file is compressed independently, so answering one frame +// decompresses exactly one file, not the archive. +// - O(log n) lookup: the index is a sorted, fixed-width array probed by binary +// search over the string table. +// - Bounded: the reader validates every offset/length against the buffer, so a +// truncated or hostile bundle yields misses instead of panics. +// +// # Binary layout (little-endian) +// +// Header (32 bytes): +// 0 magic [4]byte "SRCB" +// 4 version u16 +// 6 flags u16 (bit 0: payload entries are gzip-compressed) +// 8 nFiles u32 +// 12 indexOff u32 +// 16 strtabOff u32 +// 20 strtabLen u32 +// 24 payloadOff u32 +// 28 payloadLen u32 +// +// index[] (nFiles × 16 bytes, sorted by path string, ascending): +// 0 pathOff u32 (into strtab) +// 4 dataOff u32 (into payload) +// 8 compLen u32 (stored bytes) +// 12 rawLen u32 (decompressed bytes) +// +// strtab: NUL-terminated UTF-8 paths; offset 0 is the empty string. +// payload: per-file gzip streams, concatenated. +package srcbundle + +import ( + "bytes" + "compress/gzip" + "encoding/binary" + "io" + "sort" + "sync" + + e "github.com/pkg/errors" +) + +const ( + // Magic identifies the file format. + Magic = "SRCB" + // Version is bumped on any incompatible layout change; the reader rejects + // versions it does not understand. + Version = uint16(1) + + headerSize = 32 + indexRecSize = 16 + + // flagGzip marks payload entries as individually gzip-compressed. + flagGzip = uint16(1) +) + +var le = binary.LittleEndian + +// --- Builder (encode side) --- + +// Builder accumulates source files for one image and encodes them. Add is +// keyed by the same path string the sibling .dsymmap records. +type Builder struct { + files map[string][]byte +} + +// Add registers one source file's contents under path. Empty paths and repeats +// are ignored, so callers can add freely while walking DWARF. +func (b *Builder) Add(path string, content []byte) { + if path == "" { + return + } + if b.files == nil { + b.files = make(map[string][]byte) + } + if _, ok := b.files[path]; ok { + return + } + b.files[path] = content +} + +// Len is the number of distinct files added. +func (b *Builder) Len() int { return len(b.files) } + +// Encode writes the bundle. Paths are sorted so the reader can binary search. +func (b *Builder) Encode(w io.Writer) error { + paths := make([]string, 0, len(b.files)) + for p := range b.files { + paths = append(paths, p) + } + sort.Strings(paths) + + var strtab bytes.Buffer + strtab.WriteByte(0) // offset 0 == "" + var payload bytes.Buffer + index := make([]byte, 0, len(paths)*indexRecSize) + + for _, p := range paths { + pathOff := uint32(strtab.Len()) + strtab.WriteString(p) + strtab.WriteByte(0) + + raw := b.files[p] + dataOff := uint32(payload.Len()) + zw := gzip.NewWriter(&payload) + if _, err := zw.Write(raw); err != nil { + return e.Wrapf(err, "srcbundle: compressing %s", p) + } + if err := zw.Close(); err != nil { + return e.Wrapf(err, "srcbundle: finishing %s", p) + } + + rec := make([]byte, indexRecSize) + le.PutUint32(rec[0:], pathOff) + le.PutUint32(rec[4:], dataOff) + le.PutUint32(rec[8:], uint32(payload.Len())-dataOff) + le.PutUint32(rec[12:], uint32(len(raw))) + index = append(index, rec...) + } + + indexOff := uint32(headerSize) + strtabOff := indexOff + uint32(len(index)) + payloadOff := strtabOff + uint32(strtab.Len()) + + hdr := make([]byte, headerSize) + copy(hdr[0:4], Magic) + le.PutUint16(hdr[4:], Version) + le.PutUint16(hdr[6:], flagGzip) + le.PutUint32(hdr[8:], uint32(len(paths))) + le.PutUint32(hdr[12:], indexOff) + le.PutUint32(hdr[16:], strtabOff) + le.PutUint32(hdr[20:], uint32(strtab.Len())) + le.PutUint32(hdr[24:], payloadOff) + le.PutUint32(hdr[28:], uint32(payload.Len())) + + for _, chunk := range [][]byte{hdr, index, strtab.Bytes(), payload.Bytes()} { + if _, err := w.Write(chunk); err != nil { + return e.Wrap(err, "srcbundle: writing") + } + } + return nil +} + +// --- Bundle (decode side) --- + +// Bundle is a read-only view over encoded bundle bytes. Decompressed files are +// memoized, so several frames in one file cost a single inflate. +type Bundle struct { + nFiles int + index []byte + strtab []byte + payload []byte + gzipped bool + + mu sync.Mutex + memo map[string][]byte +} + +// Open validates the header and returns a view over raw. It does not copy: raw +// must stay alive (and unmodified) for the lifetime of the Bundle. +func Open(raw []byte) (*Bundle, error) { + if len(raw) < headerSize { + return nil, e.New("srcbundle: buffer shorter than header") + } + if string(raw[0:4]) != Magic { + return nil, e.New("srcbundle: bad magic") + } + if v := le.Uint16(raw[4:]); v != Version { + return nil, e.Errorf("srcbundle: unsupported version %d", v) + } + + flags := le.Uint16(raw[6:]) + nFiles := le.Uint32(raw[8:]) + // Every section is bounds-checked against the buffer up front so lookups can + // slice without re-validating (and a truncated bundle fails loudly here + // rather than panicking mid-query). + slice := func(off, length uint32) ([]byte, bool) { + hi := uint64(off) + uint64(length) + if hi > uint64(len(raw)) { + return nil, false + } + return raw[off:hi], true + } + + indexLen := uint64(nFiles) * indexRecSize + if indexLen > uint64(len(raw)) { + return nil, e.New("srcbundle: index length out of range") + } + index, ok := slice(le.Uint32(raw[12:]), uint32(indexLen)) + if !ok { + return nil, e.New("srcbundle: index out of range") + } + strtab, ok := slice(le.Uint32(raw[16:]), le.Uint32(raw[20:])) + if !ok { + return nil, e.New("srcbundle: strtab out of range") + } + payload, ok := slice(le.Uint32(raw[24:]), le.Uint32(raw[28:])) + if !ok { + return nil, e.New("srcbundle: payload out of range") + } + + return &Bundle{ + nFiles: int(nFiles), + index: index, + strtab: strtab, + payload: payload, + gzipped: flags&flagGzip != 0, + }, nil +} + +// Len is the number of files in the bundle. +func (b *Bundle) Len() int { return b.nFiles } + +func (b *Bundle) str(off uint32) string { + if int(off) >= len(b.strtab) { + return "" + } + s := b.strtab[off:] + if i := bytes.IndexByte(s, 0); i >= 0 { + return string(s[:i]) + } + return string(s) +} + +func (b *Bundle) rec(i int) (pathOff, dataOff, compLen, rawLen uint32) { + r := b.index[i*indexRecSize:] + return le.Uint32(r[0:]), le.Uint32(r[4:]), le.Uint32(r[8:]), le.Uint32(r[12:]) +} + +// find binary searches the sorted index for an exact path match. +func (b *Bundle) find(path string) (int, bool) { + i := sort.Search(b.nFiles, func(i int) bool { + pathOff, _, _, _ := b.rec(i) + return b.str(pathOff) >= path + }) + if i < b.nFiles { + pathOff, _, _, _ := b.rec(i) + if b.str(pathOff) == path { + return i, true + } + } + return 0, false +} + +// File returns the decompressed contents of one source file. Misses (including +// a corrupt entry) are memoized too, so a bad path is looked up once. +func (b *Bundle) File(path string) ([]byte, bool) { + b.mu.Lock() + defer b.mu.Unlock() + if v, ok := b.memo[path]; ok { + return v, v != nil + } + + out, ok := b.readFile(path) + if b.memo == nil { + b.memo = make(map[string][]byte) + } + if !ok { + b.memo[path] = nil + return nil, false + } + b.memo[path] = out + return out, true +} + +func (b *Bundle) readFile(path string) ([]byte, bool) { + i, ok := b.find(path) + if !ok { + return nil, false + } + _, dataOff, compLen, rawLen := b.rec(i) + hi := uint64(dataOff) + uint64(compLen) + if hi > uint64(len(b.payload)) { + return nil, false + } + blob := b.payload[dataOff:hi] + if !b.gzipped { + return blob, true + } + + zr, err := gzip.NewReader(bytes.NewReader(blob)) + if err != nil { + return nil, false + } + defer zr.Close() + // rawLen comes from the file, so cap the inflate to it: a corrupt/hostile + // entry can't expand without bound. + out, err := io.ReadAll(io.LimitReader(zr, int64(rawLen))) + if err != nil { + return nil, false + } + return out, true +} + +// Window returns the source context around a 1-based line: up to ctx lines +// before, the line itself, and up to ctx lines after. Every returned line keeps +// its trailing newline, matching how the JavaScript source-map path fills +// linesBefore/lineContent/linesAfter. ok is false when the file isn't bundled or +// the line is out of range. +func (b *Bundle) Window(path string, line, ctx int) (before, content, after string, ok bool) { + if line <= 0 || ctx < 0 { + return "", "", "", false + } + data, found := b.File(path) + if !found { + return "", "", "", false + } + + starts := lineStarts(data) + idx := line - 1 + if idx >= len(starts) { + return "", "", "", false + } + lo := idx - ctx + if lo < 0 { + lo = 0 + } + hi := idx + 1 + ctx + if hi > len(starts) { + hi = len(starts) + } + // Lines are contiguous in data, so a window is just two offsets. + at := func(i int) int { + if i >= len(starts) { + return len(data) + } + return starts[i] + } + return string(data[at(lo):at(idx)]), string(data[at(idx):at(idx+1)]), string(data[at(idx+1):at(hi)]), true +} + +// lineStarts returns the byte offset of each line. A trailing newline does not +// begin a new line, so counts match what an editor shows. +func lineStarts(data []byte) []int { + starts := make([]int, 1, 1+bytes.Count(data, []byte{'\n'})) + for i, c := range data { + if c == '\n' { + starts = append(starts, i+1) + } + } + if n := len(starts); n > 1 && starts[n-1] == len(data) { + starts = starts[:n-1] + } + return starts +} diff --git a/internal/symbols/srcbundle/srcbundle_test.go b/internal/symbols/srcbundle/srcbundle_test.go new file mode 100644 index 00000000..101609ee --- /dev/null +++ b/internal/symbols/srcbundle/srcbundle_test.go @@ -0,0 +1,145 @@ +package srcbundle + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const sampleSwift = "import Foundation\n" + // 1 + "\n" + // 2 + "struct Cart {\n" + // 3 + " func total() -> Int {\n" + // 4 + " let items = [1, 2, 3]\n" + // 5 + " return items[9]\n" + // 6 <- crash line + " }\n" + // 7 + "}\n" // 8 + +func encode(t *testing.T, files map[string]string) *Bundle { + t.Helper() + b := &Builder{} + for p, c := range files { + b.Add(p, []byte(c)) + } + var buf bytes.Buffer + require.NoError(t, b.Encode(&buf)) + bundle, err := Open(buf.Bytes()) + require.NoError(t, err) + return bundle +} + +func TestRoundTrip(t *testing.T) { + bundle := encode(t, map[string]string{ + "/src/Cart.swift": sampleSwift, + "/src/Other.swift": "let x = 1\n", + }) + assert.Equal(t, 2, bundle.Len()) + + got, ok := bundle.File("/src/Cart.swift") + require.True(t, ok) + assert.Equal(t, sampleSwift, string(got)) + + _, ok = bundle.File("/src/Missing.swift") + assert.False(t, ok) +} + +func TestWindow(t *testing.T) { + bundle := encode(t, map[string]string{"/src/Cart.swift": sampleSwift}) + + before, content, after, ok := bundle.Window("/src/Cart.swift", 6, 2) + require.True(t, ok) + // Every line keeps its trailing newline, matching the JS source-map path. + assert.Equal(t, " func total() -> Int {\n let items = [1, 2, 3]\n", before) + assert.Equal(t, " return items[9]\n", content) + assert.Equal(t, " }\n}\n", after) +} + +func TestWindowClampsAtFileEdges(t *testing.T) { + bundle := encode(t, map[string]string{"/src/Cart.swift": sampleSwift}) + + before, content, _, ok := bundle.Window("/src/Cart.swift", 1, 5) + require.True(t, ok) + assert.Equal(t, "", before) + assert.Equal(t, "import Foundation\n", content) + + _, content, after, ok := bundle.Window("/src/Cart.swift", 8, 5) + require.True(t, ok) + assert.Equal(t, "}\n", content) + assert.Equal(t, "", after) +} + +func TestWindowRejectsBadInput(t *testing.T) { + bundle := encode(t, map[string]string{"/src/Cart.swift": sampleSwift}) + + _, _, _, ok := bundle.Window("/src/Cart.swift", 0, 2) + assert.False(t, ok, "lines are 1-based") + + _, _, _, ok = bundle.Window("/src/Cart.swift", 999, 2) + assert.False(t, ok, "line past EOF") + + _, _, _, ok = bundle.Window("/src/Nope.swift", 1, 2) + assert.False(t, ok, "unbundled file") +} + +func TestFileWithoutTrailingNewline(t *testing.T) { + bundle := encode(t, map[string]string{"/a.swift": "one\ntwo"}) + _, content, after, ok := bundle.Window("/a.swift", 2, 1) + require.True(t, ok) + assert.Equal(t, "two", content) + assert.Equal(t, "", after) +} + +func TestEmptyBundle(t *testing.T) { + bundle := encode(t, nil) + assert.Equal(t, 0, bundle.Len()) + _, ok := bundle.File("/anything") + assert.False(t, ok) +} + +func TestOpenRejectsGarbage(t *testing.T) { + _, err := Open(nil) + assert.Error(t, err) + + _, err = Open([]byte("not a bundle at all.............")) + assert.Error(t, err) + + // Right magic, wrong version. + b := &Builder{} + b.Add("/a.swift", []byte("x\n")) + var buf bytes.Buffer + require.NoError(t, b.Encode(&buf)) + raw := buf.Bytes() + raw[4] = 0xFF + _, err = Open(raw) + assert.Error(t, err) +} + +// A truncated bundle must fail to open rather than panic during lookup. +func TestOpenRejectsTruncated(t *testing.T) { + b := &Builder{} + b.Add("/a.swift", []byte(sampleSwift)) + var buf bytes.Buffer + require.NoError(t, b.Encode(&buf)) + raw := buf.Bytes() + + for _, n := range []int{headerSize, headerSize + 4, len(raw) - 1} { + _, err := Open(raw[:n]) + assert.Error(t, err, "truncated to %d bytes should not open", n) + } +} + +func TestManyFilesBinarySearch(t *testing.T) { + files := map[string]string{} + for _, p := range []string{"/z.swift", "/a.swift", "/m.swift", "/b.swift", "/y.swift"} { + files[p] = "// " + p + "\n" + } + bundle := encode(t, files) + assert.Equal(t, 5, bundle.Len()) + for p := range files { + got, ok := bundle.File(p) + require.True(t, ok, "expected %s", p) + assert.Equal(t, "// "+p+"\n", string(got)) + } +}