fix(parser): avoid quadratic diagnostic accumulation

Sources/XCLogParser/parser/Notice+Parser.swift

@@ -42,9 +42,8 @@ extension Notice {
         let clangWarnings = self.parseClangWarnings(clangFlags: clangWarningsFlags, logSection: logSection)
 
         // Remove the messages that were categorized as clangWarnings
-        let remainingLogMessages = logSection.messages.filter { message in
-            return clangWarnings.contains { $0.title == message.title } == false
-        }
+        let clangWarningTitles = Set(clangWarnings.map { $0.title })
+        let remainingLogMessages = logSection.messages.filter { !clangWarningTitles.contains($0.title) }
         // parse details for Swift issues
         let swiftErrorDetails = parseSwiftIssuesDetailsByLocation(logSection.text)
         // we look for analyzer warnings, swift warnings, notes and errors
@@ -100,9 +99,7 @@ extension Notice {
                 return [notice]
             }
             return nil
-        }.reduce([Notice]()) { flatten, notices -> [Notice] in
-            flatten + notices
-        }
+        }.flatMap { $0 }
     }
 
     /// Xcode reports the details of Swift errors and warnings as a mixed text with all the errors in a
@@ -111,36 +108,32 @@ extension Notice {
     /// - returns: A Dictionary where the keys are the error location in the form pathToFile:line:column:
     /// and the values are the error details for that location
     public static func parseSwiftIssuesDetailsByLocation(_ text: String) -> [String: String] {
-        return text
-            .split(separator: "\r")
-            .reduce([]) { (details, line) -> [String] in
-                var details = details
-                if line.contains(": error:") || line.contains(": warning:") {
-                    details.append(String(line))
-                } else {
-                    guard let current = details.last else {
-                        return details
-                    }
-                    details.removeLast()
-                    details.append("\(current)\n\(line)")
-                }
-                return details
-        }
-        .reduce([String: String]()) { (detailsByLoc, detail) -> [String: String] in
-            let range: Range<String.Index>?
-            if detail.contains(": error:") {
-                range = detail.range(of: ": error:")
-            } else {
-                range = detail.range(of: ": warning:")
-            }
+        var detailsByLocation = [String: String]()
+        var lines = [Substring]()
+
+        func storeDetail() {
+            guard !lines.isEmpty else { return }
+            // Join once per diagnostic. Rebuilding the accumulated string for every
+            // continuation line copies a quadratic number of bytes on large logs.
+            let detail = lines.joined(separator: "\n")
+            let range = detail.range(of: ": error:") ?? detail.range(of: ": warning:")
             if let range = range {
                 let location = detail[...range.lowerBound]
-                var detailsByLoc = detailsByLoc
-                detailsByLoc[String(location)] = detail
-                return detailsByLoc
+                detailsByLocation[String(location)] = detail
+            }
+        }
+
+        for line in text.split(separator: "\r") {
+            if line.contains(": error:") || line.contains(": warning:") {
+                storeDetail()
+                lines.removeAll(keepingCapacity: true)
+                lines.append(line)
+            } else if !lines.isEmpty {
+                lines.append(line)
             }
-            return detailsByLoc
         }
+        storeDetail()
+        return detailsByLocation
     }
 
     /// Parses the text of a IDELogSection looking for the pattern [-Wwarning-type]

Tests/XCLogParserTests/ParserTests.swift

@@ -206,6 +206,98 @@ note: use 'updatedDoSomething' instead\r doSomething()\r        ^~~~~~~~~~~\r
         XCTAssertEqual(detailsByLocation.count, 4)
     }
 
+    func testSwiftDiagnosticDetailsPreserveContinuationLinesAndUnicode() {
+        let warning = "/project/🦊.swift:3:4: warning: café"
+        let error = "/project/other.swift:5:6: error: invalid type"
+        let text = "ignored preamble\r\r\(warning)\r  中文\r\r  ^~\r\(error)\r  source\r"
+
+        XCTAssertEqual(Notice.parseSwiftIssuesDetailsByLocation(text), [
+            "/project/🦊.swift:3:4:": "\(warning)\n  中文\n  ^~",
+            "/project/other.swift:5:6:": "\(error)\n  source"
+        ])
+        XCTAssertTrue(Notice.parseSwiftIssuesDetailsByLocation("").isEmpty)
+        XCTAssertTrue(Notice.parseSwiftIssuesDetailsByLocation("preamble\rplain output").isEmpty)
+    }
+
+    func testSwiftDiagnosticDetailsKeepLastOccurrenceAtLocation() {
+        let first = "/project/file.swift:1:2: warning: first"
+        let last = "/project/file.swift:1:2: error: last"
+        XCTAssertEqual(Notice.parseSwiftIssuesDetailsByLocation("\(first)\rsource\r\(last)\rtail"), [
+            "/project/file.swift:1:2:": "\(last)\ntail"
+        ])
+    }
+
+    func testSwiftDiagnosticDetailsPreserveErrorMarkerPrecedence() {
+        // Existing parsing prefers an error marker even when a warning marker appears first.
+        let text = "/project/file.swift:1:2: warning: quoted : error: message\rcontinuation"
+        XCTAssertEqual(Notice.parseSwiftIssuesDetailsByLocation(text), [
+            "/project/file.swift:1:2: warning: quoted :": text.replacingOccurrences(of: "\r", with: "\n")
+        ])
+    }
+
+    func testSwiftDiagnosticDetailsWithLargeContinuation() {
+        let header = "/project/file.swift:1:2: error: failed"
+        let line = String(repeating: "x", count: 1024)
+        let text = header + "\r" + Array(repeating: line, count: 40_000).joined(separator: "\r") + "\rtail"
+        let start = Date()
+        let details = Notice.parseSwiftIssuesDetailsByLocation(text)
+        let elapsed = Date().timeIntervalSince(start)
+
+        XCTAssertEqual(details.count, 1)
+        XCTAssertEqual(details["/project/file.swift:1:2:"], text.replacingOccurrences(of: "\r", with: "\n"))
+        // A coarse regression guard with ample headroom, not a microbenchmark.
+        // Repeatedly copying this growing diagnostic processes hundreds of GB.
+        XCTAssertLessThan(elapsed, 10, "Large diagnostics must not repeatedly copy their accumulated text")
+    }
+
+    func testSwiftDiagnosticDetailsWithManyLocations() {
+        let count = 40_000
+        let text = (0..<count).map { "/project/file.swift:\($0):1: error: failure\rsource" }.joined(separator: "\r")
+        let start = Date()
+        let details = Notice.parseSwiftIssuesDetailsByLocation(text)
+        let elapsed = Date().timeIntervalSince(start)
+
+        XCTAssertEqual(details.count, count)
+        XCTAssertEqual(details["/project/file.swift:0:1:"], "/project/file.swift:0:1: error: failure\nsource")
+        XCTAssertEqual(details["/project/file.swift:39999:1:"], "/project/file.swift:39999:1: error: failure\nsource")
+        XCTAssertLessThan(elapsed, 10, "Accumulating locations must not copy the dictionary for every diagnostic")
+    }
+
+    func testManyNoticesPreserveOrderWithoutTruncation() {
+        let messages = (0..<60_000).map { diagnosticMessage(title: "note \($0)") }
+        let section = getFakeIDEActivityLogWithMessages(messages, andText: "").mainSection
+        let start = Date()
+        let notices = Notice.parseFromLogSection(section, forType: .other, truncLargeIssues: false)
+        let elapsed = Date().timeIntervalSince(start)
+
+        XCTAssertEqual(notices.map { $0.title }, messages.map { $0.title })
+        XCTAssertTrue(notices.allSatisfy { $0.type == .note })
+        XCTAssertLessThan(elapsed, 10, "Flattening notices must not copy the growing array for every message")
+    }
+
+    func testManyClangWarningsFilterDuplicateTitlesAndKeepUnmatchedNotes() {
+        let messages = (0..<40_000).map { diagnosticMessage(title: "warning \($0)") }
+        let text = Array(repeating: "[-Wunused-variable]", count: messages.count).joined(separator: "\r")
+        let allMessages = messages + [diagnosticMessage(title: "warning 0"), diagnosticMessage(title: "remaining note")]
+        let section = getFakeIDEActivityLogWithMessages(allMessages, andText: text).mainSection
+        let start = Date()
+        let notices = Notice.parseFromLogSection(section, forType: .cCompilation, truncLargeIssues: false)
+        let elapsed = Date().timeIntervalSince(start)
+
+        XCTAssertEqual(notices.map { $0.title }, messages.map { $0.title } + ["remaining note"])
+        XCTAssertTrue(notices.dropLast().allSatisfy { $0.type == .clangWarning })
+        XCTAssertEqual(notices.last?.type, .note)
+        XCTAssertLessThan(elapsed, 10, "Filtering messages must not scan all Clang warning titles per message")
+    }
+
+    private func diagnosticMessage(title: String) -> IDEActivityLogMessage {
+        return IDEActivityLogMessage(title: title, shortTitle: "", timeEmitted: 0,
+                                     rangeEndInSectionText: 0, rangeStartInSectionText: 0,
+                                     subMessages: [], severity: 1, type: "",
+                                     location: DVTDocumentLocation(documentURLString: "", timestamp: 0),
+                                     categoryIdent: "Notice", secondaryLocations: [], additionalDescription: "")
+    }
+
     func testParseInterfaceBuilderWarning() throws {
         let timestamp = Date().timeIntervalSinceReferenceDate
         let memberId = IBMemberID(memberIdentifier: "ABC")
@@ -587,3 +679,5 @@ CompileSwift normal x86_64 (in target 'Alamofire' from project 'Pods')
                                      unknown: 0)
     }()
 }
+
+// swiftlint:enable type_body_length file_length line_length

Tests/XCLogParserTests/XCTestManifests.swift

@@ -109,6 +109,13 @@ extension ParserTests {
     //   `swift test --generate-linuxmain`
     // to regenerate.
     static let __allTests__ParserTests = [
+        ("testManyClangWarningsFilterDuplicateTitlesAndKeepUnmatchedNotes", testManyClangWarningsFilterDuplicateTitlesAndKeepUnmatchedNotes),
+        ("testManyNoticesPreserveOrderWithoutTruncation", testManyNoticesPreserveOrderWithoutTruncation),
+        ("testSwiftDiagnosticDetailsKeepLastOccurrenceAtLocation", testSwiftDiagnosticDetailsKeepLastOccurrenceAtLocation),
+        ("testSwiftDiagnosticDetailsPreserveContinuationLinesAndUnicode", testSwiftDiagnosticDetailsPreserveContinuationLinesAndUnicode),
+        ("testSwiftDiagnosticDetailsPreserveErrorMarkerPrecedence", testSwiftDiagnosticDetailsPreserveErrorMarkerPrecedence),
+        ("testSwiftDiagnosticDetailsWithLargeContinuation", testSwiftDiagnosticDetailsWithLargeContinuation),
+        ("testSwiftDiagnosticDetailsWithManyLocations", testSwiftDiagnosticDetailsWithManyLocations),
         ("testBuildIdentifierShouldUseMachineName", testBuildIdentifierShouldUseMachineName),
         ("testDateFormatterUsesJSONFormat", testDateFormatterUsesJSONFormat),
         ("testGetIndividualSteps", testGetIndividualSteps),