Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,49 @@ actor ToolExecutionObserver: ToolExecutionDelegate {
session.toolExecutionDelegate = ToolExecutionObserver()
```

### Reasoning in the transcript

Reasoning is transcript content, separate from the answer in `response.content`.
A provider can emit `Transcript.Entry.reasoning` through the existing cumulative
`transcriptEntries` on responses and streaming snapshots. Each `Transcript.Reasoning`
contains a stable `id`, display `segments`, opaque `signature: Data?`, and metadata.
Treat successive snapshots as updates to the same entries, not new history rows.

The built-in Anthropic provider populates these entries for thinking and redacted
thinking, in streaming and nonstreaming responses, including tool rounds:

```swift
let model = AnthropicLanguageModel(apiKey: apiKey, model: modelID)
let session = LanguageModelSession(model: model)
var options = GenerationOptions(maximumResponseTokens: 4096)
options[custom: AnthropicLanguageModel.self] = .init(thinking: .init(budgetTokens: 1024))
for try await snapshot in session.streamResponse(to: "Explain your approach", options: options) {
let reasoning = snapshot.transcriptEntries.compactMap { entry -> String? in
guard case .reasoning(let value) = entry else { return nil }
return value.segments.compactMap { segment -> String? in
guard case .text(let text) = segment else { return nil }
return text.content
}.joined()
}.joined()
// Replace the displayed reasoning and answer independently.
print(reasoning)
print(snapshot.content)
}
let savedTranscript = try JSONEncoder().encode(session.transcript)
```

Choose an Anthropic model and thinking budget that support this configuration.
Redacted thinking has no display segments. Signatures and metadata are opaque
replay state; preserve them with the transcript, and do not display them as text.
The Anthropic adapter can replay its own reasoning entries after Codable restoration.
When switching providers, adapters that cannot replay reasoning omit those entries
from their requests; Anthropic likewise skips reasoning from other providers.
The original reasoning remains in the transcript for display and persistence.
Anthropic still validates its own replay signatures. CoreML keeps its existing
prompt-only behavior and does not send transcript history. For structured scalar outputs that
cannot represent an absent partial value, reasoning updates wait until a valid
partial answer is available. Cancellation behavior is unchanged.

### Token Usage

Inspect token counts with `response.usage`
Expand Down
4 changes: 2 additions & 2 deletions Sources/AnyLanguageModel/LanguageModelSession.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1081,7 +1081,7 @@ extension LanguageModelSession {
/// The raw content produced so far by the model.
public var rawContent: GeneratedContent

/// Transcript entries (tool calls and outputs) produced so far while streaming.
/// Transcript entries (reasoning, tool calls and outputs) produced so far while streaming.
/// Cumulative across tool rounds;
/// empty for providers that don't stream tool activity.
public var transcriptEntries: ArraySlice<Transcript.Entry>
Expand All @@ -1096,7 +1096,7 @@ extension LanguageModelSession {
/// - Parameters:
/// - content: The partially generated content.
/// - rawContent: The raw content produced by the model.
/// - transcriptEntries: Transcript entries accumulated so far (tool calls/outputs).
/// - transcriptEntries: Transcript entries accumulated so far (reasoning/tool calls/outputs).
/// - usage: Provider-reported token usage so far.
public init(
content: Content.PartiallyGenerated,
Expand Down
118 changes: 100 additions & 18 deletions Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ public struct AnthropicLanguageModel: LanguageModel {

/// How thinking should be returned by the API.
///
/// Thinking content is not currently exposed in session responses or snapshots.
/// Thinking content is exposed as reasoning transcript entries.
public var display: ThinkingDisplay?

/// The type of thinking mode.
Expand Down Expand Up @@ -452,7 +452,7 @@ public struct AnthropicLanguageModel: LanguageModel {
let params = try createMessageParams(
model: model,
system: nil,
messages: session.transcript.toAnthropicMessages(),
messages: try session.transcript.toAnthropicMessages(),
tools: anthropicTools.isEmpty ? nil : anthropicTools,
responseSchema: responseSchema,
options: options
Expand All @@ -467,7 +467,13 @@ public struct AnthropicLanguageModel: LanguageModel {
body: body
)

var entries: [Transcript.Entry] = []
var entries: [Transcript.Entry] = message.content.compactMap { block in
switch block {
case .thinking(let thinking): return .reasoning(thinking.transcriptReasoning())
case .redactedThinking(let redacted): return .reasoning(redacted.transcriptReasoning())
default: return nil
}
}
let usage = message.usage?.reportedUsage?.value ?? .zero

// Handle tool calls, if present
Expand Down Expand Up @@ -583,7 +589,7 @@ public struct AnthropicLanguageModel: LanguageModel {

let responseSchema =
type == String.self ? nil : try convertSchemaToAnthropicFormat(schema)
var messages = session.transcript.toAnthropicMessages()
var messages = try session.transcript.toAnthropicMessages()
var state = StreamingResponseState<Content>()
var toolRounds = ToolRoundLimit(provider: "Anthropic")
while true {
Expand Down Expand Up @@ -618,6 +624,10 @@ public struct AnthropicLanguageModel: LanguageModel {
switch event {
case .contentBlockStart(let start):
blocks[start.index] = AnthropicStreamBlock(start.contentBlock)
if let reasoning = blocks[start.index]?.reasoningEntry {
state.entries.append(.reasoning(reasoning))
if let current = snapshot() { lastSnapshot = current; continuation.yield(current) }
}
case .contentBlockDelta(let delta):
switch delta.delta {
case .textDelta(let textDelta):
Expand All @@ -631,8 +641,20 @@ public struct AnthropicLanguageModel: LanguageModel {
blocks[delta.index]?.arguments += input.partialJson
case .thinkingDelta(let thinking):
blocks[delta.index]?.thinking += thinking.thinking
if let reasoning = blocks[delta.index]?.reasoningEntry,
let index = state.entries.firstIndex(where: { $0.id == reasoning.id })
{
state.entries[index] = .reasoning(reasoning)
}
if let current = snapshot() { lastSnapshot = current; continuation.yield(current) }
case .signatureDelta(let signature):
blocks[delta.index]?.signature += signature.signature
if let reasoning = blocks[delta.index]?.reasoningEntry,
let index = state.entries.firstIndex(where: { $0.id == reasoning.id })
{
state.entries[index] = .reasoning(reasoning)
}
if let current = snapshot() { lastSnapshot = current; continuation.yield(current) }
case .ignored:
break
}
Expand Down Expand Up @@ -980,8 +1002,15 @@ private func convertToolToAnthropicFormat(_ tool: any Tool) throws -> AnthropicT
// MARK: - Supporting Types

extension Transcript {
fileprivate func toAnthropicMessages() -> [AnthropicMessage] {
fileprivate func toAnthropicMessages() throws -> [AnthropicMessage] {
var messages = [AnthropicMessage]()
func appendAssistant(_ content: [AnthropicContent]) {
if let last = messages.last, last.role == .assistant {
messages[messages.count - 1] = .init(role: .assistant, content: last.content + content)
} else {
messages.append(.init(role: .assistant, content: content))
}
}
Comment on lines +1007 to +1013
for item in self {
switch item {
case .instructions(let instructions):
Expand All @@ -998,13 +1027,29 @@ extension Transcript {
content: convertSegmentsToAnthropicContent(prompt.segments)
)
)
case .reasoning(let reasoning):
guard reasoning.metadata["provider"] == GeneratedContent("anthropic") else {
// Foreign reasoning remains display history, not Anthropic replay state.
continue
}
guard let data = reasoning.signature, let signature = String(data: data, encoding: .utf8),
!signature.isEmpty
else {
throw Transcript.ReasoningReplayError.invalidSignature
}
if reasoning.metadata["isRedacted"] == GeneratedContent(true) {
appendAssistant([.redactedThinking(.init(data: signature))])
continue
}
let text = try reasoning.segments.map { segment -> String in
guard case .text(let text) = segment else {
throw Transcript.ReasoningReplayError.unsupportedProvider("Anthropic reasoning segment")
}
return text.content
}.joined()
appendAssistant([.thinking(.init(thinking: text, signature: signature))])
case .response(let response):
messages.append(
.init(
role: .assistant,
content: convertSegmentsToAnthropicContent(response.segments)
)
)
appendAssistant(convertSegmentsToAnthropicContent(response.segments))
case .toolCalls(let toolCalls):
// Add assistant message with tool use blocks
let toolUseBlocks: [AnthropicContent] = toolCalls.map { call in
Expand All @@ -1017,12 +1062,7 @@ extension Transcript {
)
)
}
messages.append(
.init(
role: .assistant,
content: toolUseBlocks
)
)
appendAssistant(toolUseBlocks)
case .toolOutput(let toolOutput):
// Add user message with tool result
messages.append(
Expand Down Expand Up @@ -1069,11 +1109,13 @@ private enum AnthropicContent: Codable, Sendable {
case toolUse(AnthropicToolUse)
case toolResult(AnthropicToolResult)
case thinking(AnthropicThinking)
case redactedThinking(AnthropicRedactedThinking)

enum CodingKeys: String, CodingKey { case type }

enum ContentType: String, Codable {
case text = "text", image = "image", toolUse = "tool_use", toolResult = "tool_result", thinking = "thinking"
case text = "text", image = "image", toolUse = "tool_use", toolResult = "tool_result", thinking = "thinking",
redactedThinking = "redacted_thinking"
}

init(from decoder: any Decoder) throws {
Expand All @@ -1090,6 +1132,8 @@ private enum AnthropicContent: Codable, Sendable {
self = .toolResult(try AnthropicToolResult(from: decoder))
case .thinking:
self = .thinking(try AnthropicThinking(from: decoder))
case .redactedThinking:
self = .redactedThinking(try AnthropicRedactedThinking(from: decoder))
}
}

Expand All @@ -1100,10 +1144,25 @@ private enum AnthropicContent: Codable, Sendable {
case .toolUse(let u): try u.encode(to: encoder)
case .toolResult(let r): try r.encode(to: encoder)
case .thinking(let h): try h.encode(to: encoder)
case .redactedThinking(let value): try value.encode(to: encoder)
}
}
}

private struct AnthropicRedactedThinking: Codable, Sendable {
let type: String
let data: String
init(data: String) { self.type = "redacted_thinking"; self.data = data }
func transcriptReasoning(id: String = UUID().uuidString) -> Transcript.Reasoning {
.init(
id: id,
metadata: ["provider": GeneratedContent("anthropic"), "isRedacted": GeneratedContent(true)],
segments: [],
signature: Data(data.utf8)
)
}
}

private struct AnthropicThinking: Codable, Sendable {
let type: String
let thinking: String
Expand Down Expand Up @@ -1155,6 +1214,17 @@ private struct AnthropicImage: Codable, Sendable {
}
}

private extension AnthropicThinking {
func transcriptReasoning(id: String = UUID().uuidString) -> Transcript.Reasoning {
.init(
id: id,
metadata: ["provider": GeneratedContent("anthropic")],
segments: [.text(.init(id: id + ":text", content: thinking))],
signature: signature.isEmpty ? nil : Data(signature.utf8)
)
}
}

private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) -> [AnthropicContent] {
var blocks: [AnthropicContent] = []
blocks.reserveCapacity(segments.count)
Expand Down Expand Up @@ -1242,6 +1312,14 @@ private struct AnthropicErrorDetail: Codable {
// MARK: - Streaming Event Types

private struct AnthropicStreamBlock {
let reasoningID = UUID().uuidString
var reasoningEntry: Transcript.Reasoning? {
if start.type == "redacted_thinking", let data = start.data {
return AnthropicRedactedThinking(data: data).transcriptReasoning(id: reasoningID)
}
guard start.type == "thinking" else { return nil }
return AnthropicThinking(thinking: thinking, signature: signature).transcriptReasoning(id: reasoningID)
}
let start: AnthropicStreamEvent.ContentBlockStartEvent.ContentBlock
var text: String
var arguments = ""
Expand All @@ -1259,6 +1337,9 @@ private struct AnthropicStreamBlock {
switch start.type {
case "text": return .text(.init(text: text))
case "thinking": return .thinking(.init(thinking: thinking, signature: signature))
case "redacted_thinking":
guard let data = start.data else { throw Transcript.ReasoningReplayError.invalidSignature }
return .redactedThinking(.init(data: data))
case "tool_use":
guard let id = start.id, let name = start.name else { return nil }
let input =
Expand Down Expand Up @@ -1349,6 +1430,7 @@ private enum AnthropicStreamEvent: Codable, Sendable {
let input: [String: JSONValue]?
let thinking: String?
let signature: String?
let data: String?
}
}

Expand Down
3 changes: 3 additions & 0 deletions Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -872,6 +872,9 @@ extension Transcript {
parts: convertSegmentsToGeminiParts(prompt.segments)
)
)
case .reasoning:
// Keep display history in the transcript without sending unsupported replay state.
continue
case .response(let response):
messages.append(
.init(
Expand Down
3 changes: 3 additions & 0 deletions Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -2320,6 +2320,9 @@ import Foundation
messages.append(("user", text))
}

case .reasoning:
// Keep display history in the transcript without sending unsupported replay state.
return
case .response(let response):
let text = try extractContent(
from: response.segments,
Expand Down
3 changes: 3 additions & 0 deletions Sources/AnyLanguageModel/Models/MLXLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1561,6 +1561,9 @@ import Foundation
case .prompt(let prompt):
chat.append(makeMLXChatMessage(from: prompt.segments, role: .user))

case .reasoning:
// Keep display history in the transcript without sending unsupported replay state.
continue
case .response(let response):
let content = response.segments.map { extractText(from: $0) }.joined(separator: "\n")
chat.append(.assistant(content))
Expand Down
5 changes: 4 additions & 1 deletion Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -602,7 +602,7 @@ struct OllamaMessage: Hashable, Codable, Sendable {

private extension Transcript {
func toOllamaMessages() throws -> [OllamaMessage] {
try map { entry in
try compactMap { entry -> OllamaMessage? in
let role: OllamaMessage.Role
let segments: [Transcript.Segment]
switch entry {
Expand All @@ -612,6 +612,9 @@ private extension Transcript {
case .prompt(let prompt):
role = .user
segments = prompt.segments
case .reasoning:
// Keep display history in the transcript without sending unsupported replay state.
return nil
case .response(let response):
role = .assistant
segments = response.segments
Expand Down
3 changes: 3 additions & 0 deletions Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1338,6 +1338,9 @@ extension Transcript {
content: .blocks(convertSegmentsToOpenAIBlocks(prompt.segments))
)
)
case .reasoning:
// Keep display history in the transcript without sending unsupported replay state.
continue
case .response(let response):
messages.append(
.init(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -922,6 +922,9 @@ extension Transcript {
content: .blocks(openResponsesConvertSegmentsToBlocks(prompt.segments))
)
)
case .reasoning:
// Keep display history in the transcript without sending unsupported replay state.
continue
case .response(let response):
list.append(
OpenResponsesMessage(
Expand Down
3 changes: 3 additions & 0 deletions Sources/AnyLanguageModel/Models/SystemLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -606,6 +606,9 @@
)
fmEntries.append(.prompt(fmPrompt))

case .reasoning:
// Keep display history in the transcript without sending unsupported replay state.
continue
case .response(let response):
let fmSegments = response.segments.toFoundationModels()
let fmResponse = FoundationModels.Transcript.Response(
Expand Down
Loading
Loading