diff --git a/README.md b/README.md index bd439ec6..cf718c31 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ session.toolExecutionDelegate = ToolExecutionObserver() ### Supported Providers - [x] [Apple Foundation Models](https://developer.apple.com/documentation/FoundationModels) +- [x] Apple [Private Cloud Compute](https://developer.apple.com/documentation/FoundationModels/PrivateCloudComputeLanguageModel) and any [`FoundationModels.LanguageModel`](https://developer.apple.com/documentation/FoundationModels/LanguageModel) conformer, including [Core AI](https://github.com/apple/coreai-models) models (OS 27) - [x] [Core ML](https://developer.apple.com/documentation/coreml) models - [x] [MLX](https://github.com/ml-explore/mlx-swift) models - [x] [llama.cpp](https://github.com/ggml-org/llama.cpp) (GGUF models) @@ -514,6 +515,56 @@ let response = try await session.respond { } ``` +### Apple Private Cloud Compute + +Uses Apple's [Private Cloud Compute model](https://developer.apple.com/documentation/FoundationModels/PrivateCloudComputeLanguageModel), +a larger server-hosted model behind the same privacy architecture as the on-device one +(requires macOS 27 / iOS 27 or later and the Private Cloud Compute entitlement). + +```swift +let model = PrivateCloudComputeLanguageModel.default +let session = LanguageModelSession(model: model) + +let response = try await session.respond { + Prompt("Summarize the attached report in three bullet points") +} +``` + +### Any Foundation Models Conformer + +On OS 27, Foundation Models accepts any type that conforms to its `LanguageModel` protocol. +`FoundationLanguageModel` wraps such a model so it works with everything in this package. +Construct the model yourself, or hand the wrapper an async factory so an expensive load +happens on the first request and you control when it is released. + +```swift +let model = FoundationLanguageModel { + try await MyModel(resourcesAt: url) +} +let session = LanguageModelSession(model: model) + +let response = try await session.respond(to: "Hello") +await model.unload() +``` + +#### Core AI Models + +Apple's [coreai-models](https://github.com/apple/coreai-models) package exports language +models for the Core AI engine, and its `CoreAILanguageModel` conforms to the Foundation Models +protocol. Add the `CoreAILM` product from that package to your app and wrap the model: + +```swift +import CoreAILanguageModels + +let model = FoundationLanguageModel { + try await CoreAILanguageModels.CoreAILanguageModel(resourcesAt: resourcesURL) +} +let session = LanguageModelSession(model: model) +``` + +The package declares a platform floor of OS 27. Apps with an earlier deployment target can +build it as an XCFramework and link it weakly, guarding every use with an availability check. + ### Core ML Runs [Core ML](https://developer.apple.com/documentation/coreml) models diff --git a/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift b/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift new file mode 100644 index 00000000..4df8c667 --- /dev/null +++ b/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift @@ -0,0 +1,137 @@ +#if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) + import Foundation + import FoundationModels + + /// A language model backed by any type that conforms to + /// `FoundationModels.LanguageModel`. + /// + /// On OS 27, Apple's Foundation Models framework opens its `LanguageModel` + /// protocol to third-party models, and `LanguageModelSession` accepts any + /// conformer. This wrapper hands such a model to a Foundation Models session + /// and bridges the result back into AnyLanguageModel, so a model built on + /// Apple's protocol can be used alongside every other provider here. + /// + /// Models that are expensive to construct can be supplied through an async + /// factory. The factory runs once, on the first request or on an explicit + /// call to ``load()``, and the caller owns the lifetime through ``unload()``. + /// + /// ```swift + /// let model = FoundationLanguageModel { + /// try await MyModel(resourcesAt: url) + /// } + /// let session = LanguageModelSession(model: model) + /// let response = try await session.respond(to: "Hello") + /// await model.unload() + /// ``` + @available(macOS 27.0, iOS 27.0, visionOS 27.0, watchOS 27.0, *) + public actor FoundationLanguageModel: LanguageModel { + public typealias UnavailableReason = Never + + private let makeModel: @Sendable () async throws -> Model + private var model: Model? + + /// Creates a language model around an already constructed model. + /// + /// - Parameter model: The Foundation Models conformer to use for generation. + public init(_ model: Model) { + self.makeModel = { model } + self.model = model + } + + /// Creates a language model whose underlying model is constructed on demand. + /// + /// - Parameter makeModel: A factory that constructs the model. It runs once, + /// on the first request or on ``load()``, and again after ``unload()``. + public init(loading makeModel: @escaping @Sendable () async throws -> Model) { + self.makeModel = makeModel + } + + /// Whether the underlying model is currently constructed. + public var isLoaded: Bool { + model != nil + } + + /// The capabilities of the underlying model, once it is loaded. + public var capabilities: FoundationModels.LanguageModelCapabilities? { + model?.capabilities + } + + /// Constructs the underlying model if it is not loaded yet. + public func load() async throws { + _ = try await loadedModel() + } + + /// Releases the underlying model. The next request constructs it again. + public func unload() { + model = nil + } + + private func loadedModel() async throws -> Model { + if let model { + return model + } + let model = try await makeModel() + self.model = model + return model + } + + private func makeSession( + tools: [any FoundationModels.Tool], + transcript: FoundationModels.Transcript + ) async throws -> FoundationModels.LanguageModelSession { + FoundationModels.LanguageModelSession( + model: try await loadedModel(), + tools: tools, + transcript: transcript + ) + } + + nonisolated public func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + let fmTools = session.tools.toFoundationModels() + let fmTranscript = fmTranscriptDroppingDuplicatePrompt(session.transcript, prompt: prompt) + .toFoundationModels( + instructions: session.instructions, + toolDefinitions: session.tools + .filter(\.includesSchemaInInstructions) + .map { Transcript.ToolDefinition(tool: $0) } + ) + return try await fmRespond( + makeSession: { try await self.makeSession(tools: fmTools, transcript: fmTranscript) }, + fmPrompt: prompt.toFoundationModels(), + fmOptions: options.toFoundationModels(), + type: type, + includeSchemaInPrompt: includeSchemaInPrompt + ) + } + + nonisolated public func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let fmTools = session.tools.toFoundationModels() + let fmTranscript = fmTranscriptDroppingDuplicatePrompt(session.transcript, prompt: prompt) + .toFoundationModels( + instructions: session.instructions, + toolDefinitions: session.tools + .filter(\.includesSchemaInInstructions) + .map { Transcript.ToolDefinition(tool: $0) } + ) + return fmStreamResponse( + makeSession: { try await self.makeSession(tools: fmTools, transcript: fmTranscript) }, + fmPrompt: prompt.toFoundationModels(), + fmOptions: options.toFoundationModels(), + type: type, + includeSchemaInPrompt: includeSchemaInPrompt + ) + } + } +#endif diff --git a/Sources/AnyLanguageModel/Models/PrivateCloudComputeLanguageModel.swift b/Sources/AnyLanguageModel/Models/PrivateCloudComputeLanguageModel.swift new file mode 100644 index 00000000..04a53c7b --- /dev/null +++ b/Sources/AnyLanguageModel/Models/PrivateCloudComputeLanguageModel.swift @@ -0,0 +1,109 @@ +#if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) + import Foundation + import FoundationModels + + /// A language model that uses Apple's Private Cloud Compute. + /// + /// Use this model to generate text with Apple's larger server-hosted models, + /// running on Apple silicon servers under the Private Cloud Compute privacy + /// architecture. Requests are stateless and cryptographically attested, and + /// no data is retained. + /// + /// Apps need the Private Cloud Compute entitlement to use this model. + /// + /// ```swift + /// let model = PrivateCloudComputeLanguageModel.default + /// let session = LanguageModelSession(model: model) + /// ``` + @available(macOS 27.0, iOS 27.0, visionOS 27.0, watchOS 27.0, *) + public struct PrivateCloudComputeLanguageModel: LanguageModel { + /// The reason the model is unavailable. + public typealias UnavailableReason = FoundationModels.PrivateCloudComputeLanguageModel.Availability + .UnavailableReason + + let pccModel: FoundationModels.PrivateCloudComputeLanguageModel + private let wrapped: FoundationLanguageModel + + /// The default Private Cloud Compute language model. + public static var `default`: PrivateCloudComputeLanguageModel { + PrivateCloudComputeLanguageModel() + } + + /// Creates the default Private Cloud Compute language model. + public init() { + let pccModel = FoundationModels.PrivateCloudComputeLanguageModel() + self.pccModel = pccModel + self.wrapped = FoundationLanguageModel(pccModel) + } + + /// The current quota usage for Private Cloud Compute requests. + public var quotaUsage: FoundationModels.PrivateCloudComputeLanguageModel.QuotaUsage { + pccModel.quotaUsage + } + + /// Whether the model accepts image input. + public var supportsImageInput: Bool { + pccModel.capabilities.contains(.vision) + } + + /// The availability status for the Private Cloud Compute language model. + public var availability: Availability { + switch pccModel.availability { + case .available: + .available + case .unavailable(let reason): + .unavailable(reason) + } + } + + public func respond( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) async throws -> LanguageModelSession.Response where Content: Generable { + try await wrapped.respond( + within: session, + to: prompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + } + + public func streamResponse( + within session: LanguageModelSession, + to prompt: Prompt, + generating type: Content.Type, + includeSchemaInPrompt: Bool, + options: GenerationOptions + ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + wrapped.streamResponse( + within: session, + to: prompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + } + + public func logFeedbackAttachment( + within session: LanguageModelSession, + sentiment: LanguageModelFeedback.Sentiment?, + issues: [LanguageModelFeedback.Issue], + desiredOutput: Transcript.Entry? + ) -> Data { + let fmSession = FoundationModels.LanguageModelSession( + model: pccModel, + tools: session.tools.toFoundationModels(), + instructions: session.instructions?.toFoundationModels() + ) + return fmSession.logFeedbackAttachment( + sentiment: sentiment?.toFoundationModels(), + issues: issues.map { $0.toFoundationModels() }, + desiredOutput: nil + ) + } + } +#endif diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 85ccd14e..920b8cc5 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -105,64 +105,13 @@ ) ) - if type == String.self { - let fmResponse = try await fmSession.respond(to: fmPrompt, options: fmOptions) - let generatedContent = GeneratedContent(fmResponse.content) - return LanguageModelSession.Response( - content: fmResponse.content as! Content, - rawContent: generatedContent, - transcriptEntries: [] - ) - } else { - // For non-String types, use schema-based generation - let schema = FoundationModels.GenerationSchema(type.generationSchema) - let fmResponse = try await fmSession.respond( - to: fmPrompt, - schema: schema, - includeSchemaInPrompt: includeSchemaInPrompt, - options: fmOptions - ) - - func finalize(content: Content) -> LanguageModelSession.Response { - let normalizedRaw = content.generatedContent - if let jsonValue = try? JSONValue(normalizedRaw), - case .array(let values) = jsonValue, - values.isEmpty, - let placeholder = placeholderContent(for: type) - { - return LanguageModelSession.Response( - content: placeholder.content, - rawContent: placeholder.rawContent, - transcriptEntries: [] - ) - } - return LanguageModelSession.Response( - content: content, - rawContent: normalizedRaw, - transcriptEntries: [] - ) - } - - do { - let generatedContent = try GeneratedContent(fmResponse.content) - let content = try type.init(generatedContent) - - return finalize(content: content) - } catch { - // Attempt partial JSON decoding before surfacing an error. - let decoder = PartialJSONDecoder() - let jsonString = fmResponse.content.jsonString - if let partialContent = try? decoder.decode(GeneratedContent.self, from: jsonString).value, - let content = try? type.init(partialContent) - { - return finalize(content: content) - } - if let placeholder = placeholderContent(for: type) { - return finalize(content: placeholder.content) - } - throw error - } - } + return try await fmRespond( + makeSession: { fmSession }, + fmPrompt: fmPrompt, + fmOptions: fmOptions, + type: type, + includeSchemaInPrompt: includeSchemaInPrompt + ) } nonisolated public func streamResponse( @@ -186,177 +135,13 @@ ) ) - let stream: AsyncThrowingStream.Snapshot, Error> = - AsyncThrowingStream { continuation in - - func accumulateText( - _ chunkText: String, - accumulatedText: inout String, - lastLength: inout Int - ) { - if chunkText.count >= lastLength, chunkText.hasPrefix(accumulatedText) { - let startIdx = chunkText.index(chunkText.startIndex, offsetBy: lastLength) - let delta = String(chunkText[startIdx...]) - accumulatedText += delta - lastLength = chunkText.count - } else if chunkText.hasPrefix(accumulatedText) { - accumulatedText = chunkText - lastLength = chunkText.count - } else if accumulatedText.hasPrefix(chunkText) { - accumulatedText = chunkText - lastLength = chunkText.count - } else { - accumulatedText += chunkText - lastLength = accumulatedText.count - } - } - - func processStringStream() async { - let fmStream: FoundationModels.LanguageModelSession.ResponseStream = - fmSession.streamResponse(to: fmPrompt, options: fmOptions) - - var accumulatedText = "" - do { - var lastLength = 0 - for try await snapshot in fmStream { - var chunkText: String = snapshot.content - - // Handle "null" from FoundationModels as first temp result - if chunkText == "null" && accumulatedText == "" { - chunkText = "" - } - - accumulateText( - chunkText, - accumulatedText: &accumulatedText, - lastLength: &lastLength - ) - - let raw = GeneratedContent(accumulatedText) - let snapshotContent = (accumulatedText as! Content).asPartiallyGenerated() - continuation.yield(.init(content: snapshotContent, rawContent: raw)) - } - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - - func processStructuredStream() async { - let schema = FoundationModels.GenerationSchema(type.generationSchema) - let partialDecoder = PartialJSONDecoder() - let fmStream = fmSession.streamResponse( - to: fmPrompt, - schema: schema, - includeSchemaInPrompt: includeSchemaInPrompt, - options: fmOptions - ) - - func processTextFallback() async { - let fmTextStream: FoundationModels.LanguageModelSession.ResponseStream = - fmSession.streamResponse(to: fmPrompt, options: fmOptions) - - var accumulatedText = "" - var didYield = false - do { - var lastLength = 0 - for try await snapshot in fmTextStream { - var chunkText: String = snapshot.content - if chunkText == "null" && accumulatedText.isEmpty { - chunkText = "" - } - - accumulateText( - chunkText, - accumulatedText: &accumulatedText, - lastLength: &lastLength - ) - - let jsonString = accumulatedText - if let partialContent = try? partialDecoder.decode( - GeneratedContent.self, - from: jsonString - ) - .value { - let partial: Content.PartiallyGenerated? = try? .init(partialContent) - if let partial { - continuation.yield(.init(content: partial, rawContent: partialContent)) - didYield = true - } - } - } - if !didYield, let placeholder = placeholderPartialContent(for: type) { - continuation.yield( - .init(content: placeholder.content, rawContent: placeholder.rawContent) - ) - } - continuation.finish() - } catch { - if !didYield, let placeholder = placeholderPartialContent(for: type) { - continuation.yield( - .init(content: placeholder.content, rawContent: placeholder.rawContent) - ) - } - continuation.finish(throwing: error) - } - } - - var didYield = false - do { - for try await snapshot in fmStream { - let jsonString = snapshot.content.jsonString - let raw = - (try? GeneratedContent(snapshot.content)) - ?? (try? GeneratedContent(json: jsonString)) - ?? GeneratedContent(jsonString) - - // Prefer partial decoding so we can surface intermediate snapshots. - if let partialContent = try? partialDecoder.decode( - GeneratedContent.self, - from: jsonString - ) - .value { - let partial: Content.PartiallyGenerated? = try? .init(partialContent) - if let partial { - continuation.yield(.init(content: partial, rawContent: partialContent)) - didYield = true - continue - } - } - - // Fallback to full conversion when partial decoding isn't possible. - if let value = try? type.init(raw) { - let snapshotContent = value.asPartiallyGenerated() - continuation.yield(.init(content: snapshotContent, rawContent: raw)) - didYield = true - } - } - if !didYield, let placeholder = placeholderPartialContent(for: type) { - continuation.yield( - .init(content: placeholder.content, rawContent: placeholder.rawContent) - ) - } - continuation.finish() - } catch { - if didYield { - continuation.finish(throwing: error) - } else { - await processTextFallback() - } - } - } - - let streamingTask: _Concurrency.Task = _Concurrency.Task(priority: nil) { - if type == String.self { - await processStringStream() - } else { - await processStructuredStream() - } - } - continuation.onTermination = { _ in streamingTask.cancel() } - } - - return LanguageModelSession.ResponseStream(stream: stream) + return fmStreamResponse( + makeSession: { fmSession }, + fmPrompt: fmPrompt, + fmOptions: fmOptions, + type: type, + includeSchemaInPrompt: includeSchemaInPrompt + ) } nonisolated public func logFeedbackAttachment( @@ -403,14 +188,14 @@ @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) extension Prompt { - fileprivate func toFoundationModels() -> FoundationModels.Prompt { + func toFoundationModels() -> FoundationModels.Prompt { FoundationModels.Prompt(self.description) } } @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) extension Instructions { - fileprivate func toFoundationModels() -> FoundationModels.Instructions { + func toFoundationModels() -> FoundationModels.Instructions { FoundationModels.Instructions(self.description) } } @@ -440,7 +225,7 @@ @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) extension LanguageModelFeedback.Sentiment { - fileprivate func toFoundationModels() -> FoundationModels.LanguageModelFeedback.Sentiment { + func toFoundationModels() -> FoundationModels.LanguageModelFeedback.Sentiment { switch self { case .positive: .positive case .negative: .negative @@ -451,7 +236,7 @@ @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) extension LanguageModelFeedback.Issue { - fileprivate func toFoundationModels() -> FoundationModels.LanguageModelFeedback.Issue { + func toFoundationModels() -> FoundationModels.LanguageModelFeedback.Issue { FoundationModels.LanguageModelFeedback.Issue( category: self.category.toFoundationModels(), explanation: self.explanation @@ -461,7 +246,7 @@ @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) extension LanguageModelFeedback.Issue.Category { - fileprivate func toFoundationModels() -> FoundationModels.LanguageModelFeedback.Issue.Category { + func toFoundationModels() -> FoundationModels.LanguageModelFeedback.Issue.Category { switch self { case .unhelpful: .unhelpful case .tooVerbose: .tooVerbose @@ -477,7 +262,7 @@ @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) extension Array where Element == (any Tool) { - fileprivate func toFoundationModels() -> [any FoundationModels.Tool] { + func toFoundationModels() -> [any FoundationModels.Tool] { map { AnyToolWrapper($0) } } } @@ -701,7 +486,7 @@ @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) extension Transcript { - fileprivate func toFoundationModels( + func toFoundationModels( instructions: AnyLanguageModel.Instructions?, toolDefinitions: [Transcript.ToolDefinition] ) -> FoundationModels.Transcript { @@ -781,7 +566,7 @@ @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) extension Array where Element == Transcript.Segment { - fileprivate func toFoundationModels() -> [FoundationModels.Transcript.Segment] { + func toFoundationModels() -> [FoundationModels.Transcript.Segment] { compactMap { segment -> FoundationModels.Transcript.Segment? in switch segment { case .text(let textSegment): @@ -843,7 +628,7 @@ @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) extension Array where Element == Transcript.ToolDefinition { - fileprivate func toFoundationModels() -> [FoundationModels.Transcript.ToolDefinition] { + func toFoundationModels() -> [FoundationModels.Transcript.ToolDefinition] { map { toolDef in FoundationModels.Transcript.ToolDefinition( name: toolDef.name, @@ -854,6 +639,265 @@ } } + // MARK: - Shared FoundationModels Session Bridging + + @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) + func fmRespond( + makeSession: @Sendable () async throws -> FoundationModels.LanguageModelSession, + fmPrompt: FoundationModels.Prompt, + fmOptions: FoundationModels.GenerationOptions, + type: Content.Type, + includeSchemaInPrompt: Bool + ) async throws -> LanguageModelSession.Response where Content: Generable { + let fmSession = try await makeSession() + if type == String.self { + let fmResponse = try await fmSession.respond(to: fmPrompt, options: fmOptions) + let generatedContent = GeneratedContent(fmResponse.content) + return LanguageModelSession.Response( + content: fmResponse.content as! Content, + rawContent: generatedContent, + transcriptEntries: [] + ) + } else { + // For non-String types, use schema-based generation + let schema = FoundationModels.GenerationSchema(type.generationSchema) + let fmResponse = try await fmSession.respond( + to: fmPrompt, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + options: fmOptions + ) + + func finalize(content: Content) -> LanguageModelSession.Response { + let normalizedRaw = content.generatedContent + if let jsonValue = try? JSONValue(normalizedRaw), + case .array(let values) = jsonValue, + values.isEmpty, + let placeholder = placeholderContent(for: type) + { + return LanguageModelSession.Response( + content: placeholder.content, + rawContent: placeholder.rawContent, + transcriptEntries: [] + ) + } + return LanguageModelSession.Response( + content: content, + rawContent: normalizedRaw, + transcriptEntries: [] + ) + } + + do { + let generatedContent = try GeneratedContent(fmResponse.content) + let content = try type.init(generatedContent) + + return finalize(content: content) + } catch { + // Attempt partial JSON decoding before surfacing an error. + let decoder = PartialJSONDecoder() + let jsonString = fmResponse.content.jsonString + if let partialContent = try? decoder.decode(GeneratedContent.self, from: jsonString).value, + let content = try? type.init(partialContent) + { + return finalize(content: content) + } + if let placeholder = placeholderContent(for: type) { + return finalize(content: placeholder.content) + } + throw error + } + } + } + + @available(macOS 26.0, iOS 26.0, watchOS 26.0, tvOS 26.0, visionOS 26.0, *) + func fmStreamResponse( + makeSession: @escaping @Sendable () async throws -> FoundationModels.LanguageModelSession, + fmPrompt: FoundationModels.Prompt, + fmOptions: FoundationModels.GenerationOptions, + type: Content.Type, + includeSchemaInPrompt: Bool + ) -> LanguageModelSession.ResponseStream where Content: Generable { + let stream: AsyncThrowingStream.Snapshot, Error> = + AsyncThrowingStream { continuation in + + func accumulateText( + _ chunkText: String, + accumulatedText: inout String, + lastLength: inout Int + ) { + if chunkText.count >= lastLength, chunkText.hasPrefix(accumulatedText) { + let startIdx = chunkText.index(chunkText.startIndex, offsetBy: lastLength) + let delta = String(chunkText[startIdx...]) + accumulatedText += delta + lastLength = chunkText.count + } else if chunkText.hasPrefix(accumulatedText) { + accumulatedText = chunkText + lastLength = chunkText.count + } else if accumulatedText.hasPrefix(chunkText) { + accumulatedText = chunkText + lastLength = chunkText.count + } else { + accumulatedText += chunkText + lastLength = accumulatedText.count + } + } + + func processStringStream(_ fmSession: FoundationModels.LanguageModelSession) async { + let fmStream: FoundationModels.LanguageModelSession.ResponseStream = + fmSession.streamResponse(to: fmPrompt, options: fmOptions) + + var accumulatedText = "" + do { + var lastLength = 0 + for try await snapshot in fmStream { + var chunkText: String = snapshot.content + + // Handle "null" from FoundationModels as first temp result + if chunkText == "null" && accumulatedText == "" { + chunkText = "" + } + + accumulateText( + chunkText, + accumulatedText: &accumulatedText, + lastLength: &lastLength + ) + + let raw = GeneratedContent(accumulatedText) + let snapshotContent = (accumulatedText as! Content).asPartiallyGenerated() + continuation.yield(.init(content: snapshotContent, rawContent: raw)) + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + + func processStructuredStream(_ fmSession: FoundationModels.LanguageModelSession) async { + let schema = FoundationModels.GenerationSchema(type.generationSchema) + let partialDecoder = PartialJSONDecoder() + let fmStream = fmSession.streamResponse( + to: fmPrompt, + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + options: fmOptions + ) + + func processTextFallback() async { + let fmTextStream: FoundationModels.LanguageModelSession.ResponseStream = + fmSession.streamResponse(to: fmPrompt, options: fmOptions) + + var accumulatedText = "" + var didYield = false + do { + var lastLength = 0 + for try await snapshot in fmTextStream { + var chunkText: String = snapshot.content + if chunkText == "null" && accumulatedText.isEmpty { + chunkText = "" + } + + accumulateText( + chunkText, + accumulatedText: &accumulatedText, + lastLength: &lastLength + ) + + let jsonString = accumulatedText + if let partialContent = try? partialDecoder.decode( + GeneratedContent.self, + from: jsonString + ) + .value { + let partial: Content.PartiallyGenerated? = try? .init(partialContent) + if let partial { + continuation.yield(.init(content: partial, rawContent: partialContent)) + didYield = true + } + } + } + if !didYield, let placeholder = placeholderPartialContent(for: type) { + continuation.yield( + .init(content: placeholder.content, rawContent: placeholder.rawContent) + ) + } + continuation.finish() + } catch { + if !didYield, let placeholder = placeholderPartialContent(for: type) { + continuation.yield( + .init(content: placeholder.content, rawContent: placeholder.rawContent) + ) + } + continuation.finish(throwing: error) + } + } + + var didYield = false + do { + for try await snapshot in fmStream { + let jsonString = snapshot.content.jsonString + let raw = + (try? GeneratedContent(snapshot.content)) + ?? (try? GeneratedContent(json: jsonString)) + ?? GeneratedContent(jsonString) + + // Prefer partial decoding so we can surface intermediate snapshots. + if let partialContent = try? partialDecoder.decode( + GeneratedContent.self, + from: jsonString + ) + .value { + let partial: Content.PartiallyGenerated? = try? .init(partialContent) + if let partial { + continuation.yield(.init(content: partial, rawContent: partialContent)) + didYield = true + continue + } + } + + // Fallback to full conversion when partial decoding isn't possible. + if let value = try? type.init(raw) { + let snapshotContent = value.asPartiallyGenerated() + continuation.yield(.init(content: snapshotContent, rawContent: raw)) + didYield = true + } + } + if !didYield, let placeholder = placeholderPartialContent(for: type) { + continuation.yield( + .init(content: placeholder.content, rawContent: placeholder.rawContent) + ) + } + continuation.finish() + } catch { + if didYield { + continuation.finish(throwing: error) + } else { + await processTextFallback() + } + } + } + + let streamingTask: _Concurrency.Task = _Concurrency.Task(priority: nil) { + let fmSession: FoundationModels.LanguageModelSession + do { + fmSession = try await makeSession() + } catch { + continuation.finish(throwing: error) + return + } + if type == String.self { + await processStringStream(fmSession) + } else { + await processStructuredStream(fmSession) + } + } + continuation.onTermination = { _ in streamingTask.cancel() } + } + + return LanguageModelSession.ResponseStream(stream: stream) + } + // MARK: - Placeholder Helpers /// Generates minimal partial content when structured output is missing or invalid. diff --git a/Tests/AnyLanguageModelTests/FoundationLanguageModelTests.swift b/Tests/AnyLanguageModelTests/FoundationLanguageModelTests.swift new file mode 100644 index 00000000..9b9a7268 --- /dev/null +++ b/Tests/AnyLanguageModelTests/FoundationLanguageModelTests.swift @@ -0,0 +1,52 @@ +import Testing +@testable import AnyLanguageModel + +#if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) + import FoundationModels + + @Suite("FoundationLanguageModel") + struct FoundationLanguageModelTests { + @available(macOS 27.0, iOS 27.0, visionOS 27.0, watchOS 27.0, *) + private actor Counter { + private(set) var count = 0 + func increment() { count += 1 } + } + + @Test func factoryRunsOnFirstLoadOnly() async throws { + guard #available(macOS 27.0, iOS 27.0, visionOS 27.0, watchOS 27.0, *) else { return } + let counter = Counter() + let model = FoundationLanguageModel { + await counter.increment() + return FoundationModels.PrivateCloudComputeLanguageModel() + } + #expect(await model.isLoaded == false) + #expect(await counter.count == 0) + try await model.load() + try await model.load() + #expect(await model.isLoaded == true) + #expect(await counter.count == 1) + } + + @Test func unloadReleasesTheModelAndReloadsOnDemand() async throws { + guard #available(macOS 27.0, iOS 27.0, visionOS 27.0, watchOS 27.0, *) else { return } + let counter = Counter() + let model = FoundationLanguageModel { + await counter.increment() + return FoundationModels.PrivateCloudComputeLanguageModel() + } + try await model.load() + await model.unload() + #expect(await model.isLoaded == false) + #expect(await model.capabilities == nil) + try await model.load() + #expect(await counter.count == 2) + } + + @Test func wrappingAnExistingModelIsLoadedImmediately() async throws { + guard #available(macOS 27.0, iOS 27.0, visionOS 27.0, watchOS 27.0, *) else { return } + let model = FoundationLanguageModel(FoundationModels.PrivateCloudComputeLanguageModel()) + #expect(await model.isLoaded == true) + #expect(model.isAvailable == true) + } + } +#endif