diff --git a/go/adk/pkg/models/anthropic_adk.go b/go/adk/pkg/models/anthropic_adk.go index c51a7da98..268602132 100644 --- a/go/adk/pkg/models/anthropic_adk.go +++ b/go/adk/pkg/models/anthropic_adk.go @@ -130,10 +130,7 @@ func genaiContentsToAnthropicMessages(contents []*genai.Content, config *genai.G var textParts []string var functionCalls []*genai.FunctionCall - var imageParts []struct { - mimeType string - data []byte - } + var mediaBlocks []anthropic.ContentBlockParamUnion for _, part := range content.Parts { if part == nil { @@ -143,11 +140,31 @@ func genaiContentsToAnthropicMessages(contents []*genai.Content, config *genai.G textParts = append(textParts, part.Text) } else if part.FunctionCall != nil { functionCalls = append(functionCalls, part.FunctionCall) - } else if part.InlineData != nil && strings.HasPrefix(part.InlineData.MIMEType, "image/") { - imageParts = append(imageParts, struct { - mimeType string - data []byte - }{part.InlineData.MIMEType, part.InlineData.Data}) + } else if part.InlineData != nil { + mime := part.InlineData.MIMEType + name := blobName(part.InlineData) + if isImageMIME(mime) { + mediaBlocks = append(mediaBlocks, anthropic.NewImageBlockBase64(mime, base64.StdEncoding.EncodeToString(part.InlineData.Data))) + } else if isAnthropicPDF(mime, name) { + mediaBlocks = append(mediaBlocks, anthropic.NewDocumentBlock(anthropic.Base64PDFSourceParam{ + Data: base64.StdEncoding.EncodeToString(part.InlineData.Data), + })) + } else if isAnthropicPlainText(mime, name) { + mediaBlocks = append(mediaBlocks, anthropic.NewDocumentBlock(anthropic.PlainTextSourceParam{ + Data: string(part.InlineData.Data), + })) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } + } else if part.FileData != nil { + mime := part.FileData.MIMEType + name := fileDataName(part.FileData) + uri := part.FileData.FileURI + if uri != "" && isAnthropicPDF(mime, name) { + mediaBlocks = append(mediaBlocks, anthropic.NewDocumentBlock(anthropic.URLPDFSourceParam{URL: uri})) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } } } @@ -188,13 +205,7 @@ func genaiContentsToAnthropicMessages(contents []*genai.Content, config *genai.G } else { // Regular user message var contentBlocks []anthropic.ContentBlockParamUnion - - // Add images first - for _, img := range imageParts { - contentBlocks = append(contentBlocks, anthropic.NewImageBlockBase64(img.mimeType, base64.StdEncoding.EncodeToString(img.data))) - } - - // Add text + contentBlocks = append(contentBlocks, mediaBlocks...) if len(textParts) > 0 { contentBlocks = append(contentBlocks, anthropic.NewTextBlock(strings.Join(textParts, "\n"))) } diff --git a/go/adk/pkg/models/bedrock.go b/go/adk/pkg/models/bedrock.go index 4af7916b1..b92b9a04e 100644 --- a/go/adk/pkg/models/bedrock.go +++ b/go/adk/pkg/models/bedrock.go @@ -667,6 +667,40 @@ func convertGenaiContentsToBedrockMessages(contents []*genai.Content, nameMap ma continue } + if part.InlineData != nil { + mime := part.InlineData.MIMEType + name := blobName(part.InlineData) + if fmtStr := bedrockImageFormat(mime); fmtStr != "" { + contentBlocks = append(contentBlocks, &types.ContentBlockMemberImage{ + Value: types.ImageBlock{ + Format: types.ImageFormat(fmtStr), + Source: &types.ImageSourceMemberBytes{Value: part.InlineData.Data}, + }, + }) + } else if docFmt := bedrockDocumentFormat(mime, name); docFmt != "" { + contentBlocks = append(contentBlocks, &types.ContentBlockMemberDocument{ + Value: types.DocumentBlock{ + Format: types.DocumentFormat(docFmt), + Name: aws.String(bedrockSafeDocName(name)), + Source: &types.DocumentSourceMemberBytes{Value: part.InlineData.Data}, + }, + }) + } else { + contentBlocks = append(contentBlocks, &types.ContentBlockMemberText{ + Value: unsupportedFileNote(name, mime), + }) + } + continue + } + + if part.FileData != nil { + // Bedrock Converse document/image blocks are bytes-only. + contentBlocks = append(contentBlocks, &types.ContentBlockMemberText{ + Value: unsupportedFileNote(fileDataName(part.FileData), part.FileData.MIMEType), + }) + continue + } + // Handle function call (tool use in Bedrock terminology). // Use the sanitized name from nameMap so Bedrock can correlate the // tool call with the tool spec sent in the same request. @@ -719,6 +753,10 @@ func convertGenaiContentsToBedrockMessages(contents []*genai.Content, nameMap ma } } + // Bedrock Converse: a document ContentBlock requires a text ContentBlock + // in the same message (https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Message.html). + contentBlocks = ensureBedrockDocumentHasText(contentBlocks) + if len(contentBlocks) > 0 { messages = append(messages, types.Message{Role: role, Content: contentBlocks}) } @@ -727,6 +765,25 @@ func convertGenaiContentsToBedrockMessages(contents []*genai.Content, nameMap ma return messages, mergeSystemInstructionFromConfig(systemInstruction, config) } +// ensureBedrockDocumentHasText appends a short prompt when a message has documents but no text +func ensureBedrockDocumentHasText(blocks []types.ContentBlock) []types.ContentBlock { + hasDoc, hasText := false, false + for _, b := range blocks { + switch b.(type) { + case *types.ContentBlockMemberDocument: + hasDoc = true + case *types.ContentBlockMemberText: + hasText = true + } + } + if hasDoc && !hasText { + blocks = append(blocks, &types.ContentBlockMemberText{ + Value: "Please review the attached document.", + }) + } + return blocks +} + // convertGenaiToolsToBedrock converts genai.Tool to Bedrock Tool format. // It sanitizes tool names to satisfy Bedrock's [a-zA-Z0-9_-]+ constraint and // returns the original->sanitized name mapping so callers can apply it to diff --git a/go/adk/pkg/models/file_parts.go b/go/adk/pkg/models/file_parts.go new file mode 100644 index 000000000..1d4929165 --- /dev/null +++ b/go/adk/pkg/models/file_parts.go @@ -0,0 +1,245 @@ +package models + +import ( + "encoding/base64" + "fmt" + "path" + "strings" + + "google.golang.org/genai" +) + +// unsupportedFileNote is appended as text when a file part cannot be mapped +// to a provider-native input. Prefer this over silently dropping the part. +func unsupportedFileNote(name, mime string) string { + if name == "" { + name = "unnamed" + } + if mime == "" { + mime = "unknown" + } + return fmt.Sprintf("[unsupported file: %s (%s)]", name, mime) +} + +func dataURI(mime string, data []byte) string { + return fmt.Sprintf("data:%s;base64,%s", mime, base64.StdEncoding.EncodeToString(data)) +} + +func blobName(b *genai.Blob) string { + if b == nil { + return "" + } + return b.DisplayName +} + +func fileDataName(f *genai.FileData) string { + if f == nil { + return "" + } + return f.DisplayName +} + +func isImageMIME(mime string) bool { + return strings.HasPrefix(mime, "image/") +} + +// OpenAI file support differs by API surface: +// +// - Chat Completions (`messages[].content[].type=file`, inline file_data): PDF only. +// - Responses (`input_file`): broad list (txt/md/csv/docx/xlsx/pptx/…). +func isOpenAIPDF(mime, name string) bool { + if strings.ToLower(mime) == "application/pdf" { + return true + } + return strings.ToLower(path.Ext(name)) == ".pdf" +} + +// isOpenAIResponsesFileMIME is the Responses input_file allowlist (common types). +// Link: https://platform.openai.com/docs/guides/file-inputs +func isOpenAIResponsesFileMIME(mime, name string) bool { + if isOpenAIPDF(mime, name) { + return true + } + switch strings.ToLower(mime) { + case "text/plain", "text/markdown", "text/csv", "text/tsv", "text/html", "text/xml", "text/css", + "application/json", "application/xml", "application/rtf", "application/csv", "text/rtf", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-powerpoint", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.oasis.opendocument.text": + return true + } + if strings.HasPrefix(strings.ToLower(mime), "text/") { + return true + } + // Browsers often send "" / application/octet-stream — use extension. + switch strings.ToLower(path.Ext(name)) { + case ".pdf", ".txt", ".text", ".md", ".markdown", ".csv", ".tsv", ".html", ".htm", + ".xml", ".json", ".rtf", ".odt", + ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", + ".py", ".js", ".mjs", ".ts", ".go", ".java", ".c", ".cc", ".cpp", ".h", ".rb", + ".sh", ".yaml", ".yml", ".css", ".sql": + return true + } + return false +} + +func isAnthropicPDF(mime, name string) bool { + return isOpenAIPDF(mime, name) +} + +// isTextFileMIME is true for UTF-8 text we can inline (CC fallback) or send as +// Anthropic PlainTextSource. Binary office formats are excluded. +func isTextFileMIME(mime, name string) bool { + switch strings.ToLower(mime) { + case "text/plain", "text/markdown", "text/csv", "text/tsv", "text/html", "text/xml", "text/css", + "application/json", "application/xml", "application/x-yaml", "text/yaml", "text/x-yaml": + return true + } + if strings.HasPrefix(strings.ToLower(mime), "text/") { + return true + } + switch strings.ToLower(path.Ext(name)) { + case ".txt", ".text", ".md", ".markdown", ".csv", ".tsv", ".html", ".htm", + ".xml", ".json", ".yaml", ".yml", ".css", + ".py", ".js", ".mjs", ".ts", ".go", ".java", ".c", ".cc", ".cpp", ".h", ".rb", ".sh", ".sql": + return true + } + return false +} + +func isAnthropicPlainText(mime, name string) bool { + return isTextFileMIME(mime, name) +} + +// inlineFileText wraps file bytes as a labeled text chunk for APIs that cannot +// take the file natively (Chat Completions non-PDF). +func inlineFileText(name string, data []byte) string { + if name == "" { + name = "file" + } + return fmt.Sprintf("[file: %s]\n%s", name, string(data)) +} + +// openAIFilename ensures OpenAI gets a filename (it uses the extension for type detection). +func openAIFilename(name, mime string) string { + if name != "" { + return name + } + switch strings.ToLower(mime) { + case "application/pdf": + return "document.pdf" + case "text/plain": + return "document.txt" + case "text/markdown": + return "document.md" + case "text/csv", "application/csv": + return "document.csv" + case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": + return "document.docx" + case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": + return "document.xlsx" + case "application/vnd.openxmlformats-officedocument.presentationml.presentation": + return "document.pptx" + case "application/json": + return "document.json" + default: + return "document" + } +} + +// bedrockImageFormat maps image MIME → Bedrock ImageFormat. Empty if unsupported. +func bedrockImageFormat(mime string) string { + switch strings.ToLower(mime) { + case "image/png": + return "png" + case "image/jpeg", "image/jpg": + return "jpeg" + case "image/gif": + return "gif" + case "image/webp": + return "webp" + default: + return "" + } +} + +// bedrockDocumentFormat maps document MIME → Bedrock DocumentFormat. Empty if unsupported. +func bedrockDocumentFormat(mime, name string) string { + switch strings.ToLower(mime) { + case "application/pdf": + return "pdf" + case "text/csv": + return "csv" + case "application/msword": + return "doc" + case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": + return "docx" + case "application/vnd.ms-excel": + return "xls" + case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": + return "xlsx" + case "text/html": + return "html" + case "text/plain": + return "txt" + case "text/markdown": + return "md" + } + // Fallback: extension from display name. + switch strings.ToLower(path.Ext(name)) { + case ".pdf": + return "pdf" + case ".csv": + return "csv" + case ".doc": + return "doc" + case ".docx": + return "docx" + case ".xls": + return "xls" + case ".xlsx": + return "xlsx" + case ".html", ".htm": + return "html" + case ".txt": + return "txt" + case ".md", ".markdown": + return "md" + } + return "" +} + +// bedrockSafeDocName keeps only chars Bedrock accepts in DocumentBlock.Name +// (alphanumeric, single spaces, -()[]; max 200). Neutral renaming is left to callers. +func bedrockSafeDocName(name string) string { + if name == "" { + return "document" + } + var b strings.Builder + prevSpace := false + for _, r := range name { + switch { + case (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || + r == '-' || r == '(' || r == ')' || r == '[' || r == ']': + b.WriteRune(r) + prevSpace = false + case r == ' ' || r == '_' || r == '.': + if !prevSpace && b.Len() > 0 { + b.WriteByte(' ') + prevSpace = true + } + } + } + out := strings.TrimSpace(b.String()) + if out == "" { + return "document" + } + if rs := []rune(out); len(rs) > 200 { + return string(rs[:200]) + } + return out +} diff --git a/go/adk/pkg/models/file_parts_test.go b/go/adk/pkg/models/file_parts_test.go new file mode 100644 index 000000000..dbbbd63e9 --- /dev/null +++ b/go/adk/pkg/models/file_parts_test.go @@ -0,0 +1,110 @@ +package models + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types" + "google.golang.org/genai" +) + +func TestGenaiContentsToOpenAIMessages_FileParts(t *testing.T) { + // Chat Completions: PDF → file part; text → inlined (file_data is PDF-only). + msgs, _ := genaiContentsToOpenAIMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/pdf", Data: []byte("%PDF"), DisplayName: "doc.pdf"}}, + {InlineData: &genai.Blob{MIMEType: "text/plain", Data: []byte("hello"), DisplayName: "notes.txt"}}, + }, + }}, nil) + b, err := json.Marshal(msgs[0].OfUser) + if err != nil { + t.Fatal(err) + } + s := string(b) + if !strings.Contains(s, `"type":"file"`) || !strings.Contains(s, "application/pdf") { + t.Fatalf("missing pdf file part: %s", s) + } + if !strings.Contains(s, "[file: notes.txt]") || !strings.Contains(s, "hello") { + t.Fatalf("expected inlined txt: %s", s) + } +} + +func TestGenaiContentsToResponsesInput_FileParts(t *testing.T) { + // Responses input_file accepts PDF and text. + input, _ := genaiContentsToResponsesInput([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "text/plain", Data: []byte("hello"), DisplayName: "notes.txt"}}, + }, + }}, nil) + b, _ := json.Marshal(input[0].OfMessage) + if !strings.Contains(string(b), "input_file") || !strings.Contains(string(b), "notes.txt") { + t.Fatalf("missing input_file: %s", b) + } +} + +func TestGenaiContentsToAnthropicMessages_FileParts(t *testing.T) { + msgs, _ := genaiContentsToAnthropicMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/pdf", Data: []byte("%PDF"), DisplayName: "doc.pdf"}}, + }, + }}, nil) + b, _ := json.Marshal(msgs[0]) + if !strings.Contains(string(b), `"type":"document"`) { + t.Fatalf("missing document: %s", b) + } +} + +func TestConvertGenaiContentsToBedrockMessages_FileParts(t *testing.T) { + // Document-only user turns must also carry a text block (Bedrock Converse API). + msgs, _ := convertGenaiContentsToBedrockMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/pdf", Data: []byte("%PDF"), DisplayName: "doc.pdf"}}, + }, + }}, nil, nil) + if len(msgs) != 1 { + t.Fatalf("msgs=%#v", msgs) + } + var hasDoc, hasText bool + for _, block := range msgs[0].Content { + switch block.(type) { + case *types.ContentBlockMemberDocument: + hasDoc = true + case *types.ContentBlockMemberText: + hasText = true + } + } + if !hasDoc || !hasText { + t.Fatalf("want document+text, got %#v", msgs[0].Content) + } +} + +func TestConvertGenaiContentsToOllamaMessages_FileParts(t *testing.T) { + msgs, _ := convertGenaiContentsToOllamaMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {Text: "what"}, + {InlineData: &genai.Blob{MIMEType: "image/png", Data: []byte{0x89, 0x50}}}, + }, + }}, nil) + if len(msgs) != 1 || len(msgs[0].Images) != 1 { + t.Fatalf("msgs=%#v", msgs) + } +} + +func TestGenaiContentsToOrchTemplate_FileParts(t *testing.T) { + msgs, _ := genaiContentsToOrchTemplate([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/pdf", Data: []byte("%PDF"), DisplayName: "a.pdf"}}, + }, + }}, nil) + content, _ := msgs[0]["content"].(string) + if !strings.Contains(content, "unsupported file: a.pdf") { + t.Fatalf("content=%q", content) + } +} diff --git a/go/adk/pkg/models/ollama_adk.go b/go/adk/pkg/models/ollama_adk.go index 82f443536..8f049e6d3 100644 --- a/go/adk/pkg/models/ollama_adk.go +++ b/go/adk/pkg/models/ollama_adk.go @@ -255,6 +255,7 @@ func convertGenaiContentsToOllamaMessages(contents []*genai.Content, config *gen } var textParts []string + var images []api.ImageData var toolCalls []api.ToolCall var toolResults []struct { content string @@ -296,6 +297,22 @@ func convertGenaiContentsToOllamaMessages(contents []*genai.Content, config *gen }{content: content}) continue } + + if part.InlineData != nil { + mime := part.InlineData.MIMEType + name := blobName(part.InlineData) + if isImageMIME(mime) { + images = append(images, api.ImageData(part.InlineData.Data)) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } + continue + } + + if part.FileData != nil { + textParts = append(textParts, unsupportedFileNote(fileDataName(part.FileData), part.FileData.MIMEType)) + continue + } } // Build message based on what we found @@ -319,8 +336,7 @@ func convertGenaiContentsToOllamaMessages(contents []*genai.Content, config *gen } } - if len(textParts) > 0 { - // Regular text message + if len(textParts) > 0 || len(images) > 0 { // Check if this is a system message if content.Role == "system" { systemInstruction = strings.Join(textParts, "\n") @@ -328,6 +344,7 @@ func convertGenaiContentsToOllamaMessages(contents []*genai.Content, config *gen msg := api.Message{ Role: role, Content: strings.Join(textParts, "\n"), + Images: images, } messages = append(messages, msg) } diff --git a/go/adk/pkg/models/openai_adk.go b/go/adk/pkg/models/openai_adk.go index f1bcce565..30b2112e2 100644 --- a/go/adk/pkg/models/openai_adk.go +++ b/go/adk/pkg/models/openai_adk.go @@ -248,7 +248,7 @@ func genaiContentsToOpenAIMessages(contents []*genai.Content, config *genai.Gene role := strings.TrimSpace(content.Role) var textParts []string var functionCalls []*genai.FunctionCall - var imageParts []openai.ChatCompletionContentPartImageImageURLParam + var contentParts []openai.ChatCompletionContentPartUnionParam for _, part := range content.Parts { if part == nil { @@ -258,10 +258,29 @@ func genaiContentsToOpenAIMessages(contents []*genai.Content, config *genai.Gene textParts = append(textParts, part.Text) } else if part.FunctionCall != nil { functionCalls = append(functionCalls, part.FunctionCall) - } else if part.InlineData != nil && strings.HasPrefix(part.InlineData.MIMEType, "image/") { - imageParts = append(imageParts, openai.ChatCompletionContentPartImageImageURLParam{ - URL: fmt.Sprintf("data:%s;base64,%s", part.InlineData.MIMEType, base64.StdEncoding.EncodeToString(part.InlineData.Data)), - }) + } else if part.InlineData != nil { + mime := part.InlineData.MIMEType + name := blobName(part.InlineData) + if isImageMIME(mime) { + contentParts = append(contentParts, openai.ImageContentPart(openai.ChatCompletionContentPartImageImageURLParam{ + URL: dataURI(mime, part.InlineData.Data), + })) + } else if isOpenAIPDF(mime, name) { + // Chat Completions file_data accepts PDF only (not txt/docx/…). + file := openai.ChatCompletionContentPartFileFileParam{ + FileData: param.NewOpt(dataURI("application/pdf", part.InlineData.Data)), + Filename: param.NewOpt(openAIFilename(name, "application/pdf")), + } + contentParts = append(contentParts, openai.FileContentPart(file)) + } else if isTextFileMIME(mime, name) { + // Fallback: inline text so .txt still reaches the model on CC. + textParts = append(textParts, inlineFileText(name, part.InlineData.Data)) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } + } else if part.FileData != nil { + // Chat Completions has no file_url; only Responses does. + textParts = append(textParts, unsupportedFileNote(fileDataName(part.FileData), part.FileData.MIMEType)) } } @@ -311,14 +330,12 @@ func genaiContentsToOpenAIMessages(contents []*genai.Content, config *genai.Gene messages = append(messages, openai.ChatCompletionMessageParamUnion{OfAssistant: &asst}) messages = append(messages, toolResponseMessages...) } else { - if len(imageParts) > 0 { - parts := make([]openai.ChatCompletionContentPartUnionParam, 0, len(textParts)+len(imageParts)) + if len(contentParts) > 0 { + parts := make([]openai.ChatCompletionContentPartUnionParam, 0, len(textParts)+len(contentParts)) for _, t := range textParts { parts = append(parts, openai.TextContentPart(t)) } - for _, img := range imageParts { - parts = append(parts, openai.ImageContentPart(img)) - } + parts = append(parts, contentParts...) messages = append(messages, openai.UserMessage(parts)) } else if len(textParts) > 0 { messages = append(messages, openai.UserMessage(strings.Join(textParts, "\n"))) diff --git a/go/adk/pkg/models/openai_responses.go b/go/adk/pkg/models/openai_responses.go index 523797277..aad6dbe1b 100644 --- a/go/adk/pkg/models/openai_responses.go +++ b/go/adk/pkg/models/openai_responses.go @@ -3,7 +3,6 @@ package models import ( "context" - "encoding/base64" "encoding/json" "fmt" "maps" @@ -97,7 +96,7 @@ func genaiContentsToResponsesInput(contents []*genai.Content, config *genai.Gene role := strings.TrimSpace(content.Role) var textParts []string var functionCalls []*genai.FunctionCall - var imageURLs []string + var contentParts responses.ResponseInputMessageContentListParam for _, part := range content.Parts { if part == nil { @@ -107,12 +106,42 @@ func genaiContentsToResponsesInput(contents []*genai.Content, config *genai.Gene textParts = append(textParts, part.Text) } else if part.FunctionCall != nil { functionCalls = append(functionCalls, part.FunctionCall) - } else if part.InlineData != nil && strings.HasPrefix(part.InlineData.MIMEType, "image/") { - imageURLs = append(imageURLs, fmt.Sprintf( - "data:%s;base64,%s", - part.InlineData.MIMEType, - base64.StdEncoding.EncodeToString(part.InlineData.Data), - )) + } else if part.InlineData != nil { + mime := part.InlineData.MIMEType + name := blobName(part.InlineData) + if isImageMIME(mime) { + img := responses.ResponseInputContentParamOfInputImage(responses.ResponseInputImageDetailAuto) + img.OfInputImage.ImageURL = param.NewOpt(dataURI(mime, part.InlineData.Data)) + contentParts = append(contentParts, img) + } else if isOpenAIResponsesFileMIME(mime, name) { + // Responses input_file: PDF + txt/md/csv/docx/xlsx/pptx/… + fileMIME := mime + if isOpenAIPDF(mime, name) { + fileMIME = "application/pdf" + } + fname := openAIFilename(name, fileMIME) + file := responses.ResponseInputFileParam{ + FileData: param.NewOpt(dataURI(fileMIME, part.InlineData.Data)), + Filename: param.NewOpt(fname), + } + contentParts = append(contentParts, responses.ResponseInputContentUnionParam{OfInputFile: &file}) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } + } else if part.FileData != nil { + mime := part.FileData.MIMEType + name := fileDataName(part.FileData) + uri := part.FileData.FileURI + // file_url is Responses-only (Chat Completions docs: not supported). + if uri != "" && isOpenAIResponsesFileMIME(mime, name) { + file := responses.ResponseInputFileParam{ + FileURL: param.NewOpt(uri), + Filename: param.NewOpt(openAIFilename(name, mime)), + } + contentParts = append(contentParts, responses.ResponseInputContentUnionParam{OfInputFile: &file}) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } } } @@ -139,7 +168,7 @@ func genaiContentsToResponsesInput(contents []*genai.Content, config *genai.Gene continue } - if len(textParts) == 0 && len(imageURLs) == 0 { + if len(textParts) == 0 && len(contentParts) == 0 { continue } @@ -148,16 +177,12 @@ func genaiContentsToResponsesInput(contents []*genai.Content, config *genai.Gene msgRole = responses.EasyInputMessageRoleAssistant } - if len(imageURLs) > 0 { - parts := make(responses.ResponseInputMessageContentListParam, 0, len(textParts)+len(imageURLs)) + if len(contentParts) > 0 { + parts := make(responses.ResponseInputMessageContentListParam, 0, len(textParts)+len(contentParts)) for _, t := range textParts { parts = append(parts, responses.ResponseInputContentParamOfInputText(t)) } - for _, url := range imageURLs { - img := responses.ResponseInputContentParamOfInputImage(responses.ResponseInputImageDetailAuto) - img.OfInputImage.ImageURL = param.NewOpt(url) - parts = append(parts, img) - } + parts = append(parts, contentParts...) input = append(input, responses.ResponseInputItemParamOfMessage(parts, msgRole)) } else { input = append(input, responses.ResponseInputItemParamOfMessage(strings.Join(textParts, "\n"), msgRole)) diff --git a/go/adk/pkg/models/sapaicore_adk.go b/go/adk/pkg/models/sapaicore_adk.go index ae4598b42..193788d39 100644 --- a/go/adk/pkg/models/sapaicore_adk.go +++ b/go/adk/pkg/models/sapaicore_adk.go @@ -204,6 +204,10 @@ func genaiContentsToOrchTemplate(contents []*genai.Content, config *genai.Genera textParts = append(textParts, part.Text) } else if part.FunctionCall != nil { functionCalls = append(functionCalls, part.FunctionCall) + } else if part.InlineData != nil { + textParts = append(textParts, unsupportedFileNote(blobName(part.InlineData), part.InlineData.MIMEType)) + } else if part.FileData != nil { + textParts = append(textParts, unsupportedFileNote(fileDataName(part.FileData), part.FileData.MIMEType)) } } diff --git a/ui/src/components/chat/ChatInterface.tsx b/ui/src/components/chat/ChatInterface.tsx index b3c92b3af..43d463e45 100644 --- a/ui/src/components/chat/ChatInterface.tsx +++ b/ui/src/components/chat/ChatInterface.tsx @@ -2,7 +2,7 @@ import type React from "react"; import { useState, useRef, useEffect, useMemo, useCallback } from "react"; -import { ArrowBigUp, X, Loader2, Mic, Square } from "lucide-react"; +import { ArrowBigUp, X, Loader2, Mic, Square, Paperclip, FileIcon, ImageIcon } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Tooltip, @@ -45,6 +45,63 @@ import { type SessionGuardOptions, } from "@/lib/chatSessionGuard"; +// Soft client caps aligned with the strictest common provider (Bedrock Converse): +// images ≤ 3.75 MB and ≤ 8000×8000 px; documents ≤ 4.5 MB; ≤ 5 files per message. +const MAX_CHAT_IMAGE_BYTES = Math.floor(3.75 * 1024 * 1024); +const MAX_CHAT_DOC_BYTES = Math.floor(4.5 * 1024 * 1024); +const MAX_CHAT_IMAGE_PX = 8000; +const MAX_CHAT_FILES = 5; + +type PendingChatFile = { + id: string; + name: string; + mimeType: string; + bytes: string; // raw base64 (no data: prefix) +}; + +function readFileAsDataURL(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(String(reader.result ?? "")); + reader.onerror = () => reject(reader.error ?? new Error("read failed")); + reader.readAsDataURL(file); + }); +} + +/** Returns false when dimensions exceed maxPx. Unreadable images pass (provider rejects). */ +async function imageWithinPixelLimit(file: File, maxPx: number): Promise { + try { + const bmp = await createImageBitmap(file); + const ok = bmp.width <= maxPx && bmp.height <= maxPx; + bmp.close(); + return ok; + } catch { + return true; + } +} + +/** Build A2A message parts: optional text + FileParts (wire shape a2a-go accepts). */ +function buildUserMessageParts(text: string, files: PendingChatFile[]): Message["parts"] { + // SDK Part typings are protobuf-shaped; chat already sends JSON {kind,text} parts. + const parts: Message["parts"] = []; + const push = (part: unknown) => parts.push(part as Message["parts"][number]); + const trimmed = text.trim(); + if (trimmed) { + push({ kind: "text", text: trimmed }); + } + for (const file of files) { + push({ + kind: "file", + file: { + bytes: file.bytes, + mimeType: file.mimeType, + name: file.name, + }, + }); + } + return parts; +} + interface ChatInterfaceProps { selectedAgentName: string; selectedNamespace: string; @@ -61,7 +118,15 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se const substrateSandbox = useChatSubstrateSandbox(); const router = useRouter(); const containerRef = useRef(null); + const fileInputRef = useRef(null); + const fileDragDepthRef = useRef(0); const [currentInputMessage, setCurrentInputMessage] = useState(""); + const [pendingFiles, setPendingFiles] = useState([]); + const [isDraggingFiles, setIsDraggingFiles] = useState(false); + // File attach is Go declarative agents only (Python adapters still drop non-images). + const supportsFileAttach = + currentAgent.agent?.spec?.type === "Declarative" && + currentAgent.agent?.spec?.declarative?.runtime === "go"; const [chatStatus, setChatStatus] = useState("ready"); @@ -299,12 +364,16 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se userMessageText: string, options: { clearInput?: boolean; + /** When false, do not attach/clear staged files (MCP ui/message path). */ + attachPendingFiles?: boolean; restoreInputOnError?: boolean; errorLabel?: string; rethrowOnError?: boolean; } = {}, ) => { - if (!userMessageText.trim() || !selectedAgentName || !selectedNamespace) { + const attachPendingFiles = options.attachPendingFiles ?? true; + const filesToSend = attachPendingFiles ? pendingFiles : []; + if ((!userMessageText.trim() && filesToSend.length === 0) || !selectedAgentName || !selectedNamespace) { return; } if (chatStatus !== "ready") { @@ -316,10 +385,6 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se return; } - if (options.clearInput ?? true) { - setCurrentInputMessage(""); - } - // Cross-tab guard: fetch the latest session state before mutating anything. // Two cases: (1) another tab is still streaming — reconnect instead of sending; // (2) another tab completed a turn we haven't loaded — reload so the user sees @@ -333,10 +398,16 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se staleOrChanged: "New messages loaded — please review before sending", }, }); + // Don't clear staged files/input if the send was blocked. if (guardResult === "blocked") return; } - setCurrentInputMessage(""); + if (options.clearInput ?? true) { + setCurrentInputMessage(""); + } + if (attachPendingFiles) { + setPendingFiles([]); + } setChatStatus("thinking"); setStoredMessages(prev => [...prev, ...streamingMessages]); setStreamingMessages([]); @@ -347,16 +418,14 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se pendingTurnStatsRef.current = undefined; const messageId = uuidv4(); + const messageParts = buildUserMessageParts(userMessageText, filesToSend); // For new sessions or when no stored messages exist, show the user message immediately const userMessage: Message = { kind: "message", messageId, role: "user", - parts: [{ - kind: "text", - text: userMessageText - }], + parts: messageParts, contextId: guardSessionId, metadata: { timestamp: Date.now() @@ -385,7 +454,10 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se isCreatingSessionRef.current = true; setIsFirstMessage(true); - const sessionName = deriveSessionTitle(userMessageText); + const sessionName = + deriveSessionTitle(userMessageText) || + deriveSessionTitle(filesToSend[0]?.name ?? "") || + "New Chat"; const newSessionResponse = await createSession({ agent_ref: `${selectedNamespace}/${selectedAgentName}`, name: sessionName, @@ -395,6 +467,7 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se toast.error("Failed to create session"); setChatStatus("error"); setCurrentInputMessage(userMessageText); + if (attachPendingFiles) setPendingFiles(filesToSend); isCreatingSessionRef.current = false; return; } @@ -421,6 +494,7 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se toast.error("Error creating session"); setChatStatus("error"); setCurrentInputMessage(userMessageText); + if (attachPendingFiles) setPendingFiles(filesToSend); isCreatingSessionRef.current = false; return; } @@ -462,10 +536,14 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se messageId, contextId: currentSessionId, }); + a2aMessage.parts = messageParts; await streamA2AMessage(a2aMessage, { errorLabel: "Streaming failed", - onError: () => setCurrentInputMessage(userMessageText), + onError: () => { + setCurrentInputMessage(userMessageText); + if (attachPendingFiles) setPendingFiles(filesToSend); + }, sessionIdForWait: currentSessionId, }); } catch (error) { @@ -474,6 +552,7 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se setChatStatus("error"); if (options.restoreInputOnError ?? true) { setCurrentInputMessage(userMessageText); + if (attachPendingFiles) setPendingFiles(filesToSend); } if (options.rethrowOnError) { throw error; @@ -481,12 +560,106 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se } }; + const handleAttachFiles = async (fileList: FileList | null) => { + if (!fileList || fileList.length === 0) return; + // Read first; enforce MAX_CHAT_FILES in the state updater so overlapping + // picker/drop ops cannot both append past the cap. + const candidates: PendingChatFile[] = []; + for (const file of Array.from(fileList)) { + const isImage = file.type.startsWith("image/"); + const maxBytes = isImage ? MAX_CHAT_IMAGE_BYTES : MAX_CHAT_DOC_BYTES; + if (file.size > maxBytes) { + const mb = (maxBytes / (1024 * 1024)).toFixed(2); + toast.error(`${file.name} exceeds the ${mb}MB ${isImage ? "image" : "file"} limit`); + continue; + } + if (isImage && !(await imageWithinPixelLimit(file, MAX_CHAT_IMAGE_PX))) { + toast.error(`${file.name} exceeds the ${MAX_CHAT_IMAGE_PX}×${MAX_CHAT_IMAGE_PX}px image limit`); + continue; + } + try { + const dataUrl = await readFileAsDataURL(file); + const comma = dataUrl.indexOf(","); + const bytes = comma >= 0 ? dataUrl.slice(comma + 1) : dataUrl; + candidates.push({ + id: uuidv4(), + name: file.name, + mimeType: file.type || "application/octet-stream", + bytes, + }); + } catch { + toast.error(`Failed to read ${file.name}`); + } + } + if (candidates.length > 0) { + let overflow = false; + setPendingFiles(prev => { + const room = MAX_CHAT_FILES - prev.length; + if (room <= 0) { + overflow = true; + return prev; + } + if (candidates.length > room) { + overflow = true; + return [...prev, ...candidates.slice(0, room)]; + } + return [...prev, ...candidates]; + }); + if (overflow) { + toast.error(`You can attach at most ${MAX_CHAT_FILES} files per message`); + } + } + if (fileInputRef.current) { + fileInputRef.current.value = ""; + } + }; + + const resetFileDrag = () => { + fileDragDepthRef.current = 0; + setIsDraggingFiles(false); + }; + + const handleComposerDragEnter = (e: React.DragEvent) => { + if (!supportsFileAttach || chatStatus !== "ready") return; + if (![...e.dataTransfer.types].includes("Files")) return; + e.preventDefault(); + e.stopPropagation(); + fileDragDepthRef.current += 1; + setIsDraggingFiles(true); + }; + + const handleComposerDragLeave = (e: React.DragEvent) => { + if (!supportsFileAttach) return; + e.preventDefault(); + e.stopPropagation(); + fileDragDepthRef.current = Math.max(0, fileDragDepthRef.current - 1); + if (fileDragDepthRef.current === 0) { + setIsDraggingFiles(false); + } + }; + + const handleComposerDragOver = (e: React.DragEvent) => { + if (!supportsFileAttach || chatStatus !== "ready") return; + if (![...e.dataTransfer.types].includes("Files")) return; + e.preventDefault(); + e.stopPropagation(); + e.dataTransfer.dropEffect = "copy"; + }; + + const handleComposerDrop = (e: React.DragEvent) => { + if (!supportsFileAttach || chatStatus !== "ready") return; + e.preventDefault(); + e.stopPropagation(); + resetFileDrag(); + void handleAttachFiles(e.dataTransfer.files); + }; + const handleSendMessage = async (e: React.FormEvent) => { e.preventDefault(); if (isListening) { stopListening(); } - if (!currentInputMessage.trim()) { + if (!currentInputMessage.trim() && pendingFiles.length === 0) { return; } @@ -499,6 +672,7 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se const handleMcpAppSendMessage = async (text: string) => { await sendChatMessageText(text, { clearInput: false, + attachPendingFiles: false, // don't send/clear user-staged attachments restoreInputOnError: false, errorLabel: "MCP app message failed", rethrowOnError: true, @@ -986,7 +1160,12 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se const handleKeyDown = (e: React.KeyboardEvent) => { if ((e.metaKey || e.ctrlKey) && e.key === "Enter") { e.preventDefault(); - if (currentInputMessage.trim() && selectedAgentName && selectedNamespace && chatStatus === "ready") { + if ( + (currentInputMessage.trim() || pendingFiles.length > 0) && + selectedAgentName && + selectedNamespace && + chatStatus === "ready" + ) { handleSendMessage(e); } } @@ -1100,18 +1279,92 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se -
+ + {isDraggingFiles && ( +
+ Drop files to attach +
+ )}