From 4d978028fba90379bd7d30bc784169ba597e86c1 Mon Sep 17 00:00:00 2001 From: Jet Chiang Date: Tue, 11 Aug 2026 20:40:13 -0400 Subject: [PATCH 1/6] ui and model converters Signed-off-by: Jet Chiang --- go/adk/pkg/models/anthropic_adk.go | 43 +-- go/adk/pkg/models/bedrock.go | 34 +++ go/adk/pkg/models/file_parts.go | 145 ++++++++++ go/adk/pkg/models/file_parts_test.go | 320 +++++++++++++++++++++++ go/adk/pkg/models/ollama_adk.go | 21 +- go/adk/pkg/models/openai_adk.go | 35 ++- go/adk/pkg/models/openai_responses.go | 54 ++-- ui/src/components/chat/ChatInterface.tsx | 240 ++++++++++++++++- ui/src/components/chat/ChatMessage.tsx | 45 +++- 9 files changed, 876 insertions(+), 61 deletions(-) create mode 100644 go/adk/pkg/models/file_parts.go create mode 100644 go/adk/pkg/models/file_parts_test.go diff --git a/go/adk/pkg/models/anthropic_adk.go b/go/adk/pkg/models/anthropic_adk.go index c51a7da98..d42b66228 100644 --- a/go/adk/pkg/models/anthropic_adk.go +++ b/go/adk/pkg/models/anthropic_adk.go @@ -130,10 +130,7 @@ func genaiContentsToAnthropicMessages(contents []*genai.Content, config *genai.G var textParts []string var functionCalls []*genai.FunctionCall - var imageParts []struct { - mimeType string - data []byte - } + var mediaBlocks []anthropic.ContentBlockParamUnion for _, part := range content.Parts { if part == nil { @@ -143,11 +140,31 @@ func genaiContentsToAnthropicMessages(contents []*genai.Content, config *genai.G textParts = append(textParts, part.Text) } else if part.FunctionCall != nil { functionCalls = append(functionCalls, part.FunctionCall) - } else if part.InlineData != nil && strings.HasPrefix(part.InlineData.MIMEType, "image/") { - imageParts = append(imageParts, struct { - mimeType string - data []byte - }{part.InlineData.MIMEType, part.InlineData.Data}) + } else if part.InlineData != nil { + mime := part.InlineData.MIMEType + name := blobName(part.InlineData) + if isImageMIME(mime) { + mediaBlocks = append(mediaBlocks, anthropic.NewImageBlockBase64(mime, base64.StdEncoding.EncodeToString(part.InlineData.Data))) + } else if isAnthropicPDF(mime) { + mediaBlocks = append(mediaBlocks, anthropic.NewDocumentBlock(anthropic.Base64PDFSourceParam{ + Data: base64.StdEncoding.EncodeToString(part.InlineData.Data), + })) + } else if isAnthropicPlainText(mime) { + mediaBlocks = append(mediaBlocks, anthropic.NewDocumentBlock(anthropic.PlainTextSourceParam{ + Data: string(part.InlineData.Data), + })) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } + } else if part.FileData != nil { + mime := part.FileData.MIMEType + name := fileDataName(part.FileData) + uri := part.FileData.FileURI + if uri != "" && isAnthropicPDF(mime) { + mediaBlocks = append(mediaBlocks, anthropic.NewDocumentBlock(anthropic.URLPDFSourceParam{URL: uri})) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } } } @@ -188,13 +205,7 @@ func genaiContentsToAnthropicMessages(contents []*genai.Content, config *genai.G } else { // Regular user message var contentBlocks []anthropic.ContentBlockParamUnion - - // Add images first - for _, img := range imageParts { - contentBlocks = append(contentBlocks, anthropic.NewImageBlockBase64(img.mimeType, base64.StdEncoding.EncodeToString(img.data))) - } - - // Add text + contentBlocks = append(contentBlocks, mediaBlocks...) if len(textParts) > 0 { contentBlocks = append(contentBlocks, anthropic.NewTextBlock(strings.Join(textParts, "\n"))) } diff --git a/go/adk/pkg/models/bedrock.go b/go/adk/pkg/models/bedrock.go index 4af7916b1..994d60c98 100644 --- a/go/adk/pkg/models/bedrock.go +++ b/go/adk/pkg/models/bedrock.go @@ -667,6 +667,40 @@ func convertGenaiContentsToBedrockMessages(contents []*genai.Content, nameMap ma continue } + if part.InlineData != nil { + mime := part.InlineData.MIMEType + name := blobName(part.InlineData) + if fmtStr := bedrockImageFormat(mime); fmtStr != "" { + contentBlocks = append(contentBlocks, &types.ContentBlockMemberImage{ + Value: types.ImageBlock{ + Format: types.ImageFormat(fmtStr), + Source: &types.ImageSourceMemberBytes{Value: part.InlineData.Data}, + }, + }) + } else if docFmt := bedrockDocumentFormat(mime, name); docFmt != "" { + contentBlocks = append(contentBlocks, &types.ContentBlockMemberDocument{ + Value: types.DocumentBlock{ + Format: types.DocumentFormat(docFmt), + Name: aws.String(bedrockSafeDocName(name)), + Source: &types.DocumentSourceMemberBytes{Value: part.InlineData.Data}, + }, + }) + } else { + contentBlocks = append(contentBlocks, &types.ContentBlockMemberText{ + Value: unsupportedFileNote(name, mime), + }) + } + continue + } + + if part.FileData != nil { + // Bedrock Converse document/image blocks are bytes-only. + contentBlocks = append(contentBlocks, &types.ContentBlockMemberText{ + Value: unsupportedFileNote(fileDataName(part.FileData), part.FileData.MIMEType), + }) + continue + } + // Handle function call (tool use in Bedrock terminology). // Use the sanitized name from nameMap so Bedrock can correlate the // tool call with the tool spec sent in the same request. diff --git a/go/adk/pkg/models/file_parts.go b/go/adk/pkg/models/file_parts.go new file mode 100644 index 000000000..2eaca9b92 --- /dev/null +++ b/go/adk/pkg/models/file_parts.go @@ -0,0 +1,145 @@ +package models + +import ( + "encoding/base64" + "fmt" + "path" + "strings" + + "google.golang.org/genai" +) + +// unsupportedFileNote is appended as text when a file part cannot be mapped +// to a provider-native input. Prefer this over silently dropping the part. +func unsupportedFileNote(name, mime string) string { + if name == "" { + name = "unnamed" + } + if mime == "" { + mime = "unknown" + } + return fmt.Sprintf("[unsupported file: %s (%s)]", name, mime) +} + +func dataURI(mime string, data []byte) string { + return fmt.Sprintf("data:%s;base64,%s", mime, base64.StdEncoding.EncodeToString(data)) +} + +func blobName(b *genai.Blob) string { + if b == nil { + return "" + } + return b.DisplayName +} + +func fileDataName(f *genai.FileData) string { + if f == nil { + return "" + } + return f.DisplayName +} + +func isImageMIME(mime string) bool { + return strings.HasPrefix(mime, "image/") +} + +func isOpenAIFileMIME(mime string) bool { + return mime == "application/pdf" +} + +func isAnthropicPDF(mime string) bool { + return mime == "application/pdf" +} + +func isAnthropicPlainText(mime string) bool { + return mime == "text/plain" || mime == "text/markdown" +} + +// bedrockImageFormat maps image MIME → Bedrock ImageFormat. Empty if unsupported. +func bedrockImageFormat(mime string) string { + switch strings.ToLower(mime) { + case "image/png": + return "png" + case "image/jpeg", "image/jpg": + return "jpeg" + case "image/gif": + return "gif" + case "image/webp": + return "webp" + default: + return "" + } +} + +// bedrockDocumentFormat maps document MIME → Bedrock DocumentFormat. Empty if unsupported. +func bedrockDocumentFormat(mime, name string) string { + switch strings.ToLower(mime) { + case "application/pdf": + return "pdf" + case "text/csv": + return "csv" + case "application/msword": + return "doc" + case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": + return "docx" + case "application/vnd.ms-excel": + return "xls" + case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": + return "xlsx" + case "text/html": + return "html" + case "text/plain": + return "txt" + case "text/markdown": + return "md" + } + // Fallback: extension from display name. + switch strings.ToLower(path.Ext(name)) { + case ".pdf": + return "pdf" + case ".csv": + return "csv" + case ".doc": + return "doc" + case ".docx": + return "docx" + case ".xls": + return "xls" + case ".xlsx": + return "xlsx" + case ".html", ".htm": + return "html" + case ".txt": + return "txt" + case ".md", ".markdown": + return "md" + } + return "" +} + +// bedrockSafeDocName keeps only chars Bedrock accepts in DocumentBlock.Name. +func bedrockSafeDocName(name string) string { + if name == "" { + return "document" + } + var b strings.Builder + prevSpace := false + for _, r := range name { + switch { + case (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || + r == '-' || r == '(' || r == ')' || r == '[' || r == ']': + b.WriteRune(r) + prevSpace = false + case r == ' ' || r == '_' || r == '.': + if !prevSpace && b.Len() > 0 { + b.WriteByte(' ') + prevSpace = true + } + } + } + out := strings.TrimSpace(b.String()) + if out == "" { + return "document" + } + return out +} diff --git a/go/adk/pkg/models/file_parts_test.go b/go/adk/pkg/models/file_parts_test.go new file mode 100644 index 000000000..43d99ba6e --- /dev/null +++ b/go/adk/pkg/models/file_parts_test.go @@ -0,0 +1,320 @@ +package models + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types" + "google.golang.org/genai" +) + +func TestUnsupportedFileNote(t *testing.T) { + got := unsupportedFileNote("report.pdf", "application/pdf") + if got != "[unsupported file: report.pdf (application/pdf)]" { + t.Fatalf("got %q", got) + } + if unsupportedFileNote("", "") != "[unsupported file: unnamed (unknown)]" { + t.Fatalf("empty defaults: %q", unsupportedFileNote("", "")) + } +} + +func TestBedrockDocumentFormat(t *testing.T) { + tests := []struct { + mime, name, want string + }{ + {"application/pdf", "a.pdf", "pdf"}, + {"text/plain", "a.txt", "txt"}, + {"application/zip", "a.zip", ""}, + {"", "notes.md", "md"}, + } + for _, tt := range tests { + if got := bedrockDocumentFormat(tt.mime, tt.name); got != tt.want { + t.Errorf("bedrockDocumentFormat(%q,%q)=%q want %q", tt.mime, tt.name, got, tt.want) + } + } +} + +func TestGenaiContentsToOpenAIMessages_FileParts(t *testing.T) { + t.Run("pdf", func(t *testing.T) { + msgs, _ := genaiContentsToOpenAIMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {Text: "summarize"}, + {InlineData: &genai.Blob{MIMEType: "application/pdf", Data: []byte("%PDF"), DisplayName: "doc.pdf"}}, + }, + }}, nil) + if len(msgs) != 1 || msgs[0].OfUser == nil { + t.Fatalf("msgs = %#v", msgs) + } + b, err := json.Marshal(msgs[0].OfUser) + if err != nil { + t.Fatal(err) + } + s := string(b) + if !strings.Contains(s, `"type":"file"`) || !strings.Contains(s, "file_data") || !strings.Contains(s, "application/pdf") { + t.Fatalf("missing file part: %s", s) + } + }) + + t.Run("image still works", func(t *testing.T) { + msgs, _ := genaiContentsToOpenAIMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "image/png", Data: []byte{0x89, 0x50}}}, + }, + }}, nil) + b, _ := json.Marshal(msgs[0].OfUser) + if !strings.Contains(string(b), "image_url") || !strings.Contains(string(b), "data:image/png;base64,") { + t.Fatalf("missing image: %s", b) + } + }) + + t.Run("unsupported mime becomes note", func(t *testing.T) { + msgs, _ := genaiContentsToOpenAIMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/zip", Data: []byte("PK"), DisplayName: "a.zip"}}, + }, + }}, nil) + b, _ := json.Marshal(msgs[0].OfUser) + if strings.Contains(string(b), `"type":"file"`) { + t.Fatalf("should not send zip as file: %s", b) + } + if !strings.Contains(string(b), "unsupported file: a.zip") { + t.Fatalf("missing note: %s", b) + } + }) + + t.Run("filedata uri unsupported on chat completions", func(t *testing.T) { + msgs, _ := genaiContentsToOpenAIMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {FileData: &genai.FileData{MIMEType: "application/pdf", FileURI: "https://example.com/a.pdf", DisplayName: "a.pdf"}}, + }, + }}, nil) + b, _ := json.Marshal(msgs[0].OfUser) + if !strings.Contains(string(b), "unsupported file: a.pdf") { + t.Fatalf("missing note: %s", b) + } + }) +} + +func TestGenaiContentsToResponsesInput_FileParts(t *testing.T) { + t.Run("pdf inline", func(t *testing.T) { + input, _ := genaiContentsToResponsesInput([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {Text: "read this"}, + {InlineData: &genai.Blob{MIMEType: "application/pdf", Data: []byte("%PDF"), DisplayName: "doc.pdf"}}, + }, + }}, nil) + b, err := json.Marshal(input[0].OfMessage) + if err != nil { + t.Fatal(err) + } + s := string(b) + if !strings.Contains(s, "input_file") || !strings.Contains(s, "file_data") { + t.Fatalf("missing input_file: %s", s) + } + }) + + t.Run("pdf uri", func(t *testing.T) { + input, _ := genaiContentsToResponsesInput([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {FileData: &genai.FileData{MIMEType: "application/pdf", FileURI: "https://example.com/a.pdf", DisplayName: "a.pdf"}}, + }, + }}, nil) + b, _ := json.Marshal(input[0].OfMessage) + if !strings.Contains(string(b), "file_url") || !strings.Contains(string(b), "https://example.com/a.pdf") { + t.Fatalf("missing file_url: %s", b) + } + }) + + t.Run("unsupported mime", func(t *testing.T) { + input, _ := genaiContentsToResponsesInput([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/zip", Data: []byte("PK"), DisplayName: "a.zip"}}, + }, + }}, nil) + b, _ := json.Marshal(input[0].OfMessage) + if strings.Contains(string(b), "input_file") { + t.Fatalf("should not send zip: %s", b) + } + if !strings.Contains(string(b), "unsupported file: a.zip") { + t.Fatalf("missing note: %s", b) + } + }) +} + +func TestGenaiContentsToAnthropicMessages_FileParts(t *testing.T) { + t.Run("pdf", func(t *testing.T) { + msgs, _ := genaiContentsToAnthropicMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {Text: "summarize"}, + {InlineData: &genai.Blob{MIMEType: "application/pdf", Data: []byte("%PDF")}}, + }, + }}, nil) + if len(msgs) != 1 { + t.Fatalf("len=%d", len(msgs)) + } + b, err := json.Marshal(msgs[0]) + if err != nil { + t.Fatal(err) + } + s := string(b) + if !strings.Contains(s, `"type":"document"`) || !strings.Contains(s, "application/pdf") { + t.Fatalf("missing document: %s", s) + } + }) + + t.Run("plain text document", func(t *testing.T) { + msgs, _ := genaiContentsToAnthropicMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "text/plain", Data: []byte("hello")}}, + }, + }}, nil) + b, _ := json.Marshal(msgs[0]) + if !strings.Contains(string(b), `"type":"document"`) { + t.Fatalf("missing document: %s", b) + } + }) + + t.Run("pdf uri", func(t *testing.T) { + msgs, _ := genaiContentsToAnthropicMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {FileData: &genai.FileData{MIMEType: "application/pdf", FileURI: "https://example.com/a.pdf"}}, + }, + }}, nil) + b, _ := json.Marshal(msgs[0]) + if !strings.Contains(string(b), "https://example.com/a.pdf") { + t.Fatalf("missing url: %s", b) + } + }) + + t.Run("image still works", func(t *testing.T) { + msgs, _ := genaiContentsToAnthropicMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "image/png", Data: []byte{0x89, 0x50}}}, + }, + }}, nil) + b, _ := json.Marshal(msgs[0]) + if !strings.Contains(string(b), `"type":"image"`) { + t.Fatalf("missing image: %s", b) + } + }) + + t.Run("unsupported mime", func(t *testing.T) { + msgs, _ := genaiContentsToAnthropicMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/zip", Data: []byte("PK"), DisplayName: "a.zip"}}, + }, + }}, nil) + b, _ := json.Marshal(msgs[0]) + if strings.Contains(string(b), `"type":"document"`) { + t.Fatalf("should not send zip: %s", b) + } + if !strings.Contains(string(b), "unsupported file: a.zip") { + t.Fatalf("missing note: %s", b) + } + }) +} + +func TestConvertGenaiContentsToBedrockMessages_FileParts(t *testing.T) { + t.Run("pdf", func(t *testing.T) { + msgs, _ := convertGenaiContentsToBedrockMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/pdf", Data: []byte("%PDF"), DisplayName: "doc.pdf"}}, + }, + }}, nil, nil) + if len(msgs) != 1 || len(msgs[0].Content) != 1 { + t.Fatalf("msgs=%#v", msgs) + } + doc, ok := msgs[0].Content[0].(*types.ContentBlockMemberDocument) + if !ok { + t.Fatalf("want document, got %T", msgs[0].Content[0]) + } + if doc.Value.Format != types.DocumentFormatPdf { + t.Fatalf("format=%q", doc.Value.Format) + } + }) + + t.Run("image", func(t *testing.T) { + msgs, _ := convertGenaiContentsToBedrockMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "image/png", Data: []byte{0x89, 0x50}}}, + }, + }}, nil, nil) + if _, ok := msgs[0].Content[0].(*types.ContentBlockMemberImage); !ok { + t.Fatalf("want image, got %T", msgs[0].Content[0]) + } + }) + + t.Run("unsupported", func(t *testing.T) { + msgs, _ := convertGenaiContentsToBedrockMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/zip", Data: []byte("PK"), DisplayName: "a.zip"}}, + }, + }}, nil, nil) + text, ok := msgs[0].Content[0].(*types.ContentBlockMemberText) + if !ok || !strings.Contains(text.Value, "unsupported file: a.zip") { + t.Fatalf("want text note, got %#v", msgs[0].Content[0]) + } + }) +} + +func TestConvertGenaiContentsToOllamaMessages_FileParts(t *testing.T) { + t.Run("image", func(t *testing.T) { + msgs, _ := convertGenaiContentsToOllamaMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {Text: "what"}, + {InlineData: &genai.Blob{MIMEType: "image/png", Data: []byte{0x89, 0x50}}}, + }, + }}, nil) + if len(msgs) != 1 || len(msgs[0].Images) != 1 { + t.Fatalf("msgs=%#v", msgs) + } + }) + + t.Run("pdf note", func(t *testing.T) { + msgs, _ := convertGenaiContentsToOllamaMessages([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/pdf", Data: []byte("%PDF"), DisplayName: "a.pdf"}}, + }, + }}, nil) + if len(msgs) != 1 || !strings.Contains(msgs[0].Content, "unsupported file: a.pdf") { + t.Fatalf("msgs=%#v", msgs) + } + if len(msgs[0].Images) != 0 { + t.Fatalf("should not attach pdf as image") + } + }) +} + +func TestGenaiContentsToOrchTemplate_FileParts(t *testing.T) { + msgs, _ := genaiContentsToOrchTemplate([]*genai.Content{{ + Role: "user", + Parts: []*genai.Part{ + {InlineData: &genai.Blob{MIMEType: "application/pdf", Data: []byte("%PDF"), DisplayName: "a.pdf"}}, + }, + }}, nil) + if len(msgs) != 1 { + t.Fatalf("msgs=%#v", msgs) + } + content, _ := msgs[0]["content"].(string) + if !strings.Contains(content, "unsupported file: a.pdf") { + t.Fatalf("content=%q", content) + } +} diff --git a/go/adk/pkg/models/ollama_adk.go b/go/adk/pkg/models/ollama_adk.go index 82f443536..8f049e6d3 100644 --- a/go/adk/pkg/models/ollama_adk.go +++ b/go/adk/pkg/models/ollama_adk.go @@ -255,6 +255,7 @@ func convertGenaiContentsToOllamaMessages(contents []*genai.Content, config *gen } var textParts []string + var images []api.ImageData var toolCalls []api.ToolCall var toolResults []struct { content string @@ -296,6 +297,22 @@ func convertGenaiContentsToOllamaMessages(contents []*genai.Content, config *gen }{content: content}) continue } + + if part.InlineData != nil { + mime := part.InlineData.MIMEType + name := blobName(part.InlineData) + if isImageMIME(mime) { + images = append(images, api.ImageData(part.InlineData.Data)) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } + continue + } + + if part.FileData != nil { + textParts = append(textParts, unsupportedFileNote(fileDataName(part.FileData), part.FileData.MIMEType)) + continue + } } // Build message based on what we found @@ -319,8 +336,7 @@ func convertGenaiContentsToOllamaMessages(contents []*genai.Content, config *gen } } - if len(textParts) > 0 { - // Regular text message + if len(textParts) > 0 || len(images) > 0 { // Check if this is a system message if content.Role == "system" { systemInstruction = strings.Join(textParts, "\n") @@ -328,6 +344,7 @@ func convertGenaiContentsToOllamaMessages(contents []*genai.Content, config *gen msg := api.Message{ Role: role, Content: strings.Join(textParts, "\n"), + Images: images, } messages = append(messages, msg) } diff --git a/go/adk/pkg/models/openai_adk.go b/go/adk/pkg/models/openai_adk.go index f1bcce565..581041d55 100644 --- a/go/adk/pkg/models/openai_adk.go +++ b/go/adk/pkg/models/openai_adk.go @@ -248,7 +248,7 @@ func genaiContentsToOpenAIMessages(contents []*genai.Content, config *genai.Gene role := strings.TrimSpace(content.Role) var textParts []string var functionCalls []*genai.FunctionCall - var imageParts []openai.ChatCompletionContentPartImageImageURLParam + var contentParts []openai.ChatCompletionContentPartUnionParam for _, part := range content.Parts { if part == nil { @@ -258,10 +258,27 @@ func genaiContentsToOpenAIMessages(contents []*genai.Content, config *genai.Gene textParts = append(textParts, part.Text) } else if part.FunctionCall != nil { functionCalls = append(functionCalls, part.FunctionCall) - } else if part.InlineData != nil && strings.HasPrefix(part.InlineData.MIMEType, "image/") { - imageParts = append(imageParts, openai.ChatCompletionContentPartImageImageURLParam{ - URL: fmt.Sprintf("data:%s;base64,%s", part.InlineData.MIMEType, base64.StdEncoding.EncodeToString(part.InlineData.Data)), - }) + } else if part.InlineData != nil { + mime := part.InlineData.MIMEType + name := blobName(part.InlineData) + if isImageMIME(mime) { + contentParts = append(contentParts, openai.ImageContentPart(openai.ChatCompletionContentPartImageImageURLParam{ + URL: dataURI(mime, part.InlineData.Data), + })) + } else if isOpenAIFileMIME(mime) { + file := openai.ChatCompletionContentPartFileFileParam{ + FileData: param.NewOpt(dataURI(mime, part.InlineData.Data)), + } + if name != "" { + file.Filename = param.NewOpt(name) + } + contentParts = append(contentParts, openai.FileContentPart(file)) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } + } else if part.FileData != nil { + // Chat Completions file parts have no URL field. + textParts = append(textParts, unsupportedFileNote(fileDataName(part.FileData), part.FileData.MIMEType)) } } @@ -311,14 +328,12 @@ func genaiContentsToOpenAIMessages(contents []*genai.Content, config *genai.Gene messages = append(messages, openai.ChatCompletionMessageParamUnion{OfAssistant: &asst}) messages = append(messages, toolResponseMessages...) } else { - if len(imageParts) > 0 { - parts := make([]openai.ChatCompletionContentPartUnionParam, 0, len(textParts)+len(imageParts)) + if len(contentParts) > 0 { + parts := make([]openai.ChatCompletionContentPartUnionParam, 0, len(textParts)+len(contentParts)) for _, t := range textParts { parts = append(parts, openai.TextContentPart(t)) } - for _, img := range imageParts { - parts = append(parts, openai.ImageContentPart(img)) - } + parts = append(parts, contentParts...) messages = append(messages, openai.UserMessage(parts)) } else if len(textParts) > 0 { messages = append(messages, openai.UserMessage(strings.Join(textParts, "\n"))) diff --git a/go/adk/pkg/models/openai_responses.go b/go/adk/pkg/models/openai_responses.go index 523797277..a336b87ce 100644 --- a/go/adk/pkg/models/openai_responses.go +++ b/go/adk/pkg/models/openai_responses.go @@ -3,7 +3,6 @@ package models import ( "context" - "encoding/base64" "encoding/json" "fmt" "maps" @@ -97,7 +96,7 @@ func genaiContentsToResponsesInput(contents []*genai.Content, config *genai.Gene role := strings.TrimSpace(content.Role) var textParts []string var functionCalls []*genai.FunctionCall - var imageURLs []string + var contentParts responses.ResponseInputMessageContentListParam for _, part := range content.Parts { if part == nil { @@ -107,12 +106,39 @@ func genaiContentsToResponsesInput(contents []*genai.Content, config *genai.Gene textParts = append(textParts, part.Text) } else if part.FunctionCall != nil { functionCalls = append(functionCalls, part.FunctionCall) - } else if part.InlineData != nil && strings.HasPrefix(part.InlineData.MIMEType, "image/") { - imageURLs = append(imageURLs, fmt.Sprintf( - "data:%s;base64,%s", - part.InlineData.MIMEType, - base64.StdEncoding.EncodeToString(part.InlineData.Data), - )) + } else if part.InlineData != nil { + mime := part.InlineData.MIMEType + name := blobName(part.InlineData) + if isImageMIME(mime) { + img := responses.ResponseInputContentParamOfInputImage(responses.ResponseInputImageDetailAuto) + img.OfInputImage.ImageURL = param.NewOpt(dataURI(mime, part.InlineData.Data)) + contentParts = append(contentParts, img) + } else if isOpenAIFileMIME(mime) { + file := responses.ResponseInputFileParam{ + FileData: param.NewOpt(dataURI(mime, part.InlineData.Data)), + } + if name != "" { + file.Filename = param.NewOpt(name) + } + contentParts = append(contentParts, responses.ResponseInputContentUnionParam{OfInputFile: &file}) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } + } else if part.FileData != nil { + mime := part.FileData.MIMEType + name := fileDataName(part.FileData) + uri := part.FileData.FileURI + if uri != "" && isOpenAIFileMIME(mime) { + file := responses.ResponseInputFileParam{ + FileURL: param.NewOpt(uri), + } + if name != "" { + file.Filename = param.NewOpt(name) + } + contentParts = append(contentParts, responses.ResponseInputContentUnionParam{OfInputFile: &file}) + } else { + textParts = append(textParts, unsupportedFileNote(name, mime)) + } } } @@ -139,7 +165,7 @@ func genaiContentsToResponsesInput(contents []*genai.Content, config *genai.Gene continue } - if len(textParts) == 0 && len(imageURLs) == 0 { + if len(textParts) == 0 && len(contentParts) == 0 { continue } @@ -148,16 +174,12 @@ func genaiContentsToResponsesInput(contents []*genai.Content, config *genai.Gene msgRole = responses.EasyInputMessageRoleAssistant } - if len(imageURLs) > 0 { - parts := make(responses.ResponseInputMessageContentListParam, 0, len(textParts)+len(imageURLs)) + if len(contentParts) > 0 { + parts := make(responses.ResponseInputMessageContentListParam, 0, len(textParts)+len(contentParts)) for _, t := range textParts { parts = append(parts, responses.ResponseInputContentParamOfInputText(t)) } - for _, url := range imageURLs { - img := responses.ResponseInputContentParamOfInputImage(responses.ResponseInputImageDetailAuto) - img.OfInputImage.ImageURL = param.NewOpt(url) - parts = append(parts, img) - } + parts = append(parts, contentParts...) input = append(input, responses.ResponseInputItemParamOfMessage(parts, msgRole)) } else { input = append(input, responses.ResponseInputItemParamOfMessage(strings.Join(textParts, "\n"), msgRole)) diff --git a/ui/src/components/chat/ChatInterface.tsx b/ui/src/components/chat/ChatInterface.tsx index b3c92b3af..32510aebf 100644 --- a/ui/src/components/chat/ChatInterface.tsx +++ b/ui/src/components/chat/ChatInterface.tsx @@ -2,7 +2,7 @@ import type React from "react"; import { useState, useRef, useEffect, useMemo, useCallback } from "react"; -import { ArrowBigUp, X, Loader2, Mic, Square } from "lucide-react"; +import { ArrowBigUp, X, Loader2, Mic, Square, Paperclip, FileIcon, ImageIcon } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Tooltip, @@ -45,6 +45,47 @@ import { type SessionGuardOptions, } from "@/lib/chatSessionGuard"; +/** Soft client cap so base64 A2A FileParts stay reasonable. */ +const MAX_CHAT_FILE_BYTES = 10 * 1024 * 1024; + +type PendingChatFile = { + id: string; + name: string; + mimeType: string; + bytes: string; // raw base64 (no data: prefix) +}; + +function readFileAsDataURL(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(String(reader.result ?? "")); + reader.onerror = () => reject(reader.error ?? new Error("read failed")); + reader.readAsDataURL(file); + }); +} + +/** Build A2A message parts: optional text + FileParts (wire shape a2a-go accepts). */ +function buildUserMessageParts(text: string, files: PendingChatFile[]): Message["parts"] { + // SDK Part typings are protobuf-shaped; chat already sends JSON {kind,text} parts. + const parts: Message["parts"] = []; + const push = (part: unknown) => parts.push(part as Message["parts"][number]); + const trimmed = text.trim(); + if (trimmed) { + push({ kind: "text", text: trimmed }); + } + for (const file of files) { + push({ + kind: "file", + file: { + bytes: file.bytes, + mimeType: file.mimeType, + name: file.name, + }, + }); + } + return parts; +} + interface ChatInterfaceProps { selectedAgentName: string; selectedNamespace: string; @@ -61,7 +102,15 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se const substrateSandbox = useChatSubstrateSandbox(); const router = useRouter(); const containerRef = useRef(null); + const fileInputRef = useRef(null); + const fileDragDepthRef = useRef(0); const [currentInputMessage, setCurrentInputMessage] = useState(""); + const [pendingFiles, setPendingFiles] = useState([]); + const [isDraggingFiles, setIsDraggingFiles] = useState(false); + // File attach is Go declarative agents only (Python adapters still drop non-images). + const supportsFileAttach = + currentAgent.agent?.spec?.type === "Declarative" && + currentAgent.agent?.spec?.declarative?.runtime === "go"; const [chatStatus, setChatStatus] = useState("ready"); @@ -304,7 +353,8 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se rethrowOnError?: boolean; } = {}, ) => { - if (!userMessageText.trim() || !selectedAgentName || !selectedNamespace) { + const filesToSend = pendingFiles; + if ((!userMessageText.trim() && filesToSend.length === 0) || !selectedAgentName || !selectedNamespace) { return; } if (chatStatus !== "ready") { @@ -318,6 +368,7 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se if (options.clearInput ?? true) { setCurrentInputMessage(""); + setPendingFiles([]); } // Cross-tab guard: fetch the latest session state before mutating anything. @@ -337,6 +388,7 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se } setCurrentInputMessage(""); + setPendingFiles([]); setChatStatus("thinking"); setStoredMessages(prev => [...prev, ...streamingMessages]); setStreamingMessages([]); @@ -347,16 +399,14 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se pendingTurnStatsRef.current = undefined; const messageId = uuidv4(); + const messageParts = buildUserMessageParts(userMessageText, filesToSend); // For new sessions or when no stored messages exist, show the user message immediately const userMessage: Message = { kind: "message", messageId, role: "user", - parts: [{ - kind: "text", - text: userMessageText - }], + parts: messageParts, contextId: guardSessionId, metadata: { timestamp: Date.now() @@ -385,7 +435,10 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se isCreatingSessionRef.current = true; setIsFirstMessage(true); - const sessionName = deriveSessionTitle(userMessageText); + const sessionName = + deriveSessionTitle(userMessageText) || + deriveSessionTitle(filesToSend[0]?.name ?? "") || + "New Chat"; const newSessionResponse = await createSession({ agent_ref: `${selectedNamespace}/${selectedAgentName}`, name: sessionName, @@ -395,6 +448,7 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se toast.error("Failed to create session"); setChatStatus("error"); setCurrentInputMessage(userMessageText); + setPendingFiles(filesToSend); isCreatingSessionRef.current = false; return; } @@ -421,6 +475,7 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se toast.error("Error creating session"); setChatStatus("error"); setCurrentInputMessage(userMessageText); + setPendingFiles(filesToSend); isCreatingSessionRef.current = false; return; } @@ -462,10 +517,14 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se messageId, contextId: currentSessionId, }); + a2aMessage.parts = messageParts; await streamA2AMessage(a2aMessage, { errorLabel: "Streaming failed", - onError: () => setCurrentInputMessage(userMessageText), + onError: () => { + setCurrentInputMessage(userMessageText); + setPendingFiles(filesToSend); + }, sessionIdForWait: currentSessionId, }); } catch (error) { @@ -474,6 +533,7 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se setChatStatus("error"); if (options.restoreInputOnError ?? true) { setCurrentInputMessage(userMessageText); + setPendingFiles(filesToSend); } if (options.rethrowOnError) { throw error; @@ -481,12 +541,82 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se } }; + const handleAttachFiles = async (fileList: FileList | null) => { + if (!fileList || fileList.length === 0) return; + const next: PendingChatFile[] = []; + for (const file of Array.from(fileList)) { + if (file.size > MAX_CHAT_FILE_BYTES) { + toast.error(`${file.name} exceeds the ${MAX_CHAT_FILE_BYTES / (1024 * 1024)}MB limit`); + continue; + } + try { + const dataUrl = await readFileAsDataURL(file); + const comma = dataUrl.indexOf(","); + const bytes = comma >= 0 ? dataUrl.slice(comma + 1) : dataUrl; + next.push({ + id: uuidv4(), + name: file.name, + mimeType: file.type || "application/octet-stream", + bytes, + }); + } catch { + toast.error(`Failed to read ${file.name}`); + } + } + if (next.length > 0) { + setPendingFiles(prev => [...prev, ...next]); + } + if (fileInputRef.current) { + fileInputRef.current.value = ""; + } + }; + + const resetFileDrag = () => { + fileDragDepthRef.current = 0; + setIsDraggingFiles(false); + }; + + const handleComposerDragEnter = (e: React.DragEvent) => { + if (!supportsFileAttach || chatStatus !== "ready") return; + if (![...e.dataTransfer.types].includes("Files")) return; + e.preventDefault(); + e.stopPropagation(); + fileDragDepthRef.current += 1; + setIsDraggingFiles(true); + }; + + const handleComposerDragLeave = (e: React.DragEvent) => { + if (!supportsFileAttach) return; + e.preventDefault(); + e.stopPropagation(); + fileDragDepthRef.current = Math.max(0, fileDragDepthRef.current - 1); + if (fileDragDepthRef.current === 0) { + setIsDraggingFiles(false); + } + }; + + const handleComposerDragOver = (e: React.DragEvent) => { + if (!supportsFileAttach || chatStatus !== "ready") return; + if (![...e.dataTransfer.types].includes("Files")) return; + e.preventDefault(); + e.stopPropagation(); + e.dataTransfer.dropEffect = "copy"; + }; + + const handleComposerDrop = (e: React.DragEvent) => { + if (!supportsFileAttach || chatStatus !== "ready") return; + e.preventDefault(); + e.stopPropagation(); + resetFileDrag(); + void handleAttachFiles(e.dataTransfer.files); + }; + const handleSendMessage = async (e: React.FormEvent) => { e.preventDefault(); if (isListening) { stopListening(); } - if (!currentInputMessage.trim()) { + if (!currentInputMessage.trim() && pendingFiles.length === 0) { return; } @@ -986,7 +1116,12 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se const handleKeyDown = (e: React.KeyboardEvent) => { if ((e.metaKey || e.ctrlKey) && e.key === "Enter") { e.preventDefault(); - if (currentInputMessage.trim() && selectedAgentName && selectedNamespace && chatStatus === "ready") { + if ( + (currentInputMessage.trim() || pendingFiles.length > 0) && + selectedAgentName && + selectedNamespace && + chatStatus === "ready" + ) { handleSendMessage(e); } } @@ -1100,18 +1235,92 @@ export default function ChatInterface({ selectedAgentName, selectedNamespace, se -
+ + {isDraggingFiles && ( +
+ Drop files to attach +
+ )}