fix(providers/google): keep media tool results in the Gemini prompt

.github/workflows/build.yml

@@ -12,11 +12,12 @@ jobs:
       GOTOOLCHAIN: local
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      # Keep the module baseline in go.mod, but run the scanner with Go 1.26.5
-      # so it uses a fixed standard library (GO-2026-5856).
+      # Keep the module baseline in go.mod, but run the scanner with Go 1.26.6
+      # so it uses a fixed standard library (GO-2026-5026, GO-2026-5972,
+      # GO-2026-6088, GO-2026-6090, GO-2026-6218).
       - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
         with:
-          go-version: "1.26.5"
+          go-version: "1.26.6"
           cache: true
           check-latest: true
       - run: |

go.mod

@@ -111,10 +111,10 @@ require (
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v4 v4.0.0-rc.3 // indirect
 	golang.org/x/crypto v0.54.0 // indirect
-	golang.org/x/image v0.44.0 // indirect
+	golang.org/x/image v0.45.0 // indirect
 	golang.org/x/sync v0.22.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect
-	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/text v0.41.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
 	google.golang.org/api v0.291.0 // indirect
 	google.golang.org/genproto v0.0.0-20260729162451-8efbd57d26e0 // indirect

go.sum

@@ -245,8 +245,8 @@ go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
 go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
 golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
 golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
-golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I=
-golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY=
+golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
+golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
 golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
 golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
@@ -255,8 +255,8 @@ golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
 golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
 golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
-golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
+golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=

providers/google/google.go

@@ -3,6 +3,7 @@ package google
 import (
 	"cmp"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -348,7 +349,7 @@ func (g languageModel) prepareParams(call fantasy.Call) (*genai.GenerateContentC
 	return config, content, warnings, nil
 }
 
-func toGooglePrompt(prompt fantasy.Prompt, isVertexAI bool) (*genai.Content, []*genai.Content, []fantasy.CallWarning) { //nolint: unparam
+func toGooglePrompt(prompt fantasy.Prompt, isVertexAI bool) (*genai.Content, []*genai.Content, []fantasy.CallWarning) {
 	var systemInstructions *genai.Content
 	var content []*genai.Content
 	var warnings []fantasy.CallWarning
@@ -545,6 +546,50 @@ func toGooglePrompt(prompt fantasy.Prompt, isVertexAI bool) (*genai.Content, []*
 						parts = append(parts, &genai.Part{
 							FunctionResponse: functionResponse,
 						})
+
+					case fantasy.ToolResultContentTypeMedia:
+						content, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](result.Output)
+						if !ok {
+							continue
+						}
+						// A function response is JSON only, and nested
+						// functionResponse.parts are accepted by Gemini 3+
+						// models only. Keep the tool call paired with a text
+						// response and attach the media as a sibling inline
+						// part of the same turn, which every Gemini
+						// generation accepts.
+						text := content.Text
+						if text == "" {
+							text = fmt.Sprintf("The tool returned %s content; see the attached media.", content.MediaType)
+						}
+						functionResponse := &genai.FunctionResponse{
+							ID:       result.ToolCallID,
+							Response: map[string]any{"result": text},
+							Name:     toolCall.ToolName,
+						}
+
+						// Vertex breaks with a 400 if this field be present.
+						if isVertexAI {
+							functionResponse.ID = ""
+						}
+						parts = append(parts, &genai.Part{
+							FunctionResponse: functionResponse,
+						})
+
+						data, err := base64.StdEncoding.DecodeString(content.Data)
+						if err != nil {
+							warnings = append(warnings, fantasy.CallWarning{
+								Type:    fantasy.CallWarningTypeOther,
+								Message: fmt.Sprintf("tool result media for %s is not valid base64, sending text only", result.ToolCallID),
+							})
+							continue
+						}
+						parts = append(parts, &genai.Part{
+							InlineData: &genai.Blob{
+								Data:     data,
+								MIMEType: content.MediaType,
+							},
+						})
 					}
 				}
 			}

providers/google/tool_result_media_test.go

@@ -0,0 +1,103 @@
+package google
+
+import (
+	"encoding/base64"
+	"testing"
+
+	"charm.land/fantasy"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/genai"
+)
+
+// Gemini function responses are JSON only, so media tool results are
+// delivered as a text function response paired with a sibling inline
+// data part in the same turn.
+
+func mediaToolResultPrompt(output fantasy.ToolResultOutputContentMedia) fantasy.Prompt {
+	return fantasy.Prompt{
+		{
+			Role: fantasy.MessageRoleAssistant,
+			Content: []fantasy.MessagePart{
+				fantasy.ToolCallPart{ToolCallID: "call-1", ToolName: "screenshot", Input: "{}"},
+			},
+		},
+		{
+			Role: fantasy.MessageRoleTool,
+			Content: []fantasy.MessagePart{
+				fantasy.ToolResultPart{ToolCallID: "call-1", Output: output},
+			},
+		},
+	}
+}
+
+func TestToGooglePrompt_MediaToolResult_ImageWithText(t *testing.T) {
+	t.Parallel()
+
+	raw := []byte{0, 1, 2, 3}
+	prompt := mediaToolResultPrompt(fantasy.ToolResultOutputContentMedia{
+		Data:      base64.StdEncoding.EncodeToString(raw),
+		MediaType: "image/png",
+		Text:      "Screenshot of the login page.",
+	})
+
+	_, contents, warnings := toGooglePrompt(prompt, false)
+
+	require.Empty(t, warnings)
+	require.Len(t, contents, 2)
+	toolTurn := contents[1]
+	require.Equal(t, genai.RoleUser, toolTurn.Role)
+	require.Len(t, toolTurn.Parts, 2)
+
+	response := toolTurn.Parts[0].FunctionResponse
+	require.NotNil(t, response)
+	require.Equal(t, "call-1", response.ID)
+	require.Equal(t, "screenshot", response.Name)
+	require.Equal(t, map[string]any{"result": "Screenshot of the login page."}, response.Response)
+
+	inline := toolTurn.Parts[1].InlineData
+	require.NotNil(t, inline)
+	require.Equal(t, "image/png", inline.MIMEType)
+	require.Equal(t, raw, inline.Data)
+}
+
+func TestToGooglePrompt_MediaToolResult_ImageWithoutText(t *testing.T) {
+	t.Parallel()
+
+	prompt := mediaToolResultPrompt(fantasy.ToolResultOutputContentMedia{
+		Data:      base64.StdEncoding.EncodeToString([]byte{9, 9, 9}),
+		MediaType: "image/jpeg",
+	})
+
+	_, contents, warnings := toGooglePrompt(prompt, true)
+
+	require.Empty(t, warnings)
+	require.Len(t, contents, 2)
+	require.Len(t, contents[1].Parts, 2)
+
+	response := contents[1].Parts[0].FunctionResponse
+	require.NotNil(t, response)
+	// Vertex rejects function response IDs.
+	require.Empty(t, response.ID)
+	require.Contains(t, response.Response["result"], "image/jpeg")
+	require.NotNil(t, contents[1].Parts[1].InlineData)
+}
+
+func TestToGooglePrompt_MediaToolResult_InvalidBase64(t *testing.T) {
+	t.Parallel()
+
+	prompt := mediaToolResultPrompt(fantasy.ToolResultOutputContentMedia{
+		Data:      "not base64!",
+		MediaType: "image/png",
+		Text:      "Screenshot text.",
+	})
+
+	_, contents, warnings := toGooglePrompt(prompt, false)
+
+	require.Len(t, warnings, 1)
+	require.Contains(t, warnings[0].Message, "not valid base64")
+	require.Len(t, contents, 2)
+	// The text function response still pairs with the tool call.
+	require.Len(t, contents[1].Parts, 1)
+	require.NotNil(t, contents[1].Parts[0].FunctionResponse)
+	require.Equal(t, map[string]any{"result": "Screenshot text."}, contents[1].Parts[0].FunctionResponse.Response)
+}