From 45406d76e7e8a4e80657574797d770f3c2ee8b26 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:26:14 +0000 Subject: [PATCH 1/2] fix(providers/google): keep media tool results in the Gemini prompt The Google provider had no case for media tool results, so a tool that returned an image (with or without accompanying text) emitted no function response at all and Gemini rejected the unpaired function call. Function responses are JSON only and nested functionResponse parts are accepted by Gemini 3+ models only, so send the accompanying text (or a placeholder) as the function response and attach the media as a sibling inline data part of the same turn. --- providers/google/google.go | 47 +++++++++- providers/google/tool_result_media_test.go | 103 +++++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 providers/google/tool_result_media_test.go diff --git a/providers/google/google.go b/providers/google/google.go index 5930ba93d..c6b593c13 100644 --- a/providers/google/google.go +++ b/providers/google/google.go @@ -3,6 +3,7 @@ package google import ( "cmp" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -348,7 +349,7 @@ func (g languageModel) prepareParams(call fantasy.Call) (*genai.GenerateContentC return config, content, warnings, nil } -func toGooglePrompt(prompt fantasy.Prompt, isVertexAI bool) (*genai.Content, []*genai.Content, []fantasy.CallWarning) { //nolint: unparam +func toGooglePrompt(prompt fantasy.Prompt, isVertexAI bool) (*genai.Content, []*genai.Content, []fantasy.CallWarning) { var systemInstructions *genai.Content var content []*genai.Content var warnings []fantasy.CallWarning @@ -545,6 +546,50 @@ func toGooglePrompt(prompt fantasy.Prompt, isVertexAI bool) (*genai.Content, []* parts = append(parts, &genai.Part{ FunctionResponse: functionResponse, }) + + case fantasy.ToolResultContentTypeMedia: + content, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](result.Output) + if !ok { + continue + } + // A function response is JSON only, and nested + // functionResponse.parts are accepted by Gemini 3+ + // models only. Keep the tool call paired with a text + // response and attach the media as a sibling inline + // part of the same turn, which every Gemini + // generation accepts. + text := content.Text + if text == "" { + text = fmt.Sprintf("The tool returned %s content; see the attached media.", content.MediaType) + } + functionResponse := &genai.FunctionResponse{ + ID: result.ToolCallID, + Response: map[string]any{"result": text}, + Name: toolCall.ToolName, + } + + // Vertex breaks with a 400 if this field be present. + if isVertexAI { + functionResponse.ID = "" + } + parts = append(parts, &genai.Part{ + FunctionResponse: functionResponse, + }) + + data, err := base64.StdEncoding.DecodeString(content.Data) + if err != nil { + warnings = append(warnings, fantasy.CallWarning{ + Type: fantasy.CallWarningTypeOther, + Message: fmt.Sprintf("tool result media for %s is not valid base64, sending text only", result.ToolCallID), + }) + continue + } + parts = append(parts, &genai.Part{ + InlineData: &genai.Blob{ + Data: data, + MIMEType: content.MediaType, + }, + }) } } } diff --git a/providers/google/tool_result_media_test.go b/providers/google/tool_result_media_test.go new file mode 100644 index 000000000..0240d7719 --- /dev/null +++ b/providers/google/tool_result_media_test.go @@ -0,0 +1,103 @@ +package google + +import ( + "encoding/base64" + "testing" + + "charm.land/fantasy" + "github.com/stretchr/testify/require" + "google.golang.org/genai" +) + +// Gemini function responses are JSON only, so media tool results are +// delivered as a text function response paired with a sibling inline +// data part in the same turn. + +func mediaToolResultPrompt(output fantasy.ToolResultOutputContentMedia) fantasy.Prompt { + return fantasy.Prompt{ + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ToolCallID: "call-1", ToolName: "screenshot", Input: "{}"}, + }, + }, + { + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ToolCallID: "call-1", Output: output}, + }, + }, + } +} + +func TestToGooglePrompt_MediaToolResult_ImageWithText(t *testing.T) { + t.Parallel() + + raw := []byte{0, 1, 2, 3} + prompt := mediaToolResultPrompt(fantasy.ToolResultOutputContentMedia{ + Data: base64.StdEncoding.EncodeToString(raw), + MediaType: "image/png", + Text: "Screenshot of the login page.", + }) + + _, contents, warnings := toGooglePrompt(prompt, false) + + require.Empty(t, warnings) + require.Len(t, contents, 2) + toolTurn := contents[1] + require.Equal(t, genai.RoleUser, toolTurn.Role) + require.Len(t, toolTurn.Parts, 2) + + response := toolTurn.Parts[0].FunctionResponse + require.NotNil(t, response) + require.Equal(t, "call-1", response.ID) + require.Equal(t, "screenshot", response.Name) + require.Equal(t, map[string]any{"result": "Screenshot of the login page."}, response.Response) + + inline := toolTurn.Parts[1].InlineData + require.NotNil(t, inline) + require.Equal(t, "image/png", inline.MIMEType) + require.Equal(t, raw, inline.Data) +} + +func TestToGooglePrompt_MediaToolResult_ImageWithoutText(t *testing.T) { + t.Parallel() + + prompt := mediaToolResultPrompt(fantasy.ToolResultOutputContentMedia{ + Data: base64.StdEncoding.EncodeToString([]byte{9, 9, 9}), + MediaType: "image/jpeg", + }) + + _, contents, warnings := toGooglePrompt(prompt, true) + + require.Empty(t, warnings) + require.Len(t, contents, 2) + require.Len(t, contents[1].Parts, 2) + + response := contents[1].Parts[0].FunctionResponse + require.NotNil(t, response) + // Vertex rejects function response IDs. + require.Empty(t, response.ID) + require.Contains(t, response.Response["result"], "image/jpeg") + require.NotNil(t, contents[1].Parts[1].InlineData) +} + +func TestToGooglePrompt_MediaToolResult_InvalidBase64(t *testing.T) { + t.Parallel() + + prompt := mediaToolResultPrompt(fantasy.ToolResultOutputContentMedia{ + Data: "not base64!", + MediaType: "image/png", + Text: "Screenshot text.", + }) + + _, contents, warnings := toGooglePrompt(prompt, false) + + require.Len(t, warnings, 1) + require.Contains(t, warnings[0].Message, "not valid base64") + require.Len(t, contents, 2) + // The text function response still pairs with the tool call. + require.Len(t, contents[1].Parts, 1) + require.NotNil(t, contents[1].Parts[0].FunctionResponse) + require.Equal(t, map[string]any{"result": "Screenshot text."}, contents[1].Parts[0].FunctionResponse.Response) +} From ed7cb7feb44aa49044a487b14176bec274c1e899 Mon Sep 17 00:00:00 2001 From: Michael Suchacz <203725896+ibetitsmike@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:37:36 +0000 Subject: [PATCH 2/2] ci: run govulncheck on Go 1.26.6 and bump golang.org/x/image The scanner reported standard library vulnerabilities fixed in Go 1.26.6 and GO-2026-6222 in golang.org/x/image v0.44.0. Keep the module baseline at go 1.26.5 for consumers and move only the scanner toolchain, and take x/image v0.45.0. --- .github/workflows/build.yml | 7 ++++--- go.mod | 4 ++-- go.sum | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c15e5118..35b979c1d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,11 +12,12 @@ jobs: GOTOOLCHAIN: local steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - # Keep the module baseline in go.mod, but run the scanner with Go 1.26.5 - # so it uses a fixed standard library (GO-2026-5856). + # Keep the module baseline in go.mod, but run the scanner with Go 1.26.6 + # so it uses a fixed standard library (GO-2026-5026, GO-2026-5972, + # GO-2026-6088, GO-2026-6090, GO-2026-6218). - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: - go-version: "1.26.5" + go-version: "1.26.6" cache: true check-latest: true - run: | diff --git a/go.mod b/go.mod index 259ea4158..bef7b594d 100644 --- a/go.mod +++ b/go.mod @@ -111,10 +111,10 @@ require ( go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.3 // indirect golang.org/x/crypto v0.54.0 // indirect - golang.org/x/image v0.44.0 // indirect + golang.org/x/image v0.45.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/api v0.291.0 // indirect google.golang.org/genproto v0.0.0-20260729162451-8efbd57d26e0 // indirect diff --git a/go.sum b/go.sum index 75e400d6b..69e526000 100644 --- a/go.sum +++ b/go.sum @@ -245,8 +245,8 @@ go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= -golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= +golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= +golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= @@ -255,8 +255,8 @@ golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=