diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c15e5118..35b979c1d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,11 +12,12 @@ jobs: GOTOOLCHAIN: local steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - # Keep the module baseline in go.mod, but run the scanner with Go 1.26.5 - # so it uses a fixed standard library (GO-2026-5856). + # Keep the module baseline in go.mod, but run the scanner with Go 1.26.6 + # so it uses a fixed standard library (GO-2026-5026, GO-2026-5972, + # GO-2026-6088, GO-2026-6090, GO-2026-6218). - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: - go-version: "1.26.5" + go-version: "1.26.6" cache: true check-latest: true - run: | diff --git a/go.mod b/go.mod index 259ea4158..bef7b594d 100644 --- a/go.mod +++ b/go.mod @@ -111,10 +111,10 @@ require ( go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.3 // indirect golang.org/x/crypto v0.54.0 // indirect - golang.org/x/image v0.44.0 // indirect + golang.org/x/image v0.45.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/api v0.291.0 // indirect google.golang.org/genproto v0.0.0-20260729162451-8efbd57d26e0 // indirect diff --git a/go.sum b/go.sum index 75e400d6b..69e526000 100644 --- a/go.sum +++ b/go.sum @@ -245,8 +245,8 @@ go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= -golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= +golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= +golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= @@ -255,8 +255,8 @@ golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= diff --git a/providers/google/google.go b/providers/google/google.go index 5930ba93d..c6b593c13 100644 --- a/providers/google/google.go +++ b/providers/google/google.go @@ -3,6 +3,7 @@ package google import ( "cmp" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -348,7 +349,7 @@ func (g languageModel) prepareParams(call fantasy.Call) (*genai.GenerateContentC return config, content, warnings, nil } -func toGooglePrompt(prompt fantasy.Prompt, isVertexAI bool) (*genai.Content, []*genai.Content, []fantasy.CallWarning) { //nolint: unparam +func toGooglePrompt(prompt fantasy.Prompt, isVertexAI bool) (*genai.Content, []*genai.Content, []fantasy.CallWarning) { var systemInstructions *genai.Content var content []*genai.Content var warnings []fantasy.CallWarning @@ -545,6 +546,50 @@ func toGooglePrompt(prompt fantasy.Prompt, isVertexAI bool) (*genai.Content, []* parts = append(parts, &genai.Part{ FunctionResponse: functionResponse, }) + + case fantasy.ToolResultContentTypeMedia: + content, ok := fantasy.AsToolResultOutputType[fantasy.ToolResultOutputContentMedia](result.Output) + if !ok { + continue + } + // A function response is JSON only, and nested + // functionResponse.parts are accepted by Gemini 3+ + // models only. Keep the tool call paired with a text + // response and attach the media as a sibling inline + // part of the same turn, which every Gemini + // generation accepts. + text := content.Text + if text == "" { + text = fmt.Sprintf("The tool returned %s content; see the attached media.", content.MediaType) + } + functionResponse := &genai.FunctionResponse{ + ID: result.ToolCallID, + Response: map[string]any{"result": text}, + Name: toolCall.ToolName, + } + + // Vertex breaks with a 400 if this field be present. + if isVertexAI { + functionResponse.ID = "" + } + parts = append(parts, &genai.Part{ + FunctionResponse: functionResponse, + }) + + data, err := base64.StdEncoding.DecodeString(content.Data) + if err != nil { + warnings = append(warnings, fantasy.CallWarning{ + Type: fantasy.CallWarningTypeOther, + Message: fmt.Sprintf("tool result media for %s is not valid base64, sending text only", result.ToolCallID), + }) + continue + } + parts = append(parts, &genai.Part{ + InlineData: &genai.Blob{ + Data: data, + MIMEType: content.MediaType, + }, + }) } } } diff --git a/providers/google/tool_result_media_test.go b/providers/google/tool_result_media_test.go new file mode 100644 index 000000000..0240d7719 --- /dev/null +++ b/providers/google/tool_result_media_test.go @@ -0,0 +1,103 @@ +package google + +import ( + "encoding/base64" + "testing" + + "charm.land/fantasy" + "github.com/stretchr/testify/require" + "google.golang.org/genai" +) + +// Gemini function responses are JSON only, so media tool results are +// delivered as a text function response paired with a sibling inline +// data part in the same turn. + +func mediaToolResultPrompt(output fantasy.ToolResultOutputContentMedia) fantasy.Prompt { + return fantasy.Prompt{ + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ToolCallID: "call-1", ToolName: "screenshot", Input: "{}"}, + }, + }, + { + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ToolCallID: "call-1", Output: output}, + }, + }, + } +} + +func TestToGooglePrompt_MediaToolResult_ImageWithText(t *testing.T) { + t.Parallel() + + raw := []byte{0, 1, 2, 3} + prompt := mediaToolResultPrompt(fantasy.ToolResultOutputContentMedia{ + Data: base64.StdEncoding.EncodeToString(raw), + MediaType: "image/png", + Text: "Screenshot of the login page.", + }) + + _, contents, warnings := toGooglePrompt(prompt, false) + + require.Empty(t, warnings) + require.Len(t, contents, 2) + toolTurn := contents[1] + require.Equal(t, genai.RoleUser, toolTurn.Role) + require.Len(t, toolTurn.Parts, 2) + + response := toolTurn.Parts[0].FunctionResponse + require.NotNil(t, response) + require.Equal(t, "call-1", response.ID) + require.Equal(t, "screenshot", response.Name) + require.Equal(t, map[string]any{"result": "Screenshot of the login page."}, response.Response) + + inline := toolTurn.Parts[1].InlineData + require.NotNil(t, inline) + require.Equal(t, "image/png", inline.MIMEType) + require.Equal(t, raw, inline.Data) +} + +func TestToGooglePrompt_MediaToolResult_ImageWithoutText(t *testing.T) { + t.Parallel() + + prompt := mediaToolResultPrompt(fantasy.ToolResultOutputContentMedia{ + Data: base64.StdEncoding.EncodeToString([]byte{9, 9, 9}), + MediaType: "image/jpeg", + }) + + _, contents, warnings := toGooglePrompt(prompt, true) + + require.Empty(t, warnings) + require.Len(t, contents, 2) + require.Len(t, contents[1].Parts, 2) + + response := contents[1].Parts[0].FunctionResponse + require.NotNil(t, response) + // Vertex rejects function response IDs. + require.Empty(t, response.ID) + require.Contains(t, response.Response["result"], "image/jpeg") + require.NotNil(t, contents[1].Parts[1].InlineData) +} + +func TestToGooglePrompt_MediaToolResult_InvalidBase64(t *testing.T) { + t.Parallel() + + prompt := mediaToolResultPrompt(fantasy.ToolResultOutputContentMedia{ + Data: "not base64!", + MediaType: "image/png", + Text: "Screenshot text.", + }) + + _, contents, warnings := toGooglePrompt(prompt, false) + + require.Len(t, warnings, 1) + require.Contains(t, warnings[0].Message, "not valid base64") + require.Len(t, contents, 2) + // The text function response still pairs with the tool call. + require.Len(t, contents[1].Parts, 1) + require.NotNil(t, contents[1].Parts[0].FunctionResponse) + require.Equal(t, map[string]any{"result": "Screenshot text."}, contents[1].Parts[0].FunctionResponse.Response) +}