From ba2a71733272b75e948c28a6fb41ffa506a40edc Mon Sep 17 00:00:00 2001 From: wucm667 Date: Mon, 12 Oct 2026 18:52:00 +0800 Subject: [PATCH 1/2] fix(gemini): preserve images from tool results --- .../service/gemini_messages_compat_service.go | 44 ++++++++++- .../gemini_messages_compat_service_test.go | 78 +++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/backend/internal/service/gemini_messages_compat_service.go b/backend/internal/service/gemini_messages_compat_service.go index 278cbfa6fa..a82ef45fd7 100644 --- a/backend/internal/service/gemini_messages_compat_service.go +++ b/backend/internal/service/gemini_messages_compat_service.go @@ -3440,6 +3440,7 @@ func convertClaudeMessagesToGeminiContents(messages any, toolUseIDToName map[str case []any: // 如果只有一个 block，不过滤空白（让上游 API 报错） singleBlock := len(content) == 1 + var toolImageParts []any for _, block := range content { bm, ok := block.(map[string]any) @@ -3480,14 +3481,20 @@ func convertClaudeMessagesToGeminiContents(messages any, toolUseIDToName map[str if name == "" { name = "tool" } + text := extractClaudeContentText(bm["content"]) + images := extractClaudeToolResultImageParts(bm["content"]) + if text == "" && len(images) > 0 { + text = "[image attached below]" + } parts = append(parts, map[string]any{ "functionResponse": map[string]any{ "name": name, "response": map[string]any{ - "content": extractClaudeContentText(bm["content"]), + "content": text, }, }, }) + toolImageParts = append(toolImageParts, images...) case "image": if src, ok := bm["source"].(map[string]any); ok { if srcType, _ := src["type"].(string); srcType == "base64" { @@ -3510,6 +3517,7 @@ func convertClaudeMessagesToGeminiContents(messages any, toolUseIDToName map[str } } } + parts = append(parts, toolImageParts...) default: // ignore } @@ -3546,6 +3554,40 @@ func extractClaudeContentText(v any) string { } } +func extractClaudeToolResultImageParts(v any) []any { + arr, ok := v.([]any) + if !ok { + return nil + } + + parts := make([]any, 0) + for _, part := range arr { + pm, ok := part.(map[string]any) + if !ok || pm["type"] != "image" { + continue + } + source, ok := pm["source"].(map[string]any) + if !ok { + continue + } + if sourceType, _ := source["type"].(string); sourceType != "base64" { + continue + } + mediaType, _ := source["media_type"].(string) + data, _ := source["data"].(string) + if mediaType == "" || data == "" { + continue + } + parts = append(parts, map[string]any{ + "inlineData": map[string]any{ + "mimeType": mediaType, + "data": data, + }, + }) + } + return parts +} + func convertClaudeToolsToGeminiTools(tools any) []any { arr, ok := tools.([]any) if !ok || len(arr) == 0 { diff --git a/backend/internal/service/gemini_messages_compat_service_test.go b/backend/internal/service/gemini_messages_compat_service_test.go index 07fafa9493..cc464aa620 100644 --- a/backend/internal/service/gemini_messages_compat_service_test.go +++ b/backend/internal/service/gemini_messages_compat_service_test.go @@ -704,6 +704,84 @@ func TestConvertClaudeMessagesToGeminiGenerateContent_AddsThoughtSignatureForToo } } +func TestConvertClaudeMessagesToGeminiGenerateContent_ExtractsToolResultImages(t *testing.T) { + claudeReq := map[string]any{ + "messages": []any{ + map[string]any{ + "role": "assistant", + "content": []any{ + map[string]any{ + "type": "tool_use", + "id": "toolu_123", + "name": "inspect", + "input": map[string]any{ + "path": "image.png", + }, + }, + }, + }, + map[string]any{ + "role": "user", + "content": []any{ + map[string]any{ + "type": "tool_result", + "tool_use_id": "toolu_123", + "content": []any{ + map[string]any{"type": "text", "text": "found it"}, + map[string]any{ + "type": "image", + "source": map[string]any{ + "type": "base64", + "media_type": "image/png", + "data": "aGVsbG8=", + }, + }, + }, + }, + map[string]any{ + "type": "tool_result", + "tool_use_id": "toolu_123", + "content": []any{ + map[string]any{ + "type": "image", + "source": map[string]any{ + "type": "base64", + "media_type": "image/jpeg", + "data": "d29ybGQ=", + }, + }, + }, + }, + }, + }, + }, + } + body, err := json.Marshal(claudeReq) + require.NoError(t, err) + + converted, err := convertClaudeMessagesToGeminiGenerateContent(body) + require.NoError(t, err) + + var geminiReq map[string]any + require.NoError(t, json.Unmarshal(converted, &geminiReq)) + contents := geminiReq["contents"].([]any) + parts := contents[1].(map[string]any)["parts"].([]any) + require.Len(t, parts, 4) + + firstResponse := parts[0].(map[string]any)["functionResponse"].(map[string]any) + require.Equal(t, "inspect", firstResponse["name"]) + require.Equal(t, "found it", firstResponse["response"].(map[string]any)["content"]) + secondResponse := parts[1].(map[string]any)["functionResponse"].(map[string]any) + require.Equal(t, "[image attached below]", secondResponse["response"].(map[string]any)["content"]) + + firstImage := parts[2].(map[string]any)["inlineData"].(map[string]any) + require.Equal(t, "image/png", firstImage["mimeType"]) + require.Equal(t, "aGVsbG8=", firstImage["data"]) + secondImage := parts[3].(map[string]any)["inlineData"].(map[string]any) + require.Equal(t, "image/jpeg", secondImage["mimeType"]) + require.Equal(t, "d29ybGQ=", secondImage["data"]) +} + func TestEnsureGeminiFunctionCallThoughtSignatures_InsertsWhenMissing(t *testing.T) { geminiReq := map[string]any{ "contents": []any{ From 492c63059980ea5797cc2c34ba642ffee9a56aee Mon Sep 17 00:00:00 2001 From: wucm667 Date: Mon, 12 Oct 2026 19:13:17 +0800 Subject: [PATCH 2/2] fix(gemini): satisfy test lint --- .../gemini_messages_compat_service_test.go | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/backend/internal/service/gemini_messages_compat_service_test.go b/backend/internal/service/gemini_messages_compat_service_test.go index cc464aa620..3e6c3539d9 100644 --- a/backend/internal/service/gemini_messages_compat_service_test.go +++ b/backend/internal/service/gemini_messages_compat_service_test.go @@ -764,20 +764,41 @@ func TestConvertClaudeMessagesToGeminiGenerateContent_ExtractsToolResultImages(t var geminiReq map[string]any require.NoError(t, json.Unmarshal(converted, &geminiReq)) - contents := geminiReq["contents"].([]any) - parts := contents[1].(map[string]any)["parts"].([]any) + contents, ok := geminiReq["contents"].([]any) + require.True(t, ok) + content, ok := contents[1].(map[string]any) + require.True(t, ok) + parts, ok := content["parts"].([]any) + require.True(t, ok) require.Len(t, parts, 4) - firstResponse := parts[0].(map[string]any)["functionResponse"].(map[string]any) + firstPart, ok := parts[0].(map[string]any) + require.True(t, ok) + firstResponse, ok := firstPart["functionResponse"].(map[string]any) + require.True(t, ok) require.Equal(t, "inspect", firstResponse["name"]) - require.Equal(t, "found it", firstResponse["response"].(map[string]any)["content"]) - secondResponse := parts[1].(map[string]any)["functionResponse"].(map[string]any) - require.Equal(t, "[image attached below]", secondResponse["response"].(map[string]any)["content"]) + firstResponseBody, ok := firstResponse["response"].(map[string]any) + require.True(t, ok) + require.Equal(t, "found it", firstResponseBody["content"]) - firstImage := parts[2].(map[string]any)["inlineData"].(map[string]any) + secondPart, ok := parts[1].(map[string]any) + require.True(t, ok) + secondResponse, ok := secondPart["functionResponse"].(map[string]any) + require.True(t, ok) + secondResponseBody, ok := secondResponse["response"].(map[string]any) + require.True(t, ok) + require.Equal(t, "[image attached below]", secondResponseBody["content"]) + + firstImagePart, ok := parts[2].(map[string]any) + require.True(t, ok) + firstImage, ok := firstImagePart["inlineData"].(map[string]any) + require.True(t, ok) require.Equal(t, "image/png", firstImage["mimeType"]) require.Equal(t, "aGVsbG8=", firstImage["data"]) - secondImage := parts[3].(map[string]any)["inlineData"].(map[string]any) + secondImagePart, ok := parts[3].(map[string]any) + require.True(t, ok) + secondImage, ok := secondImagePart["inlineData"].(map[string]any) + require.True(t, ok) require.Equal(t, "image/jpeg", secondImage["mimeType"]) require.Equal(t, "d29ybGQ=", secondImage["data"]) }