From 2bf9c6d56be6130fe863e03c9defd7eaa3ab83b7 Mon Sep 17 00:00:00 2001 From: tudoujun <131859738+tudoujunha@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:36:47 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B7=A5=E5=85=B7=E8=BE=93?= =?UTF-8?q?=E5=87=BA=E5=9B=BE=E7=89=87=E6=A1=A5=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../chatcompletions_responses_bridge.go | 198 ++++++++++- ...etions_responses_tool_output_media_test.go | 318 ++++++++++++++++++ 2 files changed, 504 insertions(+), 12 deletions(-) create mode 100644 backend/internal/pkg/apicompat/chatcompletions_responses_tool_output_media_test.go diff --git a/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go b/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go index 0d2de969dd..b5c5cf0aa1 100644 --- a/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go +++ b/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go @@ -1,6 +1,7 @@ package apicompat import ( + "bytes" "crypto/sha256" "encoding/hex" "encoding/json" @@ -9,6 +10,13 @@ import ( "time" ) +const ( + toolOutputMediaMarker = "[Tool output media moved to the following user message]" + toolOutputMediaAttribution = "[Tool output media for call %s]" +) + +type toolOutputMediaByCallID map[string][]ChatContentPart + // ResponsesToChatCompletionsRequest converts a Responses API request into a // Chat Completions request for upstreams that only implement // /v1/chat/completions. @@ -215,16 +223,16 @@ func responsesInputToChatMessages(instructions string, inputRaw json.RawMessage) return nil, fmt.Errorf("parse responses input: %w", err) } - built, err := buildChatMessagesFromItems(messages, rawItems) + built, mediaByCallID, err := buildChatMessagesFromItems(messages, rawItems) if err != nil { return nil, err } - return normalizeChatMessages(built), nil + return normalizeChatMessagesWithToolOutputMedia(built, mediaByCallID), nil } // buildChatMessagesFromItems walks the Responses input items and appends the // corresponding Chat messages. -func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessage) ([]ChatMessage, error) { +func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessage) ([]ChatMessage, toolOutputMediaByCallID, error) { // pendingReasoning holds the reasoning text from a reasoning item until the // assistant message it belongs to is emitted. DeepSeek's thinking mode // requires the reasoning_content that produced a tool call to be passed back @@ -232,6 +240,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa // across an assistant message (so a following tool call in the same turn // still receives it); any other role ends the thinking span. var pendingReasoning string + mediaByCallID := make(toolOutputMediaByCallID) for _, raw := range rawItems { raw = bytesTrimSpace(raw) @@ -248,7 +257,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa pendingReasoning = "" continue } - return nil, fmt.Errorf("parse responses input item: %w", err) + return nil, nil, fmt.Errorf("parse responses input item: %w", err) } role := chatCompletionsBridgeRole(rawString(item["role"])) @@ -320,15 +329,25 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa continue case "function_call_output", "custom_tool_call_output", "tool_search_output": outputRaw := bytesTrimSpace(item["output"]) - outputText := rawString(outputRaw) - if outputText == "" && len(outputRaw) > 0 && string(outputRaw) != "null" && string(outputRaw) != `""` { - // 对象/数组形式的输出（如 tool_search 的结果列表）整体字符串化。 - outputText = string(outputRaw) + callID := rawString(item["call_id"]) + delete(mediaByCallID, callID) + + outputText, media, rewritten := extractToolOutputMedia(outputRaw) + if rewritten { + if callID != "" { + mediaByCallID[callID] = media + } + } else { + outputText = rawString(outputRaw) + if outputText == "" && len(outputRaw) > 0 && string(outputRaw) != "null" && string(outputRaw) != `""` { + // 对象/数组形式的输出（如 tool_search 的结果列表）整体字符串化。 + outputText = string(outputRaw) + } } content, _ := json.Marshal(outputText) messages = append(messages, ChatMessage{ Role: "tool", - ToolCallID: rawString(item["call_id"]), + ToolCallID: callID, Content: content, }) pendingReasoning = "" @@ -341,7 +360,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa case "input_image": content, err := chatContentFromSingleResponsesPart(itemType, item) if err != nil { - return nil, err + return nil, nil, err } messages = append(messages, ChatMessage{Role: "user", Content: content}) pendingReasoning = "" @@ -367,7 +386,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa } chatContent, err := responsesContentToChatContent(content, role) if err != nil { - return nil, err + return nil, nil, err } messages = append(messages, ChatMessage{Role: role, Content: chatContent}) // Reasoning only survives across an assistant text message. @@ -376,7 +395,141 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa } } - return messages, nil + return messages, mediaByCallID, nil +} + +// extractToolOutputMedia rewrites only recognized image nodes. Media-free +// outputs return rewritten=false so the caller can preserve their original +// bytes and prompt-cache prefix. +func extractToolOutputMedia(outputRaw json.RawMessage) (string, []ChatContentPart, bool) { + outputRaw = bytesTrimSpace(outputRaw) + if len(outputRaw) == 0 || string(outputRaw) == "null" { + return "", nil, false + } + + var outputString string + if err := json.Unmarshal(outputRaw, &outputString); err == nil { + if isToolOutputImageDataURL(outputString) { + return toolOutputMediaMarker, []ChatContentPart{toolOutputImagePart(outputString)}, true + } + + nested, ok := decodeToolOutputJSON([]byte(outputString)) + if !ok { + return "", nil, false + } + rewritten, media, changed := rewriteToolOutputMediaValue(nested) + if !changed { + return "", nil, false + } + encoded, err := json.Marshal(rewritten) + if err != nil { + return "", nil, false + } + return string(encoded), media, true + } + + value, ok := decodeToolOutputJSON(outputRaw) + if !ok { + return "", nil, false + } + rewritten, media, changed := rewriteToolOutputMediaValue(value) + if !changed { + return "", nil, false + } + encoded, err := json.Marshal(rewritten) + if err != nil { + return "", nil, false + } + return string(encoded), media, true +} + +func decodeToolOutputJSON(raw []byte) (any, bool) { + if !json.Valid(raw) { + return nil, false + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + return nil, false + } + return value, true +} + +func rewriteToolOutputMediaValue(value any) (any, []ChatContentPart, bool) { + switch typed := value.(type) { + case []any: + var media []ChatContentPart + changed := false + for i, item := range typed { + rewritten, itemMedia, itemChanged := rewriteToolOutputMediaValue(item) + if !itemChanged { + continue + } + typed[i] = rewritten + media = append(media, itemMedia...) + changed = true + } + return typed, media, changed + case map[string]any: + if imageURL, ok := recognizedToolOutputImageURL(typed); ok { + return map[string]any{ + "type": "input_text", + "text": toolOutputMediaMarker, + }, []ChatContentPart{toolOutputImagePart(imageURL)}, true + } + + content, ok := typed["content"] + if !ok { + return typed, nil, false + } + rewritten, media, changed := rewriteToolOutputMediaValue(content) + if !changed { + return typed, nil, false + } + typed["content"] = rewritten + return typed, media, true + default: + return value, nil, false + } +} + +func recognizedToolOutputImageURL(value map[string]any) (string, bool) { + partType, _ := value["type"].(string) + if partType != "input_image" && partType != "image_url" { + return "", false + } + + switch imageURL := value["image_url"].(type) { + case string: + return imageURL, strings.TrimSpace(imageURL) != "" + case map[string]any: + url, _ := imageURL["url"].(string) + return url, strings.TrimSpace(url) != "" + default: + return "", false + } +} + +func isToolOutputImageDataURL(value string) bool { + const prefix = "data:image/" + const separator = ";base64," + if !strings.HasPrefix(value, prefix) { + return false + } + separatorIndex := strings.Index(value[len(prefix):], separator) + if separatorIndex <= 0 { + return false + } + payloadIndex := len(prefix) + separatorIndex + len(separator) + return payloadIndex < len(value) +} + +func toolOutputImagePart(imageURL string) ChatContentPart { + return ChatContentPart{ + Type: "image_url", + ImageURL: &ChatImageURL{URL: imageURL}, + } } // appendAssistantToolCall merges a tool call into the chat message list. @@ -417,6 +570,10 @@ func appendAssistantToolCall(messages []ChatMessage, toolCall ChatToolCall, pend // tool replies and intervening messages are emitted in their natural position // but never between an assistant tool_calls message and its replies. func normalizeChatMessages(messages []ChatMessage) []ChatMessage { + return normalizeChatMessagesWithToolOutputMedia(messages, nil) +} + +func normalizeChatMessagesWithToolOutputMedia(messages []ChatMessage, mediaByCallID toolOutputMediaByCallID) []ChatMessage { // Index every tool reply by its tool_call_id (last wins on duplicates). replies := make(map[string]ChatMessage) for _, m := range messages { @@ -463,6 +620,23 @@ func normalizeChatMessages(messages []ChatMessage) []ChatMessage { for _, tc := range kept { out = append(out, replies[tc.ID]) } + + var mediaParts []ChatContentPart + for _, tc := range kept { + media := mediaByCallID[tc.ID] + if len(media) == 0 { + continue + } + mediaParts = append(mediaParts, ChatContentPart{ + Type: "text", + Text: fmt.Sprintf(toolOutputMediaAttribution, tc.ID), + }) + mediaParts = append(mediaParts, media...) + } + if len(mediaParts) > 0 { + content, _ := json.Marshal(mediaParts) + out = append(out, ChatMessage{Role: "user", Content: content}) + } default: out = append(out, m) } diff --git a/backend/internal/pkg/apicompat/chatcompletions_responses_tool_output_media_test.go b/backend/internal/pkg/apicompat/chatcompletions_responses_tool_output_media_test.go new file mode 100644 index 0000000000..46c9c8b4cd --- /dev/null +++ b/backend/internal/pkg/apicompat/chatcompletions_responses_tool_output_media_test.go @@ -0,0 +1,318 @@ +package apicompat + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + testToolImageDataURL = "data:image/png;base64,AQID" + testToolImageRemote = "https://example.com/tool-output.png" +) + +func TestResponsesToolOutputMedia_ExtractsSupportedShapes(t *testing.T) { + tests := []struct { + name string + call string + outputType string + output string + imageURL string + toolText string + }{ + { + name: "image-only array", + call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`, + outputType: "function_call_output", + output: `[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]`, + imageURL: testToolImageDataURL, + }, + { + name: "text and nested image URL", + call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`, + outputType: "function_call_output", + output: `[{"type":"input_text","text":"render complete"},{"type":"image_url","image_url":{"url":"https://example.com/tool-output.png"}}]`, + imageURL: testToolImageRemote, + toolText: "render complete", + }, + { + name: "top-level image object", + call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`, + outputType: "function_call_output", + output: `{"type":"input_image","image_url":"data:image/png;base64,AQID"}`, + imageURL: testToolImageDataURL, + }, + { + name: "content wrapper", + call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`, + outputType: "function_call_output", + output: `{"status":"ok","content":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}],"unknown":{"score":0.9,"large":9007199254740993}}`, + imageURL: testToolImageDataURL, + toolText: `"large":9007199254740993`, + }, + { + name: "JSON string output", + call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`, + outputType: "function_call_output", + output: `"[{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,AQID\"}]"`, + imageURL: testToolImageDataURL, + }, + { + name: "bare data URL", + call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`, + outputType: "function_call_output", + output: `"data:image/png;base64,AQID"`, + imageURL: testToolImageDataURL, + }, + { + name: "custom tool output", + call: `{"type":"custom_tool_call","call_id":"call_image","name":"view_image","input":"{}"}`, + outputType: "custom_tool_call_output", + output: `[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]`, + imageURL: testToolImageDataURL, + }, + { + name: "tool search output", + call: `{"type":"tool_search_call","call_id":"call_image","arguments":{"query":"image"}}`, + outputType: "tool_search_output", + output: `[{"type":"image_url","image_url":{"url":"https://example.com/tool-output.png"}}]`, + imageURL: testToolImageRemote, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input := fmt.Sprintf(`[%s,{"type":%q,"call_id":"call_image","output":%s}]`, tt.call, tt.outputType, tt.output) + messages := convertToolOutputMedia(t, input) + + require.Len(t, messages, 3) + require.Equal(t, []string{"assistant", "tool", "user"}, chatMessageRoles(messages)) + require.Equal(t, "call_image", messages[1].ToolCallID) + + toolText := chatToolContentString(t, messages[1]) + require.Contains(t, toolText, "[Tool output media moved to the following user message]") + require.NotContains(t, toolText, tt.imageURL) + if tt.toolText != "" { + require.Contains(t, toolText, tt.toolText) + } + + parts := chatContentParts(t, messages[2]) + require.Len(t, parts, 2) + require.Equal(t, "text", parts[0].Type) + require.Equal(t, "[Tool output media for call call_image]", parts[0].Text) + require.Equal(t, "image_url", parts[1].Type) + require.NotNil(t, parts[1].ImageURL) + require.Equal(t, tt.imageURL, parts[1].ImageURL.URL) + }) + } +} + +func TestResponsesToolOutputMedia_ParallelBatchUsesCallOrder(t *testing.T) { + messages := convertToolOutputMedia(t, `[ + {"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"}, + {"type":"function_call","call_id":"call_B","name":"view_image","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_B","output":[{"type":"input_image","image_url":{"url":"https://example.com/b.png"}}]}, + {"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":{"url":"https://example.com/a.png"}}]} + ]`) + + require.Len(t, messages, 4) + require.Equal(t, []string{"assistant", "tool", "tool", "user"}, chatMessageRoles(messages)) + require.Equal(t, []string{"call_A", "call_B"}, []string{messages[1].ToolCallID, messages[2].ToolCallID}) + + parts := chatContentParts(t, messages[3]) + require.Len(t, parts, 4) + require.Equal(t, "[Tool output media for call call_A]", parts[0].Text) + require.Equal(t, "https://example.com/a.png", parts[1].ImageURL.URL) + require.Equal(t, "[Tool output media for call call_B]", parts[2].Text) + require.Equal(t, "https://example.com/b.png", parts[3].ImageURL.URL) +} + +func TestResponsesToolOutputMedia_PreservesRichSiblingWhenRewriting(t *testing.T) { + messages := convertToolOutputMedia(t, `[ + {"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_image","output":[ + {"type":"result","url":"https://example.com/result","score":0.9,"text":"complete","extra":{"count":2}}, + {"type":"input_image","image_url":"data:image/png;base64,AQID"} + ]} + ]`) + + toolText := chatToolContentString(t, messages[1]) + require.JSONEq(t, `[ + {"type":"result","url":"https://example.com/result","score":0.9,"text":"complete","extra":{"count":2}}, + {"type":"input_text","text":"[Tool output media moved to the following user message]"} + ]`, toolText) +} + +func TestResponsesToolOutputMedia_InterleavedMessagesFollowMediaBatch(t *testing.T) { + messages := convertToolOutputMedia(t, `[ + {"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"}, + {"type":"message","role":"developer","content":[{"type":"input_text","text":"approval saved"}]}, + {"type":"message","role":"user","content":[{"type":"input_text","text":"continue"}]}, + {"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]} + ]`) + + require.Equal(t, []string{"assistant", "tool", "user", "system", "user"}, chatMessageRoles(messages)) + require.Equal(t, "[Tool output media for call call_A]", chatContentParts(t, messages[2])[0].Text) + require.JSONEq(t, `"approval saved"`, string(messages[3].Content)) + require.JSONEq(t, `"continue"`, string(messages[4].Content)) +} + +func TestResponsesToolOutputMedia_DropsOrphanAndUnansweredCallMedia(t *testing.T) { + t.Run("orphan", func(t *testing.T) { + messages := convertToolOutputMedia(t, `[ + {"type":"function_call_output","call_id":"call_ghost","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]} + ]`) + require.Empty(t, messages) + }) + + t.Run("unanswered parallel call", func(t *testing.T) { + messages := convertToolOutputMedia(t, `[ + {"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"}, + {"type":"function_call","call_id":"call_B","name":"view_image","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]} + ]`) + + require.Len(t, messages, 3) + require.Len(t, messages[0].ToolCalls, 1) + require.Equal(t, "call_A", messages[0].ToolCalls[0].ID) + parts := chatContentParts(t, messages[2]) + require.Len(t, parts, 2) + require.NotContains(t, string(messages[2].Content), "call_B") + }) +} + +func TestResponsesToolOutputMedia_PreservesMediaFreeOutputBytes(t *testing.T) { + tests := []struct { + name string + output string + }{ + { + name: "rich unknown object", + output: `{"type":"result","url":"https://example.com/result","score":0.9,"text":"complete","extra":{"count":2}}`, + }, + { + name: "no-image array", + output: `[ { "type": "input_text", "text": "ok" }, {"unknown":true} ]`, + }, + { + name: "plain string", + output: `"plain output"`, + }, + { + name: "JSON string without image", + output: `"{ \"ok\": true }"`, + }, + { + name: "embedded data URL text", + output: `"prefix data:image/png;base64,AQID suffix"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input := fmt.Sprintf(`[ + {"type":"function_call","call_id":"call_text","name":"exec","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_text","output":%s} + ]`, tt.output) + messages := convertToolOutputMedia(t, input) + require.Len(t, messages, 2) + + var expected string + if err := json.Unmarshal([]byte(tt.output), &expected); err != nil { + expected = tt.output + } + expectedContent, err := json.Marshal(expected) + require.NoError(t, err) + require.Equal(t, string(expectedContent), string(messages[1].Content)) + }) + } +} + +func TestResponsesToolOutputMedia_DuplicateCallIDIsLastWins(t *testing.T) { + t.Run("later media replaces earlier media", func(t *testing.T) { + messages := convertToolOutputMedia(t, `[ + {"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":{"url":"https://example.com/first.png"}}]}, + {"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":{"url":"https://example.com/last.png"}}]} + ]`) + + require.Len(t, messages, 3) + require.NotContains(t, string(messages[1].Content), "first.png") + require.NotContains(t, string(messages[2].Content), "first.png") + require.Contains(t, string(messages[2].Content), "last.png") + }) + + t.Run("later text clears earlier media", func(t *testing.T) { + messages := convertToolOutputMedia(t, `[ + {"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}, + {"type":"function_call_output","call_id":"call_image","output":"latest text"} + ]`) + + require.Len(t, messages, 2) + require.Equal(t, "latest text", chatToolContentString(t, messages[1])) + }) +} + +func TestResponsesToChatCompletionsRequest_ToolContentNeverContainsExtractedMedia(t *testing.T) { + req := &ResponsesRequest{ + Model: "vision-model", + Input: json.RawMessage(`[ + {"type":"function_call","call_id":"call_function","name":"view_image","arguments":"{}"}, + {"type":"custom_tool_call","call_id":"call_custom","name":"custom_image","input":"{}"}, + {"type":"tool_search_call","call_id":"call_search","arguments":{"query":"image"}}, + {"type":"function_call_output","call_id":"call_function","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}, + {"type":"custom_tool_call_output","call_id":"call_custom","output":{"content":[{"type":"image_url","image_url":{"url":"https://example.com/custom.png"}}]}}, + {"type":"tool_search_output","call_id":"call_search","output":"data:image/jpeg;base64,BAUG"} + ]`), + } + + out, err := ResponsesToChatCompletionsRequest(req) + require.NoError(t, err) + assertChatInvariants(t, out.Messages) + + var toolCount int + for _, message := range out.Messages { + if message.Role != "tool" { + continue + } + toolCount++ + content := string(message.Content) + require.NotContains(t, content, "data:image/") + require.NotContains(t, content, "https://example.com/custom.png") + } + require.Equal(t, 3, toolCount) + require.Equal(t, []string{"assistant", "tool", "tool", "tool", "user"}, chatMessageRoles(out.Messages)) +} + +func convertToolOutputMedia(t *testing.T, input string) []ChatMessage { + t.Helper() + messages, err := responsesInputToChatMessages("", json.RawMessage(input)) + require.NoError(t, err) + assertChatInvariants(t, messages) + return messages +} + +func chatToolContentString(t *testing.T, message ChatMessage) string { + t.Helper() + require.Equal(t, "tool", message.Role) + var content string + require.NoError(t, json.Unmarshal(message.Content, &content)) + return content +} + +func chatContentParts(t *testing.T, message ChatMessage) []ChatContentPart { + t.Helper() + var parts []ChatContentPart + require.NoError(t, json.Unmarshal(message.Content, &parts)) + for _, part := range parts { + if part.Type == "image_url" { + require.NotNil(t, part.ImageURL) + require.False(t, strings.TrimSpace(part.ImageURL.URL) == "") + } + } + return parts +}