修复工具输出图片桥接

This commit is contained in:
tudoujun
2026-07-31 19:36:47 +08:00
parent 94df1fffc2
commit 2bf9c6d56b
2 changed files with 504 additions and 12 deletions
@@ -1,6 +1,7 @@
package apicompat
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
@@ -9,6 +10,13 @@ import (
"time"
)
const (
toolOutputMediaMarker = "[Tool output media moved to the following user message]"
toolOutputMediaAttribution = "[Tool output media for call %s]"
)
type toolOutputMediaByCallID map[string][]ChatContentPart
// ResponsesToChatCompletionsRequest converts a Responses API request into a
// Chat Completions request for upstreams that only implement
// /v1/chat/completions.
@@ -215,16 +223,16 @@ func responsesInputToChatMessages(instructions string, inputRaw json.RawMessage)
return nil, fmt.Errorf("parse responses input: %w", err)
}
built, err := buildChatMessagesFromItems(messages, rawItems)
built, mediaByCallID, err := buildChatMessagesFromItems(messages, rawItems)
if err != nil {
return nil, err
}
return normalizeChatMessages(built), nil
return normalizeChatMessagesWithToolOutputMedia(built, mediaByCallID), nil
}
// buildChatMessagesFromItems walks the Responses input items and appends the
// corresponding Chat messages.
func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessage) ([]ChatMessage, error) {
func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessage) ([]ChatMessage, toolOutputMediaByCallID, error) {
// pendingReasoning holds the reasoning text from a reasoning item until the
// assistant message it belongs to is emitted. DeepSeek's thinking mode
// requires the reasoning_content that produced a tool call to be passed back
@@ -232,6 +240,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
// across an assistant message (so a following tool call in the same turn
// still receives it); any other role ends the thinking span.
var pendingReasoning string
mediaByCallID := make(toolOutputMediaByCallID)
for _, raw := range rawItems {
raw = bytesTrimSpace(raw)
@@ -248,7 +257,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
pendingReasoning = ""
continue
}
return nil, fmt.Errorf("parse responses input item: %w", err)
return nil, nil, fmt.Errorf("parse responses input item: %w", err)
}
role := chatCompletionsBridgeRole(rawString(item["role"]))
@@ -320,15 +329,25 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
continue
case "function_call_output", "custom_tool_call_output", "tool_search_output":
outputRaw := bytesTrimSpace(item["output"])
outputText := rawString(outputRaw)
if outputText == "" && len(outputRaw) > 0 && string(outputRaw) != "null" && string(outputRaw) != `""` {
// 对象/数组形式的输出(如 tool_search 的结果列表)整体字符串化。
outputText = string(outputRaw)
callID := rawString(item["call_id"])
delete(mediaByCallID, callID)
outputText, media, rewritten := extractToolOutputMedia(outputRaw)
if rewritten {
if callID != "" {
mediaByCallID[callID] = media
}
} else {
outputText = rawString(outputRaw)
if outputText == "" && len(outputRaw) > 0 && string(outputRaw) != "null" && string(outputRaw) != `""` {
// 对象/数组形式的输出(如 tool_search 的结果列表)整体字符串化。
outputText = string(outputRaw)
}
}
content, _ := json.Marshal(outputText)
messages = append(messages, ChatMessage{
Role: "tool",
ToolCallID: rawString(item["call_id"]),
ToolCallID: callID,
Content: content,
})
pendingReasoning = ""
@@ -341,7 +360,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
case "input_image":
content, err := chatContentFromSingleResponsesPart(itemType, item)
if err != nil {
return nil, err
return nil, nil, err
}
messages = append(messages, ChatMessage{Role: "user", Content: content})
pendingReasoning = ""
@@ -367,7 +386,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
}
chatContent, err := responsesContentToChatContent(content, role)
if err != nil {
return nil, err
return nil, nil, err
}
messages = append(messages, ChatMessage{Role: role, Content: chatContent})
// Reasoning only survives across an assistant text message.
@@ -376,7 +395,141 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
}
}
return messages, nil
return messages, mediaByCallID, nil
}
// extractToolOutputMedia rewrites only recognized image nodes. Media-free
// outputs return rewritten=false so the caller can preserve their original
// bytes and prompt-cache prefix.
func extractToolOutputMedia(outputRaw json.RawMessage) (string, []ChatContentPart, bool) {
outputRaw = bytesTrimSpace(outputRaw)
if len(outputRaw) == 0 || string(outputRaw) == "null" {
return "", nil, false
}
var outputString string
if err := json.Unmarshal(outputRaw, &outputString); err == nil {
if isToolOutputImageDataURL(outputString) {
return toolOutputMediaMarker, []ChatContentPart{toolOutputImagePart(outputString)}, true
}
nested, ok := decodeToolOutputJSON([]byte(outputString))
if !ok {
return "", nil, false
}
rewritten, media, changed := rewriteToolOutputMediaValue(nested)
if !changed {
return "", nil, false
}
encoded, err := json.Marshal(rewritten)
if err != nil {
return "", nil, false
}
return string(encoded), media, true
}
value, ok := decodeToolOutputJSON(outputRaw)
if !ok {
return "", nil, false
}
rewritten, media, changed := rewriteToolOutputMediaValue(value)
if !changed {
return "", nil, false
}
encoded, err := json.Marshal(rewritten)
if err != nil {
return "", nil, false
}
return string(encoded), media, true
}
func decodeToolOutputJSON(raw []byte) (any, bool) {
if !json.Valid(raw) {
return nil, false
}
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.UseNumber()
var value any
if err := decoder.Decode(&value); err != nil {
return nil, false
}
return value, true
}
func rewriteToolOutputMediaValue(value any) (any, []ChatContentPart, bool) {
switch typed := value.(type) {
case []any:
var media []ChatContentPart
changed := false
for i, item := range typed {
rewritten, itemMedia, itemChanged := rewriteToolOutputMediaValue(item)
if !itemChanged {
continue
}
typed[i] = rewritten
media = append(media, itemMedia...)
changed = true
}
return typed, media, changed
case map[string]any:
if imageURL, ok := recognizedToolOutputImageURL(typed); ok {
return map[string]any{
"type": "input_text",
"text": toolOutputMediaMarker,
}, []ChatContentPart{toolOutputImagePart(imageURL)}, true
}
content, ok := typed["content"]
if !ok {
return typed, nil, false
}
rewritten, media, changed := rewriteToolOutputMediaValue(content)
if !changed {
return typed, nil, false
}
typed["content"] = rewritten
return typed, media, true
default:
return value, nil, false
}
}
func recognizedToolOutputImageURL(value map[string]any) (string, bool) {
partType, _ := value["type"].(string)
if partType != "input_image" && partType != "image_url" {
return "", false
}
switch imageURL := value["image_url"].(type) {
case string:
return imageURL, strings.TrimSpace(imageURL) != ""
case map[string]any:
url, _ := imageURL["url"].(string)
return url, strings.TrimSpace(url) != ""
default:
return "", false
}
}
func isToolOutputImageDataURL(value string) bool {
const prefix = "data:image/"
const separator = ";base64,"
if !strings.HasPrefix(value, prefix) {
return false
}
separatorIndex := strings.Index(value[len(prefix):], separator)
if separatorIndex <= 0 {
return false
}
payloadIndex := len(prefix) + separatorIndex + len(separator)
return payloadIndex < len(value)
}
func toolOutputImagePart(imageURL string) ChatContentPart {
return ChatContentPart{
Type: "image_url",
ImageURL: &ChatImageURL{URL: imageURL},
}
}
// appendAssistantToolCall merges a tool call into the chat message list.
@@ -417,6 +570,10 @@ func appendAssistantToolCall(messages []ChatMessage, toolCall ChatToolCall, pend
// tool replies and intervening messages are emitted in their natural position
// but never between an assistant tool_calls message and its replies.
func normalizeChatMessages(messages []ChatMessage) []ChatMessage {
return normalizeChatMessagesWithToolOutputMedia(messages, nil)
}
func normalizeChatMessagesWithToolOutputMedia(messages []ChatMessage, mediaByCallID toolOutputMediaByCallID) []ChatMessage {
// Index every tool reply by its tool_call_id (last wins on duplicates).
replies := make(map[string]ChatMessage)
for _, m := range messages {
@@ -463,6 +620,23 @@ func normalizeChatMessages(messages []ChatMessage) []ChatMessage {
for _, tc := range kept {
out = append(out, replies[tc.ID])
}
var mediaParts []ChatContentPart
for _, tc := range kept {
media := mediaByCallID[tc.ID]
if len(media) == 0 {
continue
}
mediaParts = append(mediaParts, ChatContentPart{
Type: "text",
Text: fmt.Sprintf(toolOutputMediaAttribution, tc.ID),
})
mediaParts = append(mediaParts, media...)
}
if len(mediaParts) > 0 {
content, _ := json.Marshal(mediaParts)
out = append(out, ChatMessage{Role: "user", Content: content})
}
default:
out = append(out, m)
}
@@ -0,0 +1,318 @@
package apicompat
import (
"encoding/json"
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
const (
testToolImageDataURL = "data:image/png;base64,AQID"
testToolImageRemote = "https://example.com/tool-output.png"
)
func TestResponsesToolOutputMedia_ExtractsSupportedShapes(t *testing.T) {
tests := []struct {
name string
call string
outputType string
output string
imageURL string
toolText string
}{
{
name: "image-only array",
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
outputType: "function_call_output",
output: `[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]`,
imageURL: testToolImageDataURL,
},
{
name: "text and nested image URL",
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
outputType: "function_call_output",
output: `[{"type":"input_text","text":"render complete"},{"type":"image_url","image_url":{"url":"https://example.com/tool-output.png"}}]`,
imageURL: testToolImageRemote,
toolText: "render complete",
},
{
name: "top-level image object",
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
outputType: "function_call_output",
output: `{"type":"input_image","image_url":"data:image/png;base64,AQID"}`,
imageURL: testToolImageDataURL,
},
{
name: "content wrapper",
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
outputType: "function_call_output",
output: `{"status":"ok","content":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}],"unknown":{"score":0.9,"large":9007199254740993}}`,
imageURL: testToolImageDataURL,
toolText: `"large":9007199254740993`,
},
{
name: "JSON string output",
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
outputType: "function_call_output",
output: `"[{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,AQID\"}]"`,
imageURL: testToolImageDataURL,
},
{
name: "bare data URL",
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
outputType: "function_call_output",
output: `"data:image/png;base64,AQID"`,
imageURL: testToolImageDataURL,
},
{
name: "custom tool output",
call: `{"type":"custom_tool_call","call_id":"call_image","name":"view_image","input":"{}"}`,
outputType: "custom_tool_call_output",
output: `[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]`,
imageURL: testToolImageDataURL,
},
{
name: "tool search output",
call: `{"type":"tool_search_call","call_id":"call_image","arguments":{"query":"image"}}`,
outputType: "tool_search_output",
output: `[{"type":"image_url","image_url":{"url":"https://example.com/tool-output.png"}}]`,
imageURL: testToolImageRemote,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
input := fmt.Sprintf(`[%s,{"type":%q,"call_id":"call_image","output":%s}]`, tt.call, tt.outputType, tt.output)
messages := convertToolOutputMedia(t, input)
require.Len(t, messages, 3)
require.Equal(t, []string{"assistant", "tool", "user"}, chatMessageRoles(messages))
require.Equal(t, "call_image", messages[1].ToolCallID)
toolText := chatToolContentString(t, messages[1])
require.Contains(t, toolText, "[Tool output media moved to the following user message]")
require.NotContains(t, toolText, tt.imageURL)
if tt.toolText != "" {
require.Contains(t, toolText, tt.toolText)
}
parts := chatContentParts(t, messages[2])
require.Len(t, parts, 2)
require.Equal(t, "text", parts[0].Type)
require.Equal(t, "[Tool output media for call call_image]", parts[0].Text)
require.Equal(t, "image_url", parts[1].Type)
require.NotNil(t, parts[1].ImageURL)
require.Equal(t, tt.imageURL, parts[1].ImageURL.URL)
})
}
}
func TestResponsesToolOutputMedia_ParallelBatchUsesCallOrder(t *testing.T) {
messages := convertToolOutputMedia(t, `[
{"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"},
{"type":"function_call","call_id":"call_B","name":"view_image","arguments":"{}"},
{"type":"function_call_output","call_id":"call_B","output":[{"type":"input_image","image_url":{"url":"https://example.com/b.png"}}]},
{"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":{"url":"https://example.com/a.png"}}]}
]`)
require.Len(t, messages, 4)
require.Equal(t, []string{"assistant", "tool", "tool", "user"}, chatMessageRoles(messages))
require.Equal(t, []string{"call_A", "call_B"}, []string{messages[1].ToolCallID, messages[2].ToolCallID})
parts := chatContentParts(t, messages[3])
require.Len(t, parts, 4)
require.Equal(t, "[Tool output media for call call_A]", parts[0].Text)
require.Equal(t, "https://example.com/a.png", parts[1].ImageURL.URL)
require.Equal(t, "[Tool output media for call call_B]", parts[2].Text)
require.Equal(t, "https://example.com/b.png", parts[3].ImageURL.URL)
}
func TestResponsesToolOutputMedia_PreservesRichSiblingWhenRewriting(t *testing.T) {
messages := convertToolOutputMedia(t, `[
{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},
{"type":"function_call_output","call_id":"call_image","output":[
{"type":"result","url":"https://example.com/result","score":0.9,"text":"complete","extra":{"count":2}},
{"type":"input_image","image_url":"data:image/png;base64,AQID"}
]}
]`)
toolText := chatToolContentString(t, messages[1])
require.JSONEq(t, `[
{"type":"result","url":"https://example.com/result","score":0.9,"text":"complete","extra":{"count":2}},
{"type":"input_text","text":"[Tool output media moved to the following user message]"}
]`, toolText)
}
func TestResponsesToolOutputMedia_InterleavedMessagesFollowMediaBatch(t *testing.T) {
messages := convertToolOutputMedia(t, `[
{"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"},
{"type":"message","role":"developer","content":[{"type":"input_text","text":"approval saved"}]},
{"type":"message","role":"user","content":[{"type":"input_text","text":"continue"}]},
{"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}
]`)
require.Equal(t, []string{"assistant", "tool", "user", "system", "user"}, chatMessageRoles(messages))
require.Equal(t, "[Tool output media for call call_A]", chatContentParts(t, messages[2])[0].Text)
require.JSONEq(t, `"approval saved"`, string(messages[3].Content))
require.JSONEq(t, `"continue"`, string(messages[4].Content))
}
func TestResponsesToolOutputMedia_DropsOrphanAndUnansweredCallMedia(t *testing.T) {
t.Run("orphan", func(t *testing.T) {
messages := convertToolOutputMedia(t, `[
{"type":"function_call_output","call_id":"call_ghost","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}
]`)
require.Empty(t, messages)
})
t.Run("unanswered parallel call", func(t *testing.T) {
messages := convertToolOutputMedia(t, `[
{"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"},
{"type":"function_call","call_id":"call_B","name":"view_image","arguments":"{}"},
{"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}
]`)
require.Len(t, messages, 3)
require.Len(t, messages[0].ToolCalls, 1)
require.Equal(t, "call_A", messages[0].ToolCalls[0].ID)
parts := chatContentParts(t, messages[2])
require.Len(t, parts, 2)
require.NotContains(t, string(messages[2].Content), "call_B")
})
}
func TestResponsesToolOutputMedia_PreservesMediaFreeOutputBytes(t *testing.T) {
tests := []struct {
name string
output string
}{
{
name: "rich unknown object",
output: `{"type":"result","url":"https://example.com/result","score":0.9,"text":"complete","extra":{"count":2}}`,
},
{
name: "no-image array",
output: `[ { "type": "input_text", "text": "ok" }, {"unknown":true} ]`,
},
{
name: "plain string",
output: `"plain output"`,
},
{
name: "JSON string without image",
output: `"{ \"ok\": true }"`,
},
{
name: "embedded data URL text",
output: `"prefix data:image/png;base64,AQID suffix"`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
input := fmt.Sprintf(`[
{"type":"function_call","call_id":"call_text","name":"exec","arguments":"{}"},
{"type":"function_call_output","call_id":"call_text","output":%s}
]`, tt.output)
messages := convertToolOutputMedia(t, input)
require.Len(t, messages, 2)
var expected string
if err := json.Unmarshal([]byte(tt.output), &expected); err != nil {
expected = tt.output
}
expectedContent, err := json.Marshal(expected)
require.NoError(t, err)
require.Equal(t, string(expectedContent), string(messages[1].Content))
})
}
}
func TestResponsesToolOutputMedia_DuplicateCallIDIsLastWins(t *testing.T) {
t.Run("later media replaces earlier media", func(t *testing.T) {
messages := convertToolOutputMedia(t, `[
{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},
{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":{"url":"https://example.com/first.png"}}]},
{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":{"url":"https://example.com/last.png"}}]}
]`)
require.Len(t, messages, 3)
require.NotContains(t, string(messages[1].Content), "first.png")
require.NotContains(t, string(messages[2].Content), "first.png")
require.Contains(t, string(messages[2].Content), "last.png")
})
t.Run("later text clears earlier media", func(t *testing.T) {
messages := convertToolOutputMedia(t, `[
{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},
{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]},
{"type":"function_call_output","call_id":"call_image","output":"latest text"}
]`)
require.Len(t, messages, 2)
require.Equal(t, "latest text", chatToolContentString(t, messages[1]))
})
}
func TestResponsesToChatCompletionsRequest_ToolContentNeverContainsExtractedMedia(t *testing.T) {
req := &ResponsesRequest{
Model: "vision-model",
Input: json.RawMessage(`[
{"type":"function_call","call_id":"call_function","name":"view_image","arguments":"{}"},
{"type":"custom_tool_call","call_id":"call_custom","name":"custom_image","input":"{}"},
{"type":"tool_search_call","call_id":"call_search","arguments":{"query":"image"}},
{"type":"function_call_output","call_id":"call_function","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]},
{"type":"custom_tool_call_output","call_id":"call_custom","output":{"content":[{"type":"image_url","image_url":{"url":"https://example.com/custom.png"}}]}},
{"type":"tool_search_output","call_id":"call_search","output":"data:image/jpeg;base64,BAUG"}
]`),
}
out, err := ResponsesToChatCompletionsRequest(req)
require.NoError(t, err)
assertChatInvariants(t, out.Messages)
var toolCount int
for _, message := range out.Messages {
if message.Role != "tool" {
continue
}
toolCount++
content := string(message.Content)
require.NotContains(t, content, "data:image/")
require.NotContains(t, content, "https://example.com/custom.png")
}
require.Equal(t, 3, toolCount)
require.Equal(t, []string{"assistant", "tool", "tool", "tool", "user"}, chatMessageRoles(out.Messages))
}
func convertToolOutputMedia(t *testing.T, input string) []ChatMessage {
t.Helper()
messages, err := responsesInputToChatMessages("", json.RawMessage(input))
require.NoError(t, err)
assertChatInvariants(t, messages)
return messages
}
func chatToolContentString(t *testing.T, message ChatMessage) string {
t.Helper()
require.Equal(t, "tool", message.Role)
var content string
require.NoError(t, json.Unmarshal(message.Content, &content))
return content
}
func chatContentParts(t *testing.T, message ChatMessage) []ChatContentPart {
t.Helper()
var parts []ChatContentPart
require.NoError(t, json.Unmarshal(message.Content, &parts))
for _, part := range parts {
if part.Type == "image_url" {
require.NotNil(t, part.ImageURL)
require.False(t, strings.TrimSpace(part.ImageURL.URL) == "")
}
}
return parts
}