mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-14 12:02:27 +08:00
修复工具输出图片桥接
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package apicompat
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
@@ -9,6 +10,13 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
toolOutputMediaMarker = "[Tool output media moved to the following user message]"
|
||||
toolOutputMediaAttribution = "[Tool output media for call %s]"
|
||||
)
|
||||
|
||||
type toolOutputMediaByCallID map[string][]ChatContentPart
|
||||
|
||||
// ResponsesToChatCompletionsRequest converts a Responses API request into a
|
||||
// Chat Completions request for upstreams that only implement
|
||||
// /v1/chat/completions.
|
||||
@@ -215,16 +223,16 @@ func responsesInputToChatMessages(instructions string, inputRaw json.RawMessage)
|
||||
return nil, fmt.Errorf("parse responses input: %w", err)
|
||||
}
|
||||
|
||||
built, err := buildChatMessagesFromItems(messages, rawItems)
|
||||
built, mediaByCallID, err := buildChatMessagesFromItems(messages, rawItems)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return normalizeChatMessages(built), nil
|
||||
return normalizeChatMessagesWithToolOutputMedia(built, mediaByCallID), nil
|
||||
}
|
||||
|
||||
// buildChatMessagesFromItems walks the Responses input items and appends the
|
||||
// corresponding Chat messages.
|
||||
func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessage) ([]ChatMessage, error) {
|
||||
func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessage) ([]ChatMessage, toolOutputMediaByCallID, error) {
|
||||
// pendingReasoning holds the reasoning text from a reasoning item until the
|
||||
// assistant message it belongs to is emitted. DeepSeek's thinking mode
|
||||
// requires the reasoning_content that produced a tool call to be passed back
|
||||
@@ -232,6 +240,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
|
||||
// across an assistant message (so a following tool call in the same turn
|
||||
// still receives it); any other role ends the thinking span.
|
||||
var pendingReasoning string
|
||||
mediaByCallID := make(toolOutputMediaByCallID)
|
||||
|
||||
for _, raw := range rawItems {
|
||||
raw = bytesTrimSpace(raw)
|
||||
@@ -248,7 +257,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
|
||||
pendingReasoning = ""
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("parse responses input item: %w", err)
|
||||
return nil, nil, fmt.Errorf("parse responses input item: %w", err)
|
||||
}
|
||||
|
||||
role := chatCompletionsBridgeRole(rawString(item["role"]))
|
||||
@@ -320,15 +329,25 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
|
||||
continue
|
||||
case "function_call_output", "custom_tool_call_output", "tool_search_output":
|
||||
outputRaw := bytesTrimSpace(item["output"])
|
||||
outputText := rawString(outputRaw)
|
||||
if outputText == "" && len(outputRaw) > 0 && string(outputRaw) != "null" && string(outputRaw) != `""` {
|
||||
// 对象/数组形式的输出(如 tool_search 的结果列表)整体字符串化。
|
||||
outputText = string(outputRaw)
|
||||
callID := rawString(item["call_id"])
|
||||
delete(mediaByCallID, callID)
|
||||
|
||||
outputText, media, rewritten := extractToolOutputMedia(outputRaw)
|
||||
if rewritten {
|
||||
if callID != "" {
|
||||
mediaByCallID[callID] = media
|
||||
}
|
||||
} else {
|
||||
outputText = rawString(outputRaw)
|
||||
if outputText == "" && len(outputRaw) > 0 && string(outputRaw) != "null" && string(outputRaw) != `""` {
|
||||
// 对象/数组形式的输出(如 tool_search 的结果列表)整体字符串化。
|
||||
outputText = string(outputRaw)
|
||||
}
|
||||
}
|
||||
content, _ := json.Marshal(outputText)
|
||||
messages = append(messages, ChatMessage{
|
||||
Role: "tool",
|
||||
ToolCallID: rawString(item["call_id"]),
|
||||
ToolCallID: callID,
|
||||
Content: content,
|
||||
})
|
||||
pendingReasoning = ""
|
||||
@@ -341,7 +360,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
|
||||
case "input_image":
|
||||
content, err := chatContentFromSingleResponsesPart(itemType, item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
messages = append(messages, ChatMessage{Role: "user", Content: content})
|
||||
pendingReasoning = ""
|
||||
@@ -367,7 +386,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
|
||||
}
|
||||
chatContent, err := responsesContentToChatContent(content, role)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
messages = append(messages, ChatMessage{Role: role, Content: chatContent})
|
||||
// Reasoning only survives across an assistant text message.
|
||||
@@ -376,7 +395,141 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa
|
||||
}
|
||||
}
|
||||
|
||||
return messages, nil
|
||||
return messages, mediaByCallID, nil
|
||||
}
|
||||
|
||||
// extractToolOutputMedia rewrites only recognized image nodes. Media-free
|
||||
// outputs return rewritten=false so the caller can preserve their original
|
||||
// bytes and prompt-cache prefix.
|
||||
func extractToolOutputMedia(outputRaw json.RawMessage) (string, []ChatContentPart, bool) {
|
||||
outputRaw = bytesTrimSpace(outputRaw)
|
||||
if len(outputRaw) == 0 || string(outputRaw) == "null" {
|
||||
return "", nil, false
|
||||
}
|
||||
|
||||
var outputString string
|
||||
if err := json.Unmarshal(outputRaw, &outputString); err == nil {
|
||||
if isToolOutputImageDataURL(outputString) {
|
||||
return toolOutputMediaMarker, []ChatContentPart{toolOutputImagePart(outputString)}, true
|
||||
}
|
||||
|
||||
nested, ok := decodeToolOutputJSON([]byte(outputString))
|
||||
if !ok {
|
||||
return "", nil, false
|
||||
}
|
||||
rewritten, media, changed := rewriteToolOutputMediaValue(nested)
|
||||
if !changed {
|
||||
return "", nil, false
|
||||
}
|
||||
encoded, err := json.Marshal(rewritten)
|
||||
if err != nil {
|
||||
return "", nil, false
|
||||
}
|
||||
return string(encoded), media, true
|
||||
}
|
||||
|
||||
value, ok := decodeToolOutputJSON(outputRaw)
|
||||
if !ok {
|
||||
return "", nil, false
|
||||
}
|
||||
rewritten, media, changed := rewriteToolOutputMediaValue(value)
|
||||
if !changed {
|
||||
return "", nil, false
|
||||
}
|
||||
encoded, err := json.Marshal(rewritten)
|
||||
if err != nil {
|
||||
return "", nil, false
|
||||
}
|
||||
return string(encoded), media, true
|
||||
}
|
||||
|
||||
func decodeToolOutputJSON(raw []byte) (any, bool) {
|
||||
if !json.Valid(raw) {
|
||||
return nil, false
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.UseNumber()
|
||||
var value any
|
||||
if err := decoder.Decode(&value); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return value, true
|
||||
}
|
||||
|
||||
func rewriteToolOutputMediaValue(value any) (any, []ChatContentPart, bool) {
|
||||
switch typed := value.(type) {
|
||||
case []any:
|
||||
var media []ChatContentPart
|
||||
changed := false
|
||||
for i, item := range typed {
|
||||
rewritten, itemMedia, itemChanged := rewriteToolOutputMediaValue(item)
|
||||
if !itemChanged {
|
||||
continue
|
||||
}
|
||||
typed[i] = rewritten
|
||||
media = append(media, itemMedia...)
|
||||
changed = true
|
||||
}
|
||||
return typed, media, changed
|
||||
case map[string]any:
|
||||
if imageURL, ok := recognizedToolOutputImageURL(typed); ok {
|
||||
return map[string]any{
|
||||
"type": "input_text",
|
||||
"text": toolOutputMediaMarker,
|
||||
}, []ChatContentPart{toolOutputImagePart(imageURL)}, true
|
||||
}
|
||||
|
||||
content, ok := typed["content"]
|
||||
if !ok {
|
||||
return typed, nil, false
|
||||
}
|
||||
rewritten, media, changed := rewriteToolOutputMediaValue(content)
|
||||
if !changed {
|
||||
return typed, nil, false
|
||||
}
|
||||
typed["content"] = rewritten
|
||||
return typed, media, true
|
||||
default:
|
||||
return value, nil, false
|
||||
}
|
||||
}
|
||||
|
||||
func recognizedToolOutputImageURL(value map[string]any) (string, bool) {
|
||||
partType, _ := value["type"].(string)
|
||||
if partType != "input_image" && partType != "image_url" {
|
||||
return "", false
|
||||
}
|
||||
|
||||
switch imageURL := value["image_url"].(type) {
|
||||
case string:
|
||||
return imageURL, strings.TrimSpace(imageURL) != ""
|
||||
case map[string]any:
|
||||
url, _ := imageURL["url"].(string)
|
||||
return url, strings.TrimSpace(url) != ""
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func isToolOutputImageDataURL(value string) bool {
|
||||
const prefix = "data:image/"
|
||||
const separator = ";base64,"
|
||||
if !strings.HasPrefix(value, prefix) {
|
||||
return false
|
||||
}
|
||||
separatorIndex := strings.Index(value[len(prefix):], separator)
|
||||
if separatorIndex <= 0 {
|
||||
return false
|
||||
}
|
||||
payloadIndex := len(prefix) + separatorIndex + len(separator)
|
||||
return payloadIndex < len(value)
|
||||
}
|
||||
|
||||
func toolOutputImagePart(imageURL string) ChatContentPart {
|
||||
return ChatContentPart{
|
||||
Type: "image_url",
|
||||
ImageURL: &ChatImageURL{URL: imageURL},
|
||||
}
|
||||
}
|
||||
|
||||
// appendAssistantToolCall merges a tool call into the chat message list.
|
||||
@@ -417,6 +570,10 @@ func appendAssistantToolCall(messages []ChatMessage, toolCall ChatToolCall, pend
|
||||
// tool replies and intervening messages are emitted in their natural position
|
||||
// but never between an assistant tool_calls message and its replies.
|
||||
func normalizeChatMessages(messages []ChatMessage) []ChatMessage {
|
||||
return normalizeChatMessagesWithToolOutputMedia(messages, nil)
|
||||
}
|
||||
|
||||
func normalizeChatMessagesWithToolOutputMedia(messages []ChatMessage, mediaByCallID toolOutputMediaByCallID) []ChatMessage {
|
||||
// Index every tool reply by its tool_call_id (last wins on duplicates).
|
||||
replies := make(map[string]ChatMessage)
|
||||
for _, m := range messages {
|
||||
@@ -463,6 +620,23 @@ func normalizeChatMessages(messages []ChatMessage) []ChatMessage {
|
||||
for _, tc := range kept {
|
||||
out = append(out, replies[tc.ID])
|
||||
}
|
||||
|
||||
var mediaParts []ChatContentPart
|
||||
for _, tc := range kept {
|
||||
media := mediaByCallID[tc.ID]
|
||||
if len(media) == 0 {
|
||||
continue
|
||||
}
|
||||
mediaParts = append(mediaParts, ChatContentPart{
|
||||
Type: "text",
|
||||
Text: fmt.Sprintf(toolOutputMediaAttribution, tc.ID),
|
||||
})
|
||||
mediaParts = append(mediaParts, media...)
|
||||
}
|
||||
if len(mediaParts) > 0 {
|
||||
content, _ := json.Marshal(mediaParts)
|
||||
out = append(out, ChatMessage{Role: "user", Content: content})
|
||||
}
|
||||
default:
|
||||
out = append(out, m)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
package apicompat
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
testToolImageDataURL = "data:image/png;base64,AQID"
|
||||
testToolImageRemote = "https://example.com/tool-output.png"
|
||||
)
|
||||
|
||||
func TestResponsesToolOutputMedia_ExtractsSupportedShapes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
call string
|
||||
outputType string
|
||||
output string
|
||||
imageURL string
|
||||
toolText string
|
||||
}{
|
||||
{
|
||||
name: "image-only array",
|
||||
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
|
||||
outputType: "function_call_output",
|
||||
output: `[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]`,
|
||||
imageURL: testToolImageDataURL,
|
||||
},
|
||||
{
|
||||
name: "text and nested image URL",
|
||||
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
|
||||
outputType: "function_call_output",
|
||||
output: `[{"type":"input_text","text":"render complete"},{"type":"image_url","image_url":{"url":"https://example.com/tool-output.png"}}]`,
|
||||
imageURL: testToolImageRemote,
|
||||
toolText: "render complete",
|
||||
},
|
||||
{
|
||||
name: "top-level image object",
|
||||
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
|
||||
outputType: "function_call_output",
|
||||
output: `{"type":"input_image","image_url":"data:image/png;base64,AQID"}`,
|
||||
imageURL: testToolImageDataURL,
|
||||
},
|
||||
{
|
||||
name: "content wrapper",
|
||||
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
|
||||
outputType: "function_call_output",
|
||||
output: `{"status":"ok","content":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}],"unknown":{"score":0.9,"large":9007199254740993}}`,
|
||||
imageURL: testToolImageDataURL,
|
||||
toolText: `"large":9007199254740993`,
|
||||
},
|
||||
{
|
||||
name: "JSON string output",
|
||||
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
|
||||
outputType: "function_call_output",
|
||||
output: `"[{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,AQID\"}]"`,
|
||||
imageURL: testToolImageDataURL,
|
||||
},
|
||||
{
|
||||
name: "bare data URL",
|
||||
call: `{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}`,
|
||||
outputType: "function_call_output",
|
||||
output: `"data:image/png;base64,AQID"`,
|
||||
imageURL: testToolImageDataURL,
|
||||
},
|
||||
{
|
||||
name: "custom tool output",
|
||||
call: `{"type":"custom_tool_call","call_id":"call_image","name":"view_image","input":"{}"}`,
|
||||
outputType: "custom_tool_call_output",
|
||||
output: `[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]`,
|
||||
imageURL: testToolImageDataURL,
|
||||
},
|
||||
{
|
||||
name: "tool search output",
|
||||
call: `{"type":"tool_search_call","call_id":"call_image","arguments":{"query":"image"}}`,
|
||||
outputType: "tool_search_output",
|
||||
output: `[{"type":"image_url","image_url":{"url":"https://example.com/tool-output.png"}}]`,
|
||||
imageURL: testToolImageRemote,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
input := fmt.Sprintf(`[%s,{"type":%q,"call_id":"call_image","output":%s}]`, tt.call, tt.outputType, tt.output)
|
||||
messages := convertToolOutputMedia(t, input)
|
||||
|
||||
require.Len(t, messages, 3)
|
||||
require.Equal(t, []string{"assistant", "tool", "user"}, chatMessageRoles(messages))
|
||||
require.Equal(t, "call_image", messages[1].ToolCallID)
|
||||
|
||||
toolText := chatToolContentString(t, messages[1])
|
||||
require.Contains(t, toolText, "[Tool output media moved to the following user message]")
|
||||
require.NotContains(t, toolText, tt.imageURL)
|
||||
if tt.toolText != "" {
|
||||
require.Contains(t, toolText, tt.toolText)
|
||||
}
|
||||
|
||||
parts := chatContentParts(t, messages[2])
|
||||
require.Len(t, parts, 2)
|
||||
require.Equal(t, "text", parts[0].Type)
|
||||
require.Equal(t, "[Tool output media for call call_image]", parts[0].Text)
|
||||
require.Equal(t, "image_url", parts[1].Type)
|
||||
require.NotNil(t, parts[1].ImageURL)
|
||||
require.Equal(t, tt.imageURL, parts[1].ImageURL.URL)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponsesToolOutputMedia_ParallelBatchUsesCallOrder(t *testing.T) {
|
||||
messages := convertToolOutputMedia(t, `[
|
||||
{"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"},
|
||||
{"type":"function_call","call_id":"call_B","name":"view_image","arguments":"{}"},
|
||||
{"type":"function_call_output","call_id":"call_B","output":[{"type":"input_image","image_url":{"url":"https://example.com/b.png"}}]},
|
||||
{"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":{"url":"https://example.com/a.png"}}]}
|
||||
]`)
|
||||
|
||||
require.Len(t, messages, 4)
|
||||
require.Equal(t, []string{"assistant", "tool", "tool", "user"}, chatMessageRoles(messages))
|
||||
require.Equal(t, []string{"call_A", "call_B"}, []string{messages[1].ToolCallID, messages[2].ToolCallID})
|
||||
|
||||
parts := chatContentParts(t, messages[3])
|
||||
require.Len(t, parts, 4)
|
||||
require.Equal(t, "[Tool output media for call call_A]", parts[0].Text)
|
||||
require.Equal(t, "https://example.com/a.png", parts[1].ImageURL.URL)
|
||||
require.Equal(t, "[Tool output media for call call_B]", parts[2].Text)
|
||||
require.Equal(t, "https://example.com/b.png", parts[3].ImageURL.URL)
|
||||
}
|
||||
|
||||
func TestResponsesToolOutputMedia_PreservesRichSiblingWhenRewriting(t *testing.T) {
|
||||
messages := convertToolOutputMedia(t, `[
|
||||
{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},
|
||||
{"type":"function_call_output","call_id":"call_image","output":[
|
||||
{"type":"result","url":"https://example.com/result","score":0.9,"text":"complete","extra":{"count":2}},
|
||||
{"type":"input_image","image_url":"data:image/png;base64,AQID"}
|
||||
]}
|
||||
]`)
|
||||
|
||||
toolText := chatToolContentString(t, messages[1])
|
||||
require.JSONEq(t, `[
|
||||
{"type":"result","url":"https://example.com/result","score":0.9,"text":"complete","extra":{"count":2}},
|
||||
{"type":"input_text","text":"[Tool output media moved to the following user message]"}
|
||||
]`, toolText)
|
||||
}
|
||||
|
||||
func TestResponsesToolOutputMedia_InterleavedMessagesFollowMediaBatch(t *testing.T) {
|
||||
messages := convertToolOutputMedia(t, `[
|
||||
{"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"},
|
||||
{"type":"message","role":"developer","content":[{"type":"input_text","text":"approval saved"}]},
|
||||
{"type":"message","role":"user","content":[{"type":"input_text","text":"continue"}]},
|
||||
{"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}
|
||||
]`)
|
||||
|
||||
require.Equal(t, []string{"assistant", "tool", "user", "system", "user"}, chatMessageRoles(messages))
|
||||
require.Equal(t, "[Tool output media for call call_A]", chatContentParts(t, messages[2])[0].Text)
|
||||
require.JSONEq(t, `"approval saved"`, string(messages[3].Content))
|
||||
require.JSONEq(t, `"continue"`, string(messages[4].Content))
|
||||
}
|
||||
|
||||
func TestResponsesToolOutputMedia_DropsOrphanAndUnansweredCallMedia(t *testing.T) {
|
||||
t.Run("orphan", func(t *testing.T) {
|
||||
messages := convertToolOutputMedia(t, `[
|
||||
{"type":"function_call_output","call_id":"call_ghost","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}
|
||||
]`)
|
||||
require.Empty(t, messages)
|
||||
})
|
||||
|
||||
t.Run("unanswered parallel call", func(t *testing.T) {
|
||||
messages := convertToolOutputMedia(t, `[
|
||||
{"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"},
|
||||
{"type":"function_call","call_id":"call_B","name":"view_image","arguments":"{}"},
|
||||
{"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}
|
||||
]`)
|
||||
|
||||
require.Len(t, messages, 3)
|
||||
require.Len(t, messages[0].ToolCalls, 1)
|
||||
require.Equal(t, "call_A", messages[0].ToolCalls[0].ID)
|
||||
parts := chatContentParts(t, messages[2])
|
||||
require.Len(t, parts, 2)
|
||||
require.NotContains(t, string(messages[2].Content), "call_B")
|
||||
})
|
||||
}
|
||||
|
||||
func TestResponsesToolOutputMedia_PreservesMediaFreeOutputBytes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
output string
|
||||
}{
|
||||
{
|
||||
name: "rich unknown object",
|
||||
output: `{"type":"result","url":"https://example.com/result","score":0.9,"text":"complete","extra":{"count":2}}`,
|
||||
},
|
||||
{
|
||||
name: "no-image array",
|
||||
output: `[ { "type": "input_text", "text": "ok" }, {"unknown":true} ]`,
|
||||
},
|
||||
{
|
||||
name: "plain string",
|
||||
output: `"plain output"`,
|
||||
},
|
||||
{
|
||||
name: "JSON string without image",
|
||||
output: `"{ \"ok\": true }"`,
|
||||
},
|
||||
{
|
||||
name: "embedded data URL text",
|
||||
output: `"prefix data:image/png;base64,AQID suffix"`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
input := fmt.Sprintf(`[
|
||||
{"type":"function_call","call_id":"call_text","name":"exec","arguments":"{}"},
|
||||
{"type":"function_call_output","call_id":"call_text","output":%s}
|
||||
]`, tt.output)
|
||||
messages := convertToolOutputMedia(t, input)
|
||||
require.Len(t, messages, 2)
|
||||
|
||||
var expected string
|
||||
if err := json.Unmarshal([]byte(tt.output), &expected); err != nil {
|
||||
expected = tt.output
|
||||
}
|
||||
expectedContent, err := json.Marshal(expected)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, string(expectedContent), string(messages[1].Content))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponsesToolOutputMedia_DuplicateCallIDIsLastWins(t *testing.T) {
|
||||
t.Run("later media replaces earlier media", func(t *testing.T) {
|
||||
messages := convertToolOutputMedia(t, `[
|
||||
{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},
|
||||
{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":{"url":"https://example.com/first.png"}}]},
|
||||
{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":{"url":"https://example.com/last.png"}}]}
|
||||
]`)
|
||||
|
||||
require.Len(t, messages, 3)
|
||||
require.NotContains(t, string(messages[1].Content), "first.png")
|
||||
require.NotContains(t, string(messages[2].Content), "first.png")
|
||||
require.Contains(t, string(messages[2].Content), "last.png")
|
||||
})
|
||||
|
||||
t.Run("later text clears earlier media", func(t *testing.T) {
|
||||
messages := convertToolOutputMedia(t, `[
|
||||
{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},
|
||||
{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]},
|
||||
{"type":"function_call_output","call_id":"call_image","output":"latest text"}
|
||||
]`)
|
||||
|
||||
require.Len(t, messages, 2)
|
||||
require.Equal(t, "latest text", chatToolContentString(t, messages[1]))
|
||||
})
|
||||
}
|
||||
|
||||
func TestResponsesToChatCompletionsRequest_ToolContentNeverContainsExtractedMedia(t *testing.T) {
|
||||
req := &ResponsesRequest{
|
||||
Model: "vision-model",
|
||||
Input: json.RawMessage(`[
|
||||
{"type":"function_call","call_id":"call_function","name":"view_image","arguments":"{}"},
|
||||
{"type":"custom_tool_call","call_id":"call_custom","name":"custom_image","input":"{}"},
|
||||
{"type":"tool_search_call","call_id":"call_search","arguments":{"query":"image"}},
|
||||
{"type":"function_call_output","call_id":"call_function","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]},
|
||||
{"type":"custom_tool_call_output","call_id":"call_custom","output":{"content":[{"type":"image_url","image_url":{"url":"https://example.com/custom.png"}}]}},
|
||||
{"type":"tool_search_output","call_id":"call_search","output":"data:image/jpeg;base64,BAUG"}
|
||||
]`),
|
||||
}
|
||||
|
||||
out, err := ResponsesToChatCompletionsRequest(req)
|
||||
require.NoError(t, err)
|
||||
assertChatInvariants(t, out.Messages)
|
||||
|
||||
var toolCount int
|
||||
for _, message := range out.Messages {
|
||||
if message.Role != "tool" {
|
||||
continue
|
||||
}
|
||||
toolCount++
|
||||
content := string(message.Content)
|
||||
require.NotContains(t, content, "data:image/")
|
||||
require.NotContains(t, content, "https://example.com/custom.png")
|
||||
}
|
||||
require.Equal(t, 3, toolCount)
|
||||
require.Equal(t, []string{"assistant", "tool", "tool", "tool", "user"}, chatMessageRoles(out.Messages))
|
||||
}
|
||||
|
||||
func convertToolOutputMedia(t *testing.T, input string) []ChatMessage {
|
||||
t.Helper()
|
||||
messages, err := responsesInputToChatMessages("", json.RawMessage(input))
|
||||
require.NoError(t, err)
|
||||
assertChatInvariants(t, messages)
|
||||
return messages
|
||||
}
|
||||
|
||||
func chatToolContentString(t *testing.T, message ChatMessage) string {
|
||||
t.Helper()
|
||||
require.Equal(t, "tool", message.Role)
|
||||
var content string
|
||||
require.NoError(t, json.Unmarshal(message.Content, &content))
|
||||
return content
|
||||
}
|
||||
|
||||
func chatContentParts(t *testing.T, message ChatMessage) []ChatContentPart {
|
||||
t.Helper()
|
||||
var parts []ChatContentPart
|
||||
require.NoError(t, json.Unmarshal(message.Content, &parts))
|
||||
for _, part := range parts {
|
||||
if part.Type == "image_url" {
|
||||
require.NotNil(t, part.ImageURL)
|
||||
require.False(t, strings.TrimSpace(part.ImageURL.URL) == "")
|
||||
}
|
||||
}
|
||||
return parts
|
||||
}
|
||||
Reference in New Issue
Block a user