From 881ab1b0c6239a928fdccc3c87d2cd011aa337ec Mon Sep 17 00:00:00 2001 From: laoxiechuzheng Date: Thu, 17 Sep 2026 01:22:22 +0800 Subject: [PATCH 1/2] fix(deepseek): lift Responses tool output images DeepSeek native Responses rejects Codex view_image results when function_call_output.output contains input_image parts. Move media into a following user message and keep the tool output as a string so call_id pairing remains valid. --- .../apicompat/responses_tool_output_media.go | 121 ++++++++++++++++++ .../responses_tool_output_media_test.go | 57 +++++++++ backend/internal/service/cn_providers_test.go | 10 ++ .../service/openai_gateway_request_body.go | 21 ++- 4 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 backend/internal/pkg/apicompat/responses_tool_output_media.go create mode 100644 backend/internal/pkg/apicompat/responses_tool_output_media_test.go diff --git a/backend/internal/pkg/apicompat/responses_tool_output_media.go b/backend/internal/pkg/apicompat/responses_tool_output_media.go new file mode 100644 index 0000000000..9daa4068e6 --- /dev/null +++ b/backend/internal/pkg/apicompat/responses_tool_output_media.go @@ -0,0 +1,121 @@ +package apicompat + +import ( + "encoding/json" + "fmt" + "strings" +) + +type responsesToolOutputMedia struct { + callID string + imageURL string +} + +// LiftResponsesToolOutputMedia moves image parts out of Responses tool outputs +// and into a following user message. Native Responses endpoints such as +// DeepSeek accept function_call_output.output as a string, but Codex view_image +// returns an array containing input_image. Keeping the image inside the tool +// output makes the upstream report "No tool output found for tool call ...". +func LiftResponsesToolOutputMedia(input any) (any, bool) { + items, ok := input.([]any) + if !ok { + return input, false + } + + rewritten := make([]any, 0, len(items)+1) + pending := make([]responsesToolOutputMedia, 0) + changed := false + + flushMedia := func() { + if len(pending) == 0 { + return + } + + content := make([]map[string]any, 0, len(pending)*2) + lastCallID := "" + for _, media := range pending { + if media.callID != lastCallID { + text := "Tool output media" + if media.callID != "" { + text = fmt.Sprintf(toolOutputMediaAttribution, media.callID) + } + content = append(content, map[string]any{ + "type": "input_text", + "text": text, + }) + lastCallID = media.callID + } + content = append(content, map[string]any{ + "type": "input_image", + "image_url": media.imageURL, + }) + } + + rewritten = append(rewritten, map[string]any{ + "type": "message", + "role": "user", + "content": content, + }) + pending = pending[:0] + } + + for _, rawItem := range items { + item, ok := rawItem.(map[string]any) + if !ok || !isResponsesToolOutputItem(item) { + flushMedia() + rewritten = append(rewritten, rawItem) + continue + } + + output, exists := item["output"] + if !exists { + rewritten = append(rewritten, item) + continue + } + outputRaw, err := json.Marshal(output) + if err != nil { + rewritten = append(rewritten, item) + continue + } + + outputText, media, didRewrite := extractToolOutputMedia(outputRaw) + if !didRewrite { + rewritten = append(rewritten, item) + continue + } + + item["output"] = outputText + callID := strings.TrimSpace(stringValue(item["call_id"])) + for _, part := range media { + if part.ImageURL == nil { + continue + } + imageURL := strings.TrimSpace(part.ImageURL.URL) + if imageURL == "" { + continue + } + pending = append(pending, responsesToolOutputMedia{ + callID: callID, + imageURL: imageURL, + }) + } + changed = true + rewritten = append(rewritten, item) + } + + flushMedia() + if !changed { + return input, false + } + return rewritten, true +} + +func isResponsesToolOutputItem(item map[string]any) bool { + switch strings.TrimSpace(stringValue(item["type"])) { + case "function_call_output", "custom_tool_call_output", + "tool_search_output", "tool_search_call_output", "mcp_tool_call_output": + return true + default: + return false + } +} diff --git a/backend/internal/pkg/apicompat/responses_tool_output_media_test.go b/backend/internal/pkg/apicompat/responses_tool_output_media_test.go new file mode 100644 index 0000000000..37417aba77 --- /dev/null +++ b/backend/internal/pkg/apicompat/responses_tool_output_media_test.go @@ -0,0 +1,57 @@ +package apicompat + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLiftResponsesToolOutputMedia(t *testing.T) { + var input any + require.NoError(t, json.Unmarshal([]byte(`[ + {"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]} + ]`), &input)) + + lifted, changed := LiftResponsesToolOutputMedia(input) + require.True(t, changed) + + items, ok := lifted.([]any) + require.True(t, ok) + require.Len(t, items, 3) + + outputItem, ok := items[1].(map[string]any) + require.True(t, ok) + outputText, ok := outputItem["output"].(string) + require.True(t, ok) + require.Contains(t, outputText, toolOutputMediaMarker) + require.NotContains(t, outputText, "data:image/png") + + mediaMessage, ok := items[2].(map[string]any) + require.True(t, ok) + require.Equal(t, "message", mediaMessage["type"]) + require.Equal(t, "user", mediaMessage["role"]) + + parts, ok := mediaMessage["content"].([]map[string]any) + require.True(t, ok) + require.Len(t, parts, 2) + require.Equal(t, "input_text", parts[0]["type"]) + require.Equal(t, "[Tool output media for call call_image]", parts[0]["text"]) + require.Equal(t, "input_image", parts[1]["type"]) + require.Equal(t, "data:image/png;base64,AQID", parts[1]["image_url"]) +} + +func TestLiftResponsesToolOutputMediaLeavesPlainOutputUntouched(t *testing.T) { + input := []any{ + map[string]any{ + "type": "function_call_output", + "call_id": "call_text", + "output": "plain output", + }, + } + + lifted, changed := LiftResponsesToolOutputMedia(input) + require.False(t, changed) + require.Equal(t, input, lifted) +} diff --git a/backend/internal/service/cn_providers_test.go b/backend/internal/service/cn_providers_test.go index 7165b29537..44becd4e5b 100644 --- a/backend/internal/service/cn_providers_test.go +++ b/backend/internal/service/cn_providers_test.go @@ -834,6 +834,16 @@ func TestNormalizeDeepSeekResponsesRequestBody(t *testing.T) { require.False(t, gjson.GetBytes(adaptiveNormalized, "store").Bool()) require.False(t, gjson.GetBytes(adaptiveNormalized, "previous_response_id").Exists()) + mediaBody := []byte(`{"model":"deepseek-v4.1-flash","store":true,"input":[{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}]}`) + mediaNormalized := normalizeDeepSeekResponsesRequestBody(deepseekResponses, mediaBody) + require.False(t, gjson.GetBytes(mediaNormalized, "store").Bool()) + require.Equal(t, gjson.String, gjson.GetBytes(mediaNormalized, "input.1.output").Type) + require.NotContains(t, gjson.GetBytes(mediaNormalized, "input.1.output").String(), "data:image/png") + require.Equal(t, "message", gjson.GetBytes(mediaNormalized, "input.2.type").String()) + require.Equal(t, "user", gjson.GetBytes(mediaNormalized, "input.2.role").String()) + require.Equal(t, "input_image", gjson.GetBytes(mediaNormalized, "input.2.content.1.type").String()) + require.Equal(t, "data:image/png;base64,AQID", gjson.GetBytes(mediaNormalized, "input.2.content.1.image_url").String()) + // 非 responses 协议（deepseek CC 账号）原样返回 deepseekCC := &Account{Platform: PlatformDeepseek, Type: AccountTypeAPIKey} require.Equal(t, string(body), string(normalizeDeepSeekResponsesRequestBody(deepseekCC, body))) diff --git a/backend/internal/service/openai_gateway_request_body.go b/backend/internal/service/openai_gateway_request_body.go index 27306e34a4..70c10d104a 100644 --- a/backend/internal/service/openai_gateway_request_body.go +++ b/backend/internal/service/openai_gateway_request_body.go @@ -10,6 +10,7 @@ import ( "strings" "unicode/utf8" + "github.com/Wei-Shaw/sub2api/internal/pkg/apicompat" "github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey" "github.com/Wei-Shaw/sub2api/internal/util/urlvalidator" "github.com/gin-gonic/gin" @@ -141,7 +142,25 @@ func normalizeDeepSeekResponsesRequestBody(account *Account, body []byte) []byte if stripped, err := sjson.DeleteBytes(normalized, "previous_response_id"); err == nil { normalized = stripped } - return normalized + + var requestBody map[string]any + if err := decodeOpenAIJSONUseNumber(normalized, &requestBody); err != nil { + return normalized + } + input, exists := requestBody["input"] + if !exists { + return normalized + } + liftedInput, changed := apicompat.LiftResponsesToolOutputMedia(input) + if !changed { + return normalized + } + requestBody["input"] = liftedInput + rebuilt, err := marshalOpenAIUpstreamJSON(requestBody) + if err != nil { + return normalized + } + return rebuilt } func trimOpenAIEncryptedReasoningItems(reqBody map[string]any) bool { From acc05620cb9ec4e5420c1660f766db7a7d7a1e78 Mon Sep 17 00:00:00 2001 From: laoxiechuzheng Date: Thu, 17 Sep 2026 15:45:56 +0800 Subject: [PATCH 2/2] fix(deepseek): keep parallel Responses tool outputs contiguous --- .../apicompat/responses_tool_output_media.go | 178 +++++++++++------- .../responses_tool_output_media_test.go | 59 ++++++ 2 files changed, 171 insertions(+), 66 deletions(-) diff --git a/backend/internal/pkg/apicompat/responses_tool_output_media.go b/backend/internal/pkg/apicompat/responses_tool_output_media.go index 9daa4068e6..c6df71cae0 100644 --- a/backend/internal/pkg/apicompat/responses_tool_output_media.go +++ b/backend/internal/pkg/apicompat/responses_tool_output_media.go @@ -23,93 +23,139 @@ func LiftResponsesToolOutputMedia(input any) (any, bool) { } rewritten := make([]any, 0, len(items)+1) - pending := make([]responsesToolOutputMedia, 0) changed := false - flushMedia := func() { - if len(pending) == 0 { - return - } - - content := make([]map[string]any, 0, len(pending)*2) - lastCallID := "" - for _, media := range pending { - if media.callID != lastCallID { - text := "Tool output media" - if media.callID != "" { - text = fmt.Sprintf(toolOutputMediaAttribution, media.callID) - } - content = append(content, map[string]any{ - "type": "input_text", - "text": text, - }) - lastCallID = media.callID - } - content = append(content, map[string]any{ - "type": "input_image", - "image_url": media.imageURL, - }) - } - - rewritten = append(rewritten, map[string]any{ - "type": "message", - "role": "user", - "content": content, - }) - pending = pending[:0] - } - - for _, rawItem := range items { - item, ok := rawItem.(map[string]any) + // DeepSeek validates parallel tool outputs as one contiguous run. Codex can + // inject developer notices between image outputs, so keep those notices + // pending until the full batch and its lifted media have been emitted. + for index := 0; index < len(items); { + item, ok := items[index].(map[string]any) if !ok || !isResponsesToolOutputItem(item) { - flushMedia() - rewritten = append(rewritten, rawItem) + rewritten = append(rewritten, items[index]) + index++ continue } - output, exists := item["output"] - if !exists { - rewritten = append(rewritten, item) - continue - } - outputRaw, err := json.Marshal(output) - if err != nil { - rewritten = append(rewritten, item) - continue - } + batchStart := index + outputs := make([]any, 0) + trailing := make([]any, 0) + pending := make([]responsesToolOutputMedia, 0) + batchChanged := false - outputText, media, didRewrite := extractToolOutputMedia(outputRaw) - if !didRewrite { - rewritten = append(rewritten, item) - continue - } - - item["output"] = outputText - callID := strings.TrimSpace(stringValue(item["call_id"])) - for _, part := range media { - if part.ImageURL == nil { + for index < len(items) { + rawItem := items[index] + item, ok := rawItem.(map[string]any) + if !ok { + break + } + if isResponsesToolOutputItem(item) { + rewrittenItem, media, didRewrite := liftResponsesToolOutputMediaItem(item) + if didRewrite { + batchChanged = true + pending = append(pending, media...) + } + outputs = append(outputs, rewrittenItem) + index++ continue } - imageURL := strings.TrimSpace(part.ImageURL.URL) - if imageURL == "" { + if isResponsesToolBatchInstruction(item) { + trailing = append(trailing, rawItem) + index++ continue } - pending = append(pending, responsesToolOutputMedia{ - callID: callID, - imageURL: imageURL, - }) + break } + + if !batchChanged { + rewritten = append(rewritten, items[batchStart:index]...) + continue + } + + rewritten = append(rewritten, outputs...) + if len(pending) > 0 { + rewritten = append(rewritten, buildResponsesToolOutputMediaMessage(pending)) + } + rewritten = append(rewritten, trailing...) changed = true - rewritten = append(rewritten, item) } - flushMedia() if !changed { return input, false } return rewritten, true } +func liftResponsesToolOutputMediaItem(item map[string]any) (any, []responsesToolOutputMedia, bool) { + output, exists := item["output"] + if !exists { + return item, nil, false + } + outputRaw, err := json.Marshal(output) + if err != nil { + return item, nil, false + } + + outputText, media, didRewrite := extractToolOutputMedia(outputRaw) + if !didRewrite { + return item, nil, false + } + + item["output"] = outputText + callID := strings.TrimSpace(stringValue(item["call_id"])) + lifted := make([]responsesToolOutputMedia, 0, len(media)) + for _, part := range media { + if part.ImageURL == nil { + continue + } + imageURL := strings.TrimSpace(part.ImageURL.URL) + if imageURL == "" { + continue + } + lifted = append(lifted, responsesToolOutputMedia{ + callID: callID, + imageURL: imageURL, + }) + } + return item, lifted, true +} + +func buildResponsesToolOutputMediaMessage(pending []responsesToolOutputMedia) map[string]any { + content := make([]map[string]any, 0, len(pending)*2) + lastCallID := "" + for _, media := range pending { + if media.callID != lastCallID { + text := "Tool output media" + if media.callID != "" { + text = fmt.Sprintf(toolOutputMediaAttribution, media.callID) + } + content = append(content, map[string]any{ + "type": "input_text", + "text": text, + }) + lastCallID = media.callID + } + content = append(content, map[string]any{ + "type": "input_image", + "image_url": media.imageURL, + }) + } + + return map[string]any{ + "type": "message", + "role": "user", + "content": content, + } +} + +func isResponsesToolBatchInstruction(item map[string]any) bool { + itemType := strings.TrimSpace(stringValue(item["type"])) + if itemType != "" && itemType != "message" { + return false + } + role := strings.TrimSpace(stringValue(item["role"])) + return role == "developer" || role == "system" +} + func isResponsesToolOutputItem(item map[string]any) bool { switch strings.TrimSpace(stringValue(item["type"])) { case "function_call_output", "custom_tool_call_output", diff --git a/backend/internal/pkg/apicompat/responses_tool_output_media_test.go b/backend/internal/pkg/apicompat/responses_tool_output_media_test.go index 37417aba77..f6f56ba9b9 100644 --- a/backend/internal/pkg/apicompat/responses_tool_output_media_test.go +++ b/backend/internal/pkg/apicompat/responses_tool_output_media_test.go @@ -55,3 +55,62 @@ func TestLiftResponsesToolOutputMediaLeavesPlainOutputUntouched(t *testing.T) { require.False(t, changed) require.Equal(t, input, lifted) } + +func TestLiftResponsesToolOutputMediaKeepsParallelBatchContiguous(t *testing.T) { + var input any + require.NoError(t, json.Unmarshal([]byte(`[ + {"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"}, + {"type":"function_call","call_id":"call_B","name":"view_image","arguments":"{}"}, + {"type":"function_call","call_id":"call_C","name":"view_image","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":"data:image/png;base64,QQ=="}]}, + {"type":"message","role":"developer","content":[{"type":"input_text","text":"resized A"}]}, + {"type":"function_call_output","call_id":"call_B","output":[{"type":"input_image","image_url":"data:image/png;base64,Qg=="}]}, + {"type":"message","role":"developer","content":[{"type":"input_text","text":"resized B"}]}, + {"type":"function_call_output","call_id":"call_C","output":[{"type":"input_image","image_url":"data:image/png;base64,Qw=="}]}, + {"type":"message","role":"developer","content":[{"type":"input_text","text":"resized C"}]} + ]`), &input)) + + lifted, changed := LiftResponsesToolOutputMedia(input) + require.True(t, changed) + + items, ok := lifted.([]any) + require.True(t, ok) + require.Len(t, items, 10) + + for index, callID := range []string{"call_A", "call_B", "call_C"} { + item, ok := items[index+3].(map[string]any) + require.True(t, ok) + require.Equal(t, "function_call_output", item["type"]) + require.Equal(t, callID, item["call_id"]) + outputText, ok := item["output"].(string) + require.True(t, ok) + require.Contains(t, outputText, toolOutputMediaMarker) + require.NotContains(t, outputText, "data:image/png") + } + + mediaMessage, ok := items[6].(map[string]any) + require.True(t, ok) + require.Equal(t, "message", mediaMessage["type"]) + require.Equal(t, "user", mediaMessage["role"]) + parts, ok := mediaMessage["content"].([]map[string]any) + require.True(t, ok) + require.Len(t, parts, 6) + require.Equal(t, "[Tool output media for call call_A]", parts[0]["text"]) + require.Equal(t, "data:image/png;base64,QQ==", parts[1]["image_url"]) + require.Equal(t, "[Tool output media for call call_B]", parts[2]["text"]) + require.Equal(t, "data:image/png;base64,Qg==", parts[3]["image_url"]) + require.Equal(t, "[Tool output media for call call_C]", parts[4]["text"]) + require.Equal(t, "data:image/png;base64,Qw==", parts[5]["image_url"]) + + for index, text := range []string{"resized A", "resized B", "resized C"} { + notice, ok := items[index+7].(map[string]any) + require.True(t, ok) + require.Equal(t, "developer", notice["role"]) + content, ok := notice["content"].([]any) + require.True(t, ok) + require.Len(t, content, 1) + contentPart, ok := content[0].(map[string]any) + require.True(t, ok) + require.Equal(t, text, contentPart["text"]) + } +}