Merge pull request #7246 from laoxiechuzheng/fix/deepseek-native-responses-tool-output-media

fix(deepseek): lift Responses tool output images

backend/internal/pkg/apicompat/responses_tool_output_media.go

@@ -0,0 +1,167 @@
+package apicompat
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type responsesToolOutputMedia struct {
+	callID   string
+	imageURL string
+}
+
+// LiftResponsesToolOutputMedia moves image parts out of Responses tool outputs
+// and into a following user message. Native Responses endpoints such as
+// DeepSeek accept function_call_output.output as a string, but Codex view_image
+// returns an array containing input_image. Keeping the image inside the tool
+// output makes the upstream report "No tool output found for tool call ...".
+func LiftResponsesToolOutputMedia(input any) (any, bool) {
+	items, ok := input.([]any)
+	if !ok {
+		return input, false
+	}
+
+	rewritten := make([]any, 0, len(items)+1)
+	changed := false
+
+	// DeepSeek validates parallel tool outputs as one contiguous run. Codex can
+	// inject developer notices between image outputs, so keep those notices
+	// pending until the full batch and its lifted media have been emitted.
+	for index := 0; index < len(items); {
+		item, ok := items[index].(map[string]any)
+		if !ok || !isResponsesToolOutputItem(item) {
+			rewritten = append(rewritten, items[index])
+			index++
+			continue
+		}
+
+		batchStart := index
+		outputs := make([]any, 0)
+		trailing := make([]any, 0)
+		pending := make([]responsesToolOutputMedia, 0)
+		batchChanged := false
+
+		for index < len(items) {
+			rawItem := items[index]
+			item, ok := rawItem.(map[string]any)
+			if !ok {
+				break
+			}
+			if isResponsesToolOutputItem(item) {
+				rewrittenItem, media, didRewrite := liftResponsesToolOutputMediaItem(item)
+				if didRewrite {
+					batchChanged = true
+					pending = append(pending, media...)
+				}
+				outputs = append(outputs, rewrittenItem)
+				index++
+				continue
+			}
+			if isResponsesToolBatchInstruction(item) {
+				trailing = append(trailing, rawItem)
+				index++
+				continue
+			}
+			break
+		}
+
+		if !batchChanged {
+			rewritten = append(rewritten, items[batchStart:index]...)
+			continue
+		}
+
+		rewritten = append(rewritten, outputs...)
+		if len(pending) > 0 {
+			rewritten = append(rewritten, buildResponsesToolOutputMediaMessage(pending))
+		}
+		rewritten = append(rewritten, trailing...)
+		changed = true
+	}
+
+	if !changed {
+		return input, false
+	}
+	return rewritten, true
+}
+
+func liftResponsesToolOutputMediaItem(item map[string]any) (any, []responsesToolOutputMedia, bool) {
+	output, exists := item["output"]
+	if !exists {
+		return item, nil, false
+	}
+	outputRaw, err := json.Marshal(output)
+	if err != nil {
+		return item, nil, false
+	}
+
+	outputText, media, didRewrite := extractToolOutputMedia(outputRaw)
+	if !didRewrite {
+		return item, nil, false
+	}
+
+	item["output"] = outputText
+	callID := strings.TrimSpace(stringValue(item["call_id"]))
+	lifted := make([]responsesToolOutputMedia, 0, len(media))
+	for _, part := range media {
+		if part.ImageURL == nil {
+			continue
+		}
+		imageURL := strings.TrimSpace(part.ImageURL.URL)
+		if imageURL == "" {
+			continue
+		}
+		lifted = append(lifted, responsesToolOutputMedia{
+			callID:   callID,
+			imageURL: imageURL,
+		})
+	}
+	return item, lifted, true
+}
+
+func buildResponsesToolOutputMediaMessage(pending []responsesToolOutputMedia) map[string]any {
+	content := make([]map[string]any, 0, len(pending)*2)
+	lastCallID := ""
+	for _, media := range pending {
+		if media.callID != lastCallID {
+			text := "Tool output media"
+			if media.callID != "" {
+				text = fmt.Sprintf(toolOutputMediaAttribution, media.callID)
+			}
+			content = append(content, map[string]any{
+				"type": "input_text",
+				"text": text,
+			})
+			lastCallID = media.callID
+		}
+		content = append(content, map[string]any{
+			"type":      "input_image",
+			"image_url": media.imageURL,
+		})
+	}
+
+	return map[string]any{
+		"type":    "message",
+		"role":    "user",
+		"content": content,
+	}
+}
+
+func isResponsesToolBatchInstruction(item map[string]any) bool {
+	itemType := strings.TrimSpace(stringValue(item["type"]))
+	if itemType != "" && itemType != "message" {
+		return false
+	}
+	role := strings.TrimSpace(stringValue(item["role"]))
+	return role == "developer" || role == "system"
+}
+
+func isResponsesToolOutputItem(item map[string]any) bool {
+	switch strings.TrimSpace(stringValue(item["type"])) {
+	case "function_call_output", "custom_tool_call_output",
+		"tool_search_output", "tool_search_call_output", "mcp_tool_call_output":
+		return true
+	default:
+		return false
+	}
+}

backend/internal/pkg/apicompat/responses_tool_output_media_test.go

@@ -0,0 +1,116 @@
+package apicompat
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLiftResponsesToolOutputMedia(t *testing.T) {
+	var input any
+	require.NoError(t, json.Unmarshal([]byte(`[
+		{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},
+		{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}
+	]`), &input))
+
+	lifted, changed := LiftResponsesToolOutputMedia(input)
+	require.True(t, changed)
+
+	items, ok := lifted.([]any)
+	require.True(t, ok)
+	require.Len(t, items, 3)
+
+	outputItem, ok := items[1].(map[string]any)
+	require.True(t, ok)
+	outputText, ok := outputItem["output"].(string)
+	require.True(t, ok)
+	require.Contains(t, outputText, toolOutputMediaMarker)
+	require.NotContains(t, outputText, "data:image/png")
+
+	mediaMessage, ok := items[2].(map[string]any)
+	require.True(t, ok)
+	require.Equal(t, "message", mediaMessage["type"])
+	require.Equal(t, "user", mediaMessage["role"])
+
+	parts, ok := mediaMessage["content"].([]map[string]any)
+	require.True(t, ok)
+	require.Len(t, parts, 2)
+	require.Equal(t, "input_text", parts[0]["type"])
+	require.Equal(t, "[Tool output media for call call_image]", parts[0]["text"])
+	require.Equal(t, "input_image", parts[1]["type"])
+	require.Equal(t, "data:image/png;base64,AQID", parts[1]["image_url"])
+}
+
+func TestLiftResponsesToolOutputMediaLeavesPlainOutputUntouched(t *testing.T) {
+	input := []any{
+		map[string]any{
+			"type":    "function_call_output",
+			"call_id": "call_text",
+			"output":  "plain output",
+		},
+	}
+
+	lifted, changed := LiftResponsesToolOutputMedia(input)
+	require.False(t, changed)
+	require.Equal(t, input, lifted)
+}
+
+func TestLiftResponsesToolOutputMediaKeepsParallelBatchContiguous(t *testing.T) {
+	var input any
+	require.NoError(t, json.Unmarshal([]byte(`[
+		{"type":"function_call","call_id":"call_A","name":"view_image","arguments":"{}"},
+		{"type":"function_call","call_id":"call_B","name":"view_image","arguments":"{}"},
+		{"type":"function_call","call_id":"call_C","name":"view_image","arguments":"{}"},
+		{"type":"function_call_output","call_id":"call_A","output":[{"type":"input_image","image_url":"data:image/png;base64,QQ=="}]},
+		{"type":"message","role":"developer","content":[{"type":"input_text","text":"resized A"}]},
+		{"type":"function_call_output","call_id":"call_B","output":[{"type":"input_image","image_url":"data:image/png;base64,Qg=="}]},
+		{"type":"message","role":"developer","content":[{"type":"input_text","text":"resized B"}]},
+		{"type":"function_call_output","call_id":"call_C","output":[{"type":"input_image","image_url":"data:image/png;base64,Qw=="}]},
+		{"type":"message","role":"developer","content":[{"type":"input_text","text":"resized C"}]}
+	]`), &input))
+
+	lifted, changed := LiftResponsesToolOutputMedia(input)
+	require.True(t, changed)
+
+	items, ok := lifted.([]any)
+	require.True(t, ok)
+	require.Len(t, items, 10)
+
+	for index, callID := range []string{"call_A", "call_B", "call_C"} {
+		item, ok := items[index+3].(map[string]any)
+		require.True(t, ok)
+		require.Equal(t, "function_call_output", item["type"])
+		require.Equal(t, callID, item["call_id"])
+		outputText, ok := item["output"].(string)
+		require.True(t, ok)
+		require.Contains(t, outputText, toolOutputMediaMarker)
+		require.NotContains(t, outputText, "data:image/png")
+	}
+
+	mediaMessage, ok := items[6].(map[string]any)
+	require.True(t, ok)
+	require.Equal(t, "message", mediaMessage["type"])
+	require.Equal(t, "user", mediaMessage["role"])
+	parts, ok := mediaMessage["content"].([]map[string]any)
+	require.True(t, ok)
+	require.Len(t, parts, 6)
+	require.Equal(t, "[Tool output media for call call_A]", parts[0]["text"])
+	require.Equal(t, "data:image/png;base64,QQ==", parts[1]["image_url"])
+	require.Equal(t, "[Tool output media for call call_B]", parts[2]["text"])
+	require.Equal(t, "data:image/png;base64,Qg==", parts[3]["image_url"])
+	require.Equal(t, "[Tool output media for call call_C]", parts[4]["text"])
+	require.Equal(t, "data:image/png;base64,Qw==", parts[5]["image_url"])
+
+	for index, text := range []string{"resized A", "resized B", "resized C"} {
+		notice, ok := items[index+7].(map[string]any)
+		require.True(t, ok)
+		require.Equal(t, "developer", notice["role"])
+		content, ok := notice["content"].([]any)
+		require.True(t, ok)
+		require.Len(t, content, 1)
+		contentPart, ok := content[0].(map[string]any)
+		require.True(t, ok)
+		require.Equal(t, text, contentPart["text"])
+	}
+}

backend/internal/service/cn_providers_test.go

@@ -834,6 +834,16 @@ func TestNormalizeDeepSeekResponsesRequestBody(t *testing.T) {
 	require.False(t, gjson.GetBytes(adaptiveNormalized, "store").Bool())
 	require.False(t, gjson.GetBytes(adaptiveNormalized, "previous_response_id").Exists())
 
+	mediaBody := []byte(`{"model":"deepseek-v4.1-flash","store":true,"input":[{"type":"function_call","call_id":"call_image","name":"view_image","arguments":"{}"},{"type":"function_call_output","call_id":"call_image","output":[{"type":"input_image","image_url":"data:image/png;base64,AQID"}]}]}`)
+	mediaNormalized := normalizeDeepSeekResponsesRequestBody(deepseekResponses, mediaBody)
+	require.False(t, gjson.GetBytes(mediaNormalized, "store").Bool())
+	require.Equal(t, gjson.String, gjson.GetBytes(mediaNormalized, "input.1.output").Type)
+	require.NotContains(t, gjson.GetBytes(mediaNormalized, "input.1.output").String(), "data:image/png")
+	require.Equal(t, "message", gjson.GetBytes(mediaNormalized, "input.2.type").String())
+	require.Equal(t, "user", gjson.GetBytes(mediaNormalized, "input.2.role").String())
+	require.Equal(t, "input_image", gjson.GetBytes(mediaNormalized, "input.2.content.1.type").String())
+	require.Equal(t, "data:image/png;base64,AQID", gjson.GetBytes(mediaNormalized, "input.2.content.1.image_url").String())
+
 	// 非 responses 协议（deepseek CC 账号）原样返回
 	deepseekCC := &Account{Platform: PlatformDeepseek, Type: AccountTypeAPIKey}
 	require.Equal(t, string(body), string(normalizeDeepSeekResponsesRequestBody(deepseekCC, body)))

backend/internal/service/openai_gateway_request_body.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"unicode/utf8"
 
+	"github.com/Wei-Shaw/sub2api/internal/pkg/apicompat"
 	"github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey"
 	"github.com/Wei-Shaw/sub2api/internal/util/urlvalidator"
 	"github.com/gin-gonic/gin"
@@ -141,7 +142,25 @@ func normalizeDeepSeekResponsesRequestBody(account *Account, body []byte) []byte
 	if stripped, err := sjson.DeleteBytes(normalized, "previous_response_id"); err == nil {
 		normalized = stripped
 	}
-	return normalized
+
+	var requestBody map[string]any
+	if err := decodeOpenAIJSONUseNumber(normalized, &requestBody); err != nil {
+		return normalized
+	}
+	input, exists := requestBody["input"]
+	if !exists {
+		return normalized
+	}
+	liftedInput, changed := apicompat.LiftResponsesToolOutputMedia(input)
+	if !changed {
+		return normalized
+	}
+	requestBody["input"] = liftedInput
+	rebuilt, err := marshalOpenAIUpstreamJSON(requestBody)
+	if err != nil {
+		return normalized
+	}
+	return rebuilt
 }
 
 func trimOpenAIEncryptedReasoningItems(reqBody map[string]any) bool {