mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-14 12:02:27 +08:00
shaw
9eb394e3e9
- System One 校验拒绝重复键与大小写/Unicode 折叠变体键（请求顶层、
questions、问题对象）：encoding/json 大小写不敏感且取最后一个重复键，
而原始 body 原样转发，此前 {"model":"x","MODEL":"jev-latest"} 可绕过
jev-latest 限制、模型白名单与流式限制。
- Prompt 审计与旧审核同时收集对象键名、问题 ID 与未知扩展字段（不含规范
字段名与已校验的 type），此前把内容放进键名即可完全绕过审计。
- 上游成功响应的 usage/model 宽松解码（浮点、数字字符串），避免上游已
计费而网关整体 502 不记账；解码失败记录 response_error ops 事件。
- 上游 413 与 400/422 同视为请求错误：透传状态码、不切换、不改账号状态。
- TypeSafe base_url 末尾的 /v1 自动剥离，避免拼出 /v1/v1/systemone；
/v1/systemone 改挂文本请求体上限。
- Composite 静态兜底模型列表、管理端候选列表恢复为不含 jev-latest；
Codex 清单永不列出 TypeSafe 模型；TypeSafe 分组候选默认 jev-latest。
- 创建账号从 Grok 切到 TypeSafe 时重置为白名单模式；错误透传平台列表补 typesafe。
408 lines
14 KiB
Go
408 lines
14 KiB
Go
package service
|
|
|
|
import (
|
|
"crypto/rand"
|
|
"fmt"
|
|
"math/big"
|
|
"strings"
|
|
|
|
"github.com/tidwall/gjson"
|
|
)
|
|
|
|
func ExtractContentModerationText(protocol string, body []byte) string {
|
|
return ExtractContentModerationInput(protocol, body).Text
|
|
}
|
|
|
|
func ExtractContentModerationInput(protocol string, body []byte) ContentModerationInput {
|
|
return extractContentModerationInput(protocol, body, true)
|
|
}
|
|
|
|
// Keyword checks share semantic moderation's current-user boundaries, but must
|
|
// inspect client-supplied reminder blocks as ordinary user text.
|
|
func extractContentModerationKeywordText(protocol string, body []byte) string {
|
|
return extractContentModerationInput(protocol, body, false).Text
|
|
}
|
|
|
|
type moderationTextCollector struct {
|
|
filterReminders bool
|
|
}
|
|
|
|
func extractContentModerationInput(protocol string, body []byte, filterReminders bool) ContentModerationInput {
|
|
collector := moderationTextCollector{filterReminders: filterReminders}
|
|
if len(body) == 0 || !gjson.ValidBytes(body) {
|
|
return ContentModerationInput{}
|
|
}
|
|
var parts []string
|
|
var images []string
|
|
switch protocol {
|
|
case ContentModerationProtocolAnthropicMessages:
|
|
collector.collectLastAnthropicUserMessage(gjson.GetBytes(body, "messages"), &parts, &images)
|
|
case ContentModerationProtocolOpenAIChat:
|
|
collector.collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images)
|
|
case ContentModerationProtocolOpenAIResponses:
|
|
collector.collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images)
|
|
case ContentModerationProtocolGemini:
|
|
collector.collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images)
|
|
case ContentModerationProtocolOpenAIImages:
|
|
collector.addModerationText(&parts, gjson.GetBytes(body, "prompt").String())
|
|
collector.collectContentValue(gjson.GetBytes(body, "images"), &parts, &images)
|
|
case ContentModerationProtocolTypeSafeSystemOne:
|
|
// System One carries no client-harness reminder blocks, so a literal
|
|
// <system-reminder> is ordinary user text and must never be skipped.
|
|
moderationTextCollector{}.collectSystemOneInput(body, &parts)
|
|
default:
|
|
collector.collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images)
|
|
collector.collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images)
|
|
collector.collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images)
|
|
}
|
|
out := ContentModerationInput{
|
|
Text: normalizeContentModerationText(strings.Join(parts, "\n")),
|
|
Images: normalizeModerationImages(images),
|
|
}
|
|
if filterReminders {
|
|
out.Normalize()
|
|
}
|
|
return out
|
|
}
|
|
|
|
// collectSystemOneInput moderates every client-controlled text of a System One
|
|
// request: question IDs, every question field except the validated type,
|
|
// unknown top-level extension fields, and the evaluated state. Object keys are
|
|
// sent to Jev as part of the JSON, so they are moderated like values.
|
|
func (collector moderationTextCollector) collectSystemOneInput(body []byte, parts *[]string) {
|
|
root := gjson.ParseBytes(body)
|
|
questions := root.Get("questions")
|
|
if !questions.IsObject() {
|
|
collector.collectSystemOneText(questions, parts)
|
|
}
|
|
questions.ForEach(func(id, question gjson.Result) bool {
|
|
collector.addModerationText(parts, id.String())
|
|
if !question.IsObject() {
|
|
collector.collectSystemOneText(question, parts)
|
|
return true
|
|
}
|
|
question.ForEach(func(field, value gjson.Result) bool {
|
|
switch field.String() {
|
|
case "type":
|
|
return true
|
|
case "instructions", "criteria":
|
|
default:
|
|
collector.addModerationText(parts, field.String())
|
|
}
|
|
collector.collectSystemOneText(value, parts)
|
|
return true
|
|
})
|
|
return true
|
|
})
|
|
root.ForEach(func(field, value gjson.Result) bool {
|
|
switch field.String() {
|
|
case "model", "stream", "state", "questions":
|
|
return true
|
|
}
|
|
collector.addModerationText(parts, field.String())
|
|
collector.collectSystemOneText(value, parts)
|
|
return true
|
|
})
|
|
collector.collectSystemOneText(root.Get("state"), parts)
|
|
}
|
|
|
|
func (collector moderationTextCollector) collectSystemOneText(value gjson.Result, parts *[]string) {
|
|
switch {
|
|
case !value.Exists():
|
|
return
|
|
case value.Type == gjson.String:
|
|
collector.addModerationText(parts, value.String())
|
|
case value.IsArray():
|
|
value.ForEach(func(_, child gjson.Result) bool {
|
|
collector.collectSystemOneText(child, parts)
|
|
return true
|
|
})
|
|
case value.IsObject():
|
|
value.ForEach(func(key, child gjson.Result) bool {
|
|
collector.addModerationText(parts, key.String())
|
|
collector.collectSystemOneText(child, parts)
|
|
return true
|
|
})
|
|
}
|
|
}
|
|
|
|
func (collector moderationTextCollector) collectLastRoleMessage(messages gjson.Result, role string, parts *[]string, images *[]string) {
|
|
if !messages.IsArray() {
|
|
return
|
|
}
|
|
array := messages.Array()
|
|
if len(array) == 0 {
|
|
return
|
|
}
|
|
last := array[len(array)-1]
|
|
if strings.ToLower(strings.TrimSpace(last.Get("role").String())) != role {
|
|
return
|
|
}
|
|
var candidate []string
|
|
var candidateImages []string
|
|
collector.collectContentValue(last.Get("content"), &candidate, &candidateImages)
|
|
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
|
return
|
|
}
|
|
*parts = append(*parts, candidate...)
|
|
*images = append(*images, candidateImages...)
|
|
}
|
|
|
|
func (collector moderationTextCollector) collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, images *[]string) {
|
|
if !messages.IsArray() {
|
|
return
|
|
}
|
|
array := messages.Array()
|
|
if len(array) == 0 {
|
|
return
|
|
}
|
|
// Anthropic requests may append a system message after the user turn. It
|
|
// does not represent a new model/tool turn, so retain the latest user
|
|
// content while continuing to reject assistant/tool-ended loops.
|
|
lastUser := len(array) - 1
|
|
for lastUser >= 0 && strings.ToLower(strings.TrimSpace(array[lastUser].Get("role").String())) == "system" {
|
|
lastUser--
|
|
}
|
|
if lastUser < 0 || strings.ToLower(strings.TrimSpace(array[lastUser].Get("role").String())) != "user" {
|
|
return
|
|
}
|
|
var candidate []string
|
|
var candidateImages []string
|
|
collector.collectAnthropicUserContentValue(array[lastUser].Get("content"), &candidate, &candidateImages)
|
|
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
|
return
|
|
}
|
|
*parts = append(*parts, candidate...)
|
|
*images = append(*images, candidateImages...)
|
|
}
|
|
|
|
func (collector moderationTextCollector) collectAnthropicUserContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
|
switch {
|
|
case !value.Exists():
|
|
return
|
|
case value.Type == gjson.String:
|
|
if !collector.filterReminders || !isAnthropicSystemReminderText(value.String()) {
|
|
collector.addModerationText(parts, value.String())
|
|
}
|
|
case value.IsArray():
|
|
value.ForEach(func(_, item gjson.Result) bool {
|
|
collector.collectAnthropicUserContentValue(item, parts, images)
|
|
return true
|
|
})
|
|
case value.IsObject():
|
|
typ := strings.ToLower(strings.TrimSpace(value.Get("type").String()))
|
|
switch typ {
|
|
case "", "text", "input_text", "message":
|
|
if value.Get("text").Exists() && (!collector.filterReminders || !isAnthropicSystemReminderText(value.Get("text").String())) {
|
|
collector.addModerationText(parts, value.Get("text").String())
|
|
}
|
|
if value.Get("content").Exists() {
|
|
collector.collectAnthropicUserContentValue(value.Get("content"), parts, images)
|
|
}
|
|
case "image_url", "input_image", "image":
|
|
collector.collectContentValue(value, parts, images)
|
|
}
|
|
}
|
|
}
|
|
|
|
func isAnthropicSystemReminderText(text string) bool {
|
|
return strings.HasPrefix(strings.TrimSpace(text), "<system-reminder>")
|
|
}
|
|
|
|
func (collector moderationTextCollector) collectLastResponsesInput(input gjson.Result, parts *[]string, images *[]string) {
|
|
switch {
|
|
case !input.Exists():
|
|
return
|
|
case input.Type == gjson.String:
|
|
collector.addModerationText(parts, input.String())
|
|
case input.IsArray():
|
|
array := input.Array()
|
|
if len(array) == 0 {
|
|
return
|
|
}
|
|
last := array[len(array)-1]
|
|
if !collector.isResponsesUserTextItem(last) {
|
|
return
|
|
}
|
|
collector.collectContentValue(last.Get("content"), parts, images)
|
|
if last.Get("type").String() == "input_text" || last.Get("text").Exists() {
|
|
collector.collectContentValue(last, parts, images)
|
|
}
|
|
case input.IsObject():
|
|
if collector.isResponsesUserTextItem(input) {
|
|
collector.collectContentValue(input.Get("content"), parts, images)
|
|
if input.Get("type").String() == "input_text" || input.Get("text").Exists() {
|
|
collector.collectContentValue(input, parts, images)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (collector moderationTextCollector) isResponsesUserTextItem(item gjson.Result) bool {
|
|
role := strings.ToLower(strings.TrimSpace(item.Get("role").String()))
|
|
if role == "user" {
|
|
return collector.responseItemHasModerationText(item)
|
|
}
|
|
if role != "" {
|
|
return false
|
|
}
|
|
return collector.responseItemHasModerationText(item)
|
|
}
|
|
|
|
func (collector moderationTextCollector) responseItemHasModerationText(item gjson.Result) bool {
|
|
var parts []string
|
|
var images []string
|
|
collector.collectContentValue(item.Get("content"), &parts, &images)
|
|
if item.Get("type").String() == "input_text" || item.Get("text").Exists() {
|
|
collector.collectContentValue(item, &parts, &images)
|
|
}
|
|
return normalizeContentModerationText(strings.Join(parts, "\n")) != "" || len(images) > 0
|
|
}
|
|
|
|
func (collector moderationTextCollector) collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]string) {
|
|
if !contents.IsArray() {
|
|
return
|
|
}
|
|
array := contents.Array()
|
|
if len(array) == 0 {
|
|
return
|
|
}
|
|
last := array[len(array)-1]
|
|
role := strings.ToLower(strings.TrimSpace(last.Get("role").String()))
|
|
if role != "" && role != "user" {
|
|
return
|
|
}
|
|
var candidate []string
|
|
var candidateImages []string
|
|
if arr := last.Get("parts"); arr.IsArray() {
|
|
arr.ForEach(func(_, part gjson.Result) bool {
|
|
collector.addModerationText(&candidate, part.Get("text").String())
|
|
addGeminiModerationImage(&candidateImages, part)
|
|
return true
|
|
})
|
|
}
|
|
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
|
return
|
|
}
|
|
*parts = append(*parts, candidate...)
|
|
*images = append(*images, candidateImages...)
|
|
}
|
|
|
|
func (collector moderationTextCollector) collectContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
|
switch {
|
|
case !value.Exists():
|
|
return
|
|
case value.Type == gjson.String:
|
|
collector.addModerationText(parts, value.String())
|
|
case value.IsArray():
|
|
value.ForEach(func(_, item gjson.Result) bool {
|
|
collector.collectContentValue(item, parts, images)
|
|
return true
|
|
})
|
|
case value.IsObject():
|
|
typ := strings.ToLower(strings.TrimSpace(value.Get("type").String()))
|
|
addModerationImage(images, value.Get("image_url.url").String())
|
|
addModerationImage(images, value.Get("image_url").String())
|
|
addModerationImage(images, value.Get("url").String())
|
|
addModerationImageData(images, value.Get("source.media_type").String(), value.Get("source.data").String())
|
|
addModerationImageData(images, value.Get("source.mediaType").String(), value.Get("source.data").String())
|
|
addModerationImageData(images, value.Get("media_type").String(), value.Get("data").String())
|
|
addModerationImageData(images, value.Get("mime_type").String(), value.Get("data").String())
|
|
addModerationImageData(images, value.Get("mimeType").String(), value.Get("data").String())
|
|
addModerationImage(images, value.Get("source.data").String())
|
|
addModerationImage(images, value.Get("data").String())
|
|
addModerationImage(images, value.Get("base64").String())
|
|
switch typ {
|
|
case "", "text", "input_text", "message":
|
|
if value.Get("text").Exists() {
|
|
collector.addModerationText(parts, value.Get("text").String())
|
|
}
|
|
if value.Get("content").Exists() {
|
|
collector.collectContentValue(value.Get("content"), parts, images)
|
|
}
|
|
case "image_url", "input_image", "image":
|
|
}
|
|
}
|
|
}
|
|
|
|
func addGeminiModerationImage(images *[]string, part gjson.Result) {
|
|
if inlineData := part.Get("inline_data"); inlineData.IsObject() {
|
|
mimeType := strings.TrimSpace(inlineData.Get("mime_type").String())
|
|
data := strings.TrimSpace(inlineData.Get("data").String())
|
|
if mimeType != "" && data != "" {
|
|
addModerationImage(images, fmt.Sprintf("data:%s;base64,%s", mimeType, data))
|
|
}
|
|
}
|
|
if inlineData := part.Get("inlineData"); inlineData.IsObject() {
|
|
mimeType := strings.TrimSpace(inlineData.Get("mimeType").String())
|
|
data := strings.TrimSpace(inlineData.Get("data").String())
|
|
if mimeType != "" && data != "" {
|
|
addModerationImage(images, fmt.Sprintf("data:%s;base64,%s", mimeType, data))
|
|
}
|
|
}
|
|
addModerationImage(images, part.Get("file_data.file_uri").String())
|
|
addModerationImage(images, part.Get("fileData.fileUri").String())
|
|
}
|
|
|
|
func addModerationImageData(images *[]string, mimeType string, data string) {
|
|
mimeType = strings.TrimSpace(mimeType)
|
|
data = strings.TrimSpace(data)
|
|
if mimeType == "" || data == "" {
|
|
return
|
|
}
|
|
addModerationImage(images, fmt.Sprintf("data:%s;base64,%s", mimeType, data))
|
|
}
|
|
|
|
func addModerationImage(images *[]string, image string) {
|
|
image = strings.TrimSpace(image)
|
|
if image == "" {
|
|
return
|
|
}
|
|
if strings.HasPrefix(image, "data:") || strings.HasPrefix(image, "http://") || strings.HasPrefix(image, "https://") {
|
|
*images = append(*images, image)
|
|
}
|
|
}
|
|
|
|
func normalizeModerationImages(images []string) []string {
|
|
out := make([]string, 0, len(images))
|
|
seen := make(map[string]struct{}, len(images))
|
|
for _, image := range images {
|
|
image = strings.TrimSpace(image)
|
|
if image == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[image]; ok {
|
|
continue
|
|
}
|
|
seen[image] = struct{}{}
|
|
out = append(out, image)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func limitContentModerationImages(images []string) []string {
|
|
if len(images) <= maxContentModerationInputImages {
|
|
return images
|
|
}
|
|
idx, err := rand.Int(rand.Reader, big.NewInt(int64(len(images))))
|
|
if err != nil {
|
|
return images[:maxContentModerationInputImages]
|
|
}
|
|
return []string{images[int(idx.Int64())]}
|
|
}
|
|
|
|
func (collector moderationTextCollector) addModerationText(parts *[]string, text string) {
|
|
text = strings.TrimSpace(text)
|
|
if text == "" {
|
|
return
|
|
}
|
|
if collector.filterReminders && strings.Contains(text, "<system-reminder>") {
|
|
return
|
|
}
|
|
*parts = append(*parts, text)
|
|
}
|
|
|
|
func normalizeContentModerationText(text string) string {
|
|
return strings.Join(strings.Fields(strings.TrimSpace(text)), " ")
|
|
}
|