mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-14 12:02:27 +08:00
fix(moderation): prevent reminder tags from bypassing keyword checks
This commit is contained in:
@@ -909,30 +909,10 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
|
||||
"configured_models", cfg.ModelFilter.Models)
|
||||
return allow, nil
|
||||
}
|
||||
content := ExtractContentModerationInput(input.Protocol, input.Body)
|
||||
if content.IsEmpty() {
|
||||
slog.Info("content_moderation.skip_empty_input",
|
||||
"user_id", input.UserID,
|
||||
"api_key_id", input.APIKeyID,
|
||||
"group_id", contentModerationLogGroupID(input.GroupID),
|
||||
"endpoint", input.Endpoint,
|
||||
"protocol", input.Protocol,
|
||||
"body_bytes", len(input.Body))
|
||||
return allow, nil
|
||||
}
|
||||
content.Normalize()
|
||||
slog.Info("content_moderation.input_extracted",
|
||||
"user_id", input.UserID,
|
||||
"api_key_id", input.APIKeyID,
|
||||
"group_id", contentModerationLogGroupID(input.GroupID),
|
||||
"endpoint", input.Endpoint,
|
||||
"protocol", input.Protocol,
|
||||
"text_runes", len([]rune(content.Text)),
|
||||
"image_count", len(content.Images))
|
||||
hashText := content.Hash()
|
||||
if cfg.Mode == ContentModerationModePreBlock {
|
||||
if cfg.KeywordBlockingMode != ContentModerationKeywordModeAPIOnly && len(cfg.BlockedKeywords) > 0 {
|
||||
if keyword, hit := runtimeSnapshot.matchBlockedKeyword(content.Text); hit {
|
||||
keywordText := extractContentModerationKeywordText(input.Protocol, input.Body)
|
||||
if keyword, hit := runtimeSnapshot.matchBlockedKeyword(keywordText); hit {
|
||||
s.recordPreBlockSyncMetric(0, ContentModerationActionKeywordBlock)
|
||||
slog.Info("content_moderation.keyword_block",
|
||||
"user_id", input.UserID,
|
||||
@@ -943,9 +923,9 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
|
||||
"keyword_blocking_mode", cfg.KeywordBlockingMode,
|
||||
"keyword", keyword)
|
||||
scores := map[string]float64{contentModerationKeywordCategory: 1.0}
|
||||
log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, content.ExcerptText(), nil, nil, "")
|
||||
log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, keywordText, nil, nil, "")
|
||||
log.MatchedKeyword = keyword
|
||||
s.enqueueRecord(input, cfg, log, hashText, false, true)
|
||||
s.enqueueRecord(input, cfg, log, (ContentModerationInput{Text: keywordText}).Hash(), false, true)
|
||||
return &ContentModerationDecision{
|
||||
Allowed: false,
|
||||
Blocked: true,
|
||||
@@ -970,6 +950,27 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
|
||||
return allow, nil
|
||||
}
|
||||
}
|
||||
content := ExtractContentModerationInput(input.Protocol, input.Body)
|
||||
if content.IsEmpty() {
|
||||
slog.Info("content_moderation.skip_empty_input",
|
||||
"user_id", input.UserID,
|
||||
"api_key_id", input.APIKeyID,
|
||||
"group_id", contentModerationLogGroupID(input.GroupID),
|
||||
"endpoint", input.Endpoint,
|
||||
"protocol", input.Protocol,
|
||||
"body_bytes", len(input.Body))
|
||||
return allow, nil
|
||||
}
|
||||
content.Normalize()
|
||||
slog.Info("content_moderation.input_extracted",
|
||||
"user_id", input.UserID,
|
||||
"api_key_id", input.APIKeyID,
|
||||
"group_id", contentModerationLogGroupID(input.GroupID),
|
||||
"endpoint", input.Endpoint,
|
||||
"protocol", input.Protocol,
|
||||
"text_runes", len([]rune(content.Text)),
|
||||
"image_count", len(content.Images))
|
||||
hashText := content.Hash()
|
||||
if cfg.PreHashCheckEnabled && s.hashCache != nil {
|
||||
matched, err := s.hashCache.HasFlaggedInputHash(ctx, hashText)
|
||||
if err != nil {
|
||||
|
||||
@@ -14,6 +14,21 @@ func ExtractContentModerationText(protocol string, body []byte) string {
|
||||
}
|
||||
|
||||
func ExtractContentModerationInput(protocol string, body []byte) ContentModerationInput {
|
||||
return extractContentModerationInput(protocol, body, true)
|
||||
}
|
||||
|
||||
// Keyword checks share semantic moderation's current-user boundaries, but must
|
||||
// inspect client-supplied reminder blocks as ordinary user text.
|
||||
func extractContentModerationKeywordText(protocol string, body []byte) string {
|
||||
return extractContentModerationInput(protocol, body, false).Text
|
||||
}
|
||||
|
||||
type moderationTextCollector struct {
|
||||
filterReminders bool
|
||||
}
|
||||
|
||||
func extractContentModerationInput(protocol string, body []byte, filterReminders bool) ContentModerationInput {
|
||||
collector := moderationTextCollector{filterReminders: filterReminders}
|
||||
if len(body) == 0 || !gjson.ValidBytes(body) {
|
||||
return ContentModerationInput{}
|
||||
}
|
||||
@@ -21,30 +36,32 @@ func ExtractContentModerationInput(protocol string, body []byte) ContentModerati
|
||||
var images []string
|
||||
switch protocol {
|
||||
case ContentModerationProtocolAnthropicMessages:
|
||||
collectLastAnthropicUserMessage(gjson.GetBytes(body, "messages"), &parts, &images)
|
||||
collector.collectLastAnthropicUserMessage(gjson.GetBytes(body, "messages"), &parts, &images)
|
||||
case ContentModerationProtocolOpenAIChat:
|
||||
collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images)
|
||||
collector.collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images)
|
||||
case ContentModerationProtocolOpenAIResponses:
|
||||
collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images)
|
||||
collector.collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images)
|
||||
case ContentModerationProtocolGemini:
|
||||
collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images)
|
||||
collector.collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images)
|
||||
case ContentModerationProtocolOpenAIImages:
|
||||
addModerationText(&parts, gjson.GetBytes(body, "prompt").String())
|
||||
collectContentValue(gjson.GetBytes(body, "images"), &parts, &images)
|
||||
collector.addModerationText(&parts, gjson.GetBytes(body, "prompt").String())
|
||||
collector.collectContentValue(gjson.GetBytes(body, "images"), &parts, &images)
|
||||
default:
|
||||
collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images)
|
||||
collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images)
|
||||
collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images)
|
||||
collector.collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images)
|
||||
collector.collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images)
|
||||
collector.collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images)
|
||||
}
|
||||
out := ContentModerationInput{
|
||||
Text: normalizeContentModerationText(strings.Join(parts, "\n")),
|
||||
Images: normalizeModerationImages(images),
|
||||
}
|
||||
out.Normalize()
|
||||
if filterReminders {
|
||||
out.Normalize()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string, images *[]string) {
|
||||
func (collector moderationTextCollector) collectLastRoleMessage(messages gjson.Result, role string, parts *[]string, images *[]string) {
|
||||
if !messages.IsArray() {
|
||||
return
|
||||
}
|
||||
@@ -58,7 +75,7 @@ func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string,
|
||||
}
|
||||
var candidate []string
|
||||
var candidateImages []string
|
||||
collectContentValue(last.Get("content"), &candidate, &candidateImages)
|
||||
collector.collectContentValue(last.Get("content"), &candidate, &candidateImages)
|
||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
||||
return
|
||||
}
|
||||
@@ -66,7 +83,7 @@ func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string,
|
||||
*images = append(*images, candidateImages...)
|
||||
}
|
||||
|
||||
func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, images *[]string) {
|
||||
func (collector moderationTextCollector) collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, images *[]string) {
|
||||
if !messages.IsArray() {
|
||||
return
|
||||
}
|
||||
@@ -80,7 +97,7 @@ func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, ima
|
||||
}
|
||||
var candidate []string
|
||||
var candidateImages []string
|
||||
collectAnthropicUserContentValue(last.Get("content"), &candidate, &candidateImages)
|
||||
collector.collectAnthropicUserContentValue(last.Get("content"), &candidate, &candidateImages)
|
||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
||||
return
|
||||
}
|
||||
@@ -88,31 +105,31 @@ func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, ima
|
||||
*images = append(*images, candidateImages...)
|
||||
}
|
||||
|
||||
func collectAnthropicUserContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
||||
func (collector moderationTextCollector) collectAnthropicUserContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
||||
switch {
|
||||
case !value.Exists():
|
||||
return
|
||||
case value.Type == gjson.String:
|
||||
if !isAnthropicSystemReminderText(value.String()) {
|
||||
addModerationText(parts, value.String())
|
||||
if !collector.filterReminders || !isAnthropicSystemReminderText(value.String()) {
|
||||
collector.addModerationText(parts, value.String())
|
||||
}
|
||||
case value.IsArray():
|
||||
value.ForEach(func(_, item gjson.Result) bool {
|
||||
collectAnthropicUserContentValue(item, parts, images)
|
||||
collector.collectAnthropicUserContentValue(item, parts, images)
|
||||
return true
|
||||
})
|
||||
case value.IsObject():
|
||||
typ := strings.ToLower(strings.TrimSpace(value.Get("type").String()))
|
||||
switch typ {
|
||||
case "", "text", "input_text", "message":
|
||||
if value.Get("text").Exists() && !isAnthropicSystemReminderText(value.Get("text").String()) {
|
||||
addModerationText(parts, value.Get("text").String())
|
||||
if value.Get("text").Exists() && (!collector.filterReminders || !isAnthropicSystemReminderText(value.Get("text").String())) {
|
||||
collector.addModerationText(parts, value.Get("text").String())
|
||||
}
|
||||
if value.Get("content").Exists() {
|
||||
collectAnthropicUserContentValue(value.Get("content"), parts, images)
|
||||
collector.collectAnthropicUserContentValue(value.Get("content"), parts, images)
|
||||
}
|
||||
case "image_url", "input_image", "image":
|
||||
collectContentValue(value, parts, images)
|
||||
collector.collectContentValue(value, parts, images)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -121,57 +138,57 @@ func isAnthropicSystemReminderText(text string) bool {
|
||||
return strings.HasPrefix(strings.TrimSpace(text), "<system-reminder>")
|
||||
}
|
||||
|
||||
func collectLastResponsesInput(input gjson.Result, parts *[]string, images *[]string) {
|
||||
func (collector moderationTextCollector) collectLastResponsesInput(input gjson.Result, parts *[]string, images *[]string) {
|
||||
switch {
|
||||
case !input.Exists():
|
||||
return
|
||||
case input.Type == gjson.String:
|
||||
addModerationText(parts, input.String())
|
||||
collector.addModerationText(parts, input.String())
|
||||
case input.IsArray():
|
||||
array := input.Array()
|
||||
if len(array) == 0 {
|
||||
return
|
||||
}
|
||||
last := array[len(array)-1]
|
||||
if !isResponsesUserTextItem(last) {
|
||||
if !collector.isResponsesUserTextItem(last) {
|
||||
return
|
||||
}
|
||||
collectContentValue(last.Get("content"), parts, images)
|
||||
collector.collectContentValue(last.Get("content"), parts, images)
|
||||
if last.Get("type").String() == "input_text" || last.Get("text").Exists() {
|
||||
collectContentValue(last, parts, images)
|
||||
collector.collectContentValue(last, parts, images)
|
||||
}
|
||||
case input.IsObject():
|
||||
if isResponsesUserTextItem(input) {
|
||||
collectContentValue(input.Get("content"), parts, images)
|
||||
if collector.isResponsesUserTextItem(input) {
|
||||
collector.collectContentValue(input.Get("content"), parts, images)
|
||||
if input.Get("type").String() == "input_text" || input.Get("text").Exists() {
|
||||
collectContentValue(input, parts, images)
|
||||
collector.collectContentValue(input, parts, images)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isResponsesUserTextItem(item gjson.Result) bool {
|
||||
func (collector moderationTextCollector) isResponsesUserTextItem(item gjson.Result) bool {
|
||||
role := strings.ToLower(strings.TrimSpace(item.Get("role").String()))
|
||||
if role == "user" {
|
||||
return responseItemHasModerationText(item)
|
||||
return collector.responseItemHasModerationText(item)
|
||||
}
|
||||
if role != "" {
|
||||
return false
|
||||
}
|
||||
return responseItemHasModerationText(item)
|
||||
return collector.responseItemHasModerationText(item)
|
||||
}
|
||||
|
||||
func responseItemHasModerationText(item gjson.Result) bool {
|
||||
func (collector moderationTextCollector) responseItemHasModerationText(item gjson.Result) bool {
|
||||
var parts []string
|
||||
var images []string
|
||||
collectContentValue(item.Get("content"), &parts, &images)
|
||||
collector.collectContentValue(item.Get("content"), &parts, &images)
|
||||
if item.Get("type").String() == "input_text" || item.Get("text").Exists() {
|
||||
collectContentValue(item, &parts, &images)
|
||||
collector.collectContentValue(item, &parts, &images)
|
||||
}
|
||||
return normalizeContentModerationText(strings.Join(parts, "\n")) != "" || len(images) > 0
|
||||
}
|
||||
|
||||
func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]string) {
|
||||
func (collector moderationTextCollector) collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]string) {
|
||||
if !contents.IsArray() {
|
||||
return
|
||||
}
|
||||
@@ -188,7 +205,7 @@ func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]
|
||||
var candidateImages []string
|
||||
if arr := last.Get("parts"); arr.IsArray() {
|
||||
arr.ForEach(func(_, part gjson.Result) bool {
|
||||
addModerationText(&candidate, part.Get("text").String())
|
||||
collector.addModerationText(&candidate, part.Get("text").String())
|
||||
addGeminiModerationImage(&candidateImages, part)
|
||||
return true
|
||||
})
|
||||
@@ -200,15 +217,15 @@ func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]
|
||||
*images = append(*images, candidateImages...)
|
||||
}
|
||||
|
||||
func collectContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
||||
func (collector moderationTextCollector) collectContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
||||
switch {
|
||||
case !value.Exists():
|
||||
return
|
||||
case value.Type == gjson.String:
|
||||
addModerationText(parts, value.String())
|
||||
collector.addModerationText(parts, value.String())
|
||||
case value.IsArray():
|
||||
value.ForEach(func(_, item gjson.Result) bool {
|
||||
collectContentValue(item, parts, images)
|
||||
collector.collectContentValue(item, parts, images)
|
||||
return true
|
||||
})
|
||||
case value.IsObject():
|
||||
@@ -227,10 +244,10 @@ func collectContentValue(value gjson.Result, parts *[]string, images *[]string)
|
||||
switch typ {
|
||||
case "", "text", "input_text", "message":
|
||||
if value.Get("text").Exists() {
|
||||
addModerationText(parts, value.Get("text").String())
|
||||
collector.addModerationText(parts, value.Get("text").String())
|
||||
}
|
||||
if value.Get("content").Exists() {
|
||||
collectContentValue(value.Get("content"), parts, images)
|
||||
collector.collectContentValue(value.Get("content"), parts, images)
|
||||
}
|
||||
case "image_url", "input_image", "image":
|
||||
}
|
||||
@@ -303,12 +320,12 @@ func limitContentModerationImages(images []string) []string {
|
||||
return []string{images[int(idx.Int64())]}
|
||||
}
|
||||
|
||||
func addModerationText(parts *[]string, text string) {
|
||||
func (collector moderationTextCollector) addModerationText(parts *[]string, text string) {
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return
|
||||
}
|
||||
if strings.Contains(text, "<system-reminder>") {
|
||||
if collector.filterReminders && strings.Contains(text, "<system-reminder>") {
|
||||
return
|
||||
}
|
||||
*parts = append(*parts, text)
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func reminderTestBody(t *testing.T, protocol string, texts []string) []byte {
|
||||
t.Helper()
|
||||
blocks := make([]map[string]string, 0, len(texts))
|
||||
for _, text := range texts {
|
||||
blocks = append(blocks, map[string]string{"type": "text", "text": text})
|
||||
}
|
||||
message := map[string]any{"role": "user", "content": blocks}
|
||||
var body any
|
||||
switch protocol {
|
||||
case ContentModerationProtocolGemini:
|
||||
body = map[string]any{"contents": []any{map[string]any{"role": "user", "parts": blocks}}}
|
||||
case ContentModerationProtocolOpenAIResponses, "unknown":
|
||||
body = map[string]any{"input": []any{message}}
|
||||
case ContentModerationProtocolOpenAIImages:
|
||||
body = map[string]any{"prompt": strings.Join(texts, "\n")}
|
||||
default:
|
||||
body = map[string]any{"messages": []any{message}}
|
||||
}
|
||||
raw, err := json.Marshal(body)
|
||||
require.NoError(t, err)
|
||||
return raw
|
||||
}
|
||||
|
||||
func TestContentModerationCheck_ReminderKeywordModes(t *testing.T) {
|
||||
for _, protocol := range []string{ContentModerationProtocolAnthropicMessages, ContentModerationProtocolOpenAIChat, ContentModerationProtocolOpenAIResponses, ContentModerationProtocolGemini, ContentModerationProtocolOpenAIImages, "unknown"} {
|
||||
for name, texts := range map[string][]string{
|
||||
"plain": {"今晚打老虎"},
|
||||
"prefix": {"<system-reminder>context</system-reminder> 今晚打老虎"},
|
||||
"suffix": {"今晚打老虎 <system-reminder>context</system-reminder>"},
|
||||
"separate": {"<system-reminder>context</system-reminder>", "今晚打老虎"},
|
||||
"inside": {"<system-reminder>今晚打老虎</system-reminder>"},
|
||||
} {
|
||||
for _, mode := range []string{ContentModerationKeywordModeKeywordOnly, ContentModerationKeywordModeAPIOnly, ContentModerationKeywordModeKeywordAndAPI} {
|
||||
t.Run(protocol+"/"+name+"/"+mode, func(t *testing.T) {
|
||||
body := reminderTestBody(t, protocol, texts)
|
||||
semantic := ExtractContentModerationInput(protocol, body)
|
||||
wantSemantic := ""
|
||||
if name == "plain" || (name == "separate" && protocol != ContentModerationProtocolOpenAIImages) {
|
||||
wantSemantic = "今晚打老虎"
|
||||
}
|
||||
require.Equal(t, wantSemantic, semantic.Text)
|
||||
require.Contains(t, extractContentModerationKeywordText(protocol, body), "今晚打老虎")
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
var payload struct {
|
||||
Input string `json:"input"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if payload.Input != semantic.Text {
|
||||
t.Errorf("semantic input changed: %q != %q", payload.Input, semantic.Text)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{}}})
|
||||
}))
|
||||
defer server.Close()
|
||||
cfg := defaultContentModerationConfig()
|
||||
cfg.Enabled = true
|
||||
cfg.Mode = ContentModerationModePreBlock
|
||||
cfg.KeywordBlockingMode = mode
|
||||
cfg.BlockedKeywords = []string{"今晚打老虎"}
|
||||
cfg.BaseURL = server.URL
|
||||
cfg.APIKeys = []string{"test"}
|
||||
cfg.RecordNonHits = false
|
||||
cfg.PreHashCheckEnabled = true
|
||||
raw, err := json.Marshal(cfg)
|
||||
require.NoError(t, err)
|
||||
repo := &contentModerationTestRepo{}
|
||||
cache := &contentModerationTestHashCache{}
|
||||
if mode != ContentModerationKeywordModeAPIOnly {
|
||||
cache.hashes = map[string]struct{}{semantic.Hash(): {}}
|
||||
}
|
||||
svc := NewContentModerationService(&contentModerationTestSettingRepo{values: map[string]string{SettingKeyRiskControlEnabled: "true", SettingKeyContentModerationConfig: string(raw)}}, repo, cache, nil, nil, nil, nil, nil)
|
||||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{Protocol: protocol, Body: body})
|
||||
require.NoError(t, err)
|
||||
if mode == ContentModerationKeywordModeAPIOnly {
|
||||
require.True(t, decision.Allowed)
|
||||
want := int32(0)
|
||||
if !semantic.IsEmpty() {
|
||||
want = 1
|
||||
}
|
||||
require.Equal(t, want, calls.Load())
|
||||
require.Empty(t, repo.snapshotLogs())
|
||||
} else {
|
||||
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
||||
require.True(t, decision.Blocked)
|
||||
require.Zero(t, calls.Load())
|
||||
require.Empty(t, cache.snapshotChecked())
|
||||
logs := requireContentModerationLogCount(t, repo, 1)
|
||||
require.Equal(t, "今晚打老虎", logs[0].MatchedKeyword)
|
||||
require.Contains(t, logs[0].InputExcerpt, "今晚打老虎")
|
||||
require.Equal(t, int64(1), svc.asyncEnqueued.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractContentModerationKeywordText_Boundaries(t *testing.T) {
|
||||
for _, tc := range []struct{ protocol, body string }{
|
||||
{ContentModerationProtocolAnthropicMessages, `{"messages":[{"role":"user","content":"old"},{"role":"user","content":[{"type":"tool_result","content":"今晚打老虎"}]}]}`},
|
||||
{ContentModerationProtocolAnthropicMessages, `{"messages":[{"role":"user","content":"old"},{"role":"assistant","content":"今晚打老虎"}]}`},
|
||||
{ContentModerationProtocolOpenAIChat, `{"messages":[{"role":"user","content":"old"},{"role":"tool","content":"今晚打老虎"}]}`},
|
||||
{ContentModerationProtocolOpenAIChat, `{"messages":[{"role":"user","content":"old"},{"role":"assistant","content":"今晚打老虎"}]}`},
|
||||
{ContentModerationProtocolOpenAIResponses, `{"input":[{"role":"user","content":"old"},{"type":"function_call_output","output":"今晚打老虎"}]}`},
|
||||
{ContentModerationProtocolOpenAIResponses, `{"input":[{"role":"user","content":"old"},{"role":"assistant","content":"今晚打老虎"}]}`},
|
||||
{ContentModerationProtocolGemini, `{"contents":[{"role":"user","parts":[{"text":"old"}]},{"role":"user","parts":[{"functionResponse":{"response":{"text":"今晚打老虎"}}}]}]}`},
|
||||
{ContentModerationProtocolGemini, `{"contents":[{"role":"user","parts":[{"text":"old"}]},{"role":"model","parts":[{"text":"今晚打老虎"}]}]}`},
|
||||
} {
|
||||
t.Run(tc.protocol+tc.body, func(t *testing.T) {
|
||||
require.Empty(t, extractContentModerationKeywordText(tc.protocol, []byte(tc.body)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentModerationCheck_ReminderPolicyPreserved(t *testing.T) {
|
||||
for _, name := range []string{"keyword_miss", "combined_miss", "observe", "off", "disabled", "group_scope", "model_scope", "no_api_key", "zero_sample"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
var payload struct {
|
||||
Input string `json:"input"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if payload.Input != "safe text" {
|
||||
t.Errorf("unexpected semantic input: %q", payload.Input)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{}}})
|
||||
}))
|
||||
defer server.Close()
|
||||
cfg := defaultContentModerationConfig()
|
||||
cfg.Enabled = true
|
||||
cfg.Mode = ContentModerationModePreBlock
|
||||
cfg.BlockedKeywords = []string{"今晚打老虎"}
|
||||
cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordAndAPI
|
||||
cfg.BaseURL = server.URL
|
||||
cfg.APIKeys = []string{"test"}
|
||||
cfg.RecordNonHits = false
|
||||
cfg.PreHashCheckEnabled = false
|
||||
switch name {
|
||||
case "keyword_miss":
|
||||
cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly
|
||||
cfg.BlockedKeywords = []string{"absent"}
|
||||
case "combined_miss":
|
||||
cfg.BlockedKeywords = []string{"absent"}
|
||||
case "observe":
|
||||
cfg.Mode = ContentModerationModeObserve
|
||||
case "off":
|
||||
cfg.Mode = ContentModerationModeOff
|
||||
case "disabled":
|
||||
cfg.Enabled = false
|
||||
case "group_scope":
|
||||
cfg.AllGroups = false
|
||||
cfg.GroupIDs = []int64{42}
|
||||
case "model_scope":
|
||||
cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"other-model"}}
|
||||
case "no_api_key":
|
||||
cfg.APIKeys = nil
|
||||
case "zero_sample":
|
||||
cfg.SampleRate = 0
|
||||
}
|
||||
raw, err := json.Marshal(cfg)
|
||||
require.NoError(t, err)
|
||||
repo := &contentModerationTestRepo{}
|
||||
svc := NewContentModerationService(&contentModerationTestSettingRepo{values: map[string]string{SettingKeyRiskControlEnabled: "true", SettingKeyContentModerationConfig: string(raw)}}, repo, nil, nil, nil, nil, nil, nil)
|
||||
body := reminderTestBody(t, ContentModerationProtocolAnthropicMessages, []string{"<system-reminder>今晚打老虎</system-reminder>", "safe text"})
|
||||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{Protocol: ContentModerationProtocolAnthropicMessages, Body: body})
|
||||
require.NoError(t, err)
|
||||
if name == "no_api_key" || name == "zero_sample" {
|
||||
require.True(t, decision.Blocked)
|
||||
requireContentModerationLogCount(t, repo, 1)
|
||||
} else {
|
||||
require.True(t, decision.Allowed)
|
||||
}
|
||||
switch name {
|
||||
case "observe":
|
||||
require.Eventually(t, func() bool { return calls.Load() == 1 }, time.Second, time.Millisecond*10)
|
||||
case "combined_miss":
|
||||
require.Equal(t, int32(1), calls.Load())
|
||||
default:
|
||||
require.Zero(t, calls.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractContentModerationKeywordText_ResponsesInputForms(t *testing.T) {
|
||||
for _, body := range []string{
|
||||
`{"input":"<system-reminder>今晚打老虎</system-reminder>"}`,
|
||||
`{"input":{"role":"user","content":"<system-reminder>今晚打老虎</system-reminder>"}}`,
|
||||
`{"input":[{"type":"input_text","text":"<system-reminder>今晚打老虎</system-reminder>"}]}`,
|
||||
} {
|
||||
require.Contains(t, extractContentModerationKeywordText(ContentModerationProtocolOpenAIResponses, []byte(body)), "今晚打老虎")
|
||||
require.Empty(t, ExtractContentModerationText(ContentModerationProtocolOpenAIResponses, []byte(body)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractContentModerationKeywordText_LongReminder(t *testing.T) {
|
||||
// Reminder context must not consume the semantic API's text limit and hide
|
||||
// the actual user keyword from the local check.
|
||||
body := reminderTestBody(t, ContentModerationProtocolAnthropicMessages, []string{
|
||||
"<system-reminder>" + strings.Repeat("x", maxModerationInputRunes) + "</system-reminder>", "今晚打老虎",
|
||||
})
|
||||
require.Contains(t, extractContentModerationKeywordText(ContentModerationProtocolAnthropicMessages, body), "今晚打老虎")
|
||||
require.Equal(t, "今晚打老虎", ExtractContentModerationText(ContentModerationProtocolAnthropicMessages, body))
|
||||
}
|
||||
Reference in New Issue
Block a user