fix(moderation): prevent reminder tags from bypassing keyword checks

This commit is contained in:
wucm667
2026-09-18 14:54:18 +08:00
parent efe9aab1e4
commit 7b4de8b6ae
3 changed files with 313 additions and 69 deletions
+25 -24
View File
@@ -909,30 +909,10 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
"configured_models", cfg.ModelFilter.Models)
return allow, nil
}
content := ExtractContentModerationInput(input.Protocol, input.Body)
if content.IsEmpty() {
slog.Info("content_moderation.skip_empty_input",
"user_id", input.UserID,
"api_key_id", input.APIKeyID,
"group_id", contentModerationLogGroupID(input.GroupID),
"endpoint", input.Endpoint,
"protocol", input.Protocol,
"body_bytes", len(input.Body))
return allow, nil
}
content.Normalize()
slog.Info("content_moderation.input_extracted",
"user_id", input.UserID,
"api_key_id", input.APIKeyID,
"group_id", contentModerationLogGroupID(input.GroupID),
"endpoint", input.Endpoint,
"protocol", input.Protocol,
"text_runes", len([]rune(content.Text)),
"image_count", len(content.Images))
hashText := content.Hash()
if cfg.Mode == ContentModerationModePreBlock {
if cfg.KeywordBlockingMode != ContentModerationKeywordModeAPIOnly && len(cfg.BlockedKeywords) > 0 {
if keyword, hit := runtimeSnapshot.matchBlockedKeyword(content.Text); hit {
keywordText := extractContentModerationKeywordText(input.Protocol, input.Body)
if keyword, hit := runtimeSnapshot.matchBlockedKeyword(keywordText); hit {
s.recordPreBlockSyncMetric(0, ContentModerationActionKeywordBlock)
slog.Info("content_moderation.keyword_block",
"user_id", input.UserID,
@@ -943,9 +923,9 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
"keyword_blocking_mode", cfg.KeywordBlockingMode,
"keyword", keyword)
scores := map[string]float64{contentModerationKeywordCategory: 1.0}
log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, content.ExcerptText(), nil, nil, "")
log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, keywordText, nil, nil, "")
log.MatchedKeyword = keyword
s.enqueueRecord(input, cfg, log, hashText, false, true)
s.enqueueRecord(input, cfg, log, (ContentModerationInput{Text: keywordText}).Hash(), false, true)
return &ContentModerationDecision{
Allowed: false,
Blocked: true,
@@ -970,6 +950,27 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
return allow, nil
}
}
content := ExtractContentModerationInput(input.Protocol, input.Body)
if content.IsEmpty() {
slog.Info("content_moderation.skip_empty_input",
"user_id", input.UserID,
"api_key_id", input.APIKeyID,
"group_id", contentModerationLogGroupID(input.GroupID),
"endpoint", input.Endpoint,
"protocol", input.Protocol,
"body_bytes", len(input.Body))
return allow, nil
}
content.Normalize()
slog.Info("content_moderation.input_extracted",
"user_id", input.UserID,
"api_key_id", input.APIKeyID,
"group_id", contentModerationLogGroupID(input.GroupID),
"endpoint", input.Endpoint,
"protocol", input.Protocol,
"text_runes", len([]rune(content.Text)),
"image_count", len(content.Images))
hashText := content.Hash()
if cfg.PreHashCheckEnabled && s.hashCache != nil {
matched, err := s.hashCache.HasFlaggedInputHash(ctx, hashText)
if err != nil {
@@ -14,6 +14,21 @@ func ExtractContentModerationText(protocol string, body []byte) string {
}
func ExtractContentModerationInput(protocol string, body []byte) ContentModerationInput {
return extractContentModerationInput(protocol, body, true)
}
// Keyword checks share semantic moderation's current-user boundaries, but must
// inspect client-supplied reminder blocks as ordinary user text.
func extractContentModerationKeywordText(protocol string, body []byte) string {
return extractContentModerationInput(protocol, body, false).Text
}
type moderationTextCollector struct {
filterReminders bool
}
func extractContentModerationInput(protocol string, body []byte, filterReminders bool) ContentModerationInput {
collector := moderationTextCollector{filterReminders: filterReminders}
if len(body) == 0 || !gjson.ValidBytes(body) {
return ContentModerationInput{}
}
@@ -21,30 +36,32 @@ func ExtractContentModerationInput(protocol string, body []byte) ContentModerati
var images []string
switch protocol {
case ContentModerationProtocolAnthropicMessages:
collectLastAnthropicUserMessage(gjson.GetBytes(body, "messages"), &parts, &images)
collector.collectLastAnthropicUserMessage(gjson.GetBytes(body, "messages"), &parts, &images)
case ContentModerationProtocolOpenAIChat:
collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images)
collector.collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images)
case ContentModerationProtocolOpenAIResponses:
collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images)
collector.collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images)
case ContentModerationProtocolGemini:
collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images)
collector.collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images)
case ContentModerationProtocolOpenAIImages:
addModerationText(&parts, gjson.GetBytes(body, "prompt").String())
collectContentValue(gjson.GetBytes(body, "images"), &parts, &images)
collector.addModerationText(&parts, gjson.GetBytes(body, "prompt").String())
collector.collectContentValue(gjson.GetBytes(body, "images"), &parts, &images)
default:
collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images)
collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images)
collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images)
collector.collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images)
collector.collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images)
collector.collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images)
}
out := ContentModerationInput{
Text: normalizeContentModerationText(strings.Join(parts, "\n")),
Images: normalizeModerationImages(images),
}
out.Normalize()
if filterReminders {
out.Normalize()
}
return out
}
func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string, images *[]string) {
func (collector moderationTextCollector) collectLastRoleMessage(messages gjson.Result, role string, parts *[]string, images *[]string) {
if !messages.IsArray() {
return
}
@@ -58,7 +75,7 @@ func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string,
}
var candidate []string
var candidateImages []string
collectContentValue(last.Get("content"), &candidate, &candidateImages)
collector.collectContentValue(last.Get("content"), &candidate, &candidateImages)
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
return
}
@@ -66,7 +83,7 @@ func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string,
*images = append(*images, candidateImages...)
}
func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, images *[]string) {
func (collector moderationTextCollector) collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, images *[]string) {
if !messages.IsArray() {
return
}
@@ -80,7 +97,7 @@ func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, ima
}
var candidate []string
var candidateImages []string
collectAnthropicUserContentValue(last.Get("content"), &candidate, &candidateImages)
collector.collectAnthropicUserContentValue(last.Get("content"), &candidate, &candidateImages)
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
return
}
@@ -88,31 +105,31 @@ func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, ima
*images = append(*images, candidateImages...)
}
func collectAnthropicUserContentValue(value gjson.Result, parts *[]string, images *[]string) {
func (collector moderationTextCollector) collectAnthropicUserContentValue(value gjson.Result, parts *[]string, images *[]string) {
switch {
case !value.Exists():
return
case value.Type == gjson.String:
if !isAnthropicSystemReminderText(value.String()) {
addModerationText(parts, value.String())
if !collector.filterReminders || !isAnthropicSystemReminderText(value.String()) {
collector.addModerationText(parts, value.String())
}
case value.IsArray():
value.ForEach(func(_, item gjson.Result) bool {
collectAnthropicUserContentValue(item, parts, images)
collector.collectAnthropicUserContentValue(item, parts, images)
return true
})
case value.IsObject():
typ := strings.ToLower(strings.TrimSpace(value.Get("type").String()))
switch typ {
case "", "text", "input_text", "message":
if value.Get("text").Exists() && !isAnthropicSystemReminderText(value.Get("text").String()) {
addModerationText(parts, value.Get("text").String())
if value.Get("text").Exists() && (!collector.filterReminders || !isAnthropicSystemReminderText(value.Get("text").String())) {
collector.addModerationText(parts, value.Get("text").String())
}
if value.Get("content").Exists() {
collectAnthropicUserContentValue(value.Get("content"), parts, images)
collector.collectAnthropicUserContentValue(value.Get("content"), parts, images)
}
case "image_url", "input_image", "image":
collectContentValue(value, parts, images)
collector.collectContentValue(value, parts, images)
}
}
}
@@ -121,57 +138,57 @@ func isAnthropicSystemReminderText(text string) bool {
return strings.HasPrefix(strings.TrimSpace(text), "<system-reminder>")
}
func collectLastResponsesInput(input gjson.Result, parts *[]string, images *[]string) {
func (collector moderationTextCollector) collectLastResponsesInput(input gjson.Result, parts *[]string, images *[]string) {
switch {
case !input.Exists():
return
case input.Type == gjson.String:
addModerationText(parts, input.String())
collector.addModerationText(parts, input.String())
case input.IsArray():
array := input.Array()
if len(array) == 0 {
return
}
last := array[len(array)-1]
if !isResponsesUserTextItem(last) {
if !collector.isResponsesUserTextItem(last) {
return
}
collectContentValue(last.Get("content"), parts, images)
collector.collectContentValue(last.Get("content"), parts, images)
if last.Get("type").String() == "input_text" || last.Get("text").Exists() {
collectContentValue(last, parts, images)
collector.collectContentValue(last, parts, images)
}
case input.IsObject():
if isResponsesUserTextItem(input) {
collectContentValue(input.Get("content"), parts, images)
if collector.isResponsesUserTextItem(input) {
collector.collectContentValue(input.Get("content"), parts, images)
if input.Get("type").String() == "input_text" || input.Get("text").Exists() {
collectContentValue(input, parts, images)
collector.collectContentValue(input, parts, images)
}
}
}
}
func isResponsesUserTextItem(item gjson.Result) bool {
func (collector moderationTextCollector) isResponsesUserTextItem(item gjson.Result) bool {
role := strings.ToLower(strings.TrimSpace(item.Get("role").String()))
if role == "user" {
return responseItemHasModerationText(item)
return collector.responseItemHasModerationText(item)
}
if role != "" {
return false
}
return responseItemHasModerationText(item)
return collector.responseItemHasModerationText(item)
}
func responseItemHasModerationText(item gjson.Result) bool {
func (collector moderationTextCollector) responseItemHasModerationText(item gjson.Result) bool {
var parts []string
var images []string
collectContentValue(item.Get("content"), &parts, &images)
collector.collectContentValue(item.Get("content"), &parts, &images)
if item.Get("type").String() == "input_text" || item.Get("text").Exists() {
collectContentValue(item, &parts, &images)
collector.collectContentValue(item, &parts, &images)
}
return normalizeContentModerationText(strings.Join(parts, "\n")) != "" || len(images) > 0
}
func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]string) {
func (collector moderationTextCollector) collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]string) {
if !contents.IsArray() {
return
}
@@ -188,7 +205,7 @@ func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]
var candidateImages []string
if arr := last.Get("parts"); arr.IsArray() {
arr.ForEach(func(_, part gjson.Result) bool {
addModerationText(&candidate, part.Get("text").String())
collector.addModerationText(&candidate, part.Get("text").String())
addGeminiModerationImage(&candidateImages, part)
return true
})
@@ -200,15 +217,15 @@ func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]
*images = append(*images, candidateImages...)
}
func collectContentValue(value gjson.Result, parts *[]string, images *[]string) {
func (collector moderationTextCollector) collectContentValue(value gjson.Result, parts *[]string, images *[]string) {
switch {
case !value.Exists():
return
case value.Type == gjson.String:
addModerationText(parts, value.String())
collector.addModerationText(parts, value.String())
case value.IsArray():
value.ForEach(func(_, item gjson.Result) bool {
collectContentValue(item, parts, images)
collector.collectContentValue(item, parts, images)
return true
})
case value.IsObject():
@@ -227,10 +244,10 @@ func collectContentValue(value gjson.Result, parts *[]string, images *[]string)
switch typ {
case "", "text", "input_text", "message":
if value.Get("text").Exists() {
addModerationText(parts, value.Get("text").String())
collector.addModerationText(parts, value.Get("text").String())
}
if value.Get("content").Exists() {
collectContentValue(value.Get("content"), parts, images)
collector.collectContentValue(value.Get("content"), parts, images)
}
case "image_url", "input_image", "image":
}
@@ -303,12 +320,12 @@ func limitContentModerationImages(images []string) []string {
return []string{images[int(idx.Int64())]}
}
func addModerationText(parts *[]string, text string) {
func (collector moderationTextCollector) addModerationText(parts *[]string, text string) {
text = strings.TrimSpace(text)
if text == "" {
return
}
if strings.Contains(text, "<system-reminder>") {
if collector.filterReminders && strings.Contains(text, "<system-reminder>") {
return
}
*parts = append(*parts, text)
@@ -0,0 +1,226 @@
package service
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func reminderTestBody(t *testing.T, protocol string, texts []string) []byte {
t.Helper()
blocks := make([]map[string]string, 0, len(texts))
for _, text := range texts {
blocks = append(blocks, map[string]string{"type": "text", "text": text})
}
message := map[string]any{"role": "user", "content": blocks}
var body any
switch protocol {
case ContentModerationProtocolGemini:
body = map[string]any{"contents": []any{map[string]any{"role": "user", "parts": blocks}}}
case ContentModerationProtocolOpenAIResponses, "unknown":
body = map[string]any{"input": []any{message}}
case ContentModerationProtocolOpenAIImages:
body = map[string]any{"prompt": strings.Join(texts, "\n")}
default:
body = map[string]any{"messages": []any{message}}
}
raw, err := json.Marshal(body)
require.NoError(t, err)
return raw
}
func TestContentModerationCheck_ReminderKeywordModes(t *testing.T) {
for _, protocol := range []string{ContentModerationProtocolAnthropicMessages, ContentModerationProtocolOpenAIChat, ContentModerationProtocolOpenAIResponses, ContentModerationProtocolGemini, ContentModerationProtocolOpenAIImages, "unknown"} {
for name, texts := range map[string][]string{
"plain": {"今晚打老虎"},
"prefix": {"<system-reminder>context</system-reminder> 今晚打老虎"},
"suffix": {"今晚打老虎 <system-reminder>context</system-reminder>"},
"separate": {"<system-reminder>context</system-reminder>", "今晚打老虎"},
"inside": {"<system-reminder>今晚打老虎</system-reminder>"},
} {
for _, mode := range []string{ContentModerationKeywordModeKeywordOnly, ContentModerationKeywordModeAPIOnly, ContentModerationKeywordModeKeywordAndAPI} {
t.Run(protocol+"/"+name+"/"+mode, func(t *testing.T) {
body := reminderTestBody(t, protocol, texts)
semantic := ExtractContentModerationInput(protocol, body)
wantSemantic := ""
if name == "plain" || (name == "separate" && protocol != ContentModerationProtocolOpenAIImages) {
wantSemantic = "今晚打老虎"
}
require.Equal(t, wantSemantic, semantic.Text)
require.Contains(t, extractContentModerationKeywordText(protocol, body), "今晚打老虎")
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
var payload struct {
Input string `json:"input"`
}
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Error(err)
}
if payload.Input != semantic.Text {
t.Errorf("semantic input changed: %q != %q", payload.Input, semantic.Text)
}
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{}}})
}))
defer server.Close()
cfg := defaultContentModerationConfig()
cfg.Enabled = true
cfg.Mode = ContentModerationModePreBlock
cfg.KeywordBlockingMode = mode
cfg.BlockedKeywords = []string{"今晚打老虎"}
cfg.BaseURL = server.URL
cfg.APIKeys = []string{"test"}
cfg.RecordNonHits = false
cfg.PreHashCheckEnabled = true
raw, err := json.Marshal(cfg)
require.NoError(t, err)
repo := &contentModerationTestRepo{}
cache := &contentModerationTestHashCache{}
if mode != ContentModerationKeywordModeAPIOnly {
cache.hashes = map[string]struct{}{semantic.Hash(): {}}
}
svc := NewContentModerationService(&contentModerationTestSettingRepo{values: map[string]string{SettingKeyRiskControlEnabled: "true", SettingKeyContentModerationConfig: string(raw)}}, repo, cache, nil, nil, nil, nil, nil)
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{Protocol: protocol, Body: body})
require.NoError(t, err)
if mode == ContentModerationKeywordModeAPIOnly {
require.True(t, decision.Allowed)
want := int32(0)
if !semantic.IsEmpty() {
want = 1
}
require.Equal(t, want, calls.Load())
require.Empty(t, repo.snapshotLogs())
} else {
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
require.True(t, decision.Blocked)
require.Zero(t, calls.Load())
require.Empty(t, cache.snapshotChecked())
logs := requireContentModerationLogCount(t, repo, 1)
require.Equal(t, "今晚打老虎", logs[0].MatchedKeyword)
require.Contains(t, logs[0].InputExcerpt, "今晚打老虎")
require.Equal(t, int64(1), svc.asyncEnqueued.Load())
}
})
}
}
}
}
func TestExtractContentModerationKeywordText_Boundaries(t *testing.T) {
for _, tc := range []struct{ protocol, body string }{
{ContentModerationProtocolAnthropicMessages, `{"messages":[{"role":"user","content":"old"},{"role":"user","content":[{"type":"tool_result","content":"今晚打老虎"}]}]}`},
{ContentModerationProtocolAnthropicMessages, `{"messages":[{"role":"user","content":"old"},{"role":"assistant","content":"今晚打老虎"}]}`},
{ContentModerationProtocolOpenAIChat, `{"messages":[{"role":"user","content":"old"},{"role":"tool","content":"今晚打老虎"}]}`},
{ContentModerationProtocolOpenAIChat, `{"messages":[{"role":"user","content":"old"},{"role":"assistant","content":"今晚打老虎"}]}`},
{ContentModerationProtocolOpenAIResponses, `{"input":[{"role":"user","content":"old"},{"type":"function_call_output","output":"今晚打老虎"}]}`},
{ContentModerationProtocolOpenAIResponses, `{"input":[{"role":"user","content":"old"},{"role":"assistant","content":"今晚打老虎"}]}`},
{ContentModerationProtocolGemini, `{"contents":[{"role":"user","parts":[{"text":"old"}]},{"role":"user","parts":[{"functionResponse":{"response":{"text":"今晚打老虎"}}}]}]}`},
{ContentModerationProtocolGemini, `{"contents":[{"role":"user","parts":[{"text":"old"}]},{"role":"model","parts":[{"text":"今晚打老虎"}]}]}`},
} {
t.Run(tc.protocol+tc.body, func(t *testing.T) {
require.Empty(t, extractContentModerationKeywordText(tc.protocol, []byte(tc.body)))
})
}
}
func TestContentModerationCheck_ReminderPolicyPreserved(t *testing.T) {
for _, name := range []string{"keyword_miss", "combined_miss", "observe", "off", "disabled", "group_scope", "model_scope", "no_api_key", "zero_sample"} {
t.Run(name, func(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
var payload struct {
Input string `json:"input"`
}
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Error(err)
}
if payload.Input != "safe text" {
t.Errorf("unexpected semantic input: %q", payload.Input)
}
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{}}})
}))
defer server.Close()
cfg := defaultContentModerationConfig()
cfg.Enabled = true
cfg.Mode = ContentModerationModePreBlock
cfg.BlockedKeywords = []string{"今晚打老虎"}
cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordAndAPI
cfg.BaseURL = server.URL
cfg.APIKeys = []string{"test"}
cfg.RecordNonHits = false
cfg.PreHashCheckEnabled = false
switch name {
case "keyword_miss":
cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly
cfg.BlockedKeywords = []string{"absent"}
case "combined_miss":
cfg.BlockedKeywords = []string{"absent"}
case "observe":
cfg.Mode = ContentModerationModeObserve
case "off":
cfg.Mode = ContentModerationModeOff
case "disabled":
cfg.Enabled = false
case "group_scope":
cfg.AllGroups = false
cfg.GroupIDs = []int64{42}
case "model_scope":
cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"other-model"}}
case "no_api_key":
cfg.APIKeys = nil
case "zero_sample":
cfg.SampleRate = 0
}
raw, err := json.Marshal(cfg)
require.NoError(t, err)
repo := &contentModerationTestRepo{}
svc := NewContentModerationService(&contentModerationTestSettingRepo{values: map[string]string{SettingKeyRiskControlEnabled: "true", SettingKeyContentModerationConfig: string(raw)}}, repo, nil, nil, nil, nil, nil, nil)
body := reminderTestBody(t, ContentModerationProtocolAnthropicMessages, []string{"<system-reminder>今晚打老虎</system-reminder>", "safe text"})
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{Protocol: ContentModerationProtocolAnthropicMessages, Body: body})
require.NoError(t, err)
if name == "no_api_key" || name == "zero_sample" {
require.True(t, decision.Blocked)
requireContentModerationLogCount(t, repo, 1)
} else {
require.True(t, decision.Allowed)
}
switch name {
case "observe":
require.Eventually(t, func() bool { return calls.Load() == 1 }, time.Second, time.Millisecond*10)
case "combined_miss":
require.Equal(t, int32(1), calls.Load())
default:
require.Zero(t, calls.Load())
}
})
}
}
func TestExtractContentModerationKeywordText_ResponsesInputForms(t *testing.T) {
for _, body := range []string{
`{"input":"<system-reminder>今晚打老虎</system-reminder>"}`,
`{"input":{"role":"user","content":"<system-reminder>今晚打老虎</system-reminder>"}}`,
`{"input":[{"type":"input_text","text":"<system-reminder>今晚打老虎</system-reminder>"}]}`,
} {
require.Contains(t, extractContentModerationKeywordText(ContentModerationProtocolOpenAIResponses, []byte(body)), "今晚打老虎")
require.Empty(t, ExtractContentModerationText(ContentModerationProtocolOpenAIResponses, []byte(body)))
}
}
func TestExtractContentModerationKeywordText_LongReminder(t *testing.T) {
// Reminder context must not consume the semantic API's text limit and hide
// the actual user keyword from the local check.
body := reminderTestBody(t, ContentModerationProtocolAnthropicMessages, []string{
"<system-reminder>" + strings.Repeat("x", maxModerationInputRunes) + "</system-reminder>", "今晚打老虎",
})
require.Contains(t, extractContentModerationKeywordText(ContentModerationProtocolAnthropicMessages, body), "今晚打老虎")
require.Equal(t, "今晚打老虎", ExtractContentModerationText(ContentModerationProtocolAnthropicMessages, body))
}