diff --git a/backend/internal/repository/http_upstream.go b/backend/internal/repository/http_upstream.go index bdd6478cff..001e597e0f 100644 --- a/backend/internal/repository/http_upstream.go +++ b/backend/internal/repository/http_upstream.go @@ -213,7 +213,7 @@ func (s *httpUpstreamService) Do(req *http.Request, proxyURL string, accountID i } // 执行请求 - client := httpClientForUpstreamRequest(entry.client, req) + client := s.httpClientForUpstreamRequest(entry.client, req) client = httpClientWithGrokAccessDeniedFallback(client) resp, err := servertiming.Do(client, req) if err != nil { @@ -277,7 +277,7 @@ func (s *httpUpstreamService) DoWithTLS(req *http.Request, proxyURL string, acco return nil, err } - client := httpClientForUpstreamRequest(entry.client, req) + client := s.httpClientForUpstreamRequest(entry.client, req) client = httpClientWithGrokAccessDeniedFallback(client) resp, err := servertiming.Do(client, req) if err != nil { @@ -297,15 +297,27 @@ func (s *httpUpstreamService) DoWithTLS(req *http.Request, proxyURL string, acco return resp, nil } -func httpClientForUpstreamRequest(client *http.Client, req *http.Request) *http.Client { - if client == nil || req == nil || !service.HTTPUpstreamRedirectsDisabled(req.Context()) { +// httpClientForUpstreamRequest 按请求上下文的标记派生客户端：禁用重定向，或对重定向的每一跳做主机校验。 +// 派生的克隆与缓存客户端共享 Transport；未打标记时原样返回。 +func (s *httpUpstreamService) httpClientForUpstreamRequest(client *http.Client, req *http.Request) *http.Client { + if client == nil || req == nil { return client } - clone := *client - clone.CheckRedirect = func(*http.Request, []*http.Request) error { - return http.ErrUseLastResponse + ctx := req.Context() + switch { + case service.HTTPUpstreamRedirectsDisabled(ctx): + clone := *client + clone.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + return &clone + case service.HTTPUpstreamPublicHostsOnly(ctx) && client.CheckRedirect == nil: + clone := *client + clone.CheckRedirect = s.redirectChecker + return &clone + default: + return client } - return &clone } // grokAccessDeniedFallbackTransport preserves the subscription CLI proxy as @@ -587,8 +599,11 @@ func (s *httpUpstreamService) shouldValidateResolvedIP() bool { return !s.cfg.Security.URLAllowlist.AllowPrivateHosts } +// validateRequestHost 校验请求主机的解析结果不落在回环、私网、链路本地或未指定地址。 +// 是否全局启用由 security.url_allowlist 决定；带 WithHTTPUpstreamPublicHostsOnly 标记的请求无论配置如何都校验。 func (s *httpUpstreamService) validateRequestHost(req *http.Request) error { - if !s.shouldValidateResolvedIP() { + publicHostsOnly := req != nil && service.HTTPUpstreamPublicHostsOnly(req.Context()) + if !s.shouldValidateResolvedIP() && !publicHostsOnly { return nil } if req == nil || req.URL == nil { diff --git a/backend/internal/repository/http_upstream_test.go b/backend/internal/repository/http_upstream_test.go index 3c7840dc47..8750771d04 100644 --- a/backend/internal/repository/http_upstream_test.go +++ b/backend/internal/repository/http_upstream_test.go @@ -961,3 +961,63 @@ func hasEntry(svc *httpUpstreamService, target *upstreamClientEntry) bool { } return false } + +func TestHTTPUpstreamDoPublicHostsOnlyRejectsPrivateDestinationBeforeConnecting(t *testing.T) { + var calls atomic.Int64 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(target.Close) + + upstream := NewHTTPUpstream(nil) + + plain, err := http.NewRequestWithContext(t.Context(), http.MethodGet, target.URL, nil) + require.NoError(t, err) + resp, err := upstream.Do(plain, "", 1, 1) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, int64(1), calls.Load(), "loopback stays reachable for requests without the marker") + + guarded, err := http.NewRequestWithContext(service.WithHTTPUpstreamPublicHostsOnly(t.Context()), http.MethodGet, target.URL, nil) + require.NoError(t, err) + resp, err = upstream.Do(guarded, "", 1, 1) + require.Error(t, err) + require.Nil(t, resp) + require.Contains(t, err.Error(), "not allowed") + require.Equal(t, int64(1), calls.Load(), "marked request must be rejected before any connection is made") +} + +func TestHTTPUpstreamPublicHostsOnlyValidatesEveryRedirectHop(t *testing.T) { + upstream, ok := NewHTTPUpstream(nil).(*httpUpstreamService) + require.True(t, ok) + base := &http.Client{} + + plain, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "https://cdn.example.com/a.png", nil) + require.NoError(t, err) + require.Same(t, base, upstream.httpClientForUpstreamRequest(base, plain)) + + guarded, err := http.NewRequestWithContext(service.WithHTTPUpstreamPublicHostsOnly(t.Context()), http.MethodGet, "https://cdn.example.com/a.png", nil) + require.NoError(t, err) + client := upstream.httpClientForUpstreamRequest(base, guarded) + require.NotSame(t, base, client) + require.NotNil(t, client.CheckRedirect) + require.Nil(t, base.CheckRedirect, "the cached client must stay untouched") + + via := []*http.Request{guarded} + for _, hop := range []string{ + "http://127.0.0.1:8080/a.png", + "http://[::1]:8080/a.png", + "http://10.0.0.8/a.png", + "http://169.254.169.254/latest/meta-data/", + "http://0.0.0.0/a.png", + } { + hopReq, err := http.NewRequestWithContext(guarded.Context(), http.MethodGet, hop, nil) + require.NoError(t, err) + require.Error(t, client.CheckRedirect(hopReq, via), "hop=%s", hop) + } + publicHop, err := http.NewRequestWithContext(guarded.Context(), http.MethodGet, "http://93.184.216.34/a.png", nil) + require.NoError(t, err) + require.NoError(t, client.CheckRedirect(publicHop, via)) + require.Error(t, client.CheckRedirect(publicHop, make([]*http.Request, 10)), "redirect chain stays capped") +} diff --git a/backend/internal/service/http_upstream_profile.go b/backend/internal/service/http_upstream_profile.go index b2de6002cf..327813de19 100644 --- a/backend/internal/service/http_upstream_profile.go +++ b/backend/internal/service/http_upstream_profile.go @@ -14,6 +14,7 @@ const ( type httpUpstreamProfileContextKey struct{} type httpUpstreamDisableRedirectsContextKey struct{} +type httpUpstreamPublicHostsOnlyContextKey struct{} // WithHTTPUpstreamProfile injects an upstream transport profile into ctx. func WithHTTPUpstreamProfile(ctx context.Context, profile HTTPUpstreamProfile) context.Context { @@ -55,3 +56,18 @@ func WithHTTPUpstreamRedirectsDisabled(ctx context.Context) context.Context { func HTTPUpstreamRedirectsDisabled(ctx context.Context) bool { return ctx != nil && ctx.Value(httpUpstreamDisableRedirectsContextKey{}) == true } + +// WithHTTPUpstreamPublicHostsOnly marks a request whose destination, and every +// redirect hop after it, must resolve to a public address. The shared upstream +// client enforces it regardless of the security.url_allowlist configuration; +// use it for fetches whose URL comes from an untrusted upstream response. +func WithHTTPUpstreamPublicHostsOnly(ctx context.Context) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, httpUpstreamPublicHostsOnlyContextKey{}, true) +} + +func HTTPUpstreamPublicHostsOnly(ctx context.Context) bool { + return ctx != nil && ctx.Value(httpUpstreamPublicHostsOnlyContextKey{}) == true +} diff --git a/backend/internal/service/http_upstream_profile_test.go b/backend/internal/service/http_upstream_profile_test.go index 9cd4bf4ff9..ea95b92611 100644 --- a/backend/internal/service/http_upstream_profile_test.go +++ b/backend/internal/service/http_upstream_profile_test.go @@ -30,3 +30,17 @@ func TestWithHTTPUpstreamRedirectsDisabled(t *testing.T) { t.Fatal("redirects should remain enabled by default") } } + +func TestWithHTTPUpstreamPublicHostsOnly(t *testing.T) { + //nolint:staticcheck // Exercises the defensive nil-context fallback. + ctx := WithHTTPUpstreamPublicHostsOnly(nil) + if !HTTPUpstreamPublicHostsOnly(ctx) { + t.Fatal("expected public-hosts-only marker to be set") + } + if HTTPUpstreamPublicHostsOnly(context.Background()) { + t.Fatal("marker must be absent by default") + } + if HTTPUpstreamRedirectsDisabled(ctx) { + t.Fatal("public-hosts-only must not disable redirects") + } +} diff --git a/backend/internal/service/openai_images_b64_backfill.go b/backend/internal/service/openai_images_b64_backfill.go index 30fb9c20c6..3210c0fcbe 100644 --- a/backend/internal/service/openai_images_b64_backfill.go +++ b/backend/internal/service/openai_images_b64_backfill.go @@ -7,10 +7,12 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" "time" "github.com/Wei-Shaw/sub2api/internal/pkg/logger" + "github.com/Wei-Shaw/sub2api/internal/util/urlvalidator" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) @@ -90,8 +92,9 @@ func (s *OpenAIGatewayService) backfillOpenAIImagesB64JSON( } // fetchOpenAIImageURLBase64 取得图片 url 内容的标准 base64 编码。 -// data: 形式的 url 直接取其 base64 载荷；其余 url 沿用 base_url 的出站 URL 策略校验后， -// 经账户代理下载，大小上限与 OAuth 路径的单图下载一致，且必须是图片内容。 +// data: 形式的 url 直接取其 base64 载荷；其余 url 先沿用 base_url 的出站 URL 策略校验， +// 再无条件拒绝回环、私网、链路本地等目的地（含重定向的每一跳），经账户代理下载， +// 大小上限与 OAuth 路径的单图下载一致，且前 512 字节须嗅探为 png/jpeg/webp/gif。 func (s *OpenAIGatewayService) fetchOpenAIImageURLBase64(ctx context.Context, account *Account, rawURL string) (string, error) { if strings.HasPrefix(strings.ToLower(rawURL), "data:") { if encoded := normalizeOpenAIImageBase64(rawURL); encoded != "" { @@ -106,7 +109,10 @@ func (s *OpenAIGatewayService) fetchOpenAIImageURLBase64(ctx context.Context, ac if err != nil { return "", fmt.Errorf("invalid image url: %w", err) } - ctx, cancel := context.WithTimeout(ctx, openAIImageURLDownloadTimeout) + if err := rejectPrivateImageHost(downloadURL); err != nil { + return "", err + } + ctx, cancel := context.WithTimeout(WithHTTPUpstreamPublicHostsOnly(ctx), openAIImageURLDownloadTimeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) if err != nil { @@ -135,17 +141,36 @@ func (s *OpenAIGatewayService) fetchOpenAIImageURLBase64(ctx context.Context, ac if len(data) == 0 { return "", errors.New("download image: empty body") } - if !isImageContent(resp.Header.Get("Content-Type"), data) { - return "", errors.New("download image: content is not an image") + if !isBackfillImageContent(data) { + return "", errors.New("download image: content is not an allowed image format") } return base64.StdEncoding.EncodeToString(data), nil } -// isImageContent 由响应头声明或字节嗅探判定内容是否为图片。 -func isImageContent(contentType string, data []byte) bool { - declared := strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0])) - if strings.HasPrefix(declared, "image/") { - return true +// rejectPrivateImageHost 拒绝主机为 localhost 或回环、私网、链路本地、未指定地址字面量的下载 URL。 +// 不受 security.url_allowlist 配置影响；域名的解析结果与重定向的每一跳由上游客户端按 +// WithHTTPUpstreamPublicHostsOnly 标记校验。 +func rejectPrivateImageHost(downloadURL string) error { + parsed, err := url.Parse(downloadURL) + if err != nil { + return fmt.Errorf("invalid image url: %w", err) } - return detectedImageContentType(data) != "" + if host := parsed.Hostname(); urlvalidator.IsBlockedHost(host) { + return fmt.Errorf("image url host is not allowed: %s", host) + } + return nil +} + +// openAIImageBackfillContentTypes 是允许回填的图片格式，以字节嗅探结果为准，响应头不作为依据。 +var openAIImageBackfillContentTypes = map[string]struct{}{ + "image/png": {}, + "image/jpeg": {}, + "image/webp": {}, + "image/gif": {}, +} + +// isBackfillImageContent 报告 data 的前 512 字节是否嗅探为允许回填的图片格式。 +func isBackfillImageContent(data []byte) bool { + _, ok := openAIImageBackfillContentTypes[detectedImageContentType(data)] + return ok } diff --git a/backend/internal/service/openai_images_b64_backfill_test.go b/backend/internal/service/openai_images_b64_backfill_test.go index 1bbfc0b7d7..bbaa3f088b 100644 --- a/backend/internal/service/openai_images_b64_backfill_test.go +++ b/backend/internal/service/openai_images_b64_backfill_test.go @@ -149,6 +149,14 @@ func TestBackfillOpenAIImagesB64JSON(t *testing.T) { wantB64: []string{wantB64}, wantDownloads: 1, }, + { + name: "响应头声明图片但字节不是图片时保留原样", + enabled: true, + body: `{"created":1,"data":[{"url":"https://cdn.example.com/a.png"}]}`, + upstream: &httpUpstreamRecorder{resp: b64BackfillImageResponse(http.StatusOK, "image/svg+xml", []byte(``))}, + wantB64: []string{""}, + wantDownloads: 1, + }, { name: "非 http(s) 协议的 url 不下载", enabled: true, @@ -206,6 +214,67 @@ func TestBackfillOpenAIImagesB64JSON_DownloadRequestShape(t *testing.T) { require.Equal(t, "http://127.0.0.1:7890", upstream.lastProxyURL) _, hasDeadline := req.Context().Deadline() require.True(t, hasDeadline) + // 目的地与重定向各跳都必须解析到公网地址；重定向本身保持跟随。 + require.True(t, HTTPUpstreamPublicHostsOnly(req.Context())) + require.False(t, HTTPUpstreamRedirectsDisabled(req.Context())) +} + +func TestBackfillOpenAIImagesB64JSON_RejectsPrivateHosts(t *testing.T) { + upstream := &httpUpstreamRecorder{resp: b64BackfillImageResponse(http.StatusOK, "image/png", b64BackfillPNGBytes)} + cfg := &config.Config{} + cfg.Security.URLAllowlist.AllowInsecureHTTP = true + svc := &OpenAIGatewayService{cfg: cfg, httpUpstream: upstream} + account := b64BackfillAccount(true) + + for _, rawURL := range []string{ + "http://localhost:8080/api/v1/admin/accounts", + "http://Foo.LocalHost/a.png", + "http://127.0.0.1:8080/a.png", + "https://127.0.0.1/a.png", + "http://[::1]:8080/a.png", + "http://10.0.0.5/a.png", + "http://172.16.0.9:9000/bucket/a.png", + "http://192.168.1.10:9000/bucket/a.png", + "http://169.254.169.254/latest/meta-data/", + "http://0.0.0.0:8080/a.png", + "http://[fe80::1]/a.png", + "http://[::ffff:127.0.0.1]/a.png", + } { + body := `{"created":1,"data":[{"url":"` + rawURL + `"}]}` + got := svc.backfillOpenAIImagesB64JSON(context.Background(), account, nil, []byte(body)) + require.Equal(t, body, string(got), "url=%s", rawURL) + } + require.Empty(t, upstream.requests, "private destinations must never be requested") + + // 同一配置下公网主机照常下载，证明拒绝依据是目的地而非协议。 + got := svc.backfillOpenAIImagesB64JSON(context.Background(), account, nil, []byte(`{"created":1,"data":[{"url":"http://cdn.example.com/a.png"}]}`)) + require.Equal(t, base64.StdEncoding.EncodeToString(b64BackfillPNGBytes), gjson.GetBytes(got, "data.0.b64_json").String()) + require.Len(t, upstream.requests, 1) +} + +func TestIsBackfillImageContent(t *testing.T) { + webp := append([]byte("RIFF\x24\x00\x00\x00WEBPVP8 "), make([]byte, 8)...) + tests := []struct { + name string + data []byte + want bool + }{ + {name: "png", data: b64BackfillPNGBytes, want: true}, + {name: "jpeg", data: []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 'J', 'F', 'I', 'F', 0x00}, want: true}, + {name: "webp", data: webp, want: true}, + {name: "gif", data: []byte("GIF89a\x01\x00\x01\x00"), want: true}, + {name: "bmp 不在允许列表", data: []byte("BM\x00\x00\x00\x00\x00\x00\x00\x00"), want: false}, + {name: "ico 不在允许列表", data: []byte{0x00, 0x00, 0x01, 0x00, 0x01, 0x00}, want: false}, + {name: "svg 文本", data: []byte(``), want: false}, + {name: "html", data: []byte("login required"), want: false}, + {name: "json", data: []byte(`{"error":"unauthorized"}`), want: false}, + {name: "空", data: nil, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, isBackfillImageContent(tt.data)) + }) + } } func TestBackfillOpenAIImagesB64JSON_NonObjectBodies(t *testing.T) { @@ -263,6 +332,7 @@ func TestOpenAIGatewayServiceForwardImages_APIKeyBackfillsB64JSONFromURL(t *test require.Equal(t, "https://relay.example.com/v1/images/generations", upstream.requests[0].URL.String()) require.Equal(t, http.MethodGet, upstream.requests[1].Method) require.Equal(t, "https://cdn.example.com/cat.png", upstream.requests[1].URL.String()) + require.True(t, HTTPUpstreamPublicHostsOnly(upstream.requests[1].Context())) require.Equal(t, http.StatusOK, rec.Code) wantB64 := base64.StdEncoding.EncodeToString(b64BackfillPNGBytes) diff --git a/backend/internal/util/urlvalidator/validator.go b/backend/internal/util/urlvalidator/validator.go index fc2b9bc4d7..882487b73b 100644 --- a/backend/internal/util/urlvalidator/validator.go +++ b/backend/internal/util/urlvalidator/validator.go @@ -162,6 +162,12 @@ func isAllowedHost(host string, allowlist []string) bool { return false } +// IsBlockedHost 报告 host 是否为 localhost、*.localhost，或回环、私网、链路本地、未指定地址的字面量 IP。 +// 只判断字面量，域名的解析结果由 ValidateResolvedIP 校验。 +func IsBlockedHost(host string) bool { + return isBlockedHost(strings.ToLower(strings.TrimSpace(host))) +} + func isBlockedHost(host string) bool { if host == "localhost" || strings.HasSuffix(host, ".localhost") { return true diff --git a/backend/internal/util/urlvalidator/validator_test.go b/backend/internal/util/urlvalidator/validator_test.go index bec9bb2110..5f401df170 100644 --- a/backend/internal/util/urlvalidator/validator_test.go +++ b/backend/internal/util/urlvalidator/validator_test.go @@ -73,3 +73,20 @@ func TestValidateHTTPURL(t *testing.T) { t.Fatalf("expected localhost to be blocked when allow_private_hosts is false") } } + +func TestIsBlockedHost(t *testing.T) { + for _, host := range []string{ + "localhost", "LOCALHOST", "foo.localhost", " 127.0.0.1 ", + "127.0.0.1", "::1", "10.0.0.5", "172.16.0.1", "192.168.1.1", + "169.254.169.254", "0.0.0.0", "::", "fe80::1", "fc00::1", "::ffff:127.0.0.1", + } { + if !IsBlockedHost(host) { + t.Fatalf("expected %q to be blocked", host) + } + } + for _, host := range []string{"example.com", "cdn.example.com", "93.184.216.34", "8.8.8.8", "2606:4700:4700::1111", ""} { + if IsBlockedHost(host) { + t.Fatalf("expected %q to be allowed", host) + } + } +}