feat(channel-monitor-v2): 添加被动监控聚合表与模式相关迁移

引入 V2 事实表/固定 rollup、模式设置、忽略错误类、健康阈值、权限与
默认忽略/缓存阈值,以及用户端隐藏吞吐速率的系统设置键。
This commit is contained in:
IanShaw027
2026-08-07 11:05:32 +08:00
parent 93367b6db4
commit a58048ac32
11 changed files with 466 additions and 0 deletions
@@ -0,0 +1,119 @@
-- Passive channel monitor V2. This migration creates empty rollup tables only;
-- backfill is intentionally handled by a bounded background worker.
CREATE TABLE IF NOT EXISTS channel_monitor_v2_config (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
version INTEGER NOT NULL DEFAULT 1,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
refresh_interval_seconds INTEGER NOT NULL DEFAULT 60
CHECK (refresh_interval_seconds IN (60, 300)),
platforms JSONB NOT NULL DEFAULT '[{"platform":"anthropic","enabled":true,"models":[]},{"platform":"openai","enabled":true,"models":[]},{"platform":"grok","enabled":true,"models":[]},{"platform":"kiro","enabled":true,"models":[]},{"platform":"gemini","enabled":true,"models":[]},{"platform":"antigravity","enabled":true,"models":[]}]'::jsonb,
group_ids BIGINT[] NOT NULL DEFAULT '{}',
updated_by BIGINT,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
INSERT INTO channel_monitor_v2_config (id) VALUES (1)
ON CONFLICT (id) DO NOTHING;
CREATE TABLE IF NOT EXISTS channel_monitor_v2_metrics_1m (
bucket_start TIMESTAMPTZ NOT NULL,
platform TEXT NOT NULL,
group_id BIGINT NOT NULL DEFAULT 0,
model TEXT NOT NULL,
success_requests BIGINT NOT NULL DEFAULT 0,
error_requests BIGINT NOT NULL DEFAULT 0,
upstream_affected_requests BIGINT NOT NULL DEFAULT 0,
upstream_attempt_count BIGINT NOT NULL DEFAULT 0,
input_tokens BIGINT NOT NULL DEFAULT 0,
output_tokens BIGINT NOT NULL DEFAULT 0,
cache_creation_tokens BIGINT NOT NULL DEFAULT 0,
cache_read_tokens BIGINT NOT NULL DEFAULT 0,
ttft_sum_ms BIGINT NOT NULL DEFAULT 0,
ttft_count BIGINT NOT NULL DEFAULT 0,
duration_sum_ms BIGINT NOT NULL DEFAULT 0,
duration_count BIGINT NOT NULL DEFAULT 0,
computed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (bucket_start, platform, group_id, model)
);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_metrics_platform_time
ON channel_monitor_v2_metrics_1m (platform, bucket_start DESC);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_metrics_group_time
ON channel_monitor_v2_metrics_1m (group_id, bucket_start DESC);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_metrics_model_time
ON channel_monitor_v2_metrics_1m (model, bucket_start DESC);
CREATE TABLE IF NOT EXISTS channel_monitor_v2_user_metrics_1m (
bucket_start TIMESTAMPTZ NOT NULL,
platform TEXT NOT NULL,
group_id BIGINT NOT NULL DEFAULT 0,
model TEXT NOT NULL,
user_id BIGINT NOT NULL,
success_requests BIGINT NOT NULL DEFAULT 0,
error_requests BIGINT NOT NULL DEFAULT 0,
input_tokens BIGINT NOT NULL DEFAULT 0,
output_tokens BIGINT NOT NULL DEFAULT 0,
cache_creation_tokens BIGINT NOT NULL DEFAULT 0,
cache_read_tokens BIGINT NOT NULL DEFAULT 0,
ttft_sum_ms BIGINT NOT NULL DEFAULT 0,
ttft_count BIGINT NOT NULL DEFAULT 0,
duration_sum_ms BIGINT NOT NULL DEFAULT 0,
duration_count BIGINT NOT NULL DEFAULT 0,
computed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (bucket_start, platform, group_id, model, user_id)
);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_user_metrics_user_time
ON channel_monitor_v2_user_metrics_1m (user_id, bucket_start DESC);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_user_metrics_time
ON channel_monitor_v2_user_metrics_1m (bucket_start DESC);
CREATE TABLE IF NOT EXISTS channel_monitor_v2_error_metrics_1m (
bucket_start TIMESTAMPTZ NOT NULL,
platform TEXT NOT NULL,
group_id BIGINT NOT NULL DEFAULT 0,
model TEXT NOT NULL,
error_category TEXT NOT NULL,
taxonomy_version SMALLINT NOT NULL,
error_requests BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (bucket_start, platform, group_id, model, error_category, taxonomy_version)
);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_errors_time
ON channel_monitor_v2_error_metrics_1m (bucket_start DESC);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_errors_category_time
ON channel_monitor_v2_error_metrics_1m (error_category, bucket_start DESC);
CREATE TABLE IF NOT EXISTS channel_monitor_v2_latency_histograms_1m (
bucket_start TIMESTAMPTZ NOT NULL,
platform TEXT NOT NULL,
group_id BIGINT NOT NULL DEFAULT 0,
model TEXT NOT NULL,
user_id BIGINT NOT NULL DEFAULT 0,
metric TEXT NOT NULL CHECK (metric IN ('ttft', 'duration')),
upper_bound_ms INTEGER NOT NULL,
sample_count BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (bucket_start, platform, group_id, model, user_id, metric, upper_bound_ms)
);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_histograms_time
ON channel_monitor_v2_latency_histograms_1m (bucket_start DESC, metric);
CREATE TABLE IF NOT EXISTS channel_monitor_v2_watermarks (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
usage_coverage_start TIMESTAMPTZ,
error_coverage_start TIMESTAMPTZ,
data_through TIMESTAMPTZ,
last_successful_at TIMESTAMPTZ,
backfill_cursor TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
INSERT INTO channel_monitor_v2_watermarks (id) VALUES (1)
ON CONFLICT (id) DO NOTHING;
COMMENT ON TABLE channel_monitor_v2_metrics_1m IS
'One-minute passive channel health facts derived from real user requests; never from active probes.';
COMMENT ON COLUMN channel_monitor_v2_error_metrics_1m.taxonomy_version IS
'Version of the ordered error classification rules used for this aggregate.';
@@ -0,0 +1,5 @@
-- Channel monitor exclusive mode: v1 (active probes) or v2 (passive aggregation).
-- Default v2 keeps existing V2-first deployments unchanged.
INSERT INTO settings (key, value)
VALUES ('channel_monitor_mode', 'v2')
ON CONFLICT (key) DO NOTHING;
@@ -0,0 +1,3 @@
-- Categories excluded from error_rate / health scoring (still shown in error breakdown).
ALTER TABLE channel_monitor_v2_config
ADD COLUMN IF NOT EXISTS ignored_error_categories TEXT[] NOT NULL DEFAULT '{}';
@@ -0,0 +1,139 @@
-- Seed factory channel_monitor_v2_config.platforms with a popular-model allow-list
-- (high-traffic names from production usage). Presentation semantics:
-- - listed models → own dimension
-- - unlisted models → __other__
-- - empty models list (operator cleared) → show every real model name (no collapse)
-- Aggregation always stores the real model name; this only affects presentation.
--
-- Only rewrite rows whose platforms still look like the empty-models factory
-- default (every platform has models: []). Custom operator configs are left alone.
UPDATE channel_monitor_v2_config
SET
platforms = $platforms$
[
{
"platform": "anthropic",
"enabled": true,
"models": [
"claude-opus-5",
"claude-opus-4-8",
"claude-opus-4-7",
"claude-opus-4-6",
"claude-opus-4.6",
"claude-opus-4-5",
"claude-opus-4-1",
"claude-opus-4",
"claude-fable-5",
"claude-sonnet-5",
"claude-sonnet-4-6",
"claude-sonnet-4.6",
"claude-sonnet-4-5",
"claude-sonnet-4.5",
"claude-sonnet-4",
"claude-haiku-4-5",
"claude-haiku-4.5",
"claude-3-7-sonnet",
"claude-3-5-sonnet",
"claude-3-5-haiku"
]
},
{
"platform": "openai",
"enabled": true,
"models": [
"gpt-5.6-sol",
"gpt-5.6-terra",
"gpt-5.6-luna",
"gpt-5.6",
"gpt-5.5",
"gpt-5.4",
"gpt-5.4-mini",
"gpt-5.3-codex-spark",
"gpt-5.2",
"gpt-5.2-pro",
"gpt-5",
"gpt-4.1",
"gpt-4.1-mini",
"gpt-4.1-nano",
"gpt-4o",
"gpt-4o-mini",
"gpt-4-turbo",
"gpt-4",
"o3",
"o3-mini",
"o4-mini",
"codex-auto-review",
"gpt-image-2",
"gpt-image-1"
]
},
{
"platform": "grok",
"enabled": true,
"models": [
"grok-4.5",
"grok-4.5-latest",
"grok-4.3",
"grok-4.20-multi-agent",
"grok-4.20-0309-reasoning",
"grok-4.20-0309-non-reasoning",
"grok-build-0.1",
"grok-build",
"grok-3-mini",
"grok-3-mini-fast",
"grok-code-fast-1",
"grok-code-fast"
]
},
{
"platform": "gemini",
"enabled": true,
"models": [
"gemini-2.5-pro",
"gemini-2.5-flash",
"gemini-2.5-flash-lite",
"gemini-2.5-flash-image",
"gemini-2.0-flash",
"gemini-2.0-flash-lite",
"gemini-3-flash",
"gemini-3-pro-image",
"gemini-3.1-pro-high",
"gemini-3.1-pro-low",
"gemini-3.1-flash-lite",
"gemini-3.5-flash",
"gemini-3.5-flash-lite"
]
},
{
"platform": "kiro",
"enabled": true,
"models": [
"claude-opus-4-8",
"claude-opus-4-6",
"claude-opus-4-5",
"claude-sonnet-4-5",
"claude-sonnet-4-6",
"claude-haiku-4-5"
]
},
{
"platform": "antigravity",
"enabled": true,
"models": [
"claude-opus-4-6",
"claude-sonnet-4-5",
"claude-sonnet-4-6",
"gemini-2.5-pro",
"gemini-2.5-flash"
]
}
]
$platforms$::jsonb,
version = version + 1,
updated_at = NOW()
WHERE id = 1
-- Only seed the exact factory default from migration 194. A row with custom
-- enabled=false, platform set/order, or any other operator tweak must be left
-- untouched even if every models list is still empty.
AND platforms = '[{"platform":"anthropic","enabled":true,"models":[]},{"platform":"openai","enabled":true,"models":[]},{"platform":"grok","enabled":true,"models":[]},{"platform":"kiro","enabled":true,"models":[]},{"platform":"gemini","enabled":true,"models":[]},{"platform":"antigravity","enabled":true,"models":[]}]'::jsonb;
@@ -0,0 +1,33 @@
-- Configurable Channel Monitor V2 health scoring thresholds.
-- Defaults are intentionally tolerant: small amounts of real traffic errors or
-- zero cache rate should not immediately paint the user page as "abnormal".
ALTER TABLE channel_monitor_v2_config
ADD COLUMN IF NOT EXISTS health_thresholds JSONB NOT NULL DEFAULT '{
"minimum_sample": 50,
"warning_error_rate": 0.05,
"critical_error_rate": 0.20,
"target_ttft_ms": 3000,
"warning_ttft_ms": 8000,
"critical_ttft_ms": 20000,
"warning_cache_rate": 0,
"critical_cache_rate": 0,
"error_weight": 0.60,
"ttft_weight": 0.20,
"cache_weight": 0.20
}'::jsonb;
UPDATE channel_monitor_v2_config
SET health_thresholds = '{
"minimum_sample": 50,
"warning_error_rate": 0.05,
"critical_error_rate": 0.20,
"target_ttft_ms": 3000,
"warning_ttft_ms": 8000,
"critical_ttft_ms": 20000,
"warning_cache_rate": 0,
"critical_cache_rate": 0,
"error_weight": 0.60,
"ttft_weight": 0.20,
"cache_weight": 0.20
}'::jsonb || health_thresholds
WHERE id = 1;
@@ -0,0 +1,95 @@
-- Fixed-size Channel Monitor V2 rollups for the 24h / 7d / 30d views.
-- Source of truth remains the 1-minute passive rollup; these tables cache the
-- UI bucket sizes so historical buckets do not need to be re-aggregated on each
-- filter change. The background aggregator overwrites affected buckets.
CREATE TABLE IF NOT EXISTS channel_monitor_v2_metrics_rollup (
bucket_start TIMESTAMPTZ NOT NULL,
bucket_seconds INTEGER NOT NULL CHECK (bucket_seconds IN (300, 3600, 43200, 86400)),
platform TEXT NOT NULL,
group_id BIGINT NOT NULL DEFAULT 0,
model TEXT NOT NULL,
success_requests BIGINT NOT NULL DEFAULT 0,
error_requests BIGINT NOT NULL DEFAULT 0,
upstream_affected_requests BIGINT NOT NULL DEFAULT 0,
upstream_attempt_count BIGINT NOT NULL DEFAULT 0,
input_tokens BIGINT NOT NULL DEFAULT 0,
output_tokens BIGINT NOT NULL DEFAULT 0,
cache_creation_tokens BIGINT NOT NULL DEFAULT 0,
cache_read_tokens BIGINT NOT NULL DEFAULT 0,
ttft_sum_ms BIGINT NOT NULL DEFAULT 0,
ttft_count BIGINT NOT NULL DEFAULT 0,
duration_sum_ms BIGINT NOT NULL DEFAULT 0,
duration_count BIGINT NOT NULL DEFAULT 0,
computed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (bucket_seconds, bucket_start, platform, group_id, model)
);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_metrics_rollup_platform_time
ON channel_monitor_v2_metrics_rollup (bucket_seconds, platform, bucket_start DESC);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_metrics_rollup_group_time
ON channel_monitor_v2_metrics_rollup (bucket_seconds, group_id, bucket_start DESC);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_metrics_rollup_model_time
ON channel_monitor_v2_metrics_rollup (bucket_seconds, model, bucket_start DESC);
CREATE TABLE IF NOT EXISTS channel_monitor_v2_user_metrics_rollup (
bucket_start TIMESTAMPTZ NOT NULL,
bucket_seconds INTEGER NOT NULL CHECK (bucket_seconds IN (300, 3600, 43200, 86400)),
platform TEXT NOT NULL,
group_id BIGINT NOT NULL DEFAULT 0,
model TEXT NOT NULL,
user_id BIGINT NOT NULL,
success_requests BIGINT NOT NULL DEFAULT 0,
error_requests BIGINT NOT NULL DEFAULT 0,
input_tokens BIGINT NOT NULL DEFAULT 0,
output_tokens BIGINT NOT NULL DEFAULT 0,
cache_creation_tokens BIGINT NOT NULL DEFAULT 0,
cache_read_tokens BIGINT NOT NULL DEFAULT 0,
ttft_sum_ms BIGINT NOT NULL DEFAULT 0,
ttft_count BIGINT NOT NULL DEFAULT 0,
duration_sum_ms BIGINT NOT NULL DEFAULT 0,
duration_count BIGINT NOT NULL DEFAULT 0,
computed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (bucket_seconds, bucket_start, platform, group_id, model, user_id)
);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_user_rollup_user_time
ON channel_monitor_v2_user_metrics_rollup (bucket_seconds, user_id, bucket_start DESC);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_user_rollup_time
ON channel_monitor_v2_user_metrics_rollup (bucket_seconds, bucket_start DESC);
CREATE TABLE IF NOT EXISTS channel_monitor_v2_error_metrics_rollup (
bucket_start TIMESTAMPTZ NOT NULL,
bucket_seconds INTEGER NOT NULL CHECK (bucket_seconds IN (300, 3600, 43200, 86400)),
platform TEXT NOT NULL,
group_id BIGINT NOT NULL DEFAULT 0,
model TEXT NOT NULL,
error_category TEXT NOT NULL,
taxonomy_version SMALLINT NOT NULL,
error_requests BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (bucket_seconds, bucket_start, platform, group_id, model, error_category, taxonomy_version)
);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_errors_rollup_time
ON channel_monitor_v2_error_metrics_rollup (bucket_seconds, bucket_start DESC);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_errors_rollup_category_time
ON channel_monitor_v2_error_metrics_rollup (bucket_seconds, error_category, bucket_start DESC);
CREATE TABLE IF NOT EXISTS channel_monitor_v2_latency_histograms_rollup (
bucket_start TIMESTAMPTZ NOT NULL,
bucket_seconds INTEGER NOT NULL CHECK (bucket_seconds IN (300, 3600, 43200, 86400)),
platform TEXT NOT NULL,
group_id BIGINT NOT NULL DEFAULT 0,
model TEXT NOT NULL,
user_id BIGINT NOT NULL DEFAULT 0,
metric TEXT NOT NULL CHECK (metric IN ('ttft', 'duration')),
upper_bound_ms INTEGER NOT NULL,
sample_count BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (bucket_seconds, bucket_start, platform, group_id, model, user_id, metric, upper_bound_ms)
);
CREATE INDEX IF NOT EXISTS idx_channel_monitor_v2_histograms_rollup_time
ON channel_monitor_v2_latency_histograms_rollup (bucket_seconds, bucket_start DESC, metric);
COMMENT ON TABLE channel_monitor_v2_metrics_rollup IS
'Precomputed fixed UI buckets derived from channel_monitor_v2_metrics_1m for 24h/7d/30d views.';
@@ -0,0 +1,9 @@
-- Runtime aggregator reads/writes the fixed rollup caches through the role that
-- executes migrations. Use CURRENT_USER so custom DATABASE_USER values work;
-- PostgreSQL permits this form in GRANT and it remains idempotent.
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE
channel_monitor_v2_metrics_rollup,
channel_monitor_v2_user_metrics_rollup,
channel_monitor_v2_error_metrics_rollup,
channel_monitor_v2_latency_histograms_rollup
TO CURRENT_USER;
@@ -0,0 +1,12 @@
-- Channel Monitor V2 is designed around fixed UI buckets; refresh passive
-- aggregation every 5 minutes and recompute the trailing bucket for each range.
-- Only migrate the untouched factory row. A row with a newer version or an
-- updater identity reflects an operator choice and must retain its 60-second
-- interval (or any other valid setting).
UPDATE channel_monitor_v2_config
SET refresh_interval_seconds = 300,
updated_at = NOW()
WHERE id = 1
AND version = 1
AND updated_by IS NULL
AND refresh_interval_seconds = 60;
@@ -0,0 +1,15 @@
-- Extend privileges for all Channel Monitor V2 tables (base + rollup + config).
-- Grant to the migration/application role rather than the historical hard-coded
-- "sub2api" role so custom DATABASE_USER deployments work as expected.
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE
channel_monitor_v2_config,
channel_monitor_v2_watermarks,
channel_monitor_v2_metrics_1m,
channel_monitor_v2_user_metrics_1m,
channel_monitor_v2_error_metrics_1m,
channel_monitor_v2_latency_histograms_1m,
channel_monitor_v2_metrics_rollup,
channel_monitor_v2_user_metrics_rollup,
channel_monitor_v2_error_metrics_rollup,
channel_monitor_v2_latency_histograms_rollup
TO CURRENT_USER;
@@ -0,0 +1,31 @@
-- Factory presets for Channel Monitor V2 config:
-- 1) ignored_error_categories: non-ops client/policy failures that should not
-- dominate health error_rate by default (still shown greyed in breakdown).
-- 2) health_thresholds cache floors: 85% watch / 60% critical (higher is better).
--
-- Only apply when the row still looks like factory empty ignore list and/or
-- zero cache thresholds (operator customizations are left alone).
UPDATE channel_monitor_v2_config
SET ignored_error_categories = ARRAY[
'authentication',
'client_cancelled',
'content_policy',
'context_limit',
'group_access',
'model_unsupported',
'not_found',
'quota_or_balance'
]::text[]
WHERE id = 1
AND COALESCE(cardinality(ignored_error_categories), 0) = 0;
UPDATE channel_monitor_v2_config
SET health_thresholds = health_thresholds
|| jsonb_build_object(
'warning_cache_rate', 0.85,
'critical_cache_rate', 0.60
)
WHERE id = 1
AND COALESCE((health_thresholds->>'warning_cache_rate')::float8, 0) = 0
AND COALESCE((health_thresholds->>'critical_cache_rate')::float8, 0) = 0;
@@ -0,0 +1,5 @@
-- Soft switch: hide RPM/TPM throughput rates on user-facing Channel Monitor V2.
-- Default false (rates visible). Admins always see full metrics.
INSERT INTO settings (key, value)
VALUES ('channel_monitor_hide_throughput', 'false')
ON CONFLICT (key) DO NOTHING;