prometheus/rules/
├── infra/
│   ├── node.rules.yml          # 节点级基础设施
│   └── kubernetes.rules.yml    # K8s 控制平面/节点
├── platform/
│   ├── jvm.rules.yml           # JVM 通用指标
│   └── middleware.rules.yml    # Redis/Kafka/DB 等
└── business/
    ├── jvb.rules.yml           # Jitsi Videobridge 专用
    ├── janus.rules.yml         # Janus 网关专用
    └── recording.rules.yml     # 录制/转码服务

groups:
- name: jvb-video-conference
  interval: 30s
  rules:
  # 核心业务：并发会议数逼近容量上限
  - alert: JVBConferenceCapacityHigh
    expr: |
      (jitsi_concurrent_conferences / jitsi_conference_capacity_max) * 100 > 80
    for: 5m
    labels:
      severity: warning
      team: video-infra
      runbook: "https://wiki.example.com/runbook/jvb-capacity"
    annotations:
      summary: "JVB 并发会议占比超 80% (当前: {{ $value | printf "%.1f" }}%)"
      description: |
        集群 {{ $labels.cluster }} 命名空间 {{ $labels.namespace }} 
        Pod {{ $labels.pod }} 并发会议数 {{ $value | printf "%.0f" }} 
        接近容量上限，建议扩容或排查异常占用。

  # 关键体验：ICE 连接失败率飙升
  - alert: JVBICEFailureRateHigh
    expr: |
      rate(jitsi_ice_failed_total[5m]) / rate(jitsi_ice_attempted_total[5m]) > 0.1
    for: 3m
    labels:
      severity: critical
      team: video-infra
      runbook: "https://wiki.example.com/runbook/ice-failure"
    annotations:
      summary: "ICE 失败率超 10% (当前: {{ $value | printf "%.2f" }}%)"
      description: |
        可能原因：NAT 穿透受限、TURN 服务器不可达、客户端网络异常。
        请检查 TURN 服务器健康状态及防火墙策略。

  # 资源异常：媒体服务器 CPU 持续高位
  - alert: JVBMediaServerCPUHigh
    expr: |
      avg by (pod) (rate(jvb_cpu_seconds_total[3m])) * 100 > 75
    for: 3m
    labels:
      severity: warning
      team: video-infra
    annotations:
      summary: "JVB 媒体服务器 CPU 占用高 ({{ $value | printf "%.1f" }}%)"
      description: "Pod {{ $labels.pod }} 可能存在转码压力过大或死循环，建议观察 GC 与线程状态。"

  # 业务异常：参会人数突降（疑似大规模掉线）
  - alert: JVBParticipantsSuddenDrop
    expr: |
      (jitsi_participants_total - jitsi_participants_total offset 10m) < -50
    for: 2m
    labels:
      severity: critical
      team: video-infra
    annotations:
      summary: "10 分钟内参会人数骤减 50+ (当前: {{ $value | printf "%.0f" }})"
      description: "疑似大规模掉线或信令服务异常，请立即排查信令链路与网关状态。"

global:
  resolve_timeout: 5m
  smtp_smarthost: 'smtp.exmail.qq.com:465'
  smtp_auth_username: 'alert@company.com'
  smtp_auth_password: '${SMTP_PASSWORD}'
  smtp_require_tls: true

templates:
  - '/etc/alertmanager/template/*.tmpl'

route:
  group_by: ['alertname', 'cluster', 'service', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default'
  routes:
  # 视频会议业务专用路由
  - match:
      service: video-conference
    receiver: 'video-conference-oncall'
    group_by: ['alertname', 'cluster', 'namespace']
    group_wait: 10s
    group_interval: 3m
    repeat_interval: 2h
    continue: true
    # 子路由：核心会议时段提升优先级
    routes:
    - match:
        severity: critical
      receiver: 'video-conference-critical'
      group_wait: 0s
      repeat_interval: 30m

  # 基础设施告警单独路由
  - match:
      team: platform
    receiver: 'platform-oncall'

  # 抑制规则：节点宕机时抑制其上所有 Pod 告警
  - match:
      alertname: NodeDown
    receiver: 'infra-critical'

receivers:
- name: 'default'
  email_configs:
  - to: 'ops-team@company.com'
    send_resolved: true

- name: 'video-conference-oncall'
  webhook_configs:
  - url: 'https://webhook.feishu.cn/xxx/video-conference'
    send_resolved: true
  wechat_configs:
  - corp_id: '${WECHAT_CORP_ID}'
    api_secret: '${WECHAT_SECRET}'
    to_party: 'video-infra'

- name: 'video-conference-critical'
  webhook_configs:
  - url: 'https://webhook.feishu.cn/xxx/video-conference-critical'
    send_resolved: true
  pagerduty_configs:
  - service_key: '${PAGERDUTY_KEY}'
    severity: critical
    details:
      runbook: '{{ .CommonAnnotations.runbook }}'

- name: 'platform-oncall'
  webhook_configs:
  - url: 'https://webhook.feishu.cn/xxx/platform'

inhibit_rules:
- source_match:
    severity: 'critical'
    alertname: 'NodeDown'
  target_match:
    service: 'video-conference'
  equal: ['cluster', 'node']
  # 节点宕机时，抑制该节点上所有视频会议相关告警

- source_match:
    alertname: 'KubeAPIServerDown'
  target_match_re:
    alertname: '.*'
  equal: ['cluster']
  # APIServer 不可用时抑制全集群告警，避免风暴

{{ define "wechat.video_conference.card" }}
{
  "msgtype": "template_card",
  "template_card": {
    "card_type": "text_notice",
    "source": {
      "icon_url": "https://cdn.company.com/icons/prometheus.png",
      "desc": "Prometheus Alertmanager",
      "desc_color": 0
    },
    "main_title": {
      "title": "[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}",
      "desc": "集群: {{ .CommonLabels.cluster }} | 环境: {{ .CommonLabels.env }} | 严重度: {{ .CommonLabels.severity }}"
    },
    "emphasis_content": {
      "title": "{{ .CommonAnnotations.summary }}",
      "desc": "{{ .CommonAnnotations.description }}"
    },
    "sub_content_text": [
      "触发时间: {{ .StartsAt.Format "2006-01-02 15:04:05" }}",
      "当前值: {{ .CommonAnnotations.value }}",
      "持续时长: {{ .StartsAt.Sub .EndsAt | duration }}"
    ],
    "horizontal_content_list": [
      {"keyname": "命名空间", "value": "{{ .CommonLabels.namespace }}"},
      {"keyname": "Pod", "value": "{{ .CommonLabels.pod }}"},
      {"keyname": "服务", "value": "{{ .CommonLabels.service }}"}
    ],
    "jump_list": [
      {
        "type": 1,
        "url": "{{ .CommonAnnotations.runbook }}",
        "title": "📖 查看运维手册"
      },
      {
        "type": 2,
        "url": "https://grafana.company.com/d/{{ .CommonLabels.dashboard_uid }}?var-cluster={{ .CommonLabels.cluster }}&var-namespace={{ .CommonLabels.namespace }}",
        "title": "📊 跳转 Grafana 仪表盘"
      }
    ],
    "card_action": {
      "type": 1,
      "url": "https://alertmanager.company.com/#/alerts?filter=alertname%3D{{ .CommonLabels.alertname }}"
    }
  }
}
{{ end }}

# business/qoe.rules.yml
groups:
- name: video-conference-qoe
  interval: 30s
  rules:
  # 核心体验指标：基于 ITU-T G.107 E-Model 简化版 MOS 评分 < 3.5（临界可用）
  - alert: VideoConferenceMOSDegraded
    expr: |
      (
        100 
        - 30 * (rate(webrtc_packets_lost_total[1m]) / rate(webrtc_packets_received_total[1m]) * 100) 
        - 15 * clamp_max(histogram_quantile(0.95, rate(webrtc_jitter_buffer_delay_seconds_bucket[5m])) * 1000, 200) / 10
        - 10 * clamp_max(rate(webrtc_nack_sent_total[1m]) / rate(webrtc_packets_sent_total[1m]) * 100, 50)
      ) < 35
    for: 5m
    labels:
      severity: warning
      domain: qoe
      team: video-infra
    annotations:
      summary: "预估 MOS 评分跌破 3.5 (当前: {{ $value | printf "%.1f" }})"
      description: |
        会议 {{ $labels.conference_id }} 区域 {{ $labels.region }} 用户体验下降。
        主因诊断：丢包率 {{ printf "%.2f" (rate(webrtc_packets_lost_total{conference_id="{{ $labels.conference_id }}"}[1m]) / rate(webrtc_packets_received_total{conference_id="{{ $labels.conference_id }}"}[1m]) * 100) }}%,
        抖动 P95 {{ printf "%.0f" (histogram_quantile(0.95, rate(webrtc_jitter_buffer_delay_seconds_bucket{conference_id="{{ $labels.conference_id }}"}[5m])) * 1000) }}ms,
        NACK 率 {{ printf "%.2f" (rate(webrtc_nack_sent_total{conference_id="{{ $labels.conference_id }}"}[1m]) / rate(webrtc_packets_sent_total{conference_id="{{ $labels.conference_id }}"}[1m]) * 100) }}%。
        建议：检查 TURN 中继负载、客户端网络类型分布、编码器降级策略。

  # 关键风险：Jitter Buffer 耗尽导致音频卡顿/视频花屏
  - alert: VideoConferenceJitterBufferUnderrun
    expr: |
      rate(webrtc_jitter_buffer_underrun_total[2m]) / rate(webrtc_jitter_buffer_emitted_total[2m]) > 0.02
    for: 3m
    labels:
      severity: critical
      domain: qoe
      team: video-infra
    annotations:
      summary: "Jitter Buffer 耗尽率超 2% (当前: {{ $value | printf "%.2f" }}%)"
      description: |
        网络抖动超出缓冲吸收能力，媒体流出现不可逆丢帧。
        请立即排查：1) 发送端带宽估算是否准确 2) 中转服务器 CPU 抢占 3) 客户端设备性能瓶颈。

# business/large-scale.rules.yml
groups:
- name: video-conference-large-scale
  interval: 15s  # 高频评估
  rules:
  # 单房间参会人数逼近架构上限（如 SFU 单进程 1000 路转发上限）
  - alert: VideoConferenceRoomCapacityNearLimit
    expr: |
      max by (conference_id, cluster) (jitsi_participants_per_conference) 
      / 
      max by (conference_id, cluster) (jitsi_conference_capacity_limit) > 0.85
    for: 2m
    labels:
      severity: warning
      scene: large-meeting
      team: video-infra
    annotations:
      summary: "大型会议房间 {{ $labels.conference_id }} 容量水位 {{ $value | printf "%.0f" }}%"
      description: |
        当前人数: {{ $value | printf "%.0f" }}，单进程转发上限: {{ $labels.capacity_limit }}。
        自动扩容策略: 触发 HPA 扩展 JVB Pod 或 启用级联 SFU 模式。
        Runbook: https://wiki.example.com/runbook/large-meeting-scaleout

  # 级联 SFU 跨节点带宽压力
  - alert: VideoConferenceCascadeLinkSaturation
    expr: |
      rate(node_network_transmit_bytes_total{interface=~"cascade.*"}[1m]) 
      / 
      node_network_speed_bytes{interface=~"cascade.*"} > 0.8
    for: 3m
    labels:
      severity: critical
      scene: large-meeting
      team: network
    annotations:
      summary: "级联链路 {{ $labels.interface }} 带宽利用率 {{ $value | printf "%.0f" }}%"
      description: "跨可用区/跨云厂商级联链路饱和，将导致远端参会者卡顿。建议：1) 启用 BWE 降码 2) 调度新参会者至就近级联节点 3) 申请临时扩容专线带宽。"

  # 信令服务器连接数突增（防刷/防爬/合法大促）
  - alert: VideoConferenceSignalingConnectionSpike
    expr: |
      (rate(signaling_websocket_connections_total[1m]) - rate(signaling_websocket_connections_total[1m] offset 5m)) 
      > 500
    for: 1m
    labels:
      severity: warning
      scene: large-meeting
      team: video-infra
    annotations:
      summary: "信令连接数 5 分钟内激增 500+ (当前速率: {{ $value | printf "%.0f" }}/min)"
      description: "疑似大型会议入场高峰或异常流量。请核对会议日程，必要时触发信令层限流熔断。"

# business/recording-pipeline.rules.yml
groups:
- name: video-conference-recording
  interval: 1m
  rules:
  # 任务队列积压超阈值（SLA：录制文件 30 分钟内可用）
  - alert: VideoRecordingQueueLagHigh
    expr: |
      recording_job_queue_lag > 50
    for: 5m
    labels:
      severity: warning
      domain: recording
      team: media-processing
    annotations:
      summary: "录制任务积压 {{ $value | printf "%.0f" }} 个，预计延迟超 SLA"
      description: |
        积压原因排查：1) 转码集群 CPU 不足 2) 对象存储写入限流 3) 单个超大文件阻塞。
        执行 `kubectl get pods -l app=recording-worker -o wide` 观察 Worker 状态。

  # 单任务处理超时（可能卡死/死锁/OOM）
  - alert: VideoRecordingTaskTimeout
    expr: |
      time() - recording_task_start_timestamp_seconds > 3600
    for: 0m  # 即时触发
    labels:
      severity: critical
      domain: recording
      team: media-processing
    annotations:
      summary: "录制任务 {{ $labels.task_id }} 运行超 1 小时"
      description: |
        任务类型: {{ $labels.task_type }} (merge/transcode/upload)。
        请检查 Worker 日志：`kubectl logs {{ $labels.worker_pod }} -c recorder`。
        如需人工介入：`kubectl exec -it {{ $labels.worker_pod }} -- /usr/local/bin/recording-cli recover {{ $labels.task_id }}`。

  # 对象存储写入失败率（防止静默丢数据）
  - alert: VideoRecordingStorageWriteFailure
    expr: |
      rate(recording_storage_write_failed_total[5m]) > 0
    for: 1m
    labels:
      severity: critical
      domain: recording
      team: storage
    annotations:
      summary: "录制文件写入对象存储失败 (近 5 分钟: {{ $value | printf "%.0f" }} 次)"
      description: |
        可能原因：存储桶配额耗尽、KMS 密钥轮换导致权限失效、网络分区。
        立即检查 MinIO/S3 审计日志及网络连通性。

{{ define "grafana.deep_link" }}
https://grafana.company.com/d/{{ .CommonAnnotations.dashboard_uid }}?{{ range $k, $v := .CommonAnnotations }}{{ if hasPrefix $k "var-" }}{{ $k }}={{ $v }}&{{ end }}{{ end }}orgId=1&from=now-30m&to=now&refresh=30s
{{ end }}

# prometheus.yml
scrape_configs:
- job_name: 'jvb-tenants'
  kubernetes_sd_configs:
  - role: pod
  relabel_configs:
  # 从 Namespace 解析租户 ID（约定：ns-<tenant_id>-<env>）
  - source_labels: [__meta_kubernetes_namespace]
    regex: 'ns-([a-z0-9]+)-.*'
    target_label: tenant_id
  # 从 Pod Label 解析业务域
  - source_labels: [__meta_kubernetes_pod_label_business_domain]
    target_label: business_domain
  # 禁止租户覆盖核心路由标签
  - source_labels: [__tmp_prometheus_label_tenant_id]
    regex: 'platform|infra'
    action: drop  # 核心组件不归属任何业务租户

# alertmanager.yml
route:
  receiver: 'default'
  group_by: ['alertname', 'tenant_id', 'cluster']
  routes:
  # 租户 A 专属路由
  - match:
      tenant_id: 'marketing'
    receiver: 'marketing-oncall'
    group_wait: 30s
    continue: true
    routes:
    - match:
        severity: critical
      receiver: 'marketing-critical-pagerduty'

  # 租户 B 专属路由
  - match:
      tenant_id: 'education'
    receiver: 'education-oncall'
    # 教育租户要求工作日 9-22 点仅发飞书，非工作日发短信
    mute_time_intervals:
    - 'off_hours'
    receivers:
    - 'education-feishu'
    - 'education-sms'

  # 平台团队兜底路由（基础设施告警不归属租户）
  - match:
      team: 'platform'
    receiver: 'platform-oncall'

mute_time_intervals:
- name: 'off_hours'
  time_intervals:
  - times:
    - start_time: '22:00'
      end_time: '09:00'
    weekdays: ['saturday', 'sunday']

# recording/downsampling.rules.yml
groups:
- name: downsampling-1m
  interval: 1m
  rules:
  # 高基数业务指标：仅保留 Top N 与聚合值
  - record: tenant:jvb_participants_max:1m
    expr: max by (tenant_id, cluster) (jitsi_participants_total)
  - record: tenant:jvb_conferences_count:1m
    expr: count by (tenant_id, cluster) (jitsi_concurrent_conferences)
  - record: tenant:webrtc_bitrate_p95:1m
    expr: histogram_quantile(0.95, sum by (tenant_id, cluster, le) (rate(webrtc_outbound_rtp_bitrate_bucket[1m])))

- name: downsampling-1h
  interval: 1h
  rules:
  # 长周期趋势指标
  - record: tenant:jvb_participants_peak_daily:1h
    expr: max_over_time(tenant:jvb_participants_max:1m[24h])
  - record: tenant:recording_storage_bytes_daily:1h
    expr: max_over_time(recording_storage_bytes_total[24h])

{{ define "sanitize.annotations" }}
{{- $desc := .CommonAnnotations.description -}}
{{- $desc = regexReplaceAll "\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b" $desc "<IP_REDACTED>" -}}
{{- $desc = regexReplaceAll "(?i)user[_-]?id[:=]\s*\w+" $desc "user_id=<HASH>" -}}
{{- $desc = regexReplaceAll "(?i)email[:=]\s*[\w.+-]+@[\w.-]+" $desc "email=<REDACTED>" -}}
{{- print $desc -}}
{{ end }}

# Alertmanager 启动参数
--web.config.file=/etc/alertmanager/web.yml

# web.yml (启用基本认证 + 访问日志)
tls_server_config:
  cert_file: /certs/alertmanager.crt
  key_file: /certs/alertmanager.key
basic_auth_users:
  admin: "$2a$10$..."  # bcrypt hash
  viewer: "$2a$10$..."
# 通过 Nginx/Envoy 代理层记录访问日志至 Elasticsearch/Loki
# 审计字段：timestamp, remote_addr, user, method, path, user_agent, response_code

# .gitlab-ci.yml / .github/workflows/chaos-alert-test.yml
stages:
  - validate_rules
  - deploy_chaos
  - verify_alerts
  - cleanup
  - report

verify_alerts:
  script:
    - |
      # 等待告警触发
      sleep 180
      # 查询 Alertmanager API 校验
      ALERTS=$(curl -s -u $AM_USER:$AM_PASS $AM_URL/api/v2/alerts | jq -r '.[] | select(.labels.alertname=="'$EXPECTED_ALERT'")')
      if [ -z "$ALERTS" ]; then
        echo "❌ FAIL: Expected alert $EXPECTED_ALERT not fired"
        exit 1
      fi
      # 校验关键标签完整性
      echo "$ALERTS" | jq -e '.labels | has("cluster") and has("tenant_id") and has("runbook")' || exit 1
      echo "✅ PASS: Alert fired with correct labels"
  rules:
    - if: $CI_COMMIT_BRANCH == "main"

覆盖率 = (已编写规则并通过混沌验证的故障模式数) / (故障模式库总数)
目标：核心链路 100%，重要链路 90%，边缘链路 70%

graph LR
    App[应用/Exporter] --> Prom_New[Prometheus New v3.0]
    App --> Prom_Old[Prometheus Old v2.47]
    Prom_New --> AM_New[Alertmanager New]
    Prom_Old --> AM_Old[Alertmanager Old]
    AM_New --> Diff[告警差异对比工具]
    AM_Old --> Diff
    Diff --> Report[每日差异报告]

# 1. 语法检查
promtool check rules /rules/*.yml

# 2. 单元测试
promtool test rules testdata/*.yml

# 3. 语义等价性测试（针对 PromQL 重写）
# 使用 promql-parser 对比 AST 树，或使用 VictoriaMetrics `vmalert` 的 `--dry-run` 模式对比评估结果

# Alert: JVBICEFailureRateHigh
**最后更新**: 2024-11-15 | **维护者**: @zhangsan @lisi

## 1. 告警含义
ICE 连接建立失败率 > 10% 持续 3 分钟，直接影响用户入会成功率。

## 2. 常见根因排查树

A[ICE 失败率高] --> B{TURN 服务器健康?}
B -- 否 --> C[重启 TURN Pod / 扩容]
B -- 是 --> D{客户端网络类型分布}
D --> E[企业网/防火墙拦截 UDP 3478]
D --> F[运营商 QoS 限制]
D --> G[客户端版本过旧]

## 3. 关键仪表盘链接
- [JVB ICE 分析](https://grafana/d/jvb-ice?var-conference_id=)
- [TURN 服务器监控](https://grafana/d/turn-server)

## 4. 历史事故关联
- INC-20240315: 电信骨干网故障导致华东区 ICE 失败率 40%
- INC-20231102: TURN 证书过期导致全网 TLS 握手失败

## 5. 可执行修复动作
- [ ] `kubectl rollout restart deployment/turn-server -n infra`
- [ ] `ansible-playbook update-turn-firewall.yml -e region=cn-hangzhou`