┌─────────────────────────────────────────────────────┐
│                  业务决策层 (HPA 直接消费)            │
│  • media_active_streams      (当前活跃流总数)        │
│  • media_pending_tasks       (待处理转码任务队列长度) │
│  • media_gpu_utilization     (GPU 算力综合利用率)    │
│  • media_bandwidth_usage_pct (出口带宽使用率)        │
├─────────────────────────────────────────────────────┤
│                  指标聚合层 (Prometheus Rules)       │
│  • 多副本求和/求平均  •  分位数平滑  •  趋势预测      │
├─────────────────────────────────────────────────────┤
│                  采集暴露层 (Exporter / Pushgateway)  │
│  • Node Exporter + GPU Exporter  •  业务侧埋点上报    │
└─────────────────────────────────────────────────────┘

# prometheus-adapter configmap 关键片段
rules:
  custom:
    - seriesQuery: 'media_node_active_streams{namespace!="",pod!=""}'
      resources:
        overrides:
          namespace: {resource: "namespace"}
          pod: {resource: "pod"}
      name:
        matches: "^(.*)$"
        as: "active_streams"
      metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)'

    - seriesQuery: 'media_node_pending_transcode_tasks{namespace!="",pod!=""}'
      resources:
        overrides:
          namespace: {resource: "namespace"}
          pod: {resource: "pod"}
      name:
        matches: "^(.*)$"
        as: "pending_tasks"
      metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)'

    - seriesQuery: 'media_node_gpu_memory_used_bytes{namespace!="",pod!=""}'
      resources:
        overrides:
          namespace: {resource: "namespace"}
          pod: {resource: "pod"}
      name:
        matches: "^(.*)$"
        as: "gpu_mem_usage_pct"
      metricsQuery: |
        (sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
         / 
         sum(nvidia_gpu_memory_total_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
        ) * 100

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: media-transcoder-hpa
  namespace: media-prod
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: media-transcoder
  minReplicas: 3
  maxReplicas: 200
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 600   # 缩容冷却 10 分钟，防抖
      policies:
        - type: Percent
          value: 10
          periodSeconds: 120
    scaleUp:
      stabilizationWindowSeconds: 30    # 扩容快速响应
      policies:
        - type: Percent
          value: 50
          periodSeconds: 30
        - type: Pods
          value: 10
          periodSeconds: 30
      selectPolicy: Max
  metrics:
    # 核心指标：活跃流数，单 Pod 目标 50 路
    - type: Pods
      pods:
        metric:
          name: active_streams
        target:
          type: AverageValue
          averageValue: "50"
    # 兜底指标：任务队列积压，单 Pod 目标 < 20
    - type: Pods
      pods:
        metric:
          name: pending_tasks
        target:
          type: AverageValue
          averageValue: "20"
    # 硬性约束：GPU 显存不超过 85%
    - type: Pods
      pods:
        metric:
          name: gpu_mem_usage_pct
        target:
          type: AverageValue
          averageValue: "85"

# Deployment 片段
spec:
  template:
    spec:
      # 1. 调度侧：优先调度到有 GPU 资源、低负载节点
      schedulerName: default-scheduler
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 80
              preference:
                matchExpressions:
                  - key: nvidia.com/gpu.present
                    operator: Exists
            - weight: 20
              preference:
                matchExpressions:
                  - key: node.kubernetes.io/instance-type
                    operator: In
                    values: ["gpu.v100", "gpu.t4"]
      # 2. 启动侧：宽容启动期资源抖动
      containers:
        - name: transcoder
          startupProbe:
            httpGet:
              path: /health/startup
              port: 8080
            failureThreshold: 30      # 允许 5 分钟启动窗口
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /health/ready
              port: 8080
            initialDelaySeconds: 5
            periodSeconds: 5
            failureThreshold: 3
          # 3. 资源声明：显式声明 GPU，防止调度到无 GPU 节点
          resources:
            limits:
              nvidia.com/gpu: "1"
              memory: "8Gi"
              cpu: "4"
            requests:
              nvidia.com/gpu: "1"
              memory: "6Gi"
              cpu: "2"

lifecycle:
  preStop:
    exec:
      command:
        - /bin/sh
        - -c
        - |
          # 1. 标记节点进入“下线中”状态，停止接收新流
          curl -X POST localhost:8080/admin/drain
          # 2. 等待现有流自然结束或强制迁移（设定超时 300s）
          for i in {1..60}; do
            active=$(curl -s localhost:8080/metrics | grep media_node_active_streams | awk '{print $2}')
            [ "$active" -eq 0 ] && break
            sleep 5
          done
          # 3. 兜底：强制切断残留连接，上报任务重调度
          curl -X POST localhost:8080/admin/force_shutdown

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: media-transcoder-pdb
  namespace: media-prod
spec:
  minAvailable: 80%          # 任意时刻至少保留 80% 副本可用
  selector:
    matchLabels:
      app: media-transcoder

groups:
  - name: media-hpa-alerts
    rules:
      - alert: MediaHPAScaleUpStuck
        expr: |
          (kube_hpa_status_desired_replicas - kube_hpa_status_current_replicas) > 0
          and
          (time() - kube_pod_status_ready_time{condition="true"}) > 300
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "HPA 期望扩容但 Pod 长时间未 Ready"
          description: "Deployment {{ $labels.deployment }} 已连续 5 分钟未达到期望副本数，疑似镜像拉取失败或调度受阻"

      - alert: MediaHPAMetricStale
        expr: |
          time() - media_node_active_streams{job="media-transcoder"} > 60
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "媒体节点自定义指标采集中断"
          description: "Pod {{ $labels.pod }} 指标超过 60 秒未更新，HPA 可能基于陈旧数据决策"

graph TD
    A[业务流量洪峰] --> B{HPA 判断}
    B -- 副本数达上限/资源请求不足 --> C[VPA Recommender 分析历史用量]
    C -- 建议调整 Request/Limit --> D[VPA Updater 驱动滚动更新]
    B -- Pending Pods 触发 --> E[Cluster Autoscaler 扩容节点池]
    E -- 新节点 Ready --> F[HPA 调度新副本]
    G[业务低谷] --> H[HPA 缩容副本]
    H -- 节点利用率低 --> I[CA 缩容节点]
    I -- 驱逐 Pod --> J[PDB + preStop 优雅下线]

apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: media-transcoder-vpa
  namespace: media-prod
spec:
  targetRef:
    apiVersion: "apps/v1"
    kind: Deployment
    name: media-transcoder
  updatePolicy:
    updateMode: "Auto"          # 生产建议 "Initial" 或 "Off" + 手动发布
    containerPolicies:
      - containerName: transcoder
        controlledResources: ["cpu", "memory"]  # 不控制 nvidia.com/gpu
        minAllowed:
          cpu: "500m"
          memory: "2Gi"
        maxAllowed:
          cpu: "8"
          memory: "16Gi"
  resourcePolicy:
    containerPolicies:
      - containerName: transcoder
        # GPU 资源由设备插件独占，禁止 VPA 触碰
        controlledResources: ["cpu", "memory"]

# Pod Spec 片段：HAMi 细粒度申请 + 拓扑感知
spec:
  schedulerName: hami-scheduler  # 或 default-scheduler + 插件
  containers:
    - name: transcoder
      resources:
        limits:
          nvidia.com/gpu: "1"
          nvidia.com/gpumem: "4096"   # 显存 4GiB
          nvidia.com/gpucores: "50"   # 50% 算力核心
        requests:
          nvidia.com/gpu: "1"
          nvidia.com/gpumem: "4096"
          nvidia.com/gpucores: "50"
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: nvidia.com/gpu.product
                operator: In
                values: ["Tesla-T4", "A100-SXM4-40GB"]  # 指定 GPU 型号
    # 反亲和：避免同一物理机单点故障
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchExpressions:
                - key: app
                  operator: In
                  values: [media-transcoder]
            topologyKey: kubernetes.io/hostname

┌────────────────────────────────────────────────────────────┐
│                    节点池分层策略                            │
├──────────────────┬──────────────────────┬──────────────────┤
│   基础池         │   弹性池             │   超售池         │
│  (On-Demand)     │  (Spot/Preemptible)  │  (Spot + 过载保护)│
├──────────────────┼──────────────────────┼──────────────────┤
│ 比例: 30%        │ 比例: 60%            │ 比例: 10%        │
│ 角色: 兜底、长任务│ 角色: 主力转码、短任务 │ 角色: 削峰填谷   │
│ Taint: 无        │ Taint: spot=true     │ Taint: spot=true │
│                  │ Toleration: 仅弹性负载│ Toleration: 仅   │
│                  │                      │   可降级任务     │
└──────────────────┴──────────────────────┴──────────────────┘

# DaemonSet 运行在每个 Spot 节点
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: spot-termination-handler
spec:
  template:
    spec:
      containers:
        - name: handler
          image: myregistry/spot-handler:v1
          env:
            - name: PROVIDER
              value: "aliyun"  # 或 aws, tencent, huawei
          securityContext:
            privileged: true  # 需访问宿主机元数据
          volumeMounts:
            - name: host-root
              mountPath: /host
      volumes:
        - name: host-root
          hostPath:
            path: /

# AWS NTH 配置示例
--enable-spot-interruption-draining=true
--pod-termination-grace-period=120
--pre-drain-script=/scripts/save_checkpoint.sh

# PrometheusRule: 从 Ingress 聚合流量指标
groups:
  - name: ingress-media-metrics
    rules:
      - expr: |
          sum(rate(nginx_ingress_controller_requests{service=~"media-.*"}[30s])) by (service, namespace)
        record: media:ingress:qps_per_service
      - expr: |
          histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{service=~"media-.*"}[1m])) by (le, service))
        record: media:ingress:p99_latency

# APISIX / NGINX 配置片段：镜像 10% 流量到 Canary 节点池
plugin_config:
  traffic-split:
    rules:
      - match:
          - vars: [["arg_canary", "==", "true"]]
        weighted_upstreams:
          - upstream_id: media-transcoder-stable
            weight: 90
          - upstream_id: media-transcoder-canary
            weight: 10
  mirror:
    upstream_id: media-transcoder-canary
    percentage: 10  # 复制 10% 请求到新节点池，不阻塞主流程

# HPA 扩容前置条件：依赖服务健康度检查
# 通过自定义 Metrics Server 扩展或 KEDA ScaledObject 实现
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: media-transcoder-keda
spec:
  scaleTargetRef:
    name: media-transcoder
  triggers:
    - type: prometheus
      metadata:
        serverAddress: http://prometheus:9090
        metricName: media_active_streams
        query: |
          sum(media_node_active_streams) by (namespace)
          /
          sum(media_node_active_streams{job="media-transcoder"}) by (namespace)
        threshold: "50"
        authModes: "basic"
    # 熔断门控：仅当下游存储 P99 延迟 < 500ms 时允许扩容
    - type: external
      metadata:
        targetValue: "1"
        scalerAddress: http://dependency-health-checker:8080/check/storage

用户请求 -> 全局负载均衡 (GTM/GSLB) -> 就近地域集群
                                      |
                    ┌─────────────────┼─────────────────┐
                    ▼                 ▼                 ▼
              集群 A (华东)       集群 B (华南)       集群 C (海外)
              HPA 本地扩容        HPA 本地扩容        HPA 本地扩容
                    │                 │                 │
                    └─────────────────┼─────────────────┘
                                      ▼
                          Karmada 控制面 (资源聚合视图)
                          - 跨集群 Pod 调度策略
                          - 统一 HPA 联邦资源视图

# Karmada: 定义媒体转码工作负载跨集群分发策略
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: media-transcoder-pp
spec:
  resourceSelectors:
    - apiVersion: apps/v1
      kind: Deployment
      name: media-transcoder
  placement:
    clusterAffinity:
      clusterNames: ["cluster-east", "cluster-south", "cluster-oversea"]
    replicaScheduling: Weighted
    replicaSchedulingType: Divided
    weightPreference:
      staticWeightList:
        - targetCluster:
            clusterNames: ["cluster-east"]
          weight: 50
        - targetCluster:
            clusterNames: ["cluster-south"]
          weight: 30
        - targetCluster:
            clusterNames: ["cluster-oversea"]
          weight: 20
---
# OverridePolicy: 不同地域集群差异化配置 (如 GPU 型号、镜像仓库)
apiVersion: policy.karmada.io/v1alpha1
kind: OverridePolicy
metadata:
  name: media-transcoder-overrides
spec:
  resourceSelectors:
    - apiVersion: apps/v1
      kind: Deployment
      name: media-transcoder
  overriders:
    plaintext:
      - clusterNames: ["cluster-oversea"]
        patcher: json
        patch: |
          [
            {"op": "replace", "path": "/spec/template/spec/containers/0/image", "value": "registry.intl.example.com/media-transcoder:v1.2.0"},
            {"op": "replace", "path": "/spec/template/spec/nodeSelector/nvidia.com/gpu.product", "value": "A10G"}
          ]

# .gitlab-ci.yml / GitHub Actions 片段
stages:
  - validate
  - chaos
  - report

chaos_weekly:
  stage: chaos
  image: chaosiq/chaostoolkit:latest
  script:
    - chaosiq login $CHAOSIQ_TOKEN
    - chaos run experiments/hpa_scale_up.yaml --settings=env=prod
    - chaos run experiments/spot_termination.yaml --settings=env=prod
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"  # 仅定时触发
  artifacts:
    reports:
      junit: chaos-report.xml

# Kyverno ClusterPolicy: 验证镜像签名
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: verify-image-signature
spec:
  validationFailureAction: Enforce
  rules:
    - name: check-cosign-signature
      match:
        any:
        - resources:
            kinds: [Pod]
      verifyImages:
        - image: "registry.example.com/media/*"
          key: |-
            -----BEGIN PUBLIC KEY-----
            ...
            -----END PUBLIC KEY-----

# 审计策略示例 (audit-policy.yaml)
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
  - level: Metadata
    resources:
      - group: autoscaling
        resources: [horizontalpodautoscalers]
      - group: ""
        resources: [pods, nodes, events]
      - group: apps
        resources: [deployments, replicasets]
  - level: RequestResponse
    users: ["system:serviceaccount:kube-system:cluster-autoscaler"]
    resources:
      - group: ""
        resources: [nodes, pods/eviction]
  - level: RequestResponse
    verbs: ["create", "update", "patch", "delete"]
    resources:
      - group: ""
        resources: [secrets, configmaps]  # 敏感配置变更