1
# Prometheus Operator
2
apiVersion: monitoring.coreos.com/v1
3
kind: Prometheus
4
metadata:
5
  name: k8s-prometheus
6
  namespace: monitoring
7
spec:
8
  replicas: 2
9
  retention: 15d
10
  serviceAccountName: prometheus
11
  serviceMonitorSelector:
12
    matchLabels:
13
      team: monitoring
14
  resources:
15
    requests:
16
      cpu: 200m
17
      memory: 512Mi
18
    limits:
19
      cpu: 1000m
20
      memory: 2Gi
21
  storage:
22
    volumeClaimTemplate:
23
      spec:
24
        storageClassName: ssd
25
        resources:
26
          requests:
27
            storage: 50Gi

1
# CPU 使用率
2
100 - (sum(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance) * 100)
3

4
# 内存使用率
5
100 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100
6

7
# 磁盘使用率
8
100 - (node_filesystem_avail_bytes / node_filesystem_size_bytes) * 100

1
# Pod CPU 使用率
2
sum(rate(container_cpu_usage_seconds_total[5m])) by (pod, namespace)
3

4
# Pod 内存使用
5
sum(container_memory_working_set_bytes) by (pod, namespace)
6

7
# Pod 重启次数
8
increase(kube_pod_container_status_restarts_total[1h])

1
# API Server 请求延迟
2
histogram_quantile(0.99,
3
    rate(apiserver_request_duration_seconds_bucket[5m]))
4

5
# Scheduler 调度延迟
6
histogram_quantile(0.95,
7
    rate(scheduler_e2e_scheduling_duration_seconds_bucket[5m]))
8

9
# etcd 磁盘写入延迟
10
histogram_quantile(0.99,
11
    rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m]))

1
# Import dashboard via JSON
2
apiVersion: v1
3
kind: ConfigMap
4
metadata:
5
  name: grafana-dashboard-k8s
6
  namespace: monitoring
7
data:
8
  k8s-cluster.json: |
9
    {
10
      "dashboard": {
11
        "title": "Kubernetes Cluster",
12
        "uid": "k8s-cluster",
13
        "panels": [
14
          {
15
            "title": "CPU 使用率",
16
            "type": "graph",
17
            "targets": [
18
              {
19
                "expr": "sum(rate(container_cpu_usage_seconds_total[5m])) by (namespace)",
20
                "legendFormat": "{{namespace}}"
21
              }
22
            ]
23
          },
24
          {
25
            "title": "内存使用",
26
            "type": "graph",
27
            "targets": [
28
              {
29
                "expr": "sum(container_memory_working_set_bytes) by (namespace)",
30
                "legendFormat": "{{namespace}}"
31
              }
32
            ]
33
          }
34
        ]
35
      }
36
    }

1
# Kubernetes 集群总览
2
apiVersion: v1
3
kind: ConfigMap
4
metadata:
5
  name: cluster-overview
6
  namespace: monitoring
7
data:
8
  dashboard.json: |
9
    {
10
      "panels": [
11
        {
12
          "title": "节点数",
13
          "type": "stat",
14
          "gridPos": {"h": 8, "w": 6},
15
          "targets": [
16
            {"expr": "count(kube_node_info)"}
17
          ]
18
        },
19
        {
20
          "title": "Pod 总数",
21
          "type": "stat",
22
          "gridPos": {"h": 8, "w": 6},
23
          "targets": [
24
            {"expr": "sum(kube_pod_info)"}
25
          ]
26
        },
27
        {
28
          "title": "CPU 分配率",
29
          "type": "gauge",
30
          "gridPos": {"h": 8, "w": 6},
31
          "targets": [
32
            {"expr": "sum(kube_pod_container_resource_requests_cpu_cores) / sum(kube_node_status_allocatable_cpu_cores) * 100"}
33
          ]
34
        },
35
        {
36
          "title": "内存分配率",
37
          "type": "gauge",
38
          "gridPos": {"h": 8, "w": 6},
39
          "targets": [
40
            {"expr": "sum(kube_pod_container_resource_requests_memory_bytes) / sum(kube_node_status_allocatable_memory_bytes) * 100"}
41
          ]
42
        }
43
      ]
44
    }

1
apiVersion: monitoring.coreos.com/v1
2
kind: PrometheusRule
3
metadata:
4
  name: k8s-alerts
5
  namespace: monitoring
6
spec:
7
  groups:
8
    - name: kubernetes
9
      rules:
10
        # K8s 组件告警
11
        - alert: K8sApiserverDown
12
          expr: up{job="kube-apiserver"} == 0
13
          for: 5m
14
          labels:
15
            severity: critical
16
          annotations:
17
            summary: "API Server is down"
18
            description: "API Server has been down for more than 5 minutes"
19

20
        - alert: K8sNodeNotReady
21
          expr: kube_node_status_condition{condition="Ready",status="true"} == 0
22
          for: 10m
23
          labels:
24
            severity: warning
25
          annotations:
26
            summary: "Node {{ $labels.node }} is not ready"
27
            description: "Node {{ $labels.node }} has been not ready for 10 minutes"
28

29
        # 资源告警
30
        - alert: HighCPUUsage
31
          expr: sum(rate(container_cpu_usage_seconds_total[5m])) by (node) > 0.8
32
          for: 5m
33
          labels:
34
            severity: warning
35
          annotations:
36
            summary: "High CPU usage on {{ $labels.node }}"
37
            description: "Node {{ $labels.node }} CPU usage is above 80%"
38

39
        - alert: HighMemoryUsage
40
          expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.85
41
          for: 5m
42
          labels:
43
            severity: warning
44
          annotations:
45
            summary: "High Memory usage on {{ $labels.node }}"
46

47
        # Pod 告警
48
        - alert: PodRestartingTooMuch
49
          expr: rate(kube_pod_container_status_restarts_total[1h]) > 0.05
50
          for: 5m
51
          labels:
52
            severity: warning
53
          annotations:
54
            summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} restarting too much"

1
apiVersion: monitoring.coreos.com/v1
2
kind: AlertmanagerConfig
3
metadata:
4
  name: team-config
5
  namespace: monitoring
6
spec:
7
  receivers:
8
    - name: "default"
9
      emailConfigs:
10
        - to: "ops-team@example.com"
11
          headers:
12
            subject: "[{{ .Status | toUpper }}] {{ .GroupLabels.alertname }}"
13
    - name: "slack"
14
      slackConfigs:
15
        - channel: "#alerts"
16
          apiUrl: "https://hooks.slack.com/xxx"
17
          title: "[{{ .Status | toUpper }}] {{ .GroupLabels.alertname }}"
18
          text: |
19
            {{ range .Alerts }}
20
            *Alert:* {{ .Annotations.summary }}
21
            *Description:* {{ .Annotations.description }}
22
            {{ end }}

1
# ServiceMonitor
2
apiVersion: monitoring.coreos.com/v1
3
kind: ServiceMonitor
4
metadata:
5
  name: my-app
6
  namespace: monitoring
7
  labels:
8
    team: monitoring
9
spec:
10
  selector:
11
    matchLabels:
12
      app: my-app
13
  endpoints:
14
    - port: metrics
15
      interval: 30s
16
      path: /metrics
17
  namespaceSelector:
18
    matchNames:
19
      - production

1
# Python 应用暴露 Prometheus 指标
2
from prometheus_client import Counter, Histogram, Gauge
3

4
# 定义指标
5
REQUEST_COUNT = Counter(
6
    'http_requests_total',
7
    'Total HTTP requests',
8
    ['method', 'endpoint', 'status']
9
)
10

11
REQUEST_LATENCY = Histogram(
12
    'http_request_duration_seconds',
13
    'HTTP request latency',
14
    ['method', 'endpoint']
15
)
16

17
ACTIVE_CONNECTIONS = Gauge(
18
    'http_active_connections',
19
    'Active HTTP connections'
20
)
21

22
# 使用指标
23
@app.route('/api/users')
24
def get_users():
25
    REQUEST_COUNT.labels(method='GET', endpoint='/api/users', status='200').inc()
26
    with REQUEST_LATENCY.labels(method='GET', endpoint='/api/users').time():
27
        # 处理请求
28
        pass
29
    return jsonify(users)

1
# 查看相关 Pod 日志
2
kubectl logs -n <namespace> <pod-name> --tail=100
3

4
# 查看链路追踪 (Jaeger)
5
# 访问 Jaeger UI，根据 trace ID 查询完整调用链