1
# 可用性 SLI：成功请求比例
2
sum(rate(http_requests_total{status!~"5.."}[5m]))
3
/
4
sum(rate(http_requests_total[5m]))
5

6
# 延迟 SLI：P99 延迟
7
histogram_quantile(0.99, sum(rate(http_request_duration_bucket[5m])) by (le, service))
8

9
# 正确性 SLI：数据一致性检查通过率
10
sum(rate(consistency_check_total{result="pass"}[5m]))
11
/
12
sum(rate(consistency_check_total[5m]))

1
# SLO 定义
2
apiVersion: sloth.slok.dev/v1
3
kind: PrometheusSLO
4
spec:
5
  service: order-service
6
  slo:
7
    - name: "availability"
8
      objective: 99.9
9
      description: "99.9% 的请求成功返回"
10
      sli:
11
        events:
12
          error_query: sum(rate(http_requests_total{service="order-service",status=~"5.."}[{{.window}}]))
13
          total_query: sum(rate(http_requests_total{service="order-service"}[{{.window}}]))
14
      alerting:
15
        name: OrderServiceAvailabilitySLO
16
        labels:
17
          team: order-team
18
        page_alert:
19
          labels:
20
            severity: page
21
        ticket_alert:
22
          labels:
23
            severity: ticket
24

25
    - name: "latency"
26
      objective: 99.0
27
      description: "99% 的请求在 500ms 内完成"
28
      sli:
29
        events:
30
          error_query: |
31
            sum(rate(http_request_duration_seconds_bucket{service="order-service",le="0.5"}[{{.window}}]))
32
            /
33
            sum(rate(http_request_duration_seconds_count{service="order-service"}[{{.window}}]))
34
          total_query: "1"

1
错误预算 = 1 - SLO
2
30 天错误预算 = 30 × 24 × 60 × 60 × 0.1% = 2592 秒 ≈ 43.2 分钟

1
30 天总秒数 = 30 × 86400 = 2,592,000
2
错误预算秒数 = 2,592,000 × 0.1% = 2,592 秒 ≈ 43.2 分钟
3

4
30 天总请求数 = 1000 × 2,592,000 = 2,592,000,000
5
允许失败请求数 = 2,592,000,000 × 0.1% = 2,592,000

1
燃烧率 = 实际错误率 / 允许错误率

1
# 完整 PromQL
2
(
3
  sum(rate(http_requests_total{status=~"5.."}[1h])) by (service)
4
  /
5
  sum(rate(http_requests_total[1h])) by (service)
6
)
7
/
8
(1 - 0.999)  # 允许错误率 = 1 - SLO

1
# 多窗口告警规则
2
groups:
3
  - name: slo-burn-rate
4
    rules:
5
      # 快速燃烧：14.4x 燃烧率，1h/5m 窗口
6
      - alert: SLOFastBurnRate
7
        expr: |
8
          (
9
            sum(rate(http_requests_total{status=~"5.."}[1h])) by (service)
10
            /
11
            sum(rate(http_requests_total[1h])) by (service)
12
          )
13
          /
14
          (
15
            sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
16
            /
17
            sum(rate(http_requests_total[5m])) by (service)
18
          )
19
          > 14.4
20
        for: 5m
21
        labels:
22
          severity: page
23

24
      # 慢速燃烧：6x 燃烧率，6h/30m 窗口
25
      - alert: SLOSlowBurnRate
26
        expr: |
27
          (
28
            sum(rate(http_requests_total{status=~"5.."}[6h])) by (service)
29
            /
30
            sum(rate(http_requests_total[6h])) by (service)
31
          )
32
          /
33
          (
34
            sum(rate(http_requests_total{status=~"5.."}[30m])) by (service)
35
            /
36
            sum(rate(http_requests_total[30m])) by (service)
37
          )
38
          > 6
39
        for: 30m
40
        labels:
41
          severity: ticket

1
# 快速燃烧（Page 级）
2
# 条件：14.4x 燃烧率在 1 小时窗口内持续
3
# 错误预算消耗：14.4 × (1h / 720h) = 2%
4
# 含义：1 小时内消耗 2% 的 30 天错误预算
5
# 30 天错误预算在 2 天内耗尽 → 需要立即处理
6

7
# 慢速燃烧（Ticket 级）
8
# 条件：6x 燃烧率在 6 小时窗口内持续
9
# 错误预算消耗：6 × (6h / 720h) = 5%
10
# 含义：6 小时内消耗 5% 的 30 天错误预算
11
# 30 天错误预算在 5 天内耗尽 → 工作时间处理

1
# SLO 告警规则模板
2
# 替换 {{SERVICE}} 和 {{SLO_THRESHOLD}} 即可使用
3
groups:
4
  - name: slo-{{SERVICE}}-burn-rate
5
    rules:
6
      # ===== 可用性 SLO =====
7
      - alert: {{SERVICE}}AvailabilityFastBurn
8
        expr: |
9
          (
10
            sum(rate(http_requests_total{service="{{SERVICE}}",status=~"5.."}[1h]))
11
            /
12
            sum(rate(http_requests_total{service="{{SERVICE}}"}[1h]))
13
          ) > (1 - {{SLO_THRESHOLD}}) * 14.4
14
        for: 5m
15
        labels:
16
          severity: page
17
          slo: availability
18
          service: "{{SERVICE}}"
19
        annotations:
20
          summary: "{{SERVICE}} 可用性 SLO 快速燃烧"
21
          runbook_url: "https://wiki/runbooks/{{SERVICE}}-availability"
22

23
      - alert: {{SERVICE}}AvailabilitySlowBurn
24
        expr: |
25
          (
26
            sum(rate(http_requests_total{service="{{SERVICE}}",status=~"5.."}[6h]))
27
            /
28
            sum(rate(http_requests_total{service="{{SERVICE}}"}[6h]))
29
          ) > (1 - {{SLO_THRESHOLD}}) * 6
30
        for: 30m
31
        labels:
32
          severity: ticket
33
          slo: availability
34
          service: "{{SERVICE}}"
35
        annotations:
36
          summary: "{{SERVICE}} 可用性 SLO 慢速燃烧"
37

38
      # ===== 延迟 SLO =====
39
      - alert: {{SERVICE}}LatencyFastBurn
40
        expr: |
41
          (
42
            1 -
43
            sum(rate(http_request_duration_bucket{service="{{SERVICE}}",le="0.5"}[1h]))
44
            /
45
            sum(rate(http_request_duration_count{service="{{SERVICE}}"}[1h]))
46
          ) > (1 - 0.99) * 14.4
47
        for: 5m
48
        labels:
49
          severity: page
50
          slo: latency
51
          service: "{{SERVICE}}"
52
        annotations:
53
          summary: "{{SERVICE}} 延迟 SLO 快速燃烧"

1
spec:
2
  service: payment-service
3
  slo:
4
    - name: availability
5
      objective: 99.95
6
      description: "99.95% 的支付请求成功完成"
7
      sli:
8
        events:
9
          error_query: sum(rate(payment_requests_total{status!="success"}[{{.window}}]))
10
          total_query: sum(rate(payment_requests_total[{{.window}}]))
11

12
    - name: latency
13
      objective: 99.9
14
      description: "99.9% 的支付请求在 3s 内完成"
15
      sli:
16
        events:
17
          error_query: |
18
            sum(rate(payment_duration_bucket{le="3"}[{{.window}}]))
19
            /
20
            sum(rate(payment_duration_count[{{.window}}]))
21
          total_query: "1"
22

23
    - name: correctness
24
      objective: 99.99
25
      description: "99.99% 的支付金额一致"
26
      sli:
27
        events:
28
          error_query: sum(rate(payment_reconciliation_errors_total[{{.window}}]))
29
          total_query: sum(rate(payment_requests_total[{{.window}}]))

1
spec:
2
  service: search-service
3
  slo:
4
    - name: availability
5
      objective: 99.9
6
      description: "99.9% 的搜索请求返回结果"
7
      sli:
8
        events:
9
          error_query: sum(rate(search_requests_total{status=~"5.."}[{{.window}}]))
10
          total_query: sum(rate(search_requests_total[{{.window}}]))
11

12
    - name: latency
13
      objective: 99.0
14
      description: "99% 的搜索请求在 500ms 内完成"
15
      sli:
16
        events:
17
          error_query: |
18
            sum(rate(search_duration_bucket{le="0.5"}[{{.window}}]))
19
            /
20
            sum(rate(search_duration_count[{{.window}}]))
21
          total_query: "1"

1
spec:
2
  service: notification-service
3
  slo:
4
    - name: delivery
5
      objective: 99.5
6
      description: "99.5% 的通知在 5 分钟内送达"
7
      sli:
8
        events:
9
          error_query: sum(rate(notifications_total{delivered="false"}[{{.window}}]))
10
          total_query: sum(rate(notifications_total[{{.window}}]))

1
# 1. 当前 SLI 值
2
1 - (
3
  sum(rate(http_requests_total{status=~"5.."}[30d]))
4
  /
5
  sum(rate(http_requests_total[30d]))
6
)
7

8
# 2. 剩余错误预算
9
1 - (
10
  (1 - sum(rate(http_requests_total{status=~"5.."}[30d])) / sum(rate(http_requests_total[30d])))
11
  /
12
  0.999  # SLO 目标
13
)
14

15
# 3. 燃烧率
16
(
17
  sum(rate(http_requests_total{status=~"5.."}[1h]))
18
  /
19
  sum(rate(http_requests_total[1h]))
20
)
21
/
22
(1 - 0.999)
23

24
# 4. 错误预算消耗趋势（30 天窗口）
25
1 - (
26
  sum_over_time((1 - (sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m]))))[30d:])
27
  /
28
  sum_over_time((1)[30d:])
29
)

1
## 事件复盘
2

3
### 基本信息
4
- 事件时间：YYYY-MM-DD HH:MM - HH:MM
5
- 影响时长：X 分钟
6
- SLO 影响：可用性 SLO 从 99.9% 降至 99.X%
7
- 错误预算消耗：X%
8

9
### 时间线
10
- HH:MM - SLO 告警触发
11
- HH:MM - 值班工程师响应
12
- HH:MM - 定位根因
13
- HH:MM - 实施缓解
14
- HH:MM - SLO 恢复
15

16
### 根因分析
17
- 直接原因：
18
- 贡献因素：
19
- 根本原因：
20

21
### 行动项
22
- [ ] 修复根因（负责人，截止日期）
23
- [ ] 更新 SLO 告警规则（负责人，截止日期）
24
- [ ] 增加防御性检测（负责人，截止日期）

1
# 安装 Sloth
2
go install github.com/slok/sloth/cmd/sloth@latest
3

4
# 生成 Prometheus 规则
5
sloth generate -i slo.yaml -o prometheus-rules.yaml
6

7
# 应用规则
8
kubectl apply -f prometheus-rules.yaml

1
# Sloth 自动生成的 Recording Rules
2
groups:
3
  - name: sloth-slo-rules-order-service-availability
4
    rules:
5
      # SLI 当前值
6
      - record: slo:sli_error:ratio_rate5m
7
        expr: |
8
          sum(rate(http_requests_total{service="order-service",status=~"5.."}[5m]))
9
          /
10
          sum(rate(http_requests_total{service="order-service"}[5m]))
11

12
      # 30 天错误预算剩余
13
      - record: slo:error_budget:remaining
14
        expr: |
15
          1 - (
16
            slo:sli_error:ratio_rate30d
17
            /
18
            (1 - 0.999)
19
          )
20

21
      # 燃烧率
22
      - record: slo:burn_rate:rate1h
23
        expr: |
24
          slo:sli_error:ratio_rate1h
25
          /
26
          (1 - 0.999)

1
# slo.yaml — 订单服务 SLO
2
spec:
3
  service: order-service
4
  slo:
5
    - name: availability
6
      objective: 99.9
7
      sli:
8
        events:
9
          error_query: sum(rate(http_requests_total{service="order-service",status=~"5.."}[{{.window}}]))
10
          total_query: sum(rate(http_requests_total{service="order-service"}[{{.window}}]))

1
# 生成 Prometheus 规则
2
sloth generate -i slo.yaml -o rules.yaml
3

4
# 应用规则
5
# 将 rules.yaml 添加到 Prometheus 配置

1
# 注入错误，模拟 SLO 违规
2
for i in $(seq 1 100); do
3
  curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/api/v1/error
4
done
5

6
# 观察燃烧率告警是否触发
7
curl http://localhost:9090/api/v1/alerts