1
http_request_duration_seconds{method="GET", status="200", endpoint="/api/v1/orders"} 0.150
2
│                                   │                                                │
3
│                                   │                                                └─ 数值
4
│                                   └─ 标签（Labels / 维度）
5
└─ 指标名称（Metric Name）

1
// Go: 使用 OTel SDK 定义 Counter
2
import (
3
    "go.opentelemetry.io/otel/metric"
4
    "go.opentelemetry.io/otel"
5
)
6

7
var requestCounter metric.Int64Counter
8

9
func init() {
10
    meter := otel.Meter("order-service")
11
    requestCounter, _ = meter.Int64Counter(
12
        "http.server.request.total",
13
        metric.WithDescription("Total number of HTTP requests"),
14
    )
15
}
16

17
func handleRequest(w http.ResponseWriter, r *http.Request) {
18
    // 递增 Counter
19
    requestCounter.Add(r.Context(), 1,
20
        metric.WithAttributes(
21
            semconv.HTTPMethodKey.String(r.Method),
22
            semconv.HTTPStatusCodeKey.Int(200),
23
            semconv.HTTPRouteKey.String("/api/v1/orders"),
24
        ),
25
    )
26
}

1
# 正确：查询请求速率
2
rate(http_server_request_total{service="order-service"}[5m])
3

4
# 错误：查询 Counter 绝对值
5
http_server_request_total{service="order-service"}

1
var activeConnections metric.Int64UpDownCounter
2

3
func handleConnection(conn net.Conn) {
4
    activeConnections.Add(context.Background(), 1)
5
    defer activeConnections.Add(context.Background(), -1)
6

7
    // 处理连接...
8
}

1
// 定义 Histogram
2
var requestDuration metric.Float64Histogram
3

4
func init() {
5
    meter := otel.Meter("order-service")
6
    requestDuration, _ = meter.Float64Histogram(
7
        "http.server.request.duration",
8
        metric.WithUnit("s"),
9
        metric.WithDescription("HTTP request duration in seconds"),
10
        // 自定义分桶（默认分桶通常不够用）
11
        metric.WithExplicitBucketBoundaries(
12
            0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
13
        ),
14
    )
15
}
16

17
func handleRequest(w http.ResponseWriter, r *http.Request) {
18
    start := time.Now()
19
    // 处理请求...
20
    duration := time.Since(start).Seconds()
21

22
    requestDuration.Record(r.Context(), duration,
23
        metric.WithAttributes(
24
            semconv.HTTPMethodKey.String(r.Method),
25
            semconv.HTTPRouteKey.String("/api/v1/orders"),
26
        ),
27
    )
28
}

1
# 一个 Histogram 产生 N+2 个时间序列（N = 桶数）
2
http_server_request_duration_bucket{le="0.005"} 10     # ≤5ms: 10 个请求
3
http_server_request_duration_bucket{le="0.01"}  25     # ≤10ms: 25 个请求
4
http_server_request_duration_bucket{le="0.025"} 50     # ≤25ms: 50 个请求
5
http_server_request_duration_bucket{le="0.05"}  80     # ≤50ms: 80 个请求
6
http_server_request_duration_bucket{le="0.1"}   95     # ≤100ms: 95 个请求
7
http_server_request_duration_bucket{le="0.25"}  99     # ≤250ms: 99 个请求
8
http_server_request_duration_bucket{le="0.5"}   100    # ≤500ms: 100 个请求
9
http_server_request_duration_bucket{le="+Inf"}  100    # 总请求数
10
http_server_request_duration_sum              3.5      # 总耗时
11
http_server_request_duration_count            100      # 总请求数

1
# 使用 histogram_quantile 计算 P99
2
histogram_quantile(0.99,
3
  sum(rate(http_server_request_duration_bucket[5m])) by (le, service, endpoint)
4
)

1
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, +Inf

1
总序列数 = 指标数 × ∏(每个标签的唯一值数)

1
# OTel Collector: 只允许低基数标签通过
2
processors:
3
  filter:
4
    error_mode: ignore
5
    metrics:
6
      metric:
7
        - 'name == "http.server.request.duration" and attributes["user_id"] != nil'
8
        # 移除高基数标签
9
  transform:
10
    error_mode: ignore
11
    metric_statements:
12
      - context: datapoint
13
        statements:
14
          - delete_key(attributes, "user_id")
15
          - delete_key(attributes, "request_id")
16
          - delete_key(attributes, "ip_address")

1
// 将 IP 地址降维为地区
2
func ipToRegion(ip string) string {
3
    // 使用 GeoIP 数据库将 IP 映射为地区
4
    region := geoip.Lookup(ip).Region
5
    return region  // "us-east-1", "eu-west-1" 等
6
}
7

8
// 使用降维后的标签
9
requestDuration.Record(ctx, duration,
10
    metric.WithAttributes(
11
        semconv.HTTPMethodKey.String(method),
12
        attribute.String("client_region", ipToRegion(clientIP)),
13
    ),
14
)

1
<namespace>_<subsystem>_<name>_<unit>

1
# 1. 请求速率（QPS）
2
sum(rate(http_server_request_total[5m])) by (service)
3

4
# 2. 错误率
5
sum(rate(http_server_request_total{status=~"5.."}[5m])) by (service)
6
/
7
sum(rate(http_server_request_total[5m])) by (service)
8

9
# 3. P50 / P90 / P99 延迟
10
histogram_quantile(0.50, sum(rate(http_server_request_duration_bucket[5m])) by (le, service))
11
histogram_quantile(0.90, sum(rate(http_server_request_duration_bucket[5m])) by (le, service))
12
histogram_quantile(0.99, sum(rate(http_server_request_duration_bucket[5m])) by (le, service))
13

14
# 4. 延迟 SLO 达成率（P99 < 500ms）
15
sum(rate(http_server_request_duration_bucket{le="0.5"}[5m])) by (service)
16
/
17
sum(rate(http_server_request_duration_count[5m])) by (service)
18

19
# 5. RED 方法（Rate-Errors-Duration）
20
# Rate
21
sum(rate(http_server_request_total[5m])) by (service, endpoint)
22
# Errors
23
sum(rate(http_server_request_total{status=~"5.."}[5m])) by (service, endpoint)
24
# Duration
25
histogram_quantile(0.99, sum(rate(http_server_request_duration_bucket[5m])) by (le, service, endpoint))
26

27
# 6. USE 方法（Utilization-Saturation-Errors）— 适用于资源指标
28
# Utilization: CPU 使用率
29
1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m]))
30
# Saturation: 运行队列长度
31
avg(node_load5)
32
# Errors: 设备错误
33
rate(node_disk_io_time_seconds_total[5m])

1
SLO: 99.9% 的请求在 500ms 内完成
2
  └─ SLI: 请求延迟 < 500ms 的比例
3
       └─ 指标: histogram_quantile(0.999, rate(http_server_request_duration_bucket[5m]))

1
# 使用 Docker Compose 启动 Prometheus
2
docker compose up -d prometheus
3

4
# 验证 Prometheus 运行
5
curl http://localhost:9090/api/v1/query?query=up

1
# recording_rules.yaml — 预计算常用查询
2
groups:
3
  - name: service:red
4
    interval: 30s
5
    rules:
6
      # 请求速率
7
      - record: service:http_requests:rate5m
8
        expr: sum(rate(http_server_request_total[5m])) by (service, endpoint)
9

10
      # 错误率
11
      - record: service:http_errors:rate5m
12
        expr: sum(rate(http_server_request_total{status=~"5.."}[5m])) by (service, endpoint)
13

14
      # P99 延迟
15
      - record: service:http_request_duration:p99
16
        expr: histogram_quantile(0.99, sum(rate(http_server_request_duration_bucket[5m])) by (le, service, endpoint))

1
# 在 Prometheus UI 中验证
2
open http://localhost:9090
3

4
# 查询示例
5
# 1. 所有服务的请求速率
6
curl 'http://localhost:9090/api/v1/query?query=sum(rate(http_server_request_total[5m]))%20by%20(service)'
7

8
# 2. P99 延迟
9
curl 'http://localhost:9090/api/v1/query?query=histogram_quantile(0.99,sum(rate(http_server_request_duration_bucket[5m]))%20by%20(le,service))'
10

11
# 3. 基数检查
12
curl 'http://localhost:9090/api/v1/query?query=count(http_server_request_duration_bucket)'