1
# 查看延迟分布
2
histogram_quantile(0.50, sum(rate(http_request_duration_bucket[5m])) by (le, service))
3
histogram_quantile(0.90, sum(rate(http_request_duration_bucket[5m])) by (le, service))
4
histogram_quantile(0.99, sum(rate(http_request_duration_bucket[5m])) by (le, service))

1
# 按服务分解 P99
2
histogram_quantile(0.99, sum(rate(http_request_duration_bucket[5m])) by (le, service))

1
# 按服务分解
2
histogram_quantile(0.99, sum(rate(http_request_duration_bucket[5m])) by (le, service))
3

4
# 按路由分解
5
histogram_quantile(0.99, sum(rate(http_request_duration_bucket[5m])) by (le, route))
6

7
# 按方法分解
8
histogram_quantile(0.99, sum(rate(http_request_duration_bucket[5m])) by (le, method))

1
# 查看内存使用趋势
2
process_runtime_go_memory_heap_inuse_bytes{service="order-service"}
3
process_runtime_go_memory_heap_alloc_bytes{service="order-service"}

1
# GC 频率和耗时
2
rate(go_gc_duration_seconds_count[5m])
3
histogram_quantile(0.99, rate(go_gc_duration_seconds_bucket[5m]))

1
# Go: 生成堆 profile
2
curl http://localhost:6060/debug/pprof/heap > heap.prof
3
go tool pprof -http=:8080 heap.prof
4

5
# 查看分配最多的函数
6
(pprof) top20 -cum

1
# 查看 Goroutine 数量
2
curl http://localhost:6060/debug/pprof/goroutine?debug=1
3

4
# 输出示例：
5
# goroutine 12345 [chan receive, 10 minutes]:
6
# main.processOrder(...)
7
#   /app/order.go:42
8
#
9
# goroutine 12346 [chan receive, 10 minutes]:
10
# main.processOrder(...)
11
#   /app/order.go:42
12

13
# 如果大量 Goroutine 卡在同一个位置，说明是泄漏

1
// 常见的 Goroutine 泄漏模式
2
// 错误：没有退出条件
3
func processOrders(ch <-chan Order) {
4
    for order := range ch {
5
        process(order)  // 如果 ch 永远不关闭，Goroutine 永远不会退出
6
    }
7
}
8

9
// 正确：使用 context 控制退出
10
func processOrders(ctx context.Context, ch <-chan Order) {
11
    for {
12
        select {
13
        case order, ok := <-ch:
14
            if !ok {
15
                return  // channel 关闭时退出
16
            }
17
            process(order)
18
        case <-ctx.Done():
19
            return  // context 取消时退出
20
        }
21
    }
22
}

1
# 查看错误率趋势
2
sum(rate(http_requests_total{status=~"5.."}[5m])) by (service, status)
3
/
4
sum(rate(http_requests_total[5m])) by (service, status)

1
{service="order-service"} | json | level="ERROR" | error_code="TIMEOUT"

1
# 收集所有错误日志（带 TraceID）
2
{service="order-service"} | json | level="ERROR"
3
| line_format "{{.timestamp}} {{.trace_id}} {{.error_code}} {{.message}}"

1
# 验证 GC 停顿与错误的关联
2
# 计算 GC P99 与错误率的相关性
3
(
4
  histogram_quantile(0.99, rate(go_gc_duration_seconds_bucket[5m]))
5
  > 0.01  # GC 停顿超过 10ms
6
)
7
and
8
(
9
  sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m]))
10
  > 0.001  # 错误率超过 0.1%
11
)

1
# GC 耗时
2
histogram_quantile(0.99, rate(go_gc_duration_seconds_bucket[5m]))
3

4
# P99 延迟
5
histogram_quantile(0.99, sum(rate(http_request_duration_bucket[5m])) by (le))

1
# 查看 GC trace
2
GODEBUG=gctrace=1 ./my-app 2>gc.log
3

4
# 输出示例：
5
# gc 1 @0.005s 2%: 0.018+0.52+0.023 ms clock, 0.14+0.26/1.0/0.37+0.18 ms cpu, 4->4->1 MB, 5 MB goal, 8 P

1
// 优化 1：使用 sync.Pool 复用对象
2
var bufPool = sync.Pool{
3
    New: func() interface{} {
4
        return new(bytes.Buffer)
5
    },
6
}
7

8
func processRequest() {
9
    buf := bufPool.Get().(*bytes.Buffer)
10
    defer bufPool.Put(buf)
11
    buf.Reset()
12
    // 使用 buf...
13
}
14

15
// 优化 2：预分配切片
16
// 错误：动态扩容
17
items := []Item{}
18
for _, item := range data {
19
    items = append(items, item)  // 每次扩容都会分配新内存
20
}
21

22
// 正确：预分配
23
items := make([]Item, 0, len(data))
24
for _, item := range data {
25
    items = append(items, item)  // 无需扩容
26
}
27

28
// 优化 3：调整 GOGC
29
// 默认 GOGC=100，表示堆增长 100% 时触发 GC
30
// 对于延迟敏感的服务，可以降低 GOGC 以更频繁地 GC
31
// GOGC=50：堆增长 50% 时触发 GC，GC 更频繁但停顿更短

1
// 修复前
2
rdb := redis.NewClient(&redis.Options{
3
    Addr:         "redis:6379",
4
    PoolSize:     20,     // 太小
5
    MinIdleConns: 5,
6
})
7

8
// 修复后
9
rdb := redis.NewClient(&redis.Options{
10
    Addr:         "redis:6379",
11
    PoolSize:     100,    // 增加连接池
12
    MinIdleConns: 25,     // 增加空闲连接
13
    ReadTimeout:  500 * time.Millisecond,  // 设置读超时
14
    WriteTimeout: 500 * time.Millisecond,  // 设置写超时
15
})

1
// 修复前：原生 Map，无过期
2
var paymentCache = make(map[string]*Payment)
3

4
func cachePayment(p *Payment) {
5
    paymentCache[p.ID] = p  // 永远不会删除
6
}
7

8
// 修复后：带 TTL 的缓存
9
var paymentCache = ttlcache.NewCache[string, *Payment]()
10
paymentCache.SetTTL(24 * time.Hour)  // 24 小时过期
11

12
func cachePayment(p *Payment) {
13
    paymentCache.Set(p.ID, p)
14
}

1
# Elasticsearch 线程池配置修复
2
# 修复前
3
thread_pool.search.size: 5
4
thread_pool.search.queue_size: 100
5

6
# 修复后
7
thread_pool.search.size: 10
8
thread_pool.search.queue_size: 500

1
// 客户端重试
2
func searchWithRetry(ctx context.Context, query string, maxRetries int) (*SearchResult, error) {
3
    var lastErr error
4
    for i := 0; i < maxRetries; i++ {
5
        result, err := esClient.Search(ctx, query)
6
        if err == nil {
7
            return result, nil
8
        }
9
        lastErr = err
10
        // 指数退避
11
        time.Sleep(time.Duration(math.Pow(2, float64(i))) * 100 * time.Millisecond)
12
    }
13
    return nil, lastErr
14
}

1
# RED 方法
2
# Rate
3
sum(rate(http_requests_total[5m])) by (service)
4
# Errors
5
sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
6
# Duration
7
histogram_quantile(0.99, sum(rate(http_request_duration_bucket[5m])) by (le, service))
8

9
# USE 方法
10
# Utilization
11
1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m]))
12
# Saturation
13
avg(node_load5)
14
# Errors
15
rate(node_disk_io_time_seconds_total[5m])

1
# 注入延迟
2
curl -X POST http://localhost:8080/api/v1/orders -d '{"delay_ms": 2000}'

1
# 1. 在 Grafana 中查看 P99 延迟飙升
2
# 2. 点击 Exemplar 跳转到追踪
3
# 3. 在追踪中定位慢 Span
4
# 4. 查看关联日志
5
# 5. 确认根因

1
# 注入内存泄漏
2
curl -X POST http://localhost:8080/api/v1/leak -d '{"size_mb": 100}'
3

4
# 观察内存增长
5
curl http://localhost:9090/api/v1/query?query=process_resident_memory_bytes
6

7
# 生成堆 profile
8
curl http://localhost:6060/debug/pprof/heap > heap.prof
9
go tool pprof -http=:8080 heap.prof