1
# tempo.yaml — 生产级配置
2
server:
3
  http_listen_port: 3200
4

5
distributor:
6
  receivers:
7
    otlp:
8
      protocols:
9
        grpc:
10
          endpoint: 0.0.0.0:4319
11
        http:
12
          endpoint: 0.0.0.0:4320
13

14
ingester:
15
  max_block_duration: 30m
16
  trace_idle_period: 30s
17
  max_block_bytes: 100MB
18

19
storage:
20
  trace:
21
    backend: s3
22
    s3:
23
      bucket: tempo-traces
24
      endpoint: s3.amazonaws.com
25
    wal:
26
      path: /var/tempo/wal
27

28
compactor:
29
  compaction:
30
    block_retention: 720h  # 30 天保留
31

32
metrics_generator:
33
  registry:
34
    external_labels:
35
      source: tempo
36
  storage:
37
    path: /var/tempo/generator/wal
38
    remote_write:
39
      - url: http://mimir:9009/api/v1/push

1
# 基础查询：按服务名查找
2
{resource.service.name = "order-service"}
3

4
# 按状态码查找错误
5
{status = error}
6

7
# 按延迟查找慢请求
8
{duration > 1s}
9

10
# 组合查询：特定服务的慢请求
11
{resource.service.name = "order-service" && duration > 500ms}
12

13
# 按 Span 属性查询
14
{span.http.method = "POST" && span.http.status_code >= 500}
15

16
# 嵌套查询：调用链关系
17
{resource.service.name = "frontend"} -> {resource.service.name = "order-service" && duration > 1s}

1
# 基础查询
2
{service="order-service"} |= "ERROR"
3

4
# 结构化查询
5
{service="order-service"} | json | level="ERROR" | error_code="DB_TIMEOUT"
6

7
# 聚合查询
8
sum(count_over_time({service="order-service"} | json | level="ERROR" [5m])) by (error_code)
9

10
# 延迟分析
11
quantile_over_time(0.99, {service="order-service"} | json | unwrap duration_ms [5m])
12

13
# 追踪关联
14
{service="order-service"} | json | trace_id="abc123"
15

16
# Top N 热点路径
17
topk(10, sum(rate({service="order-service"} | json | unwrap duration_ms [5m])) by (http_route))
18

19
# 错误率趋势
20
sum(rate({service="order-service"} | json | level="ERROR" [5m])) / sum(rate({service="order-service"} | json [5m])) * 100

1
# 差：全量扫描
2
rate(http_request_duration_seconds_sum[5m])
3

4
# 好：按服务过滤
5
rate(http_request_duration_seconds_sum{service="order-service"}[5m])
6

7
# 更好：按服务 + 方法过滤
8
rate(http_request_duration_seconds_sum{service="order-service",method="POST"}[5m])
9

10
# 差：高基数 Group By
11
sum(rate(http_request_duration_seconds_sum[5m])) by (user_id)
12

13
# 好：低基数 Group By
14
sum(rate(http_request_duration_seconds_sum[5m])) by (service, method)

1
# 差：全扫描
2
{duration > 100ms}
3

4
# 好：按服务 + 时间范围
5
{resource.service.name = "order-service" && duration > 100ms}
6

7
# 更好：按 TraceID 直接查询
8
# 从指标/日志中获取 TraceID，直接查询

1
# 差：全量扫描 + 正则匹配
2
{service="order-service"} |~ "error|timeout"
3

4
# 好：结构化查询
5
{service="order-service"} | json | level="ERROR"
6

7
# 更好：精确标签过滤
8
{service="order-service", level="ERROR"}

1
# VictoriaMetrics 保留配置
2
-retentionPeriod=90d           # 全局保留 90 天
3
-downsampling.period=5m:30d    # 5 分钟降采样保留 30 天
4
-downsampling.period=1h:90d    # 1 小时降采样保留 90 天

1
# Tempo 保留配置
2
compactor:
3
  compaction:
4
    block_retention: 720h      # 30 天保留
5
    compaction_window: 1h      # 1 小时合并窗口
6
    max_block_bytes: 100MB
7
    compacted_block_retention: 1h  # 合并后旧块保留 1 小时

1
# Loki 保留配置
2
limits_config:
3
  - max_query_length: 721h    # 最大查询范围 30 天
4
  retention_period: 744h       # 保留 31 天
5
compactor:
6
  working_directory: /var/loki/compactor
7
  compaction_interval: 10m
8
  retention_enabled: true
9
  retention_delete_delay: 2h
10
  delete_request_cancel_period: 24h

1
# VictoriaMetrics 降采样配置
2
downsampling:
3
  - resolution: 5m
4
    retention: 30d
5
  - resolution: 1h
6
    retention: 90d
7
  - resolution: 1d
8
    retention: 365d

1
docker compose up -d prometheus tempo loki
2

3
# 验证各组件
4
curl http://localhost:9090/api/v1/query?query=up
5
curl http://localhost:3200/ready
6
curl http://localhost:3100/ready

1
# 发送追踪 → Tempo
2
curl -X POST http://localhost:4318/v1/traces \
3
  -H "Content-Type: application/json" \
4
  -d '{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"test"}}]},"scopeSpans":[{"spans":[{"traceId":"01234567890123456789012345678901","spanId":"0123456789012345","name":"test","kind":1,"startTimeUnixNano":"1700000000000000000","endTimeUnixNano":"1700000001000000000"}]}]}]}'
5

6
# 在 Grafana 中验证
7
open http://localhost:3000/explore