1
# 传统监控：只知道"出错了"
2
curl -s http://api.example.com/health
3
# → HTTP 500
4

5
# 可观测性：知道"为什么出错了"
6
# 1. 指标：P99 延迟从 50ms 飙升到 2s
7
# 2. 追踪：发现 DB 查询耗时 1.8s
8
# 3. 日志：日志显示 "connection pool exhausted, waiting for connection"
9
# → 根因：连接池配置过小，高并发时耗尽

1
// 非结构化日志 — 难以搜索和分析
2
2026-06-18 03:14:22 ERROR Failed to process request for user 12345
3

4
// 结构化日志 — 可搜索、可分析、可关联
5
{
6
  "timestamp": "2026-06-18T03:14:22.456Z",
7
  "level": "ERROR",
8
  "message": "Failed to process request",
9
  "service": "order-service",
10
  "trace_id": "abc123def456",
11
  "span_id": "789ghi",
12
  "user_id": "12345",
13
  "error_code": "DB_CONNECTION_TIMEOUT",
14
  "duration_ms": 5000
15
}

1
# 查询过去 5 分钟的 P99 延迟
2
histogram_quantile(0.99,
3
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
4
)
5

6
# 查询错误率
7
sum(rate(http_requests_total{status=~"5.."}[5m]))
8
/
9
sum(rate(http_requests_total[5m]))

1
// 使用 OTel SDK 创建 Counter 和 Histogram
2
func initMetrics(meter metric.Meter) {
3
    // 请求计数器
4
    requestCounter, _ := meter.Int64Counter("http_requests_total",
5
        metric.WithDescription("HTTP 请求总数"),
6
    )
7
    // 请求延迟直方图
8
    latencyHist, _ := meter.Float64Histogram("http_request_duration_seconds",
9
        metric.WithDescription("HTTP 请求延迟"),
10
    )
11

12
    // 记录指标
13
    requestCounter.Add(ctx, 1, metric.WithAttributes(
14
        attribute.String("method", "GET"),
15
        attribute.Int("status", 200),
16
    ))
17
    latencyHist.Record(ctx, 0.15, metric.WithAttributes(
18
        attribute.String("method", "GET"),
19
    ))
20
}

1
{
2
  "trace_id": "abc123def456",
3
  "spans": [
4
    {
5
      "span_id": "span1",
6
      "operation": "API Gateway",
7
      "duration_ms": 2000,
8
      "children": [
9
        {
10
          "span_id": "span2",
11
          "operation": "order-service.process",
12
          "duration_ms": 1800,
13
          "children": [
14
            {
15
              "span_id": "span3",
16
              "operation": "db.query",
17
              "duration_ms": 1700
18
            }
19
          ]
20
        }
21
      ]
22
    }
23
  ]
24
}

1
                    ┌──────────┐
2
                    │ func_d() │
3
                ┌───┴──────────┴───┐
4
                │    func_b()      │
5
            ┌───┴──────────────────┴───┐
6
            │       func_a()           │
7
        ┌───┴──────────────────────────┴───┐
8
        │          main()                  │
9
        └──────────────────────────────────┘

1
# 使用系列提供的 Docker Compose 环境
2
cd observability-lab
3
docker compose up -d
4

5
# 验证所有组件
6
docker compose ps

1
# 发送一条测试追踪
2
curl -X POST http://localhost:4318/v1/traces \
3
  -H "Content-Type: application/json" \
4
  -d '{
5
    "resourceSpans": [{
6
      "resource": {
7
        "attributes": [
8
          {"key": "service.name", "value": {"stringValue": "demo-api"}},
9
          {"key": "service.version", "value": {"stringValue": "1.0.0"}}
10
        ]
11
      },
12
      "scopeSpans": [{
13
        "scope": {"name": "manual-test"},
14
        "spans": [{
15
          "traceId": "01234567890123456789012345678901",
16
          "spanId": "0123456789012345",
17
          "name": "test-request",
18
          "kind": 1,
19
          "startTimeUnixNano": "1700000000000000000",
20
          "endTimeUnixNano": "1700000001000000000",
21
          "attributes": [
22
            {"key": "http.method", "value": {"stringValue": "GET"}},
23
            {"key": "http.url", "value": {"stringValue": "/api/v1/orders"}}
24
          ]
25
        }]
26
      }]
27
    }]
28
  }'

1
# 打开 Grafana
2
open http://localhost:3000
3

4
# 验证步骤：
5
# 1. 进入 Explore 页面
6
# 2. 选择 Tempo 数据源
7
# 3. 搜索 TraceID: 01234567890123456789012345678901
8
# 4. 确认能看到追踪详情