1
{
2
  "trace_id": "abc123def45678901234567890123456",
3
  "span_id": "789ghi01234567890",
4
  "parent_span_id": "0123456789abcdef0",
5
  "operation_name": "OrderService.ProcessOrder",
6
  "kind": "SERVER",
7
  "start_time": "2026-06-21T03:14:22.000Z",
8
  "end_time": "2026-06-21T03:14:23.800Z",
9
  "status": {
10
    "code": "ERROR",
11
    "message": "Database connection timeout"
12
  },
13
  "attributes": {
14
    "http.method": "POST",
15
    "http.url": "/api/v1/orders",
16
    "http.status_code": 500,
17
    "db.system": "postgresql",
18
    "db.operation": "INSERT",
19
    "error.type": "timeout"
20
  },
21
  "events": [
22
    {
23
      "name": "exception",
24
      "timestamp": "2026-06-21T03:14:23.790Z",
25
      "attributes": {
26
        "exception.type": "java.sql.SQLException",
27
        "exception.message": "Connection pool exhausted"
28
      }
29
    }
30
  ],
31
  "links": [],
32
  "resource": {
33
    "service.name": "order-service",
34
    "service.version": "v2.3.1",
35
    "host.name": "pod-order-7b8c9"
36
  }
37
}

1
traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01
2
              │  │                       │                │         │
3
              │  │                       │                │         └─ trace-flags
4
              │  │                       │                └─ parent-id
5
              │  │                       └─ trace-id
6
              │  └─ version
7
              └─ 格式前缀

1
// Go: W3C TraceContext 传播
2
import (
3
    "go.opentelemetry.io/otel"
4
    "go.opentelemetry.io/otel/propagation"
5
    "go.opentelemetry.io/otel/trace"
6
)
7

8
func makeHTTPRequest(ctx context.Context, url string) (*http.Response, error) {
9
    req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
10
    if err != nil {
11
        return nil, err
12
    }
13

14
    // 将当前 Span 上下文注入 HTTP 头部
15
    // 这会自动设置 traceparent 和 tracestate 头部
16
    otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(req.Header))
17

18
    return http.DefaultClient.Do(req)
19
}
20

21
func handleHTTPRequest(w http.ResponseWriter, r *http.Request) {
22
    // 从 HTTP 头部提取 Span 上下文
23
    ctx := otel.GetTextMapPropagator().Extract(
24
        r.Context(),
25
        propagation.HeaderCarrier(r.Header),
26
    )
27

28
    // 创建子 Span
29
    ctx, span := otel.Tracer("order-service").Start(ctx, "ProcessOrder")
30
    defer span.End()
31

32
    // 处理请求...
33
}

1
tracestate: vendor1=value1,vendor2=value2

1
# OTel Collector: 概率采样
2
processors:
3
  probabilistic_sampler:
4
    sampling_percentage: 10  # 10% 采样率

1
# OTel Collector: 尾部采样
2
processors:
3
  tail_sampling:
4
    decision_wait: 10s        # 等待 10s 收集所有 Span
5
    num_traces: 100000        # 缓冲区大小
6
    expected_new_traces_per_sec: 1000
7
    policies:
8
      # 策略 1: 错误请求全量采集
9
      - name: errors
10
        type: status_code
11
        status_code:
12
          status_codes:
13
            - ERROR
14
      # 策略 2: 慢请求全量采集（> 1s）
15
      - name: slow-requests
16
        type: latency
17
        latency:
18
          threshold_ms: 1000
19
      # 策略 3: 正常请求 10% 采样
20
      - name: normal-requests
21
        type: probabilistic
22
        probabilistic:
23
          sampling_percentage: 10

1
# OTel Collector: 自适应采样（实验性）
2
processors:
3
  adaptive_sampling:
4
    sampling_rate:
5
      min: 0.01    # 最低 1%
6
      max: 1.0     # 最高 100%
7
    target_throughput: 100  # 目标: 每秒 100 条追踪

1
// 反模式：一个 Span 覆盖整个请求处理
2
ctx, span := tracer.Start(ctx, "handleRequest")
3
defer span.End()
4
// ... 100 行代码 ...
5

6
// 最佳实践：关键操作各自创建 Span
7
ctx, span := tracer.Start(ctx, "handleRequest")
8
defer span.End()
9

10
ctx, dbSpan := tracer.Start(ctx, "db.QueryOrders")
11
// 数据库查询
12
dbSpan.End()
13

14
ctx, cacheSpan := tracer.Start(ctx, "cache.Lookup")
15
// 缓存查询
16
cacheSpan.End()

1
// 反模式：每个函数调用都创建 Span
2
ctx, span1 := tracer.Start(ctx, "validateInput")
3
// 验证输入
4
span1.End()
5

6
ctx, span2 := tracer.Start(ctx, "parseRequest")
7
// 解析请求
8
span2.End()
9

10
ctx, span3 := tracer.Start(ctx, "buildResponse")
11
// 构建响应
12
span3.End()

1
// 反模式：启动 goroutine 时不传播上下文
2
go func() {
3
    // 这个 goroutine 中的 Span 不会关联到原始追踪
4
    processOrder(order)
5
}()
6

7
// 最佳实践：传播上下文
8
go func(ctx context.Context) {
9
    ctx, span := tracer.Start(ctx, "processOrder")
10
    defer span.End()
11
    processOrder(ctx, order)
12
}(ctx)

1
docker compose up -d tempo
2

3
# 验证 Tempo 运行
4
curl http://localhost:3200/ready

1
# 通过 OTel Collector 发送追踪
2
curl -X POST http://localhost:4318/v1/traces \
3
  -H "Content-Type: application/json" \
4
  -d '{
5
    "resourceSpans": [{
6
      "resource": {
7
        "attributes": [
8
          {"key": "service.name", "value": {"stringValue": "order-service"}},
9
          {"key": "service.version", "value": {"stringValue": "v2.3.1"}}
10
        ]
11
      },
12
      "scopeSpans": [{
13
        "scope": {"name": "io.opentelemetry.sdk.trace"},
14
        "spans": [{
15
          "traceId": "0af7651916cd43dd8448eb211c80319c",
16
          "spanId": "b7ad6b7169203331",
17
          "parentSpanId": "",
18
          "name": "GET /api/v1/orders",
19
          "kind": 2,
20
          "startTimeUnixNano": "1700000000000000000",
21
          "endTimeUnixNano": "1700000002000000000",
22
          "status": {"code": 1},
23
          "attributes": [
24
            {"key": "http.method", "value": {"stringValue": "GET"}},
25
            {"key": "http.status_code", "value": {"intValue": "200"}}
26
          ]
27
        }]
28
      }]
29
    }]
30
  }'

1
# 打开 Grafana Explore
2
open http://localhost:3000/explore
3

4
# 查询方式：
5
# 1. 选择 Tempo 数据源
6
# 2. Search 模式：{ service.name = "order-service" }
7
# 3. TraceID 模式：0af7651916cd43dd8448eb211c80319c