1
# 普通 Histogram 桶
2
http_request_duration_bucket{le="0.5"} 80
3

4
# 带 Exemplar 的 Histogram 桶
5
http_request_duration_bucket{le="0.5"} 80 # {trace_id="abc123"} 0.45

1
# Prometheus 启用 Exemplar 存储
2
storage:
3
  tsdb:
4
    path: /prometheus
5
    # Exemplar 最大存储量，默认 0（不存储）
6
    # 建议设置为 100000（10 万条），约占 100MB 内存
7
    max_exemplars: 100000
8

9
# 也可以通过命令行参数
10
# --storage.tsdb.max-exemplars=100000

1
// Go: OTel SDK 自动在 Histogram 中附加 Exemplar
2
// 无需额外配置，OTel SDK 默认将当前 SpanContext 作为 Exemplar
3
var requestDuration metric.Float64Histogram
4

5
func handleRequest(w http.ResponseWriter, r *http.Request) {
6
    ctx, span := tracer.Start(r.Context(), "handleRequest")
7
    defer span.End()
8

9
    start := time.Now()
10
    // 处理请求...
11
    duration := time.Since(start).Seconds()
12

13
    // Record 会自动附加当前 SpanContext 作为 Exemplar
14
    requestDuration.Record(ctx, duration,
15
        metric.WithAttributes(
16
            semconv.HTTPMethodKey.String(r.Method),
17
            semconv.HTTPRouteKey.String("/api/v1/orders"),
18
        ),
19
    )
20
}

1
# Python: OTel SDK 自动附加 Exemplar
2
from opentelemetry import metrics, trace
3
from opentelemetry.sdk.metrics import MeterProvider
4

5
# 默认行为：Histogram 记录时自动附加当前 SpanContext
6
meter = metrics.get_meter("order-service")
7
request_duration = meter.create_histogram(
8
    name="http.server.request.duration",
9
    unit="s",
10
    description="HTTP 请求延迟分布",
11
)
12

13
def handle_request(request):
14
    tracer = trace.get_tracer("order-service")
15
    with tracer.start_as_current_span("handle_request") as span:
16
        # 处理请求...
17
        request_duration.record(0.45, {"method": "GET", "route": "/api/orders"})
18
        # Exemplar 自动附加当前 span 的 trace_id

1
// Go: 自动注入 TraceID 到 slog
2
type OTelHandler struct {
3
    inner slog.Handler
4
}
5

6
func (h OTelHandler) Handle(ctx context.Context, r slog.Record) error {
7
    span := trace.SpanFromContext(ctx)
8
    if span.SpanContext().IsValid() {
9
        r.AddAttrs(
10
            slog.String("trace_id", span.SpanContext().TraceID().String()),
11
            slog.String("span_id", span.SpanContext().SpanID().String()),
12
        )
13
    }
14
    return h.inner.Handle(ctx, r)
15
}

1
// Java: 使用 MDC (Mapped Diagnostic Context) 注入 TraceID
2
// OTel Java Agent 自动将 trace_id/span_id 注入 MDC
3
// logback-spring.xml 配置
4
<configuration>
5
    <appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
6
        <encoder class="net.logstash.logback.encoder.LogstashEncoder">
7
            <!-- 自动从 MDC 提取 trace_id -->
8
            <includeMdcKeyName>trace_id</includeMdcKeyName>
9
            <includeMdcKeyName>span_id</includeMdcKeyName>
10
            <includeMdcKeyName>trace_flags</includeMdcKeyName>
11
        </encoder>
12
    </appender>
13
</configuration>

1
# Python: 使用 structlog + OTel 注入
2
import structlog
3
from opentelemetry import trace
4

5
def add_trace_id(logger, method_name, event_dict):
6
    """自定义 processor：注入 trace_id 和 span_id"""
7
    span = trace.get_current_span()
8
    if span.is_recording():
9
        ctx = span.get_span_context()
10
        event_dict["trace_id"] = format(ctx.trace_id, "032x")
11
        event_dict["span_id"] = format(ctx.span_id, "016x")
12
    return event_dict
13

14
structlog.configure(
15
    processors=[
16
        add_trace_id,          # 注入 TraceID
17
        structlog.processors.JSONRenderer(),  # JSON 输出
18
    ],
19
)

1
// Node.js: 使用 OTel API 注入 TraceID
2
import { trace, context } from '@opentelemetry/api';
3
import pino from 'pino';
4

5
const logger = pino({
6
  formatters: {
7
    log(object) {
8
      const span = trace.getSpan(context.active());
9
      if (span) {
10
        const { traceId, spanId } = span.spanContext();
11
        object.trace_id = traceId;
12
        object.span_id = spanId;
13
      }
14
      return object;
15
    },
16
  },
17
});

1
# Grafana 数据源关联配置
2
apiVersion: 1
3
datasources:
4
  - name: Tempo
5
    type: tempo
6
    uid: tempo
7
    jsonData:
8
      tracesToMetrics:
9
        datasourceUid: prometheus
10
        tags:
11
          - key: service
12
            value: service
13
      tracesToLogs:
14
        datasourceUid: loki
15
        filterByTraceID: true
16
        filterBySpanID: true
17
        tags:
18
          - key: service
19
            value: service
20
      serviceMap:
21
        datasourceUid: prometheus
22

23
  - name: Loki
24
    type: loki
25
    uid: loki
26
    jsonData:
27
      derivedFields:
28
        - name: TraceID
29
          matcherRegex: '"trace_id":"(\w+)"'
30
          url: '$${__value.raw}'
31
          datasourceUid: tempo
32

33
  - name: Prometheus
34
    type: prometheus
35
    uid: prometheus
36
    jsonData:
37
      exemplarTraceIdDestinations:
38
        - name: trace_id
39
          datasourceUid: tempo
40
          urlDisplayLabel: 'View Trace'

1
{
2
  "span_id": "span1",
3
  "attributes": {
4
    "profile_id": "prof_abc123",
5
    "profile.type": "cpu"
6
  }
7
}

1
# Tempo 数据源配置：追踪 → 性能分析
2
- name: Tempo
3
  type: tempo
4
  uid: tempo
5
  jsonData:
6
    tracesToProfiles:
7
      datasourceUid: pyroscope
8
      # Profile 类型：cpu、memory、goroutine 等
9
      profileType: process_cpu:cpu:nanoseconds:cpu:nanoseconds
10
      # 自定义标签映射
11
      customQueryParameters:
12
        service: service.name
13
      # 时间范围偏移（Profile 采集间隔可能不同）
14
      spanEndTimeShift: '-30s'
15
      spanStartTimeShift: '30s'

1
# Pyroscope 数据源配置
2
- name: Pyroscope
3
  type: grafana-pyroscope-datasource
4
  uid: pyroscope
5
  url: http://pyroscope:4040

1
{
2
  "timestamp": "2026-06-27T02:15:32.456Z",
3
  "level": "WARN",
4
  "service": "order-service",
5
  "trace_id": "7f3b2a1c4d5e6f7890abcdef12345678",
6
  "span_id": "a1b2c3d4e5f67890",
7
  "message": "connection pool exhausted, waiting for available connection",
8
  "wait_time_ms": 1820,
9
  "pool_size": 50,
10
  "active_connections": 50,
11
  "pending_requests": 30
12
}

1
// 修复前
2
db.SetMaxOpenConns(50)
3

4
// 修复后
5
db.SetMaxOpenConns(100)
6
db.SetMaxIdleConns(25)  // 同时增加空闲连接数
7
db.SetConnMaxLifetime(5 * time.Minute)  // 连接最大存活时间

1
// 服务 A：下划线格式
2
{"trace_id": "abc123", "span_id": "def456"}
3

4
// 服务 B：驼峰格式
5
{"traceId": "abc123", "spanId": "def456"}
6

7
// 服务 C：嵌套格式
8
{"context": {"trace": {"id": "abc123"}, "span": {"id": "def456"}}}

1
# OTel Collector: 统一日志中的 TraceID 格式
2
processors:
3
  transform/log_format:
4
    error_mode: ignore
5
    log_statements:
6
      - context: log
7
        statements:
8
          # 将所有格式统一为 trace_id（下划线）
9
          - set(attributes["trace_id"], attributes["traceId"]) where attributes["traceId"] != nil
10
          - set(attributes["span_id"], attributes["spanId"]) where attributes["spanId"] != nil

1
// 错误：用 Gauge 记录延迟，无法使用 Exemplar
2
latencyGauge.Set(ctx, duration)
3

4
// 正确：用 Histogram 记录延迟，自动附加 Exemplar
5
latencyHistogram.Record(ctx, duration)

1
# OTel Collector: 尾部采样配置
2
processors:
3
  tail_sampling:
4
    decision_wait: 10s
5
    num_traces: 100000
6
    policies:
7
      # 错误追踪总是保留
8
      - name: errors
9
        type: status_code
10
        status_code:
11
          status_codes:
12
            - ERROR
13
      # 慢请求追踪总是保留
14
      - name: slow-requests
15
        type: latency
16
        latency:
17
          threshold_ms: 1000
18
      # 其他追踪 10% 采样
19
      - name: default
20
        type: probabilistic
21
        probabilistic:
22
          sampling_percentage: 10

1
// Go: 自动化关联测试
2
func TestTraceIDCorrelation(t *testing.T) {
3
    // 1. 发送一个带追踪的请求
4
    ctx, span := tracer.Start(context.Background(), "test-request")
5
    traceID := span.SpanContext().TraceID().String()
6

7
    // 2. 记录指标（带 Exemplar）
8
    requestDuration.Record(ctx, 0.5)
9

10
    // 3. 记录日志（带 TraceID）
11
    slog.InfoContext(ctx, "test log", "key", "value")
12

13
    // 4. 验证 Tempo 中有该 TraceID
14
    trace, err := tempoClient.GetTrace(traceID)
15
    require.NoError(t, err)
16
    require.NotNil(t, trace)
17

18
    // 5. 验证 Loki 中有该 TraceID 的日志
19
    logs, err := lokiClient.Query(`{service="test"} |= "` + traceID + `"`)
20
    require.NoError(t, err)
21
    require.NotEmpty(t, logs)
22

23
    span.End()
24
}

1
# 更新 Grafana 数据源配置
2
docker compose restart grafana
3

4
# 打开 Grafana
5
open http://localhost:3000/explore

1
# 1. 在 Prometheus 面板中点击 Exemplar
2
# 2. 验证跳转到 Tempo 追踪
3
# 3. 在追踪中点击 Span
4
# 4. 验证查看关联日志
5
# 5. 在日志中点击 TraceID
6
# 6. 验证跳转回追踪

1
# 发送一个带追踪的请求，验证完整的关联链路
2
curl -X POST http://localhost:8080/api/v1/orders \
3
  -H "Content-Type: application/json" \
4
  -H "traceparent: 00-7f3b2a1c4d5e6f7890abcdef12345678-a1b2c3d4e5f67890-01" \
5
  -d '{"item": "test-order"}'
6

7
# 在 Tempo 中查询该 TraceID
8
curl http://localhost:3200/api/traces/7f3b2a1c4d5e6f7890abcdef12345678
9

10
# 在 Loki 中查询该 TraceID 的日志
11
curl "http://localhost:3100/loki/api/v1/query_range?query={service=\"order-service\"}+%7C%3D+%227f3b2a1c4d5e6f7890abcdef12345678%22"