1
# 搜索"用户 12345 的订单失败"——需要正则匹配
2
grep "user_id=12345.*order.*failed" /var/log/app.log
3
# 在 TB 级日志中，这可能需要数十分钟

1
# 统计各错误码的出现次数——需要复杂的正则和 awk
2
grep "error_code=" /var/log/app.log | \
3
  sed 's/.*error_code=\([^ ]*\).*/\1/' | \
4
  sort | uniq -c | sort -rn

1
# 找到某个 TraceID 的所有日志——如果日志里没有 TraceID，就做不到
2
grep "trace_id=abc123" /var/log/app.log

1
{
2
  "timestamp": "2026-06-19T03:14:22.456Z",
3
  "level": "ERROR",
4
  "service": "order-service",
5
  "trace_id": "abc123def456",
6
  "span_id": "789ghi012",
7
  "user_id": "12345",
8
  "order_id": "ORD-67890",
9
  "error_code": "DB_CONNECTION_TIMEOUT",
10
  "message": "Failed to process order",
11
  "duration_ms": 5000,
12
  "db_pool_active": 50,
13
  "db_pool_max": 50
14
}

1
# 推荐命名（遵循 OTel 语义约定）
2
service.name: order-service
3
service.version: v2.3.1
4
http.method: GET
5
http.status_code: 500
6
http.url: /api/v1/orders/12345
7
db.system: postgresql
8
db.operation: SELECT
9
error.code: DB_CONNECTION_TIMEOUT
10

11
# 不推荐命名（不一致、难搜索）
12
svc: order-service
13
ver: v2.3.1
14
method: GET
15
status: 500
16
url: /api/v1/orders/12345
17
db: pg
18
op: select
19
err: DB_TIMEOUT

1
// Go: 使用 OTel SDK 自动注入
2
import (
3
    "go.opentelemetry.io/otel"
4
    "go.opentelemetry.io/otel/trace"
5
    "log/slog"
6
)
7

8
func setupLogger() *slog.Logger {
9
    handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
10
        Level: slog.LevelInfo,
11
    })
12

13
    // 包装 handler，自动注入 TraceID 和 SpanID
14
    return slog.New(otelHandler{handler})
15
}
16

17
type otelHandler struct {
18
    inner slog.Handler
19
}
20

21
func (h otelHandler) Handle(ctx context.Context, r slog.Record) error {
22
    span := trace.SpanFromContext(ctx)
23
    if span.SpanContext().IsValid() {
24
        r.AddAttrs(
25
            slog.String("trace_id", span.SpanContext().TraceID().String()),
26
            slog.String("span_id", span.SpanContext().SpanID().String()),
27
        )
28
    }
29
    return h.inner.Handle(ctx, r)
30
}

1
# Python: 手动注入关联 ID
2
import logging
3
import json
4
from contextvars import ContextVar
5

6
trace_id_var: ContextVar[str] = ContextVar('trace_id', default='')
7
span_id_var: ContextVar[str] = ContextVar('span_id', default='')
8

9
class StructuredFormatter(logging.Formatter):
10
    def format(self, record):
11
        log_entry = {
12
            'timestamp': self.formatTime(record),
13
            'level': record.levelname,
14
            'message': record.getMessage(),
15
            'service': 'order-service',
16
            'trace_id': trace_id_var.get(''),
17
            'span_id': span_id_var.get(''),
18
        }
19
        if hasattr(record, 'props'):
20
            log_entry.update(record.props)
21
        if record.exc_info:
22
            log_entry['stack_trace'] = self.formatException(record.exc_info)
23
        return json.dumps(log_entry)
24

25
# 使用
26
def handle_request(request):
27
    trace_id_var.set(request.headers.get('traceparent', '').split('-')[1] if 'traceparent' in request.headers else '')
28
    logger.info("Processing request", extra={
29
        'props': {
30
            'user_id': request.user_id,
31
            'order_id': request.order_id,
32
        }
33
    })

1
// Java: Spring Boot 中间件注入
2
@Component
3
public class TraceIdFilter implements Filter {
4

5
    @Override
6
    public void doFilter(ServletRequest request, ServletResponse response,
7
                         FilterChain chain) throws IOException, ServletException {
8
        HttpServletRequest httpRequest = (HttpServletRequest) request;
9

10
        // 从 W3C TraceContext 头部提取 TraceID
11
        String traceparent = httpRequest.getHeader("traceparent");
12
        String traceId = extractTraceId(traceparent);
13

14
        // 注入到 MDC（Mapped Diagnostic Context）
15
        MDC.put("trace_id", traceId);
16
        MDC.put("span_id", UUID.randomUUID().toString().replace("-", "").substring(0, 16));
17

18
        try {
19
            chain.doFilter(request, response);
20
        } finally {
21
            MDC.clear();
22
        }
23
    }
24

25
    // logback.xml 配置
26
    // <pattern>{"timestamp":"%d","level":"%-5level","trace_id":"%X{trace_id}","span_id":"%X{span_id}","message":"%msg"}%n</pattern>
27
}

1
traceparent: 00-abc123def45678901234567890123456-789ghi01234567890-01
2
              │  │                       │                │         │
3
              │  │                       │                │         └─ 采样标志
4
              │  │                       │                └─ 父 Span ID
5
              │  │                       └─ Trace ID (32 hex)
6
              │  └─ 版本号
7
              └─ 格式前缀

1
package main
2

3
import (
4
    "context"
5
    "log/slog"
6
    "os"
7
    "time"
8
    "go.opentelemetry.io/otel/trace"
9
)
10

11
// 自定义 Handler，自动注入 OTel 上下文
12
type OTelHandler struct {
13
    inner slog.Handler
14
}
15

16
func (h OTelHandler) Enabled(ctx context.Context, level slog.Level) bool {
17
    return h.inner.Enabled(ctx, level)
18
}
19

20
func (h OTelHandler) Handle(ctx context.Context, r slog.Record) error {
21
    // 自动注入 TraceID 和 SpanID
22
    span := trace.SpanFromContext(ctx)
23
    if span.SpanContext().IsValid() {
24
        r.AddAttrs(
25
            slog.String("trace_id", span.SpanContext().TraceID().String()),
26
            slog.String("span_id", span.SpanContext().SpanID().String()),
27
        )
28
    }
29
    return h.inner.Handle(ctx, r)
30
}
31

32
func (h OTelHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
33
    return OTelHandler{inner: h.inner.WithAttrs(attrs)}
34
}
35

36
func (h OTelHandler) WithGroup(name string) slog.Handler {
37
    return OTelHandler{inner: h.inner.WithGroup(name)}
38
}
39

40
func main() {
41
    // 创建结构化日志器
42
    baseHandler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
43
        Level: slog.LevelInfo,
44
    })
45
    logger := slog.New(OTelHandler{inner: baseHandler})
46

47
    // 设置全局默认日志器
48
    slog.SetDefault(logger)
49

50
    // 使用
51
    slog.Info("Order processed",
52
        slog.String("order_id", "ORD-12345"),
53
        slog.String("user_id", "user-67890"),
54
        slog.Duration("duration", 150*time.Millisecond),
55
        slog.Int("items_count", 3),
56
    )
57
}

1
{
2
  "time": "2026-06-19T03:14:22.456Z",
3
  "level": "INFO",
4
  "msg": "Order processed",
5
  "trace_id": "abc123def45678901234567890123456",
6
  "span_id": "789ghi01234567890",
7
  "order_id": "ORD-12345",
8
  "user_id": "user-67890",
9
  "duration": 150000000,
10
  "items_count": 3
11
}

1
const pino = require('pino');
2

3
const logger = pino({
4
  level: process.env.LOG_LEVEL || 'info',
5
  formatters: {
6
    level: (label) => ({ level: label }),
7
  },
8
  // 默认字段
9
  base: {
10
    service: 'order-service',
11
    version: process.env.SERVICE_VERSION || 'unknown',
12
  },
13
  // 时间戳格式
14
  timestamp: pino.stdTimeFunctions.isoTime,
15
});
16

17
// OTel 上下文注入中间件
18
function injectTraceContext(logger, context) {
19
  const activeSpan = context?.activeSpan;
20
  if (activeSpan) {
21
    const spanContext = activeSpan.spanContext();
22
    return logger.child({
23
      trace_id: spanContext.traceId,
24
      span_id: spanContext.spanId,
25
    });
26
  }
27
  return logger;
28
}
29

30
// 使用
31
const reqLogger = injectTraceContext(logger, otelContext);
32
reqLogger.info({ order_id: 'ORD-12345', user_id: 'user-67890', duration_ms: 150 }, 'Order processed');

1
# Loki 配置
2
auth_enabled: false
3

4
server:
5
  http_listen_port: 3100
6

7
common:
8
  path_prefix: /loki
9
  storage:
10
    filesystem:
11
      chunks_directory: /loki/chunks
12
      rules_directory: /loki/rules
13
  replication_factor: 1
14

15
schema_config:
16
  configs:
17
    - from: 2024-01-01
18
      store: tsdb
19
      object_store: filesystem
20
      schema: v13
21
      index:
22
        prefix: loki_index_
23
        period: 24h
24

25
limits_config:
26
  # 查询限制
27
  max_query_length: 721h
28
  max_query_parallelism: 32
29
  # 写入限制
30
  ingestion_rate_mb: 20
31
  ingestion_burst_size_mb: 30
32
  # 保留期
33
  retention_period: 744h  # 31 天

1
# 基础查询：查找 order-service 的 ERROR 日志
2
{service="order-service"} |= "ERROR"
3

4
# 结构化查询：查找特定用户的所有日志
5
{service="order-service"} | json | user_id="12345"
6

7
# 追踪关联：查找某个 TraceID 的所有日志
8
{service="order-service"} | json | trace_id="abc123def456"
9

10
# 聚合查询：统计各错误码的出现次数
11
sum(count_over_time({service="order-service"} | json | level="ERROR" [5m])) by (error_code)
12

13
# 延迟分析：P99 延迟趋势
14
quantile_over_time(0.99, {service="order-service"} | json | unwrap duration_ms [5m]) by (endpoint)

1
# 不同级别的保留策略
2
retention:
3
  fatal: 90d    # 致命错误保留 90 天
4
  error: 30d    # 错误保留 30 天
5
  warn: 14d     # 警告保留 14 天
6
  info: 7d      # 信息保留 7 天
7
  debug: 1d     # 调试保留 1 天

1
// 采样策略：INFO 级别采样 10%，ERROR 级别全量
2
func shouldLog(level slog.Level) bool {
3
    switch level {
4
    case slog.LevelError, slog.LevelWarn:
5
        return true
6
    case slog.LevelInfo:
7
        return rand.Float64() < 0.1  // 10% 采样
8
    default:
9
        return false
10
    }
11
}

1
# 裁剪高基数字段
2
fields_to_drop:
3
  - request_body    # 请求体太大
4
  - response_body   # 响应体太大
5
  - session_token   # 敏感信息
6
  - x_forwarded_for # 高基数 IP 地址

1
// 反模式：每条日志都重复写上下文
2
slog.Info("Processing order",
3
    slog.String("service", "order-service"),
4
    slog.String("order_id", orderID),
5
    slog.String("user_id", userID),
6
)
7
slog.Error("Database timeout",
8
    slog.String("service", "order-service"),
9
    slog.String("order_id", orderID),
10
    slog.String("user_id", userID),
11
)
12

13
// 最佳实践：使用上下文日志
14
orderLogger := slog.With(
15
    slog.String("service", "order-service"),
16
    slog.String("order_id", orderID),
17
    slog.String("user_id", userID),
18
)
19
orderLogger.Info("Processing order")
20
orderLogger.Error("Database timeout", slog.String("db_host", dbHost))

1
// 标准错误日志模板
2
func logError(ctx context.Context, err error, msg string, attrs ...slog.Attr) {
3
    span := trace.SpanFromContext(ctx)
4

5
    logger := slog.With(
6
        slog.String("trace_id", span.SpanContext().TraceID().String()),
7
        slog.String("span_id", span.SpanContext().SpanID().String()),
8
        slog.String("error_type", fmt.Sprintf("%T", err)),
9
        slog.String("error_message", err.Error()),
10
    )
11

12
    // 如果有堆栈信息
13
    var stackErr interface{ StackTrace() errors.StackTrace }
14
    if errors.As(err, &stackErr) {
15
        logger = logger.With(slog.String("stack_trace", fmt.Sprintf("%+v", stackErr.StackTrace())))
16
    }
17

18
    logger.Error(msg, attrs...)
19
}

1
# 使用 Docker Compose 启动 Loki
2
docker compose up -d loki
3

4
# 验证 Loki 运行
5
curl http://localhost:3100/ready

1
# 通过 OTel Collector 发送日志
2
curl -X POST http://localhost:4318/v1/logs \
3
  -H "Content-Type: application/json" \
4
  -d '{
5
    "resourceLogs": [{
6
      "resource": {
7
        "attributes": [
8
          {"key": "service.name", "value": {"stringValue": "demo-api"}},
9
          {"key": "service.version", "value": {"stringValue": "1.0.0"}}
10
        ]
11
      },
12
      "scopeLogs": [{
13
        "scope": {"name": "io.opentelemetry.sdk.log"},
14
        "logRecords": [{
15
          "timeUnixNano": "1700000000000000000",
16
          "severityNumber": 17,
17
          "severityText": "ERROR",
18
          "body": {"stringValue": "Database connection timeout"},
19
          "attributes": [
20
            {"key": "trace_id", "value": {"stringValue": "abc123def45678901234567890123456"}},
21
            {"key": "span_id", "value": {"stringValue": "789ghi01234567890"}},
22
            {"key": "error_code", "value": {"stringValue": "DB_CONNECTION_TIMEOUT"}},
23
            {"key": "duration_ms", "value": {"intValue": "5000"}}
24
          ]
25
        }]
26
      }]
27
    }]
28
  }'

1
# 打开 Grafana Explore
2
open http://localhost:3000/explore
3

4
# LogQL 查询示例
5
# 1. 查找所有 ERROR 日志
6
{service="demo-api"} | json | level="ERROR"
7

8
# 2. 按 TraceID 关联查询
9
{service="demo-api"} | json | trace_id="abc123def45678901234567890123456"
10

11
# 3. 统计错误码分布
12
sum(count_over_time({service="demo-api"} | json | level="ERROR" [1h])) by (error_code)