1
receivers:
2
  otlp:
3
    protocols:
4
      grpc:
5
        endpoint: 0.0.0.0:4317
6
        max_recv_msg_size_mib: 4
7
      http:
8
        endpoint: 0.0.0.0:4318
9
        maxRequestBodySize: 4194304
10
  prometheus:
11
    config:
12
      scrape_configs:
13
        - job_name: 'otel-collector'
14
          scrape_interval: 15s
15
          static_configs:
16
            - targets: ['localhost:8888']
17
  filelog:
18
    include: [/var/log/app/*.log]
19
    parsers:
20
      json:
21
        time_key: timestamp
22
        severity_key: level

1
processors:
2
  memory_limiter:
3
    check_interval: 5s
4
    limit_mib: 400
5
    spike_limit_mib: 100
6
  k8sattributes:
7
    auth_type: serviceAccount
8
    passthrough: false
9
    filter:
10
      node_from_env_var: KUBE_NODE_NAME
11
    extract:
12
      metadata:
13
        - k8s.pod.name
14
        - k8s.namespace.name
15
        - k8s.deployment.name
16
  filter:
17
    error_mode: ignore
18
    traces:
19
      span:
20
        - 'attributes["http.route"] == "/healthz"'
21
        - 'attributes["http.route"] == "/readyz"'
22
  transform:
23
    error_mode: ignore
24
    trace_statements:
25
      - context: span
26
        statements:
27
          - set(attributes["service.name"], attributes["service"]) where attributes["service"] != nil
28
          - delete_key(attributes, "user_id")
29
          - set(attributes["env"], "production")
30
  batch:
31
    send_batch_size: 1024
32
    send_batch_max_size: 2048
33
    timeout: 5s

1
exporters:
2
  otlphttp/tempo:
3
    endpoint: http://tempo:4318
4
    sending_queue:
5
      enabled: true
6
      num_consumers: 10
7
      queue_size: 5000
8
    retry_on_failure:
9
      enabled: true
10
      initial_interval: 5s
11
      max_interval: 30s
12
      max_elapsed_time: 300s
13
  prometheus:
14
    endpoint: 0.0.0.0:8889
15
    namespace: otel
16
  loki:
17
    endpoint: http://loki:3100/loki/api/v1/push
18
    default_labels_enabled:
19
      exporter: false
20
  kafka:
21
    protocol_version: 3.3.1
22
    brokers:
23
      - kafka:9092
24
    topic: otel-traces

1
connectors:
2
  spanmetrics:
3
    histogram:
4
      explicit:
5
        buckets: [2ms, 4ms, 6ms, 8ms, 10ms, 50ms, 100ms, 200ms, 400ms, 800ms, 1s, 5s]
6
    dimensions:
7
      - name: http.method
8
      - name: http.status_code
9
      - name: http.route
10
    metrics_flush_interval: 15s
11
service:
12
  pipelines:
13
    traces:
14
      receivers: [otlp]
15
      processors: [memory_limiter, batch]
16
      exporters: [otlphttp/tempo, spanmetrics]  # spanmetrics 是 connector
17
    metrics:
18
      receivers: [otlp, spanmetrics]  # spanmetrics 的输出作为 metrics 的输入
19
      processors: [memory_limiter, batch]
20
      exporters: [prometheus]

1
service:
2
  pipelines:
3
    traces:
4
      receivers: [otlp]
5
      processors: [memory_limiter, k8sattributes, tail_sampling, batch]
6
      exporters: [otlphttp/tempo, otlphttp/debug]
7
    metrics:
8
      receivers: [otlp]
9
      processors: [memory_limiter, k8sattributes, filter, transform, batch]
10
      exporters: [prometheus, otlphttp/debug]
11
    logs:
12
      receivers: [otlp]
13
      processors: [memory_limiter, k8sattributes, filter/logs, batch]
14
      exporters: [loki, otlphttp/debug]

1
processors:
2
  tail_sampling:
3
    decision_wait: 10s          # 等待 10s 收集完整追踪
4
    num_traces: 100000          # 内存中缓存的追踪数
5
    expected_new_traces_per_sec: 1000
6
    policies:
7
      - name: error-policy
8
        type: status_code
9
        status_code:
10
          status_codes:
11
            - ERROR
12
            - UNSET
13
      - name: slow-policy
14
        type: latency
15
        latency:
16
          threshold_ms: 1000
17
      - name: critical-service-policy
18
        type: string_attribute
19
        string_attribute:
20
          key: service.name
21
          values:
22
            - payment-service
23
            - auth-service
24
      - name: fallback-policy
25
        type: probabilistic
26
        probabilistic:
27
          sampling_percentage: 5

1
processors:
2
  filter/critical:
3
    error_mode: ignore
4
    traces:
5
      span:
6
        - 'attributes["service.name"] == "payment-service"'
7
  filter/non-critical:
8
    error_mode: ignore
9
    traces:
10
      span:
11
        - 'attributes["service.name"] != "payment-service"'
12
exporters:
13
  otlphttp/critical:
14
    endpoint: http://tempo-critical:4318
15
  otlphttp/standard:
16
    endpoint: http://tempo-standard:4318
17
service:
18
  pipelines:
19
    traces/critical:
20
      receivers: [otlp]
21
      processors: [memory_limiter, filter/critical, batch]
22
      exporters: [otlphttp/critical]
23
    traces/standard:
24
      receivers: [otlp]
25
      processors: [memory_limiter, filter/non-critical, tail_sampling, batch]
26
      exporters: [otlphttp/standard]

1
processors:
2
  transform:
3
    error_mode: ignore
4
    trace_statements:
5
      - context: span
6
        statements:
7
          - set(attributes["service.name"], attributes["service"]) where attributes["service"] != nil
8
          - delete_key(attributes, "user_id")
9
          - delete_key(attributes, "request_id")
10
          - set(attributes["env"], "production")
11
    metric_statements:
12
      - context: datapoint
13
        statements:
14
          - set(attributes["service"], attributes["service.name"]) where attributes["service.name"] != nil

1
receivers:
2
  otlp:
3
    protocols:
4
      grpc:
5
        endpoint: 0.0.0.0:4317
6
exporters:
7
  kafka:
8
    protocol_version: 3.3.1
9
    brokers:
10
      - kafka:9092
11
    topic: otel-traces
12
receivers:
13
  kafka:
14
    brokers:
15
      - kafka:9092
16
    topics:
17
      - otel-traces
18
    consumer_group: otel-collector-gateway

1
service:
2
  pipelines:
3
    traces/critical:
4
      receivers: [otlp]
5
      processors: [memory_limiter, filter/critical, batch]
6
      exporters: [otlphttp/tempo-critical]
7
    traces/standard:
8
      receivers: [otlp]
9
      processors: [memory_limiter, filter/non-critical, tail_sampling, batch]
10
      exporters: [otlphttp/tempo-standard]
11
    metrics:
12
      receivers: [otlp, spanmetrics]
13
      processors: [memory_limiter, filter/metrics, transform, batch]
14
      exporters: [prometheus]
15
    logs:
16
      receivers: [otlp]
17
      processors: [memory_limiter, filter/logs, batch]
18
      exporters: [loki]

1
service:
2
  pipelines:
3
    traces:
4
      processors: [memory_limiter/traces, batch/traces]
5
    metrics:
6
      processors: [memory_limiter/metrics, batch/metrics]
7
    logs:
8
      processors: [memory_limiter/logs, batch/logs]
9
processors:
10
  memory_limiter/traces:
11
    check_interval: 5s
12
    limit_mib: 800
13
  memory_limiter/metrics:
14
    check_interval: 5s
15
    limit_mib: 200
16
  memory_limiter/logs:
17
    check_interval: 5s
18
    limit_mib: 400
19
  batch/traces:
20
    send_batch_size: 2048
21
    timeout: 5s
22
  batch/metrics:
23
    send_batch_size: 1024
24
    timeout: 10s
25
  batch/logs:
26
    send_batch_size: 4096
27
    timeout: 3s

1
groups:
2
  - name: otel-collector
3
    rules:
4
      - alert: CollectorHighRefusalRate
5
        expr: rate(otelcol_receiver_refused_spans[5m]) > 100
6
        for: 5m
7
        labels:
8
          severity: warning
9
        annotations:
10
          summary: "OTel Collector 拒绝率过高"
11
          description: "Collector {{ $labels.instance }} 拒绝率 {{ $value }} spans/s"
12
      - alert: CollectorQueueBacklog
13
        expr: otelcol_exporter_queue_size > 4000
14
        for: 5m
15
        labels:
16
          severity: warning
17
        annotations:
18
          summary: "OTel Collector 导出队列积压"
19
          description: "队列 {{ $labels.exporter }} 积压 {{ $value }} 条"

1
processors:
2
  memory_limiter:
3
    limit_mib: 200
4
  batch:
5
    send_batch_size: 1024
6
    timeout: 5s
7
exporters:
8
  otlphttp:
9
    endpoint: http://otel-gateway:4318
10
processors:
11
  memory_limiter:
12
    limit_mib: 2000
13
  k8sattributes:
14
    auth_type: serviceAccount
15
  tail_sampling:
16
    decision_wait: 10s
17
    num_traces: 100000
18
    policies: [...]
19
  transform:
20
    error_mode: ignore
21
    trace_statements: [...]
22
  batch:
23
    send_batch_size: 2048
24
    timeout: 5s

1
apiVersion: v1
2
kind: Service
3
metadata:
4
  name: otel-gateway
5
spec:
6
  type: ClusterIP
7
  selector:
8
    app: otel-gateway
9
  ports:
10
    - name: otlp-grpc
11
      port: 4317
12
      targetPort: 4317
13
---
14
apiVersion: v1
15
kind: Service
16
metadata:
17
  name: otel-gateway-headless
18
spec:
19
  clusterIP: None  # headless
20
  selector:
21
    app: otel-gateway
22
  ports:
23
    - name: otlp-grpc
24
      port: 4317
25
      targetPort: 4317