1
# VictoriaMetrics 集群部署
2
# vminsert — 写入入口
3
apiVersion: apps/v1
4
kind: Deployment
5
metadata:
6
  name: vminsert
7
spec:
8
  replicas: 5
9
  template:
10
    spec:
11
      containers:
12
        - name: vminsert
13
          image: victoriametrics/vminsert:v1.93.0
14
          args:
15
            - --storageNode=vmstorage-0:8400,vmstorage-1:8400,vmstorage-2:8400
16
            - --maxLabelsPerTimeseries=30
17
            - --maxLabelValueLen=2048
18
            - --rpc.tls=true
19
          resources:
20
            requests:
21
              memory: "2Gi"
22
              cpu: "2"
23
            limits:
24
              memory: "4Gi"
25
              cpu: "4"
26
---
27

28
apiVersion: apps/v1
29
kind: StatefulSet
30
metadata:
31
  name: vmstorage
32
spec:
33
  replicas: 3
34
  template:
35
    spec:
36
      containers:
37
        - name: vmstorage
38
          image: victoriametrics/vmstorage:v1.93.0
39
          args:
40
            - --storageDataPath=/vm-data
41
            - --retentionPeriod=6m
42
            - --downsampling.period=5m:30d,1h:180d
43
          resources:
44
            requests:
45
              memory: "32Gi"
46
              cpu: "8"
47
          volumeMounts:
48
            - name: data
49
              mountPath: /vm-data

1
# Mimir 分布式部署（Helm Values）
2
mimir:
3
  structured_config:
4
    distributor:
5
      ring:
6
        heartbeat_period: 5s
7
      rate_limiting_strategy: "global"
8

9
    ingester:
10
      ring:
11
        replication_factor: 3
12
      chunks_encoding: "snappy"
13

14
    store_gateway:
15
      sharding_ring:
16
        replication_factor: 3
17

18
    querier:
19
      max_concurrent: 20
20
      timeout: 2m
21

22
    limits:
23
      max_global_series_per_user: 5000000
24
      max_query_length: 720h
25
      max_query_parallelism: 16
26

27
# 资源配置
28
distributor:
29
  replicas: 10
30
  resources:
31
    requests:
32
      memory: 1Gi
33
      cpu: 2
34

35
ingester:
36
  replicas: 30
37
  resources:
38
    requests:
39
      memory: 16Gi
40
      cpu: 8
41

42
querier:
43
  replicas: 20
44
  resources:
45
    requests:
46
      memory: 8Gi
47
      cpu: 4
48

49
store_gateway:
50
  replicas: 15
51
  resources:
52
    requests:
53
      memory: 8Gi
54
      cpu: 4

1
# 监控每个指标的序列数
2
count by (__name__) ({__name__=~".+"})
3

4
# 找出高基数指标
5
topk(10, count by (__name__) ({__name__=~".+"}))
6

7
# 监控标签基数
8
count by (service) (http_server_request_duration_bucket)

1
# Mimir 基数限制
2
limits:
3
  max_global_series_per_user: 5000000    # 每用户最大序列数
4
  max_global_metadata_per_user: 500000   # 每用户最大元数据数
5
  max_label_names_per_series: 30         # 每序列最大标签数
6
  max_label_value_length: 2048           # 标签值最大长度
7
  creation_rate_limit: 50000             # 每秒新序列创建速率

1
# OTel Collector: 过滤高基数标签
2
processors:
3
  transform/remove_high_cardinality:
4
    error_mode: ignore
5
    metric_statements:
6
      - context: metric
7
        statements:
8
          # 移除 user_id 标签
9
          - delete_key(attributes, "user_id")
10
          # 移除 request_path 中的动态 ID
11
          - set(attributes["request_path"], Replace(attributes["request_path"], "\\d+", ":id")) where attributes["request_path"] != nil
12
          # 将 IP 地址聚合为地区
13
          - set(attributes["region"], "unknown") where attributes["client_ip"] != nil
14
          - delete_key(attributes, "client_ip")
15

16
  filter/drop_debug_metrics:
17
    error_mode: ignore
18
    traces:
19
      span:
20
        # 丢弃 debug 级别的追踪
21
        - 'attributes["log.level"] == "debug"'

1
# 预聚合：按服务统计请求率（不保留 user_id 维度）
2
groups:
3
  - name: preaggregation
4
    interval: 30s
5
    rules:
6
      - record: http:requests:rate5m
7
        expr: sum(rate(http_requests_total[5m])) by (service, method, status)
8
        # 原始指标有 user_id 标签，但预聚合后不再保留

1
# VictoriaMetrics 降采样
2
downsampling:
3
  - resolution: 5m
4
    retention: 90d
5
    filter:
6
      exclude: ['go_*', 'process_*']  # 排除不需要降采样的指标
7
  - resolution: 1h
8
    retention: 365d
9
  - resolution: 1d
10
    retention: 0  # 永久保留

1
# Mimir 降采样配置
2
compactor:
3
  compaction_interval: 30m
4
  # 降采样块配置
5
  downsampling:
6
    - resolution: 5m
7
      retention: 90d
8
    - resolution: 1h
9
      retention: 365d
10
  # 降采样时保留的指标（排除不需要降采样的）
11
  downsampling_disable:
12
    - 'go_*'
13
    - 'process_*'
14
    - 'scrape_*'

1
# 可观测性成本估算模型
2
def estimate_monthly_cost(
3
    active_series: int,        # 活跃时间序列数
4
    samples_per_second: int,   # 每秒样本数
5
    retention_days: int,       # 数据保留天数
6
    storage_cost_per_gb: float = 0.023,  # S3 存储成本 ($/GB/月)
7
):
8
    # 每个样本约 2 字节（压缩后）
9
    bytes_per_sample = 2
10

11
    # 每天存储量
12
    daily_storage_gb = (samples_per_second * 86400 * bytes_per_sample) / (1024**3)
13

14
    # 总存储量（考虑降采样）
15
    # 原始数据 7 天 + 5m 降采样 30 天 + 1h 降采样 retention_days 天
16
    raw_storage = daily_storage_gb * 7
17
    ds_5m_storage = daily_storage_gb * 0.05 * 30  # 5m 降采样约为原始的 5%
18
    ds_1h_storage = daily_storage_gb * 0.005 * retention_days  # 1h 降采样约为原始的 0.5%
19

20
    total_storage_gb = raw_storage + ds_5m_storage + ds_1h_storage
21

22
    # 月度存储成本
23
    storage_cost = total_storage_gb * storage_cost_per_gb
24

25
    # 计算成本（Mimir 集群）
26
    compute_cost = active_series * 0.0001  # 约 $0.1/千序列/月
27

28
    return {
29
        "storage_gb": total_storage_gb,
30
        "storage_cost": storage_cost,
31
        "compute_cost": compute_cost,
32
        "total_cost": storage_cost + compute_cost,
33
    }
34

35
# 示例：1000 万序列，50 万样本/秒
36
result = estimate_monthly_cost(
37
    active_series=10_000_000,
38
    samples_per_second=500_000,
39
    retention_days=365,
40
)
41
# 预估：存储 ~2TB，月度成本 ~$2000-3000

1
# 预计算常用查询
2
groups:
3
  - name: slo:rules
4
    interval: 30s
5
    rules:
6
      - record: slo:availability:rate5m
7
        expr: |
8
          1 - (
9
            sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
10
            /
11
            sum(rate(http_requests_total[5m])) by (service)
12
          )
13

14
      - record: slo:latency:p99:rate5m
15
        expr: |
16
          histogram_quantile(0.99,
17
            sum(rate(http_request_duration_bucket[5m])) by (le, service)
18
          )

1
# 格式：level:metric:operations
2
# level: 聚合级别（slo, cluster, namespace, service）
3
# metric: 指标名
4
# operations: 聚合操作
5

6
slo:availability:rate5m          # SLO 级别，可用性，5m 速率
7
cluster:cpu:utilization:rate5m   # 集群级别，CPU 利用率
8
service:latency:p99:rate5m       # 服务级别，P99 延迟

1
# 查询 Mimir 的慢查询日志
2
# 查看最近 1 小时的慢查询
3
sum(rate(cortex_query_seconds_bucket{le="30"}[1h])) by (query_type)
4
/
5
sum(rate(cortex_query_seconds_count[1h])) by (query_type)
6

7
# 查看查询队列深度
8
cortex_query_frontend_queue_length
9

10
# 查看 Ingester 查询延迟
11
histogram_quantile(0.99, rate(cortex_ingester_query_seconds_bucket[5m]))

1
# 全局 Recording Rules：聚合多个集群的数据
2
groups:
3
  - name: global:rules
4
    interval: 60s
5
    rules:
6
      # 全局请求率 = 各集群请求率之和
7
      - record: global:http:requests:rate5m
8
        expr: |
9
          sum(http:requests:rate5m) by (service, method, status)
10
        # 从多个集群的 Mimir 数据源查询
11

12
      # 全局可用性
13
      - record: global:slo:availability:rate5m
14
        expr: |
15
          1 - (
16
            sum(global:http:errors:rate5m) by (service)
17
            /
18
            sum(global:http:requests:rate5m) by (service)
19
          )

1
# 查询高基数指标
2
curl 'http://localhost:9090/api/v1/query?query=topk(10,count%20by%20(__name__)%20({__name__=~%22.%2B%22}))'

1
# VictoriaMetrics 降采样
2
downsampling:
3
  - resolution: 5m
4
    retention: 30d

1
# 预计算 SLO 指标
2
groups:
3
  - name: slo:rules
4
    interval: 30s
5
    rules:
6
      - record: slo:availability:rate5m
7
        expr: |
8
          1 - (
9
            sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
10
            /
11
            sum(rate(http_requests_total[5m])) by (service)
12
          )