1
# 查看 NUMA 节点信息
2
numactl --hardware
3
# available: 2 nodes (0-1)
4
# node 0 size: 64217 MB
5
# node 1 size: 64509 MB
6
# node 0 free: 58123 MB
7
# node 1 free: 58456 MB
8
# node distances:
9
# node   0   1
10
#   0:  10  21
11
#   1:  21  10
12

13
# 距离矩阵：10 = 本地，21 = 远端（2.1x 延迟）
14

15
# 查看进程的 NUMA 内存分布
16
numastat -p $(pidof your_program)
17

18
# 查看每个 NUMA 节点的 CPU 列表
19
lscpu | grep -i numa

1
# 绑定到 Node 0 分配内存
2
numactl --membind=0 ./your_program
3

4
# 交错分配内存（适合带宽密集型）
5
numactl --interleave=all ./your_program
6

7
# 绑定 CPU 和内存到同一节点
8
numactl --cpunodebind=0 --membind=0 ./your_program
9

10
# 查看当前 NUMA 策略
11
numactl --show

1
#include <numa.h>
2
#include <numaif.h>
3
#include <stdio.h>
4

5
void numa_aware_allocation() {
6
    if (numa_available() < 0) {
7
        printf("NUMA 不可用\n");
8
        return;
9
    }
10

11
    int num_nodes = numa_num_configured_nodes();
12
    printf("NUMA 节点数: %d\n", num_nodes);
13

14
    // 在当前线程所在的 NUMA 节点分配内存
15
    int preferred_node = numa_node_of_cpu(sched_getcpu());
16
    void *local_mem = numa_alloc_onnode(1024 * 1024, preferred_node);
17

18
    // 交错分配
19
    void *interleaved_mem = numa_alloc_interleaved(1024 * 1024 * 4);
20

21
    // 绑定到指定节点
22
    void *bound_mem = numa_alloc_onnode(1024 * 1024, 0);
23

24
    numa_free(local_mem, 1024 * 1024);
25
    numa_free(interleaved_mem, 1024 * 1024 * 4);
26
    numa_free(bound_mem, 1024 * 1024);
27
}

1
#define _GNU_SOURCE
2
#include <sched.h>
3
#include <pthread.h>
4

5
void bind_to_node(int node) {
6
    cpu_set_t cpuset;
7
    CPU_ZERO(&cpuset);
8

9
    // 获取该节点的 CPU 列表
10
    struct bitmask *cpus = numa_allocate_cpumask();
11
    numa_node_to_cpus(node, cpus);
12
    for (int i = 0; i < numa_num_configured_cpus(); i++) {
13
        if (numa_bitmask_isbitset(cpus, i)) {
14
            CPU_SET(i, &cpuset);
15
        }
16
    }
17
    numa_free_cpumask(cpus);
18

19
    pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
20
}

1
# postgresql.conf
2
# 绑定到指定 NUMA 节点
3
# 使用 numactl 启动
4
numactl --cpunodebind=0 --membind=0 pg_ctl start
5

6
# 共享缓冲区大小应考虑 NUMA 节点内存
7
shared_buffers = 32GB  # 不超过单个 NUMA 节点的内存

1
# 方案 1：交错分配
2
numactl --interleave=all mysqld
3

4
# 方案 2：每个 NUMA 节点一个 MySQL 实例
5
numactl --cpunodebind=0 --membind=0 mysqld --port=3306
6
numactl --cpunodebind=1 --membind=1 mysqld --port=3307

1
<!-- libvirt XML 配置 -->
2
<numatune>
3
    <memory mode='strict' nodeset='0'/>
4
</numatune>
5
<vcpu placement='static' cpuset='0-15'>16</vcpu>
6
<cputune>
7
    <vcpupin vcpu='0' cpuset='0'/>
8
    <vcpupin vcpu='1' cpuset='1'/>
9
    <!-- ... -->
10
</cputune>

1
# Docker 绑定到 NUMA 节点
2
docker run --cpuset-cpus=0-15 --cpuset-mems=0 your_image
3

4
# Kubernetes NUMA 感知调度
5
# 需要启用 CPU Manager 和 Device Manager
6
# kubelet 配置：
7
# --cpu-manager-policy=static
8
# --topology-manager-policy=best-effort

1
#include <stdio.h>
2
#include <stdlib.h>
3
#include <time.h>
4
#include <numa.h>
5
#include <numaif.h>
6

7
#define SIZE (64 * 1024 * 1024)  // 64M int = 256MB
8

9
int main() {
10
    if (numa_available() < 0) {
11
        printf("NUMA 不可用\n");
12
        return 1;
13
    }
14

15
    int num_nodes = numa_num_configured_nodes();
16
    printf("NUMA 节点数: %d\n", num_nodes);
17

18
    for (int node = 0; node < num_nodes; node++) {
19
        int *arr = numa_alloc_onnode(SIZE * sizeof(int), node);
20
        for (int i = 0; i < SIZE; i++) arr[i] = i;
21

22
        clock_t start = clock();
23
        long long sum = 0;
24
        for (int i = 0; i < SIZE; i++) sum += arr[i];
25
        clock_t end = clock();
26

27
        printf("Node %d 分配: %.3f 秒\n", node,
28
               (double)(end - start) / CLOCKS_PER_SEC);
29
        numa_free(arr, SIZE * sizeof(int));
30
    }
31
    return 0;
32
}

1
# 本地访问
2
numactl --cpunodebind=0 --membind=0 ./memory_benchmark
3

4
# 远端访问
5
numactl --cpunodebind=0 --membind=1 ./memory_benchmark
6

7
# 交错分配
8
numactl --interleave=all ./memory_benchmark
9

10
# 对比三种方式的延迟

1
# 查看 NUMA 内存分配统计
2
numastat
3

4
# 查看进程的 NUMA 内存分布
5
numastat -p $(pidof your_program)
6

7
# 查看 NUMA 命中/未命中
8
perf stat -e numa_hit,numa_miss,numa_foreign ./your_program

1
// NUMA 感知的线程池
2
#define _GNU_SOURCE
3
#include <sched.h>
4
#include <pthread.h>
5
#include <numa.h>
6

7
struct thread_config {
8
    int numa_node;
9
    int cpu_within_node;
10
    void* (*func)(void*);
11
    void* arg;
12
};
13

14
void* numa_thread_entry(void* arg) {
15
    struct thread_config *cfg = (struct thread_config*)arg;
16

17
    // 1. 绑定到指定 NUMA 节点的 CPU
18
    cpu_set_t cpuset;
19
    CPU_ZERO(&cpuset);
20
    struct bitmask *cpus = numa_allocate_cpumask();
21
    numa_node_to_cpus(cfg->numa_node, cpus);
22
    int cpu_list[64], n = 0;
23
    for (int i = 0; i < numa_num_configured_cpus(); i++) {
24
        if (numa_bitmask_isbitset(cpus, i)) cpu_list[n++] = i;
25
    }
26
    CPU_SET(cpu_list[cfg->cpu_within_node % n], &cpuset);
27
    pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
28
    numa_free_cpumask(cpus);
29

30
    // 2. 设置内存分配策略为本地节点
31
    struct bitmask *nodemask = numa_allocate_nodemask();
32
    numa_bitmask_setbit(nodemask, cfg->numa_node);
33
    set_mempolicy(MPOL_BIND, nodemask->maskp, nodemask->size + 1);
34
    numa_free_nodemask(nodemask);
35

36
    // 3. 执行线程函数
37
    return cfg->func(cfg->arg);
38
}

1
// NUMA 分区哈希表
2
struct numa_hash_table {
3
    int num_nodes;
4
    struct hash_bucket **partitions;  // 每个节点一个分区
5
};
6

7
// 创建：每个分区在对应 NUMA 节点分配
8
struct numa_hash_table* numa_ht_create(int capacity) {
9
    struct numa_hash_table *ht = malloc(sizeof(*ht));
10
    ht->num_nodes = numa_num_configured_nodes();
11
    ht->partitions = malloc(ht->num_nodes * sizeof(void*));
12

13
    for (int i = 0; i < ht->num_nodes; i++) {
14
        // 在 NUMA 节点 i 上分配分区
15
        ht->partitions[i] = numa_alloc_onnode(
16
            capacity / ht->num_nodes * sizeof(struct hash_bucket), i);
17
    }
18
    return ht;
19
}
20

21
// 查找：只访问本地分区（零跨节点访问）
22
struct hash_entry* numa_ht_lookup(struct numa_hash_table *ht, int key) {
23
    int node = numa_node_of_cpu(sched_getcpu());  // 当前线程所在节点
24
    int local_key = key / ht->num_nodes;  // 映射到本地分区
25
    return hash_lookup(ht->partitions[node], local_key);
26
}

1
// NUMA 延迟基准测试
2
#include <stdio.h>
3
#include <stdlib.h>
4
#include <time.h>
5
#include <numa.h>
6

7
#define SIZE (64 * 1024 * 1024)  // 256MB
8

9
int main() {
10
    if (numa_available() < 0) { printf("NUMA 不可用\n"); return 1; }
11

12
    int nn = numa_num_configured_nodes();
13
    int cpu_node = numa_node_of_cpu(sched_getcpu());
14

15
    printf("当前线程在 Node %d\n\n", cpu_node);
16
    printf("%-15s %-15s %-10s\n", "分配节点", "访问延迟(ms)", "相对延迟");
17

18
    for (int mem_node = 0; mem_node < nn; mem_node++) {
19
        int *arr = numa_alloc_onnode(SIZE * sizeof(int), mem_node);
20
        for (int i = 0; i < SIZE; i++) arr[i] = i;
21

22
        clock_t start = clock();
23
        volatile long long sum = 0;
24
        for (int i = 0; i < SIZE; i++) sum += arr[i];
25
        clock_t end = clock();
26

27
        double ms = (double)(end - start) / CLOCKS_PER_SEC * 1000;
28
        printf("Node %-8d  %-12.1f  %.1fx\n", mem_node, ms,
29
               mem_node == cpu_node ? 1.0 : ms / 100.0);
30
        numa_free(arr, SIZE * sizeof(int));
31
    }
32
    return 0;
33
}