1
# Ceph 数据映射流程
2
def ceph_map_object(object_id, pool_id, cluster_map):
3
    """将对象映射到 OSD"""
4
    # 1. 计算归置组（PG）
5
    pg_id = hash(object_id) % pg_num
6

7
    # 2. 使用 CRUSH 算法计算 OSD 集合
8
    osd_set = crush_algorithm(pg_id, cluster_map, pool_rules)
9

10
    # 3. Primary OSD 处理读写
11
    primary_osd = osd_set[0]
12

13
    return {
14
        "object_id": object_id,
15
        "pg_id": pg_id,
16
        "osd_set": osd_set,
17
        "primary": primary_osd,
18
    }

1
# CRUSH 算法简化实现
2
def crush_choose(pgid, cluster_map, rule, num_replicas):
3
    """CRUSH 选择 OSD"""
4
    result = []
5

6
    for replica in range(num_replicas):
7
        # 1. 从故障域的根开始
8
        current = cluster_map.root
9

10
        # 2. 沿故障域层级向下选择
11
        for failure_domain in rule.failure_domains:
12
            # host → rack → row → room
13
            children = get_children(current, failure_domain)
14

15
            # 3. 使用 straw 桶算法选择子节点
16
            selected = straw_bucket_select(
17
                children, pgid, replica
18
            )
19

20
            current = selected
21

22
        # 4. 检查是否已在结果中（避免同一 OSD）
23
        if current.osd not in result:
24
            result.append(current.osd)
25

26
    return result
27

28
# Straw 桶算法
29
def straw_bucket_select(items, pgid, replica):
30
    """Straw 桶选择算法"""
31
    # 为每个 item 计算一个 straw 长度
32
    # 长度与 item 的权重成正比
33
    max_straw = -1
34
    selected = None
35

36
    for item in items:
37
        # hash(pgid, replica, item_id) → straw 长度
38
        straw = hash(pgid, replica, item.id) & item.weight
39

40
        if straw > max_straw:
41
            max_straw = straw
42
            selected = item
43

44
    return selected

1
# 创建分布式副本卷（2 副本，跨 3 节点）
2
gluster volume create gv0 replica 2 \
3
    node1:/data/brick1 node2:/data/brick1 \
4
    node3:/data/brick1 node1:/data/brick2 \
5
    node2:/data/brick2 node3:/data/brick2
6

7
# 启动卷
8
gluster volume start gv0
9

10
# 查看卷信息
11
gluster volume info gv0
12
# Volume Name: gv0
13
# Type: Distributed-Replicate
14
# Number of Bricks: 3 x 2 = 6
15
# Status: Started
16

17
# 在线扩容：添加 Brick
18
gluster volume add-brick gv0 node4:/data/brick1 node5:/data/brick1
19

20
# 触发重平衡
21
gluster volume rebalance gv0 start

1
# Ceph 集群状态
2
ceph status
3
# cluster:
4
#   id:     xxx
5
#   health: HEALTH_OK
6
#   services:
7
#     mon: 3 daemons
8
#     osd: 12 osds: 12 up, 12 in
9
#   data:
10
#     pools: 3 pools, 576 pgs
11
#     objects: 1.2M objects, 450 GB
12
#     usage: 12 TiB used, 108 TiB / 120 TiB avail
13
#     pgs:  576 active+clean
14

15
# 添加新 OSD 后的自动重平衡
16
ceph osd tree
17
# ID  CLASS  WEIGHT   TYPE NAME
18
# -1         0.52340  root default
19
# -3         0.17446      host node1
20
#  0    hdd  0.08723          osd.0    up  1.00000
21
#  1    hdd  0.08723          osd.1    up  1.00000
22
# -5         0.17446      host node2
23
#  2    hdd  0.08723          osd.2    up  1.00000
24
#  3    hdd  0.08723          osd.3    up  1.00000
25

26
# 重平衡进度
27
ceph -w
28
# pgmap v1234: 576 pgs: 576 active+clean; 450 GB data, 12 TiB used

1
# 使用 cephadm 部署 Ceph
2
# 1. 安装 cephadm
3
curl --silent --remote-name \
4
    https://download.ceph.com/rpm-18.2.0/el9/noarch/cephadm
5
chmod +x cephadm
6

7
# 2. 引导集群
8
./cephadm bootstrap --mon-ip 192.168.1.1
9

10
# 3. 添加节点
11
ceph orch host add node2 192.168.1.2
12
ceph orch host add node3 192.168.1.3
13

14
# 4. 添加 OSD
15
ceph orch daemon add osd node1:/dev/sdb
16
ceph orch daemon add osd node2:/dev/sdb
17
ceph orch daemon add osd node3:/dev/sdb
18

19
# 5. 创建池
20
ceph osd pool create mypool 128 128 replicated
21
ceph osd pool set mypool size 3
22
ceph osd pool set mypool min_size 2

1
# 使用 rados bench 测试性能
2
# 写入测试
3
rados bench -p mypool 60 write --no-cleanup
4

5
# 读取测试
6
rados bench -p mypool 60 seq
7

8
# 随机读取测试
9
rados bench -p mypool 60 rand
10

11
# 使用 fio 测试 RBD 性能
12
fio --name=ceph-rbd --ioengine=rbd --pool=mypool \
13
    --rbdname=test-volume --size=10G \
14
    --rw=randwrite --bs=4k --iodepth=32 --numjobs=4

1
# 创建纠删码 Profile
2
ceph osd erasure-code-profile set my-ec-profile \
3
    k=6 m=3 \
4
    crush-failure-domain=host \
5
    plugin=jerasure \
6
    technique=reed_sol_van
7

8
# 基于该 Profile 创建池
9
ceph osd pool create ec-pool 128 128 erasure my-ec-profile
10

11
# 查看池信息
12
ceph osd pool ls detail | grep ec-pool
13
# pool 4 'ec-pool' erasure profile my-ec-profile size 9 min_size 7

1
# RBD 操作
2
# 创建镜像
3
rbd create mypool/myimage --size 1T --image-feature layering,exclusive-lock
4

5
# 创建快照
6
rbd snap create mypool/myimage@snap1
7

8
# 保护快照（克隆前必须保护）
9
rbd snap protect mypool/myimage@snap1
10

11
# 基于快照克隆
12
rbd clone mypool/myimage@snap1 mypool/clone1
13

14
# 查看克隆关系
15
rbd children mypool/myimage@snap1
16

17
# 合并克隆（flatten）
18
rbd flatten mypool/clone1