1
# S3 核心操作
2
# 创建桶
3
aws s3 mb s3://my-bucket
4

5
# 上传对象
6
aws s3 cp photo.jpg s3://my-bucket/images/
7

8
# 下载对象
9
aws s3 cp s3://my-bucket/images/photo.jpg ./
10

11
# 列出对象
12
aws s3 ls s3://my-bucket/images/
13

14
# 删除对象
15
aws s3 rm s3://my-bucket/images/photo.jpg
16

17
# 分片上传（大文件）
18
aws s3 cp large-file.tar.gz s3://my-bucket/ --storage-class STANDARD_IA
19

20
# 生命周期管理
21
aws s3api put-bucket-lifecycle-configuration \
22
    --bucket my-bucket \
23
    --lifecycle-configuration file://lifecycle.json

1
# MinIO 纠删码配置
2
# 4 节点 × 4 磁盘 = 16 磁盘
3
# 默认: 8 数据块 + 8 校验块
4
# 可容忍 8 块磁盘故障
5
# 空间利用率: 50%
6

7
# 自定义配置
8
# 16 磁盘: 12 数据 + 4 校验
9
# 可容忍 4 块磁盘故障
10
# 空间利用率: 75%
11

12
minio_ec_configs = {
13
    "默认 (8+8)": {
14
        "data_blocks": 8,
15
        "parity_blocks": 8,
16
        "fault_tolerance": 8,
17
        "utilization": "50%",
18
    },
19
    "高利用率 (12+4)": {
20
        "data_blocks": 12,
21
        "parity_blocks": 4,
22
        "fault_tolerance": 4,
23
        "utilization": "75%",
24
    },
25
    "高容错 (4+12)": {
26
        "data_blocks": 4,
27
        "parity_blocks": 12,
28
        "fault_tolerance": 12,
29
        "utilization": "25%",
30
    },
31
}

1
# 配置 S3 跨地域复制
2
# 1. 开启版本控制
3
aws s3api put-bucket-versioning \
4
    --bucket my-bucket \
5
    --versioning-configuration Status=Enabled
6

7
# 2. 创建复制配置
8
aws s3api put-bucket-replication \
9
    --bucket my-bucket \
10
    --replication-configuration file://replication.json
11

12
# replication.json
13
# {
14
#   "Role": "arn:aws:iam::123456:role/s3-replication",
15
#   "Rules": [{
16
#     "Status": "Enabled",
17
#     "Destination": {
18
#       "Bucket": "arn:aws:s3:::my-bucket-replica",
19
#       "StorageClass": "STANDARD"
20
#     }
21
#   }]
22
# }

1
# MinIO 站点复制配置
2
# 主站点
3
mc admin bucket remote add \
4
    myminio/mybucket \
5
    http://minio-replica:9000/mybucket \
6
    --name replica-site
7

8
# 启用版本控制
9
mc version enable myminio/mybucket
10

11
# 验证复制状态
12
mc admin bucket remote ls myminio/mybucket

1
{
2
  "Rules": [
3
    {
4
      "ID": "Transition to IA after 30 days",
5
      "Status": "Enabled",
6
      "Transitions": [
7
        {
8
          "Days": 30,
9
          "StorageClass": "STANDARD_IA"
10
        }
11
      ]
12
    },
13
    {
14
      "ID": "Transition to Glacier after 90 days",
15
      "Status": "Enabled",
16
      "Transitions": [
17
        {
18
          "Days": 90,
19
          "StorageClass": "GLACIER"
20
        }
21
      ]
22
    },
23
    {
24
      "ID": "Expire after 365 days",
25
      "Status": "Enabled",
26
      "Expiration": {
27
        "Days": 365
28
      }
29
    },
30
    {
31
      "ID": "Cleanup incomplete uploads",
32
      "Status": "Enabled",
33
      "AbortIncompleteMultipartUpload": {
34
        "DaysAfterInitiation": 7
35
      }
36
    }
37
  ]
38
}

1
# MinIO 分布式部署（4 节点）
2
# 每个节点 4 块磁盘
3
export MINIO_ROOT_USER=minioadmin
4
export MINIO_ROOT_PASSWORD=minioadmin
5

6
minio server \
7
    http://node{1...4}/mnt/data{1...4} \
8
    --console-address ":9001"
9

10
# 使用 Docker Compose
11
cat > docker-compose.yml << 'EOF'
12
version: "3.8"
13
services:
14
  minio1:
15
    image: minio/minio:latest
16
    command: server http://minio{1...4}/data{1...4} --console-address ":9001"
17
    environment:
18
      MINIO_ROOT_USER: minioadmin
19
      MINIO_ROOT_PASSWORD: minioadmin
20
    ports:
21
      - "9000:9000"
22
      - "9001:9001"
23
    volumes:
24
      - data1-1:/data1
25
      - data1-2:/data2
26
      - data1-3:/data3
27
      - data1-4:/data4
28
  # ... 其他节点类似
29
EOF

1
# 使用 warp 进行性能测试
2
# 写入测试
3
warp put --host=minio:9000 --access-key=minioadmin \
4
    --secret-key=minioadmin --bucket=bench --duration=60s
5

6
# 读取测试
7
warp get --host=minio:9000 --access-key=minioadmin \
8
    --secret-key=minioadmin --bucket=bench --duration=60s
9

10
# 混合测试
11
warp mixed --host=minio:9000 --access-key=minioadmin \
12
    --secret-key=minioadmin --bucket=bench --duration=60s \
13
    --get=50 --put=20 --delete=30

1
# S3 分片上传示例
2
import boto3
3

4
s3 = boto3.client('s3')
5
bucket = 'my-bucket'
6
key = 'large-file.tar.gz'
7
file_path = '/data/large-file.tar.gz'
8
part_size = 64 * 1024 * 1024  # 64MB
9

10
# 1. 初始化分片上传
11
response = s3.create_multipart_upload(Bucket=bucket, Key=key)
12
upload_id = response['UploadId']
13

14
# 2. 上传分片
15
parts = []
16
with open(file_path, 'rb') as f:
17
    part_number = 1
18
    while True:
19
        data = f.read(part_size)
20
        if not data:
21
            break
22
        resp = s3.upload_part(
23
            Bucket=bucket, Key=key, UploadId=upload_id,
24
            PartNumber=part_number, Body=data
25
        )
26
        parts.append({'PartNumber': part_number, 'ETag': resp['ETag']})
27
        part_number += 1
28

29
# 3. 完成分片上传
30
s3.complete_multipart_upload(
31
    Bucket=bucket, Key=key, UploadId=upload_id,
32
    MultipartUpload={'Parts': parts}
33
)

1
// IAM 策略：允许特定用户访问指定桶
2
{
3
  "Version": "2012-10-17",
4
  "Statement": [
5
    {
6
      "Effect": "Allow",
7
      "Action": ["s3:GetObject", "s3:PutObject"],
8
      "Resource": "arn:aws:s3:::my-bucket/*"
9
    },
10
    {
11
      "Effect": "Allow",
12
      "Action": "s3:ListBucket",
13
      "Resource": "arn:aws:s3:::my-bucket"
14
    }
15
  ]
16
}

1
// 桶策略：允许跨账号只读访问
2
{
3
  "Version": "2012-10-17",
4
  "Statement": [
5
    {
6
      "Effect": "Allow",
7
      "Principal": {"AWS": "arn:aws:iam::999999999999:root"},
8
      "Action": ["s3:GetObject"],
9
      "Resource": "arn:aws:s3:::my-bucket/public/*"
10
    }
11
  ]
12
}

1
# 热分区避免：前缀散列
2
import hashlib
3

4
def generate_key(original_path):
5
    """生成散列前缀的 Key，避免热分区"""
6
    hash_prefix = hashlib.md5(original_path.encode()).hexdigest()[:4]
7
    return f"{hash_prefix}/{original_path}"
8

9
# 原始: logs/2024/app.log → 所有请求打到同一分区
10
# 优化: a1b2/logs/2024/app.log → 请求分散到 65536 个分区

1
# MinIO Tenant CRD
2
apiVersion: minio.min.io/v2
3
kind: Tenant
4
metadata:
5
  name: my-minio
6
  namespace: minio-system
7
spec:
8
  image: minio/minio:latest
9
  imagePullPolicy: IfNotPresent
10
  pools:
11
    - name: pool-0
12
      servers: 4
13
      volumesPerServer: 4
14
      volumeClaimTemplate:
15
        metadata:
16
          name: data
17
        spec:
18
          accessModes:
19
            - ReadWriteOnce
20
          resources:
21
            requests:
22
              storage: 1Ti
23
  mountPath: /data
24
  requestAutoCert: true
25
  credsSecret:
26
    name: minio-creds-secret
27
  console:
28
    enabled: true
29
    replicas: 2