1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

#安装工具软件
yum install vim net-tools wget lsof  python3 bash-completion -y


#关闭防火墙和selinux
systemctl stop firewalld && systemctl disable firewalld
sed -i '/^SELINUX=/c SELINUX=disabled' /etc/selinux/config
setenforce 0

#配置主机地址解析
cat << EOF >> /etc/hosts
172.16.88.13 ceph-88-13
172.16.88.14 ceph-88-14
172.16.88.15 ceph-88-15
172.16.88.16 ceph-88-16
EOF


#配置互相免密
ssh-keygen -t rsa -P ''
for i in `tail -n 4 /etc/hosts | awk '{print $1}'`; do ssh-copy-id -i ~/.ssh/id_rsa.pub $i;done

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

#如果服务器以前安装过docker则需要执行一下命令进行删除
yum remove -y docker \
              docker-client \
            docker-client-latest \
            docker-common \
            docker-latest \
            docker-latest-logrotate \
            docker-logrotate \
            docker-selinux \
            docker-engine-selinux \
            docker-engine
            
#安装所需要的包
yum install -y yum-utils device-mapper-persistent-data lvm2   
#设置稳定的存储库
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
#安装docker
yum install -y docker-ce docker-ce-cli containerd.io 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

#! /bin/bash

docker pull uhub.service.ucloud.cn/dylan_prometheus/prometheus:v2.33.4
docker pull uhub.service.ucloud.cn/dylan_prometheus/node-exporter:v1.3.1
docker pull uhub.service.ucloud.cn/dylan_prometheus/alertmanager:v0.23.0
docker pull uhub.service.ucloud.cn/dylan_ceph/ceph:v17
docker pull uhub.service.ucloud.cn/dylan_ceph/ceph-grafana:8.3.5
docker pull uhub.service.ucloud.cn/dylan_ceph/haproxy:2.3
docker pull uhub.service.ucloud.cn/dylan_ceph/keepalived:2.1.5


docker tag uhub.service.ucloud.cn/dylan_ceph/ceph:v17 quay.io/ceph/ceph:v17
docker tag uhub.service.ucloud.cn/dylan_prometheus/prometheus:v2.33.4 quay.io/prometheus/prometheus:v2.33.4
docker tag uhub.service.ucloud.cn/dylan_prometheus/node-exporter:v1.3.1 quay.io/prometheus/node-exporter:v1.3.1
docker tag uhub.service.ucloud.cn/dylan_prometheus/alertmanager:v0.23.0 quay.io/prometheus/alertmanager:v0.23.0
docker tag uhub.service.ucloud.cn/dylan_ceph/ceph-grafana:8.3.5 quay.io/ceph/ceph-grafana:8.3.5
docker tag uhub.service.ucloud.cn/dylan_ceph/haproxy:2.3 quay.io/ceph/haproxy:2.3
docker tag uhub.service.ucloud.cn/dylan_ceph/keepalived:2.1.5 quay.io/ceph/keepalived:2.1.5

1
2

#注意 此命令会从quay.io下载镜像，如果服务拉取不了镜像，会报错 
cephadm bootstrap --mon-ip 172.16.88.13

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

......
Waiting for mgr epoch 9...
mgr epoch 9 is available
Generating a dashboard self-signed certificate...
Creating initial admin user...
Fetching dashboard port number...
Ceph Dashboard is now available at:

	     URL: https://ceph-88-13:8443/
	    User: admin
	Password: 3w0ujolsfi

Enabling client.admin keyring and conf on hosts with "admin" label
Saving cluster configuration to /var/lib/ceph/788b7912-4ac8-11ed-bef2-005056bc091f/config directory
Enabling autotune for osd_memory_target
You can access the Ceph CLI as following in case of multi-cluster or non-default config:

	sudo /usr/sbin/cephadm shell --fsid 788b7912-4ac8-11ed-bef2-005056bc091f -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring

Or, if you are only running a single cluster on this host:

	sudo /usr/sbin/cephadm shell

Please consider enabling telemetry to help improve Ceph:

	ceph telemetry on

For more information see:

	https://docs.ceph.com/docs/master/mgr/telemetry/

Bootstrap complete.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

#部署
ceph orch apply mgr 3
ceph orch apply mgr ceph-88-13,ceph-88-14,ceph-88-15

#查看mgr部署状态
ceph -s


  cluster:
    id:     788b7912-4ac8-11ed-bef2-005056bc091f
    health: HEALTH_WARN
            OSD count 0 < osd_pool_default_size 3

  services:
    mon: 3 daemons, quorum ceph-88-13,ceph-88-14,ceph-88-15 (age 24m)
    mgr: ceph-88-13.holxbb(active, since 82m), standbys: ceph-88-14.ssbqqi, ceph-88-15.cgprpz
    osd: 0 osds: 0 up, 0 in

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

#查看设备  AVAILABLE状态为yes的是可以添加的
ceph orch device ls


# 添加ceph-88-13的新硬盘
ceph orch daemon add osd ceph-88-13:/dev/sdb

#查看是否正常
ceph osd tree

ID  CLASS  WEIGHT   TYPE NAME            STATUS  REWEIGHT  PRI-AFF
-1         0.01949  root default
-3         0.01949      host ceph-88-13
 0    hdd  0.01949          osd.0            up   1.00000  1.00000
 
#添加剩余主机上硬盘
ceph orch daemon add osd ceph-88-14:/dev/sdb
ceph orch daemon add osd ceph-88-15:/dev/sdb
ceph orch daemon add osd ceph-88-16:/dev/sdb

#查看osd列表 是否添加正常
ceph osd tree

ID  CLASS  WEIGHT   TYPE NAME            STATUS  REWEIGHT  PRI-AFF
-1         0.07794  root default
-3         0.01949      host ceph-88-13
 0    hdd  0.01949          osd.0            up   1.00000  1.00000
-5         0.01949      host ceph-88-14
 1    hdd  0.01949          osd.1            up   1.00000  1.00000
-7         0.01949      host ceph-88-15
 2    hdd  0.01949          osd.2            up   1.00000  1.00000
-9         0.01949      host ceph-88-16
 3    hdd  0.01949          osd.3            up   1.00000  1.00000

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

#这时的集群状态就是ok了
ceph -s

  cluster:
    id:     788b7912-4ac8-11ed-bef2-005056bc091f
    health: HEALTH_OK

  services:
    mon: 3 daemons, quorum ceph-88-13,ceph-88-14,ceph-88-15 (age 38m)
    mgr: ceph-88-13.holxbb(active, since 96m), standbys: ceph-88-14.ssbqqi, ceph-88-15.cgprpz
    osd: 4 osds: 4 up (since 46s), 4 in (since 66s)

  data:
    pools:   1 pools, 1 pgs
    objects: 2 objects, 449 KiB
    usage:   482 MiB used, 80 GiB / 80 GiB avail
    pgs:     1 active+clean

1
2
3
4
5
6

#部署mds
# ceph orch apply mds <cephfs_name> ceph-88-13
ceph orch apply mds cephfs --placement="3 ceph-88-13 ceph-88-14 ceph-88-15"
ceph fs status cephfs
ceph mds stat
ceph orch ps --daemon-type mds

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

#开启删除
ceph config set mon mon_allow_pool_delete true 

#删除mycephfs
ceph fs volume rm mycephfs --yes-i-really-mean-it

#关闭删除
ceph config set mon mon_allow_pool_delete false

#查看 可以看到mycephfs已经被删除了
ceph osd pool ls

.mgr
cephfs.cephfs.meta
cephfs.cephfs.data

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

#创建rbd
ceph osd pool create rbd 16

#应用程序启用RBD
ceph osd pool application enable rbd rbd

#创建rbd存储，指定大小为10GB
rbd create rbd1 --size 10240

#查看rbd信息
rbd --image rbd1 info
......
rbd image 'rbd1':
	size 10 GiB in 2560 objects
	order 22 (4 MiB objects)
	snapshot_count: 0
	id: 39f9165d7094
	block_name_prefix: rbd_data.39f9165d7094
	format: 2
	features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
	op_features:
	flags:
	create_timestamp: Fri Oct 14 16:02:23 2022
	access_timestamp: Fri Oct 14 16:02:23 2022
	modify_timestamp: Fri Oct 14 16:02:23 2022
#size : 就是这个块的大小，即1024MB=1G，1024MB/256 = 4M,共分成了256个对象(object)，每个对象4M。
#order 22： 22是个编号，4M是22， 8M是23，也就是2^22 bytes = 4MB, 2^23 bytes = 8MB
#block_name_prefix : 这个是块的最重要的属性了，这是每个块在ceph中的唯一前缀编号，有了这个前缀，把主机上的OSD都拔下来带回家，就能复活所有的VM了
#format : 格式有两种，1和2
	
#配置
ceph osd crush tunables hammer
ceph osd crush reweight-all

#关闭一些内核默认不支持的特性
rbd feature disable rbd1 exclusive-lock object-map fast-diff deep-flatten

#查看特性是否已禁用
rbd --image rbd1 info | grep features
......
	features: layering
	op_features:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

#copy认证文件到客户端
##这里还是用ceph-88-16集群做客户端，过程中需要使用rbd命令。默认这个集群上是没有的，需要在前面<初始化cephadm环境>步骤中安装ceph-common
scp /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 172.16.88.16:/etc/ceph

#映射到客户端（在需要挂载的客户端运行）
rbd map --image rbd1

#查看映射情况
rbd showmapped 
rbd device list


#格式化
mkfs.xfs /dev/rbd0

#创建挂载目录，并将rbd挂载到指定目录
mkdir /mnt/rbd
mount /dev/rbd0 /mnt/rbd/

#查看挂载情况
df -hl | grep rbd

#观察集群变化
rados -p testpool ls|sort

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

#显示映射的块设备
rbd device list

#取消映射块设备
rbd device unmap /dev/rbd0

#删除rbd1
rbd rm rbd1

#查看rbd信息 就看不到信息了
rbd --image rbd1 info

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

### 注意，使用cephadm时，radosgw守护程序是通过监视器配置数据库而不是通过ceph.conf或命令行来配置的。 如果该配置尚未就绪（通常在client.rgw。。部分中），那么radosgw守护程序将使用默认设置（例如，绑定到端口80）启动

# 如果未创建领域，要先创建一个领域
radosgw-admin realm create --rgw-realm=myorg --default

#创建一个新的区域组
radosgw-admin zonegroup create --rgw-zonegroup=default --master --default

#创建一个区域
radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=cn-beijing  --master --default

#为特定领域和区域部署一组radosgw守护程序
# ceph orch apply rgw *<realm-name>* *<zone-name>* --placement="*<num-daemons>* [*<host1>* ...]"
ceph orch apply rgw myorg cn-beijing --placement="3 ceph-88-13 ceph-88-14 ceph-88-15"


#查看rgw是否启动
ceph orch ps --daemon-type rgw


#创建radosgw用户
radosgw-admin user create --uid="admin" --display-name="admin user"

#创建后需要把access_key和secret_key保存下来，也可以使用下面的命令来查看
radosgw-admin user info --uid=admin
......
    "keys": [
        {
            "user": "admin",
            "access_key": "C6W60VA3QUS2EZ1HCDC9",
            "secret_key": "I46OElK7umx8uIhHB2GRu3w4OAE6HuJS8kZ2g1cA"
        }
    ],
......

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

#在客户端安装s3cmd软件
yum install s3cmd -y

#生成配置文件
#注意  我这里是之前配置过一遍，所以显示的和第一次不一样，不过填写内容的地方都一样
s3cmd --configure

Enter new values or accept defaults in brackets with Enter.
Refer to user manual for detailed description of all options.

Access key and Secret key are your identifiers for Amazon S3. Leave them empty for using the env variables.
Access Key [SS09CDSWF4KYU91F17AE]: C6W60VA3QUS2EZ1HCDC9  ##服务端生成的Access Key
Secret Key [ST68lZyEcu6erKduzP4hecVkcYsPsKi9pGg8BTzi]: I46OElK7umx8uIhHB2GRu3w4OAE6HuJS8kZ2g1cA ##服务端生成的Secret Key
Default Region [US]:  #直接回车

Use "s3.amazonaws.com" for S3 Endpoint and not modify it to the target Amazon S3.
S3 Endpoint [172.16.88.13]: 172.16.88.13 #输入对象存储的ip地址

Use "%(bucket)s.s3.amazonaws.com" to the target Amazon S3. "%(bucket)s" and "%(location)s" vars can be used
if the target S3 system supports dns based buckets.
DNS-style bucket+hostname:port template for accessing a bucket [172.16.88.13]: 172.16.88.13   #输入对象存储的ip地址

Encryption password is used to protect your files from reading
by unauthorized persons while in transfer to S3
Encryption password:     #空密码回车
Path to GPG program [/usr/bin/gpg]: ##/usr/bin/gpg命令路径 回车

When using secure HTTPS protocol all communication with Amazon S3
servers is protected from 3rd party eavesdropping. This method is
slower than plain HTTP, and can only be proxied with Python 2.7 or newer
Use HTTPS protocol [No]: no  #是否使用https， 选no

On some networks all internet access must go through a HTTP proxy.
Try setting it here if you can't connect to S3 directly
HTTP Proxy server name:   #留空回车

New settings:
  Access Key: C6W60VA3QUS2EZ1HCDC9
  Secret Key: I46OElK7umx8uIhHB2GRu3w4OAE6HuJS8kZ2g1cA
  Default Region: US
  S3 Endpoint: 172.16.88.13
  DNS-style bucket+hostname:port template for accessing a bucket: 172.16.88.13
  Encryption password:
  Path to GPG program: /usr/bin/gpg
  Use HTTPS protocol: False
  HTTP Proxy server name:
  HTTP Proxy server port: 0

Test access with supplied credentials? [Y/n] y  #是否进行测试
Please wait, attempting to list all buckets...
Success. Your access key and secret key worked fine :-)

Now verifying that encryption works...
Not configured. Never mind.

Save settings? [y/N] y   # y 要保存配置文件
Configuration saved to '/root/.s3cfg' # 最后配置文件保存的位置/root.s3cfg


#创建桶mybucket
s3cmd mb s3://mybucket

#查看所有的桶
s3cmd ls

#像指定桶中上传/etc/hosts文件
s3cmd put /etc/hosts s3://mybucket
......
upload: '/etc/hosts' -> 's3://mybucket/hosts'  [1 of 1]
 158 of 158   100% in    1s    82.93 B/s  done
 
 #显示mybucket中的文件
 s3cmd ls s3://mybucket
 
 #删除mybucket中的hosts文件
 s3cmd del s3://mybucket/hosts
 
 #删除mybucket
 s3cmd rb s3://mybucket

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

#使用ceph fs命令  会自动创建相应的存储池
ceph fs volume create myfs --placement=3
ceph fs status myfs

#查看存储池的情况
ceph osd pool ls

#查看fs信息
ceph fs ls
......
name: myfs, metadata pool: cephfs.myfs.meta, data pools: [cephfs.myfs.data ]

#如果想删除存储池
ceph config set mon mon_allow_pool_delete true
#注意 --yes-i-really-mean-it 参数一定要加
ceph fs volume rm myfs --yes-i-really-mean-it

#这里是分别删除
ceph osd pool rm cephfs.myfs.data cephfs.myfs.data  --yes-i-really-really-mean-it
ceph osd pool rm cephfs.myfs.meta cephfs.myfs.meta  --yes-i-really-really-mean-it

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

#启用nfs服务  可以看到nfs模块是默认启动的
ceph mgr module enable nfs
......
module 'nfs' is already enabled

#创建NFS所需的存储池
ceph osd pool create mynfs_data

#在存储池上开启nfs应用
ceph osd pool application enable mynfs_data nfs
......
enabled application 'nfs' on pool 'mynfs_data'


#删除存储池  这里再介绍一种方式
#开启删除
ceph tell mon.* injectargs --mon_allow_pool_delete true
#删除
ceph osd pool delete mynfs_data mynfs_data --yes-i-really-really-mean-it
ceph osd pool delete .nfs .nfs --yes-i-really-really-mean-it

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

#创建nfs集群 这里的88.17为一个空的未使用的ip
ceph nfs cluster create mynfs 3 --ingress --virtual_ip 172.16.88.17

#查看集群
ceph nfs cluster ls

#修改配置的话， 可以先导出为yaml格式
ceph orch ls --service-name nfs.mynfs --export > nfs.mynfs.yaml


#如果想删除NFS集群，有以下两个方法
##方法一、使用ceph nfs cluster rm <cluster_id> 命令
ceph nfs cluster rm mynfs

##方法二 使用 ceph orch rm 删除
ceph orch rm ingress.nfs.mynfs
ceph orch rm nfs.mynfs

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

#新建CephFs的nfs导出
ceph nfs export create cephfs --cluster-id mynfs --pseudo-path /data --fsname myfs --squash no_root_squash

#查看nfs信息
#也可以在ceph Dashboard中新建NFS，但是Dashboard中和CLI中新建的，查看最后参数不一样，Dashboard会多一些
ceph nfs export info mynfs /data
......
{
  "export_id": 1,
  "path": "/",
  "cluster_id": "mynfs",
  "pseudo": "/data",
  "access_type": "RW",
  "squash": "no_root_squash",
  "security_label": true,
  "protocols": [
    4
  ],
  "transports": [
    "TCP"
  ],
  "fsal": {
    "name": "CEPH",
    "user_id": "nfs.mynfs.1",
    "fs_name": "myfs"
  },
  "clients": []
}

1
2
3
4
5
6
7
8
9

#为了破坏集群并删除该集群中存储的所有数据，请暂停cephadm以避免部署新的守护进程
ceph orch pause

#验证集群的FSID
ceph fsid

#清除集群中所有主机的ceph守护进程
# For each host:
cephadm rm-cluster --force --zap-osds --fsid <fsid>