ceph 的安装和维护

首先用ansible 来部署ceph，这个是一个开源项目，文档在这里
https://docs.ceph.com/ceph-ansible/master/

以前用ceph-deploy 部署其实现在也可以，只是现在ansible这么流行，而且可以复用，维持鲁棒性，因此都在退ansible维护。

上面的内容待补充，因为原有官方文档够详细了，不过到时候贴几个配置的例子就行了

ceph 维护

首先是对ceph所有进程的维护
启动，停止和重新启动所有Ceph守护程

systemctl start ceph.target
systemctl stop ceph.target
systemctl restart ceph.target

按类型启动，停止和重新启动Ceph守护程序
在Ceph Monitor节点上：

[root@mon ~]# systemctl start ceph-mon.target
[root@mon ~]# systemctl stop ceph-mon.target
[root@mon ~]# systemctl restart ceph-mon.target

在Ceph Manager节点上：

[root@mgr ~]# systemctl start ceph-mgr.target
[root@mgr ~]# systemctl stop ceph-mgr.target
[root@mgr ~]# systemctl restart ceph-mgr.target

在Ceph OSD节点上：
Starting

[root@osd ~]# systemctl start ceph-osd.target
Stopping

[root@osd ~]# systemctl stop ceph-osd.target
Restarting

[root@osd ~]# systemctl restart ceph-osd.target

在Ceph Object Gateway节点上：

[root@rgw ~]# systemctl start ceph-radosgw.target
Stopping

[root@rgw ~]# systemctl stop ceph-radosgw.target
Restarting

[root@rgw ~]# systemctl restart ceph-radosgw.target

关闭或者重启ceph 集群

关闭的顺序是 先设置集群状态，关闭osd 服务，最后关闭mon 服务
启动的顺序是 先开启mon服务，然后开启osd服务，最后设置集群状态

#ceph fs set <fs_name> max_mds 1
#ceph mds deactivate <fs_name>:1 # rank 2 of 2
#ceph status # wait for rank 1 to finish stopping
#ceph fs set <fs_name> cluster_down true
#ceph mds fail <fs_name>:0

然后设置ceph 集群禁止调度恢复文件。

#ceph osd set noout
#ceph osd set norecover
#ceph osd set norebalance
#ceph osd set nobackfill
#ceph osd set nodown
#ceph osd set pause

然后按顺序关闭osd服务和mon服务，
启动是先开启mon服务，osd服务，然后重新设置集群状态
Power on the monitor nodes:

[root@mon ~]# systemctl start ceph-mon.target
Power on the OSD nodes:

[root@osd ~]# systemctl start ceph-osd.target
Wait for all the nodes to come up. Verify all the services are up and the connectivity is fine between the nodes.
Unset the noout, norecover, norebalance, nobackfill, nodown and pause flags. Run the following on a node with the client keyrings, for example, the Ceph Monitor or OpenStack controller node:

如果使用cephfs 最后还要重新设置集群状态

#ceph osd unset noout
#ceph osd unset norecover
#ceph osd unset norebalance
#ceph osd unset nobackfill
#ceph osd unset nodown
#ceph osd unset pause

[root@admin~]# ceph fs set <fs_name> cluster_down false
Start the RADOS Gateway and NFS-Ganesha Gateway.

On the RADOS Gateway node:

# systemctl start ceph-radosgw.target
On the NFS-Ganesha Gateway node:

# systemctl start nfs-ganesha.service

ceph 安装好后，
看状态基本就是几个命令
ceph health detail
ceph -s
ceph -w

root@proxmox01s:~# ceph health detail
HEALTH_OK
root@proxmox01s:~# ceph -s
  cluster:
    id:     c3ad598e-2444-4d40-ba94-3001d4a0ccb7
    health: HEALTH_OK

  services:
    mon: 3 daemons, quorum proxmox01s,proxmox02s,proxmox03s
    mgr: proxmox03s(active), standbys: proxmox02s
    mds: cephfs-1/1/1 up  {0=proxmox04s=up:active}, 1 up:standby
    osd: 40 osds: 40 up, 40 in

  data:
    pools:   4 pools, 1672 pgs
    objects: 2.80M objects, 10.7TiB
    usage:   32.1TiB used, 26.3TiB / 58.4TiB avail
    pgs:     1669 active+clean
             3    active+clean+scrubbing+deep

  io:
    client:   10.8MiB/s rd, 29.6MiB/s wr, 176op/s rd, 590op/s wr

root@proxmox01s:~# ceph -w
  cluster:
    id:     c3ad598e-2444-4d40-ba94-3001d4a0ccb7
    health: HEALTH_OK

  services:
    mon: 3 daemons, quorum proxmox01s,proxmox02s,proxmox03s
    mgr: proxmox03s(active), standbys: proxmox02s
    mds: cephfs-1/1/1 up  {0=proxmox04s=up:active}, 1 up:standby
    osd: 40 osds: 40 up, 40 in

  data:
    pools:   4 pools, 1672 pgs
    objects: 2.80M objects, 10.7TiB
    usage:   32.1TiB used, 26.3TiB / 58.4TiB avail
    pgs:     1669 active+clean
             3    active+clean+scrubbing+deep

  io:
    client:   9.15MiB/s rd, 26.1MiB/s wr, 148op/s rd, 535op/s wr

查看当前的pool
ceph osd pool ls

root@proxmox01s:~# ceph osd pool ls
vm-disk
cephfs_data
cephfs_metadata
ssd

查看pool里面的内容,注意命令不同，输出也不一样

rbd ls -p vm-disk
rados ls -p vm-disk

root@proxmox01s:~# rados ls -p vm-disk |more
rbd_data.484cf86b8b4567.000000000001bd76
rbd_data.6493626b8b4567.0000000000015622
rbd_data.4aa7406b8b4567.000000000001adfb
rbd_data.4859ad6b8b4567.000000000000fba7
rbd_data.4423106b8b4567.000000000000fc0c
rbd_data.4373b76b8b4567.0000000000012016
rbd_data.438d176b8b4567.000000000000e59d

rbd ls -p vm-disk

root@proxmox01s:~# rbd ls -p vm-disk
base-200-disk-0
kubernetes-dynamic-pvc-13bb4c14-2d24-11e9-81e1-da8820d8b83a
kubernetes-dynamic-pvc-2244af0b-31cf-11e9-9e13-56913ba006a2
kubernetes-dynamic-pvc-37b0028a-1b0f-11ea-8050-ea50e0a750db
kubernetes-dynamic-pvc-38c7186d-4946-11e9-b616-def7505fd032
kubernetes-dynamic-pvc-3b0c03ed-318f-11e9-9e13-56913ba006a2
kubernetes-dynamic-pvc-3e0654c6-38cc-11e9-ba72-def7505fd032
kubernetes-dynamic-pvc-3fc43857-43c6-11e9-b616-def7505fd032
kubernetes-dynamic-pvc-438e0a98-55eb-11e9-b616-def7505fd032
kubernetes-dynamic-pvc-4e9d3e7d-90da-11e9-bc53-def7505fd032
kubernetes-dynamic-pvc-56cd18ec-43c6-11e9-b616-def7505fd032

查看集群使用空间

root@proxmox01s:~# ceph df
GLOBAL:
    SIZE        AVAIL       RAW USED     %RAW USED
    58.4TiB     26.3TiB      32.1TiB         54.93
POOLS:
    NAME                ID     USED        %USED     MAX AVAIL     OBJECTS
    vm-disk             2      10.6TiB     67.28       5.16TiB     2787627
    cephfs_data         6      47.7GiB      0.89       5.16TiB       12609
    cephfs_metadata     7       187MiB         0       5.16TiB          75
    ssd                 8           0B         0       5.16TiB           0