常用指令

服务启动

[root@prometheus prometheus]# systemctl daemon-reload
[root@prometheus prometheus]# systemctl restart node_exporter
[root@prometheus prometheus]# systemctl  restart prometheus
[root@prometheus prometheus]# systemctl restart grafana
[root@prometheus prometheus]# systemctl restart updatetime

工具检测

[root@prometheus prometheus]#  ./promtool check config ./prometheus.yml 

热加载

[root@prometheus prometheus]# ps -ef|grep prometheus
[root@prometheus prometheus]# kill -HUP 1267

时间同步

$ yum install ntpdate -y
$ ntpdate time.windows.com

相关服务

prometheus

http://192.168.153.22:9090/

-------------------------------prometheus----------------------------------------
# vi /usr/lib/systemd/system/prometheus.service
[Unit]
Description=prometheus
[Service]
ExecStart=/opt/monitor/prometheus/prometheus --config.file=/opt/monitor/prometheus/prometheus.yml
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
[Install]
WantedBy=multi-user.target
---------------------------------------------------------------------

[root@prometheus prometheus]# systemctl daemon-reload
[root@prometheus prometheus]# systemctl start prometheus

grafana

http://192.168.153.21:3000/

------------------------------------grafana----------------------------------
[root@prometheus ~]# vi /usr/lib/systemd/system/grafana.service

[Unit]
Description=grafana
[Service]
ExecStart=/opt/monitor/grafana/bin/grafana-server -homepath=/opt/monitor/grafana
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
[Install]
WantedBy=multi-user.target
--------------------------------------------------------------------------------------
[root@prometheus prometheus]# systemctl daemon-reload
[root@prometheus prometheus]# systemctl start grafana

node_exporter

[root@k8snode1 prometheus]# vi /usr/lib/systemd/system/node_exporter.service 
[Unit]
Description=node_exporter
[Service]
ExecStart=/usr/local/node_exporter/node_exporter
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
[Install]
WantedBy=multi-user.target

----------------------------------------------------------------------------

[root@prometheus ~]# systemctl daemon-reload
[root@prometheus ~]# systemctl start node_exporter

prometheus监控

配置被监控端

scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'
    static_configs:
    - targets: ['localhost:9090']

  - job_name: 'Linux Server'
    static_configs:
    - targets: ['192.168.153.20:9100']

1635429871239.png

监控Linux服务器

使用Grafana展示node_exporter数据指标，仪表盘ID： 9276

1635430281665.png 1635430316866.png

node_exporter启用HTTP认证

安装httpd-tool

[root@prometheus ~]# yum install httpd-tools –y

[root@prometheus ~]# htpasswd -nBC 12 '' | tr -d ':\n'
New password: 
Re-type new password: 
$2y$12$w78Y3R7SLaGlPY4OkmRCcu7esY.xzENZeDTj.87QhWsBRADyaRDLi

[root@prometheus node_exporter]# ls
LICENSE  node_exporter  NOTICE
[root@prometheus node_exporter]# vi config.yml
basic_auth_users:
  lql: $2y$12$w78Y3R7SLaGlPY4OkmRCcu7esY.xzENZeDTj.87QhWsBRADyaRDLi
  
  

系统服务-node_exporter

 
[root@prometheus node_exporter]# vi /usr/lib/systemd/system/node_exporter.service
[Unit]
Description=node_exporter
[Service]
ExecStart=/usr/local/node_exporter/node_exporter  --web.config=/usr/local/node_exporter/config.yml
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
[Install]
WantedBy=multi-user.target

----------------------------------------------------------------------------

[root@prometheus ~]# systemctl daemon-reload
[root@prometheus ~]# systemctl start node_exporter

primetheus配置文件

  - job_name: 'Linux Server'
    basic_auth:
      username: lql
      password: 12345678
    static_configs:
    - targets: ['192.168.153.20:9100']

监控系统服务运行状态

服务配置

 
[root@prometheus node_exporter]# vi /usr/lib/systemd/system/node_exporter.service

[Unit]
Description=node_exporter
[Service]
ExecStart=/usr/local/node_exporter/node_exporter --web.config=/usr/local/node_exporter/config.yml --collector.systemd  --collector.systemd.unit-whitelist=
(docker|sshd|nginx).service
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
[Install]
WantedBy=multi-user.target

----------------------------------------------------------------------------------

[root@prometheus prometheus]# systemctl daemon-reload
[root@prometheus prometheus]# systemctl restart node_exporter

http://192.168.153.17:9100/metrics

服务检测

1635475436994.png

查看服务状态

node_systemd_unit_state

1635475662804.png 1635475816420.png

#docker服务关闭，value则为0 （systemctl stop dokcer ）

监控docker

Docker部署cAdvisor

docker run -d \
--volume=/:/rootfs:ro \
--volume=/var/run:/var/run:ro \
--volume=/sys:/sys:ro \
--volume=/var/lib/docker/:/var/lib/docker:ro \
--volume=/dev/disk/:/dev/disk:ro \
--publish=8080:8080 \
--detach=true \
--name=cadvisor \
google/cadvisor:latest

http://192.168.153.17:8080/metrics

prometheus配置

  - job_name: 'docker'
    static_configs:
    - targets: ['192.168.153.17:8080']
    

1635476978343.png

导入granfana

仪表盘模板ID：193

1635477572914.png

增加相关变量

1635489169318.png

label_values(up,instance)
.*:8080

#修改指标【instance="$Node"】，每个指标都要修改
#比如：sum(rate(container_cpu_user_seconds_total{image!="",instance="$Node"}[5m]) * 100)

监控mysql

启动mysql

[root@ansible ~]# docker start mysql

启动mysqld_exporter服务

[root@ansible mysql_exporter]# ls
LICENSE  mysqld_exporter  NOTICE

#配置文件写入mysql的用户名和密码
[root@ansible mysql_exporter]# vi .my.cnf
[client]
user=root
password=123
#启动服务
[root@ansible mysql_exporter]# ./mysqld_exporter --config.my-cnf=".my.cnf"
http://192.168.153.17:9104/

prometheus配置

http://192.168.153.17:9104/

  - job_name: 'mysql'
    static_configs:
    - targets: ['192.168.153.17:9104']

导入granfana

仪表盘ID：7362

1635491714113.png