基于DockerCompose 搭建 ELK（Elastic S

准备一台虚拟机，（最好3G内存以上，不然会很卡，很卡很卡）

1. 准备docker环境

安装docker环境

// 安装docker 
yum install -y docker 

安装Docker Compose

// 输入命令行 
sudo curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
// 配置权限
sudo chmod +x /usr/local/bin/docker-compose
// 查看版本
docker-compose --version

01.png

2. 配置运行时需要的变量

创建容器间交互的 桥接式网络 elk_demo_network

docker network create elk_demo_network

创建网络 异常 Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?
使用sudo启动

sudo service docker restart

查看网络创建情况

docker network list

02.png

配置es需要的 变量 避免es异常

es 异常 exit code 78 解决方式：

sudo sysctl -w vm.max_map_count=524288

es需要的内存很大 ，可以通过 修改环境变量 获取配置 虚拟机内存处理

3. 配置基本完成 配置docker-compose （PS：注意文件目录问题 此处文件放在 /root/elk_demo）

创建docker-compose.yml
进入到目录 /root/elk_demo

vi docker-compose.yml

version: '2.2'
services:
# nginx -- start
  nginx01:
    image: nginx
    container_name: nginx01
    privileged: true
    ports:
      - "80:80"
    environment:
      - NGINX_HOST=foobar.com
      - NGINX_PORT=80
    networks:
      - esnet
    volumes:
#      - /root/elk_demo/nginx/mysite.template:/etc/nginx/conf.d/mysite.template 
      - /root/elk_demo/nginx/logs/:/var/log/nginx/  
      - /root/elk_demo/nginx/configs/:/etc/nginx/conf.d/   # if would open this setting, you should put a 'default.conf' file in your configs dir. 
# nginx -- end
# elasticsearch cluster -- start 
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0
    container_name: es01
    environment:
      - node.name=es01
      - discovery.seed_hosts=es02
      - cluster.initial_master_nodes=es01,es02
      - cluster.name=docker-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms128m -Xmx128m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
    networks:
      - esnet
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0
    container_name: es02
    environment:
      - node.name=es02
      - discovery.seed_hosts=es01
      - cluster.initial_master_nodes=es01,es02
      - cluster.name=docker-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms128m -Xmx128m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata02:/usr/share/elasticsearch/data
    networks:
      - esnet
# elasticsearch cluster -- end
# kibana -- start
  kibana01:
    image: docker.elastic.co/kibana/kibana:7.4.0
    container_name: kibana01
    privileged: true
    ports:
      - 5601:5601
#    environment:
#          - ELASTICSEARCH_HOSTS=http://es01:9200
#          - I18N_LOCALE=zh-CN
    networks:
      - esnet
    volumes:
      - /root/elk_demo/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
    depends_on:
      - es01
# kibana -- end
# filebeat -- start
  filebeat01:
    image: docker.elastic.co/beats/filebeat:7.4.0
    container_name: filebeat01
    privileged: true
#    environment:
#    #      - ES_HOST=es01
#    #      - ES_PORT=9200
#    #      - KIBANA_HOST=kibana
#    #      - KIBANA_PORT=5601
    networks:
      - esnet
    volumes:
      - /root/elk_demo/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /root/elk_demo/nginx/logs:/var/log/
      - /root/elk_demo/filebeat/modules.d/:/usr/share/filebeat/modules.d/
    depends_on:
      - es01
      - kibana01
# filebeat -- end
# metricbeat -- start
  metricbeat02:
    image: docker.elastic.co/beats/metricbeat:7.4.0
    container_name: metricbeat02
    privileged: true
#    environment:
#      - ES_HOST=es01
#      - ES_PORT=9200
#      - KIBANA_HOST=kibana
#      - KIBANA_PORT=5601
    networks:
      - esnet
    volumes:
      - /root/elk_demo/metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
      - /proc:/hostfs/proc:ro
      - /:/hostfs:ro
      - /root/elk_demo/metricbeat/modules.d/:/usr/share/metricbeat/modules.d/
    depends_on:
      - es01
      - kibana01
# metricbeat -- end




volumes:
  esdata01:
    driver: local
  esdata02:
    driver: local

networks:
  esnet:
    external:
      name: elk_demo_network

创建docker-compose.yml 之后。开始处理相关的配置文件 。这边试验对象 是nginx 所以需要配置nginx 的相关配置

nginx 相关配置

根据 docker-compose.yml 可以看出 nginx 的配置环境变量的目录 在 /root/elk_demo/nginx/configs/。其实此处可以不配置 ，也可以使用，配置的目的是为了更好的操作配置文件，另一个就是为了 下面的 metribeat做实验铺垫

进入到 目录 /root/elk_demo/nginx/configs
创建文件 metricbeat.conf

 vi metricbeat.conf

server {
    listen       80;
    server_name  localhost;

    #charset koi8-r;
    #access_log  /var/log/nginx/host.access.log  main;

    location / {
        root   /usr/share/nginx/html;
        index  index.html index.htm;
    }

    location /nginx-status {
        stub_status on;
    access_log off;
    }

    #error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }

}

Kibana 相关配置

同上 在目录

vi /root/elk_demo/kibana/kibana.yml

i18n.locale: zh-CN 
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://es01:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: true

filebeat 相关配置

vi /root/elk_demo/filebeat/filebeat.yml

output:
  elasticsearch:
    hosts: ["es01:9200"] 

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false


setup.template.settings:
   index.number_of_shards: 3

setup.kibana:
  host: "kibana01:5601"

此外，为了配置 nginx 仪表盘 所以还需要配置 nginx的配置文件
创建目录 /root/elk_demo/filebeat/modules.d

vi /root/elk_demo/filebeat/modules.d/nginx.yml

- module: nginx
  access:
    enabled: true
    var.paths: ["/var/log/access.log*"]

  error:
    enabled: true
    var.paths: ["/var/log/error.log*"]

metricbeat 相关配置

vi /root/elk_demo/metricbeat/metricbeat.yml

output.elasticsearch:
  # Array of hosts to connect to.
    hosts: ["es01:9200"]

setup.kibana:
  host: "kibana01:5601"

metricbeat.config.modules:
  path: ${path.config}/modules.d/*.yml

同上 配置 Nginx 仪表盘

vi /root/elk_demo/metricbeat/modules.d/nginx.yml

# Module: nginx
# Docs: https://www.elastic.co/guide/en/beats/metricbeat/7.4/metricbeat-module-nginx.html

- module: nginx
  #metricsets:
  #  - stubstatus
  period: 10s

  # Nginx hosts
  hosts: ["nginx01:80"]

  # Path to server status. Default server-status
  server_status_path: "nginx-status"

  #username: "user"
  #password: "secret"

vi /root/elk_demo/metricbeat/modules.d/system.yml

配置 system 仪表盘

# Module: system
# Docs: https://www.elastic.co/guide/en/beats/metricbeat/7.4/metricbeat-module-system.html

- module: system
  period: 10s
  metricsets:
    - cpu
    - load
    - memory
    - network
    - process
    - process_summary
    - socket_summary
    #- entropy
    #- core
    #- diskio
    #- socket
  process.include_top_n:
    by_cpu: 5      # include top 5 processes by CPU
    by_memory: 5   # include top 5 processes by memory

- module: system
  period: 1m
  metricsets:
    - filesystem
    - fsstat
  processors:
  - drop_event.when.regexp:
      system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'

- module: system
  period: 15m
  metricsets:
    - uptime

#- module: system
#  period: 5m
#  metricsets:
#    - raid
#  raid.mount_point: '/'

各项准备完成 开始最终步骤 启动

进入 /root/elk_demo

docker-compose up

03.png

---

测试

访问：http://192.168.62.144/ 验证nginx
访问：http://192.168.62.144/nginx-status 验证nginx 心跳
访问：http://192.168.62.144:5601 验证kibana

05.png

安装Nginx 仪表盘

确认之前的 beats 配置是否生效

# 查看容器信息
docker ps

# 查看模块配置信息
docker exec 容器ID ./filebeat modules list
docker exec 容器ID ./metricbeat modules list

#安装仪表盘

docker exec 容器ID ./filebeat setup --dashboards
docker exec 容器ID ./metricbeat setup --dashboards

配置仪表盘 01 08.png 10.png

以上就完成了使用 Beats+ES+Kibana 完成 Nginx 运行监控