基于DockerCompose 搭建 ELK（Elastic S

前面描述了 ES+Beats+Kibana的操作，本章主要描述Beats+Logstash+Kibana直接的操作。

1. Beats 配置，此处举例是filebeats ，基本配置与之前相同，只不过把beats的output指向 改为 logstash

output:
#  elasticsearch:
#    hosts: ["es01:9200"] 
   logstash:
     hosts: ["logstash02:5044"]      

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/*.log



filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false


setup.template.settings:
   index.number_of_shards: 3

setup.kibana:
  host: "kibana01:5601"

2. logstash 配置
   logstash.yml

xpack.monitoring.elasticsearch.hosts: http://es01:9200

logstash.conf 此处用于配置 logstash对日志的处理

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
  beats {
    port => 5044
  }
}


 filter {
     grok {
          match => { "message" => "%{IP:client} %{WORD:test_method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
      }
      geoip {
          # source => "message"
          source => "client"
    }   
 }


output {
  elasticsearch {
    hosts => ["http://es01:9200"]
    index => "logstash-test-%{+YYYY.MM.dd}-%{[@metadata][version]}"
    #user => "elastic"
    #password => "changeme"
  }
  stdout { codec => rubydebug }
}

fliter：描述：

• grok ：正则表达式插件
• geoip ：IP 位置插件 ，读取

---

汇总的docker-compose.yml

version: '2'
services:
  filebeat03:
    image: docker.elastic.co/beats/filebeat:7.4.0
    container_name: filebeat03
    privileged: true
    networks:
      - esnet
    volumes:
      - /root/elk_demo/logstash_demo/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /root/elk_demo/logstash_demo/logs:/var/log/
      - /root/elk_demo/logstash_demo/filebeat/modules/:/usr/share/filebeat/modules.d/

  logstash:
    image: docker.elastic.co/logstash/logstash:7.4.0
    container_name: logstash02    
    privileged: true
    environment:
      - f=logstash.conf
#      - ES_PORT=9200
#      - KIBANA_HOST=kibana
#      - KIBANA_PORT=5601
    networks:
      - esnet
    volumes:
      - /root/elk_demo/logstash_demo/logstash/pipeline/:/usr/share/logstash/pipeline/
      - /root/elk_demo/logstash_demo/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml
      - /root/elk_demo/logstash_demo/logstash/log/logstash-tutorial.log:/usr/local/programs/logstash/logstash-tutorial.log
#      - /root/elk_demo/logstash/config/logstash.conf:/usr/share/logstash/config/logstash.conf
#      - /root/elk_demo/nginx/logs:/var/log/
#      - /root/elk_demo/filebeat/modules/:/usr/share/filebeat/modules.d/
    depends_on:
      - filebeat03
networks:
  esnet:
    external:
      name: elk_demo_network

---

测试：

echo "183.60.88.6 POST /hello_world.html 15824 0.043"  >> test.log 

01.png 02.png 03.png

kibana中查看到的es数据

04.png