EFK日志收集分析系统

https://docs.fluentd.org/v1.0/articles/out_elasticsearch

1.png

cat >> /etc/hosts <<eof   [Node1 Node2]
10.0.0.20 master
10.0.0.22 web1
eof

# yum install java        [Node1 Node2]
# java -version

Node1

# wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.rpm
# vim /etc/elasticsearch/elasticsearch.yml
cluster.name: my-application
node.name: node-1
node.master: true
network.host: 10.0.0.20
http.port: 9200

Node2
# vim /etc/elasticsearch/elasticsearch.yml
cluster.name: my-application
node.name: node-2
node.master: false
network.host: 10.0.0.22
http.port: 9200
discovery.zen.ping.unicast.hosts: ["host1", "10.0.0.20”]

# /etc/init.d/elasticsearch start
# /etc/init.d/elasticsearch status
# curl http://10.0.0.20:9200/_cat/health
# curl http://10.0.0.20:9200/_cat/nodes


Fluentd (tdagent)  Node2

# wget http://packages.treasuredata.com.s3.amazonaws.com/3/redhat/7/x86_64/td-agent-3.2.0-0.el7.x86_64.rpm
# rpm -ivh td-agent-3.2.0-0.el7.x86_64.rpm  --force --nodeps
# yum install -y libcurl-devel
# /opt/td-agent/embedded/bin/fluent-gem install fluent-plugin-elasticsearch

# cd /etc/td-agent/
# cat td-agent.conf
<source>
  @type forward
  port 24224
</source>

####################################
<source>
  @type tail
  path /var/log/httpd/access_log
  pos_file /var/log/td-agent/httpd-access.log.pos
  tag apache.access
  <parse>
    @type apache2
  </parse>
</source>
####################################
<match debug.**>
  @type stdout
</match>
####################################

<match *.**>
  @type copy
  <store>
    @type elasticsearch
    host 10.0.0.20
    port 9200
    logstash_format true
    logstash_prefix fluentd-${tag}
    logstash_dateformat %Y%m%d
    include_tag_key true
    type_name access_log
    tag_key @log_name
    flush_interval 1s
  </store>
  <store>
    @type stdout
  </store>
</match>

# /etc/init.d/td-agent restart
# yum -y install httpd
# chmod 777 -R /var/log/httpd/
# tail -f /var/log/td-agent/td-agent.log
# curl http://10.0.0.20:9200/_cat/indices?v


Kibana    [master]

# wget https://artifacts.elastic.co/downloads/kibana/kibana-6.3.1-x86_64.rpm
# rpm -ivh kibana-6.3.1-x86_64.rpm

# vim /etc/kibana/kibana.yml
server.port: 5601
server.host: "10.0.0.20"
elasticsearch.url: "http://10.0.0.20:9200"
kibana.index: ".kibana"

# /etc/init.d/kibana restart
# tail -f /var/log/kibana/kibana.stderr

浏览器 http://10.0.0.20:5601/app/kibana#/home?_g=()

2.png 3.png 4.png 5.png 6.png