1.部署ELK

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.6.0.rpm
wget https://artifacts.elastic.co/downloads/kibana/kibana-6.6.0-x86_64.rpm
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.6.0-x86_64.rpm
yum install -y java
rpm -ivh elasticsearch-6.6.0.rpm
rpm -ivh kibana-6.6.0-x86_64.rpm
rpm -ivh filebeat-6.6.0-x86_64.rpm

vim /etc/chrony.conf
server ntp.ailiyun.com iburst
systemctl restart chronyd

[root@es1 ~]# egrep -v  "^#|^$" /etc/elasticsearch/elasticsearch.yml
node.name: node-1
path.data: /data/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
http.port: 9200
[root@es1 ~]# systemctl start elasticsearch.service

[root@es1 ~]# egrep -v "^#|^$" /etc/kibana/kibana.yml
server.port: 5601
server.host: "10.0.0.233"
server.name: "es1"
elasticsearch.hosts: ["http://10.0.0.233:9200"]
[root@es1 ~]# systemctl restart kibana.service

[root@es1 ~]# vim /etc/filebeat/filebeat.yml  里面文件是yml格式 注意格式不能错
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
[root@es1 ~]# systemctl start filebeat.service 

cat >> /etc/yum.repos.d/nginx.repo <<EOF
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/\$releasever/\$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true
EOF
yum install -y nginx httpd-tools 测试收集日志使用
[root@es1 ~]# systemctl start nginx
[root@es1 ~]# ab -n 100 -c 100 http://10.0.0.233/ 压测

样例.png
图片.png
删除样例.png
删除样例.数据.png
kibana数据库删除后自建图形全部丢失.png
图片.png

2.kibana导入elasticsearch数据

kibana导入1.png
kibana导入2.png
kibana导入3.png
kibana导入4.png
只查看日志信息.png
筛选.png
将筛选条件disable.png

3. 收集 nginx json格式日志

将nginx日志输出为json格式
vim /etc/nginx/nginx.conf
    log_format json '{ "time_local": "$time_local", '
                                        '"remote_addr": "$remote_addr", '
                                        '"referer": "$http_referer", '
                                        '"request": "$request", '
                                        '"status": $status, '
                                        '"bytes": $body_bytes_sent, '
                                        '"agent": "$http_user_agent", '
                                        '"x_forwarded": "$http_x_forwarded_for", '
                                        '"up_addr": "$upstream_addr",'
                                        '"up_host": "$upstream_http_host",'
                                        '"upstream_time": "$upstream_response_time",'
                                        '"request_time": "$request_time"'
                                        ' }';
    access_log  /var/log/nginx/access.log  json;
清除旧日志
> /var/log/nginx/access.log
检查并重启nginx
nginx -t
systemctl restart nginx

filebeat修改配置文件
[root@es1 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
[root@es1 nginx]# systemctl restart filebeat.service

删除之前数据.png
图片.png

[root@es1 ~]# vim /etc/filebeat/filebeat.yml 自定义索引名称 使用月份命名
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  index: "nginx-%{[beat.version]}-%{+yyyy.MM}"
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true

[root@es1 nginx]# systemctl restart filebeat.service

图片.png
图片.png

4. 收集nginx 访问日志 及 错误日志

https://www.elastic.co/guide/en/beats/filebeat/6.6/elasticsearch-output.html

新建一台主机 es2安装nginx filebeat
yum install -y nginx httpd-tools
scp root@10.0.0.233:/etc/nginx/nginx.conf /etc/nginx/
nginx -t
systemctl start nginx
scp root@10.0.0.233:/root/filebeat-6.6.0-x86_64.rpm  /root
rpm -ivh filebeat-6.6.0-x86_64.rpm 
scp root@10.0.0.233:/etc/filebeat/filebeat.yml /etc/filebeat/
systemctl start filebeat.service 
systemctl status filebeat.service 

图片.png

[root@es2 ~]# vim /etc/filebeat/filebeat.yml  在input打tag output调用  这个tag可以是es中的任何字段
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  #index: "nginx-%{[beat.version]}-%{+yyyy.MM}"
  indices:
    - index: "nginx-access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "access"
    - index: "nginx-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "error"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
[root@es2 ~]# systemctl restart filebeat.service 清除es中之前的数据库 kibana中添加

图片.png
图片.png

5.收集tomcat日志

[root@es1 ~]# yum install tomcat tomcat-webapps tomcat-admin-webapps tomcat-docs-webapp tomcat-javadoc -y
[root@es1 ~]# systemctl start tomcat
[root@es1 ~]# systemctl status tomcat
修改文件 使tomcat日志变成json格式
vim /etc/tomcat/server.xml 找到下面 删除 大概139行
     pattern="%h %l %u %t "%r" %s %b" />
替换成
   pattern="{"clientip":"%h","ClientUser":"%l","authenticated":"%u","AccessTime":"%t","method":"%r","status":"%s","SendBytes":"%b","Query?string":"%q","partner":"%{Referer}i","AgentVersion":"%{User-Agent}i"}"/>
systemctl restart tomcat

日志.png

[root@es1 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]
##############tomcat#################

- type: log
  enabled: true
  paths:
    - /var/log/tomcat/localhost_access_log.*.txt
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["tomcat"]

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  #index: "nginx-%{[beat.version]}-%{+yyyy.MM}"
  indices:
    - index: "nginx-access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "access"
    - index: "nginx-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "error"
    - index: "tomcat-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "tomcat"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
[root@es1 ~]# systemctl restart filebeat.service

图片.png
图片.png

5.收集java Elasticsearch多行匹配模式日志

日志多行.png

[root@es1 elasticsearch]# vim /etc/filebeat/filebeat.yml

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]
##############tomcat#################

- type: log
  enabled: true
  paths:
    - /var/log/tomcat/localhost_access_log.*.txt
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["tomcat"]

##############el#################

- type: log
  enabled: true
  paths:
    - /var/log/elasticsearch/elasticsearch.log
  tags: ["els"]
  multiline.pattern: '^\['
  multiline.negate: true
  multiline.match: after

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  #index: "nginx-%{[beat.version]}-%{+yyyy.MM}"
  indices:
    - index: "nginx-access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "access"
    - index: "nginx-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "error"
    - index: "tomcat-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "tomcat"
    - index: "els-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "els"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true

systemctl restart elasticsearch.service 
制造错误日志 修改els配置文件  任意单词随意修改一下 重启 启动不起来 查看日志

图片.png

6.收集docker日志 多个容器使用docker-compose

yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum makecache fast
yum -y install docker-ce
systemctl start docker

sudo curl -L "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
docker-compose version

[root@es1 ~]# vim docker-compose.yml 

version: '3'
services:
  nginx:
    image: nginx
    labels:
      service: nginx
    logging:
      options:
        labels: "service"
    ports:
      - "8080:80"
  db:
    image: nginx
    labels:
      service: db
    logging:
      options:
        labels: "service"
    ports:
      - "80:80"
[root@es1 ~]# docker-compose up 启动 默认是前台运行

[root@es1 containers]# vim /etc/filebeat/filebeat.yml

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/lib/docker/containers/*/*-json.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  index: "docker-%{[beat.version]}-%{+yyyy.MM}"

setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
setup.template.overwrite: true
[root@es1 containers]# systemctl restart filebeat.service

图片.png
图片.png

改进
[root@es1 containers]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/lib/docker/containers/*/*-json.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  indices:
    - index: "docker-nginx-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        attrs.service: "nginx"
    - index: "docker-db-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        attrs.service: "db"

setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
setup.template.overwrite: true
[root@es1 containers]# systemctl restart filebeat.service

图片.png

改进
[root@es1 containers]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/lib/docker/containers/*/*-json.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  indices:
    - index: "docker-nginx-access%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        attrs.service: "nginx"
        stream: "stdout"
    - index: "docker-nginx-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        attrs.service: "nginx"
        stream: "stderr"
    - index: "docker-db-access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        attrs.service: "db"
        stream: "stdout"
    - index: "docker-db-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        attrs.service: "db"
        stream: "stderr"

setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
setup.template.overwrite: true
[root@es1 containers]# systemctl restart filebeat.service

图片.png
图片.png

7.filebeat 使用收集日志docker 单docker

https://www.elastic.co/guide/en/beats/filebeat/6.6/filebeat-input-docker.html

[root@es1 ~]# docker run --name nginx -p 80:80 -d nginx
af643c87bb8fa14f790d88943f08e4329b390ae5f023608c635f93dcb71e2948
[root@es1 ~]# ll /var/lib/docker/containers/
总用量 0
drwx-----x 4 root root 237 3月   3 11:09 af643c87bb8fa14f790d88943f08e4329b390ae5f023608c635f93dcb71e2948

[root@es1 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: docker
  containers.ids:
  - 'af643c87bb8fa14f790d88943f08e4329b390ae5f023608c635f93dcb71e2948'

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  index: "docker-nginx-%{[beat.version]}-%{+yyyy.MM}"

setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
setup.template.overwrite: true
[root@es1 ~]# systemctl restart filebeat.service

图片.png

[root@es1 ~]# vim /etc/filebeat/filebeat.yml

filebeat.inputs:
- type: docker
  containers.ids:
  - 'af643c87bb8fa14f790d88943f08e4329b390ae5f023608c635f93dcb71e2948'

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  indices:
    - index: "docker-nginx-access%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        stream: "stdout"
    - index: "docker-nginx-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        stream: "stderr"

setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
setup.template.overwrite: true
[root@es1 ~]# systemctl restart filebeat.service

图片.png

8.filebeat 使用模块收集nginx 普通日志

https://www.elastic.co/guide/en/beats/filebeat/6.6/configuration-filebeat-modules.html

#查看激活相关参数命令
filebeat modules --list 
#激活filebeat的nginx模块
filebeat modules enable nginx
#显示激活的模块
filebeat modules list

vim /etc/filebeat/filebeat.yml 添加下面
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
  reload.period: 10s

[root@es1 ~]# filebeat modules list
Enabled:

Disabled:
apache2
auditd
elasticsearch
haproxy
icinga
iis
kafka
kibana
logstash
mongodb
mysql
nginx
osquery
postgresql
redis
suricata
system
traefik
[root@es1 ~]# filebeat modules enable nginx
Enabled nginx
[root@es1 ~]# ll /etc/filebeat/modules.d/  将后面后缀.disabled 删除 就激活模块了
总用量 72
-rw-r--r-- 1 root root 371 1月  24 2019 apache2.yml.disabled
-rw-r--r-- 1 root root 175 1月  24 2019 auditd.yml.disabled
-rw-r--r-- 1 root root 845 1月  24 2019 elasticsearch.yml.disabled
-rw-r--r-- 1 root root 269 1月  24 2019 haproxy.yml.disabled
-rw-r--r-- 1 root root 546 1月  24 2019 icinga.yml.disabled
-rw-r--r-- 1 root root 371 1月  24 2019 iis.yml.disabled
-rw-r--r-- 1 root root 396 1月  24 2019 kafka.yml.disabled
-rw-r--r-- 1 root root 188 1月  24 2019 kibana.yml.disabled
-rw-r--r-- 1 root root 361 1月  24 2019 logstash.yml.disabled
-rw-r--r-- 1 root root 189 1月  24 2019 mongodb.yml.disabled
-rw-r--r-- 1 root root 368 1月  24 2019 mysql.yml.disabled
-rw-r--r-- 1 root root 369 1月  24 2019 nginx.yml
-rw-r--r-- 1 root root 388 1月  24 2019 osquery.yml.disabled
-rw-r--r-- 1 root root 192 1月  24 2019 postgresql.yml.disabled
-rw-r--r-- 1 root root 463 1月  24 2019 redis.yml.disabled
-rw-r--r-- 1 root root 190 1月  24 2019 suricata.yml.disabled
-rw-r--r-- 1 root root 574 1月  24 2019 system.yml.disabled
-rw-r--r-- 1 root root 195 1月  24 2019 traefik.yml.disabled

恢复之前nginx 普通日志格式
systemctl start nginx

[root@es1 ~]# vim /etc/filebeat/filebeat.yml
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
  reload.period: 10s

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  index: "nginx-%{[beat.version]}-%{+yyyy.MM}"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true

[root@es1 ~]# vim /etc/filebeat/modules.d/nginx.yml 
- module: nginx
  # Access logs
  access:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/var/log/nginx/access.log"]

  # Error logs
  error:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/var/log/nginx/error.log"]

[root@es1 ~]# systemctl restart filebeat.service 
[root@es1 ~]# tailf /var/log/filebeat/filebeat
    sudo bin/elasticsearch-plugin install ingest-user-agent
    sudo bin/elasticsearch-plugin install ingest-geoip
[root@es1 ~]# find / -name "elasticsearch-plugin"
/usr/share/elasticsearch/bin/elasticsearch-plugin
[root@es1 ~]# cd /usr/share/elasticsearch/
[root@es1 elasticsearch]# bin/elasticsearch-plugin install ingest-user-agent
[root@es1 elasticsearch]# bin/elasticsearch-plugin install ingest-geoip
-> Downloading ingest-geoip from elastic
[=================================================] 100%   
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@     WARNING: plugin requires additional permissions     @
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
* java.lang.RuntimePermission accessDeclaredMembers
* java.lang.reflect.ReflectPermission suppressAccessChecks
See http://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html
for descriptions of what these permissions allow and the associated risks.

Continue with installation? [y/N]y  输入y
-> Installed ingest-geoip
[root@es1 ~]# systemctl restart elasticsearch.service filebeat.service 

图片.png
注意选择.png
图片.png

[root@es1 ~]# vim /etc/filebeat/filebeat.yml

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
  reload.period: 10s

output.elasticsearch:
  hosts: ["10.0.0.233:9200"]
  indices:
    - index: "nginx-access%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        fileset.name: "access"
    - index: "nginx-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        fileset.name: "error"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
[root@es1 ~]# systemctl restart filebeat.service 
/usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml 排错

图片.png
图片.png

9.kinaba 画图

图片.png
图片.png
图片.png
图片.png
图片.png
图片.png
图片.png
图片.png
图片.png
图片.png
图片.png
图片.png
图片.png
图片.png
图片.png

10.使用redis做缓存实现elk filebeat模块不支持redis redis只支持单节点

https://www.elastic.co/guide/en/beats/filebeat/6.6/redis-output.html
nginx产生日志--filebeat收集日志--redis缓存日志--logstash读日志写入--elasticsearch存放日志--kinaba展示日志

mkdir /data/soft -p
cd /data/soft 
yum install wget  gcc-c++ -y
wget http://download.redis.io/releases/redis-5.0.7.tar.gz
tar xf redis-5.0.7.tar.gz -C /opt/
ln -s /opt/redis-5.0.7 /opt/redis
cd /opt/redis
make 
make install 

mkdir -p /opt/redis_6379/{conf,pid,logs}
mkdir -p /data/redis_6379
cat >/opt/redis_6379/conf/redis_6379.conf<< EOF
daemonize yes 
bind 10.0.0.233
port 6379
pidfile /opt/redis_6379/pid/redis_6379.pid
logfile /opt/redis_6379/logs/redis_6379.log
EOF
redis-server /opt/redis_6379/conf/redis_6379.conf

ps -ef|grep redis
netstat -lntup|grep 6379
[root@es1 ~]# redis-cli -h 10.0.0.233
10.0.0.233:6379> keys *
(empty list or set)

[root@es1 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.redis:
  hosts: ["10.0.0.233"]
  key: "filebeat"
  db: 0
  timeout: 5
[root@es1 ~]# systemctl restart filebeat.service
[root@es1 ~]# redis-cli -h 10.0.0.233
10.0.0.233:6379> keys *
1) "filebeat"
10.0.0.233:6379> type filebeat
list
10.0.0.233:6379> llen filebeat
(integer) 7
10.0.0.233:6379> LRANGE filebeat 1 7
1) "{\"@timestamp\":\"2021-03-03T07:39:33.967Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"message\":\"192.168.88.1 - - [03/Mar/2021:15:39:30 +0800] \\\"GET / HTTP/1.1\\\" 304 0 \\\"-\\\" \\\"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36\\\" \\\"-\\\"\",\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"es1\"},\"offset\":63883,\"json\":{},\"prospector\":{\"type\":\"log\"},\"beat\":{\"name\":\"es1\",\"hostname\":\"es1\",\"version\":\"6.6.0\"},\"log\":{\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"source\":\"/var/log/nginx/access.log\"}"

wget https://artifacts.elastic.co/downloads/logstash/logstash-6.6.0.rpm
[root@es1 ~]# rpm -ivh logstash-6.6.0.rpm 
[root@es1 ~]# vim /etc/logstash/conf.d/logsatsh.conf
input {
  redis {
    host => "10.0.0.233"
    port => "6379"
    db => "0"
    key => "filebeat"
    data_type => "list"
  }
}
filter{
  mutate {
    convert => ["upstream_time","float"]
    convert => ["request_time","float"]
  }
}

output{
  stdout {}
    elasticsearch {
      hosts => "10.0.0.233:9200"
      manage_template => false
      index => "nginx_access-%{+yyyy.MM.dd}"
  }
}
[root@es1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logsatsh.conf 启动时间很久
[root@es1 ~]# ab -n 20000 -c 20 http://10.0.0.233/
[root@es1 ~]# redis-cli -h 10.0.0.233
10.0.0.233:6379> LLEN filebeat
(integer) 16625
10.0.0.233:6379> LLEN filebeat
(integer) 16500
10.0.0.233:6379> LLEN filebeat
(integer) 16500

图片.png

[root@es1 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

output.redis:
  hosts: ["10.0.0.233"]
  keys:
    - key: "nginx_access"
      when.contains:
        tags: "access"
    - key: "nginx_error"
      when.contains:
        tags: "error"
[root@es1 ~]# systemctl restart filebeat.service
[root@es1 ~]# ab -n 20000 -c 20 http://10.0.0.233/ss.html
[root@es1 ~]# redis-cli -h 10.0.0.233
10.0.0.233:6379> keys *
1) "nginx_error"
2) "nginx_access"

[root@es1 ~]# vim /etc/logstash/conf.d/logsatsh.conf
input {
  redis {
    host => "10.0.0.233"
    port => "6379"
    db => "0"
    key => "nginx_access"
    data_type => "list"
  }
  redis {
    host => "10.0.0.233"
    port => "6379"
    db => "0"
    key => "nginx_error"
    data_type => "list"
  }
}
filter{
  mutate {
    convert => ["upstream_time","float"]
    convert => ["request_time","float"]
  }
}

output{
  stdout {}
  if "access" in [tags]{
    elasticsearch {
      hosts => "10.0.0.233:9200"
      manage_template => false
      index => "nginx_access-%{+yyyy.MM}"
    }
  }
  if "error" in [tags]{
    elasticsearch {
      hosts => "10.0.0.233:9200"
      manage_template => false
      index => "nginx_error-%{+yyyy.MM}"
    }
  }
}

[root@es1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logsatsh.conf

图片.png

11. 优化redis缓存 过程

filebeat收集日志--redis缓存日志--logstash读日志写入
filebeat收集写入redis时候不用两个key使用一个key
logstash读入日志写入时有根据tag判断哪个是access 哪个是error

[root@es1 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

output.redis:
  hosts: ["10.0.0.233"]
  key: "nginx"
[root@es1 ~]# systemctl restart filebeat.service

[root@es1 ~]# vim /etc/logstash/conf.d/logsatsh.conf 
input {
input { 
input { 
  redis {
    host => "10.0.0.233"
    port => "6379"
    db => "0"
    key => "nginx"
    data_type => "list"
  }
}
filter{
  mutate {
    convert => ["upstream_time","float"]
    convert => ["request_time","float"]
  }
}

output{
  stdout {}
  if "access" in [tags]{
    elasticsearch {
      hosts => "10.0.0.233:9200"
      manage_template => false
      index => "nginx_access-%{+yyyy.MM}"
    }
  }
  if "error" in [tags]{
    elasticsearch {
      hosts => "10.0.0.233:9200"
      manage_template => false
      index => "nginx_error-%{+yyyy.MM}"
    }
  }
}
[root@es1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logsatsh.conf

[root@es1 ~]# redis-cli -h 10.0.0.233
10.0.0.233:6379> keys *
1) "nginx"

图片.png

11.kinaba x-pack 监控开启

图片.png
图片.png

12.使用 两个redis单节点 keepalive vip做主备

es2 es3两个节点安装redis
mkdir /data/soft -p
cd /data/soft 
yum install wget  gcc-c++ -y
wget http://download.redis.io/releases/redis-5.0.4.tar.gz
tar xf redis-5.0.4.tar.gz -C /opt/
ln -s /opt/redis-5.0.4 /opt/redis
cd /opt/redis
make 
make install 

cat >/opt/redis_6379/conf/redis_6379.conf<< EOF
daemonize yes 
bind 0.0.0.0
port 6379
pidfile /opt/redis_6379/pid/redis_6379.pid
logfile /opt/redis_6379/logs/redis_6379.log
EOF
redis-server /opt/redis_6379/conf/redis_6379.conf

es2 es3两个节点安装keepalived
yum -y install wget gcc make tar openssl openssl-devel libnl libnl-devel libnfnetlink-devel
cd /usr/local/src
wget https://www.keepalived.org/software/keepalived-2.2.1.tar.gz
tar -zxvf keepalived-2.2.1.tar.gz
cd keepalived-2.2.1
./configure --prefix=/usr/local/keepalived
make && make install
es2 es3：
vim /usr/local/keepalived/etc/keepalived/keepalived.conf
vrrp_script check_redis
{
    script "/root/check.sh"
    interval 3
    weight -20
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    nopreempt
    virtual_router_id 51
    priority 100
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass 666666
    }
    virtual_ipaddress {
        10.0.0.2/24
    }
    track_script
    {
        check_redis
    }
}

vim /root/check.sh 
#!/bin/sh
check=`ss -lntup|grep 6379|wc -l`
if [ $check -eq 1 ];then
exit 0
else
exit 1
fi
chmod +x /root/check.sh
检查配置，启动keepalived
/usr/local/keepalived/sbin/keepalived -t -f /usr/local/keepalived/etc/keepalived/keepalived.conf
/usr/local/keepalived/sbin/keepalived -f /usr/local/keepalived/etc/keepalived/keepalived.conf

[root@es1 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

output.redis:
  hosts: ["10.0.0.3"]
  key: "nginx"
[root@es1 ~]# systemctl restart filebeat.service

[root@es1 ~]# vim /etc/logstash/conf.d/logsatsh.conf 
input {
input { 
input { 
  redis {
    host => "10.0.0.3"
    port => "6379"
    db => "0"
    key => "nginx"
    data_type => "list"
  }
}
filter{
  mutate {
    convert => ["upstream_time","float"]
    convert => ["request_time","float"]
  }
}

output{
  stdout {}
  if "access" in [tags]{
    elasticsearch {
      hosts => "10.0.0.233:9200"
      manage_template => false
      index => "nginx_access-%{+yyyy.MM}"
    }
  }
  if "error" in [tags]{
    elasticsearch {
      hosts => "10.0.0.233:9200"
      manage_template => false
      index => "nginx_error-%{+yyyy.MM}"
    }
  }
}
[root@es1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logsatsh.conf

[root@es1 ~]# ab -n 20000 -c 20 http://10.0.0.233/ss.html

图片.png