1. 流程说明

2. 配置过程

2.1 nginx配置

log_format json  '{"time_local": "$time_local", '
                          '"remote_addr": "$remote_addr", '
                          '"referer": "$http_referer", '
                          '"request": "$request", '
                          '"status": $status, '
                          '"bytes": $body_bytes_sent, '
                          '"agent": "$http_user_agent", '
                          '"x_forwarded": "$http_x_forwarded_for", '
                          '"up_addr": "$upstream_addr", '
                          '"up_host": "$upstream_http_host", '
                          '"upstream_time": "$upstream_response_time", '
                          '"request_time": "$request_time"}';
# 使用json日志格式
access_log  /var/log/nginx/access.log main;

2.2 filebeat配置

filebeat.inputs:
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]
output.kafka:
  hosts: ["10.0.0.110:9092","10.0.0.111:9092","10.0.0.112:9092"]
  topic: nginx_log

2.3 logstash配置

input {
  kafka {
    bootstrap_servers => "10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092"
    topics => ["nginx_log"]
    group_id => "logstash"
    codec => "json"
  }
}

filter {
  mutate {
    convert => ["upstream_time", "float"]
    convert => ["request_time", "float"]
  }
}

output {
    if "access" in [tags] {
      elasticsearch {
        hosts => "http://10.0.0.101:9200"
        manage_template => false
        index => "nginx_access-%{+yyyy.MM}"
      }
    }
    if "error" in [tags] {
      elasticsearch {
        hosts => "http://10.0.0.101:9200"
        manage_template => false
        index => "nginx_error-%{+yyyy.MM}"
      }
    }
}

3. 测试

• 创建kafka topic

  [root@kafka01 ~]# /opt/kafka/bin/kafka-topics.sh --create --bootstrap-server 10.0.0.110:9092,10.0.0.111:9092,10.0.0.111:9092 --replication-factor 3 --partitions 3 --topic nginx_log
  

• 监听kafka topic

  [root@kafka03 ~]# /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server 10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092 --topic nginx_log --from-beginning
  

• 启动相关服务

  [root@nginx01 ~]# systemctl start nginx
  [root@es01 ~]# systemctl start elasticsearch
  [root@es01 ~]# systemctl start kibana
  [root@nginx01 ~]# systemctl start filebeat
  [root@es01 ~]# /usr/share/logstash/bin/logstash -f /root/logstash.yml
  

• 发送测试请求

  [root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/
  [root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/baidu
  

• kafka-console-consumer消费到数据

  [root@kafka03 ~]# /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server 10.0.0.110:9092,10.0.0.111:9092,10.0.0.112:9092 --topic nginx_log --from-beginning
  {"@timestamp":"2020-04-27T09:09:34.585Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.6.0","topic":"nginx_log"},"beat":{"name":"nginx01","hostname":"nginx01","version":"6.6.0"},"x_forwarded":"-","source":"/var/log/nginx/access.log","bytes":153,"request":"GET /baidu HTTP/1.0","status":404,"offset":552760,"up_host":"-","input":{"type":"log"},"time_local":"27/Apr/2020:17:09:34 +0800","tags":["access"],"host":{"name":"nginx01"},"log":{"file":{"path":"/var/log/nginx/access.log"}},"up_addr":"-","remote_addr":"10.0.0.109","request_time":"0.000","upstream_time":"-","referer":"-","agent":"ApacheBench/2.3","prospector":{"type":"log"}}
  {"@timestamp":"2020-04-27T09:09:34.585Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.6.0","topic":"nginx_log"},"agent":"ApacheBench/2.3","request_time":"0.000","prospector":{"type":"log"},"beat":{"version":"6.6.0","name":"nginx01","hostname":"nginx01"},"host":{"name":"nginx01"},"status":404,"up_host":"-","remote_addr":"10.0.0.109","tags":["access"],"source":"/var/log/nginx/access.log","log":{"file":{"path":"/var/log/nginx/access.log"}},"up_addr":"-","referer":"-","request":"GET /baidu HTTP/1.0","x_forwarded":"-","bytes":153,"time_local":"27/Apr/2020:17:09:34 +0800","input":{"type":"log"},"offset":553880,"upstream_time":"-"}
  ......
  

• 查看kibana

  GET _cat/indices
  
  yellow open nginx_access-2020.04            apimPU-QTAmP7GeE7l8evQ 5 1 2000  0   689kb   689kb
  yellow open nginx_error-2020.04             WH3Lme3gQuqBSK8MFYfsSw 5 1 1000  0 754.2kb 754.2kb
  
  GET nginx_access-2020.04/_search
  
  {
    "took" : 4,
    "timed_out" : false,
    "_shards" : {
      "total" : 5,
      "successful" : 5,
      "skipped" : 0,
      "failed" : 0
    },
    "hits" : {
      "total" : 1999,
      "max_score" : 1.0,
      "hits" : [
        {
          "_index" : "nginx_access-2020.04",
          "_type" : "doc",
          "_id" : "avBpunEBINm9vG5xGD9v",
          "_score" : 1.0,
          "_source" : {
            "tags" : [
              "access"
            ],
            "request" : "GET / HTTP/1.0",
            "offset" : 246975,
            "time_local" : "27/Apr/2020:14:49:37 +0800",
            "referer" : "-",
            "beat" : {
              "hostname" : "nginx01",
              "version" : "6.6.0",
              "name" : "nginx01"
            },
            "input" : {
              "type" : "log"
            },
            "host" : {
              "name" : "nginx01"
            },
            "status" : 200,
            "up_addr" : "-",
            "up_host" : "-",
            "prospector" : {
              "type" : "log"
            },
            "bytes" : 612,
            "@version" : "1",
            "agent" : "ApacheBench/2.3",
            "upstream_time" : 0.0,
            "request_time" : 0.0,
            "@timestamp" : "2020-04-27T06:49:45.660Z",
            "source" : "/var/log/nginx/access.log",
            "log" : {
              "file" : {
                "path" : "/var/log/nginx/access.log"
              }
            },
            "x_forwarded" : "-",
            "remote_addr" : "10.0.0.109"
          }
        }
      ]
    }
  }