美文网首页
009.ELK使用Redis做缓存收集Nginx日志

009.ELK使用Redis做缓存收集Nginx日志

作者: CoderJed | 来源:发表于2020-04-27 15:08 被阅读0次

1. 流程说明

2. 配置过程

2.1 nginx配置

log_format json  '{"time_local": "$time_local", '
                          '"remote_addr": "$remote_addr", '
                          '"referer": "$http_referer", '
                          '"request": "$request", '
                          '"status": $status, '
                          '"bytes": $body_bytes_sent, '
                          '"agent": "$http_user_agent", '
                          '"x_forwarded": "$http_x_forwarded_for", '
                          '"up_addr": "$upstream_addr", '
                          '"up_host": "$upstream_http_host", '
                          '"upstream_time": "$upstream_response_time", '
                          '"request_time": "$request_time"}';
# 使用json日志格式
access_log  /var/log/nginx/access.log main;

2.2 filebeat配置

filebeat.inputs:
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]
- type: log
  enabled: true 
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]
output.redis:
  hosts: ["10.0.0.104:6379"]
  keys:
    - key: "nginx_access"   
      when.contains:
        tags: "access"
    - key: "nginx_error"
      when.contains:
        tags: "error"

2.3 logstash配置

input {
  redis {
    host => "10.0.0.104"
    port => "6379"
    db => "0"
    key => "nginx_access"
    data_type => "list"
  }
  redis {
    host => "10.0.0.104"
    port => "6379"
    db => "0"
    key => "nginx_error"
    data_type => "list"
  }
}

filter {
  mutate {
    # 这两个字段转为float类型
    convert => ["upstream_time", "float"]
    convert => ["request_time", "float"]
  }
}

output {
    stdout {}
    if "access" in [tags] {
      elasticsearch {
        hosts => ["10.0.0.101:9200"]
        manage_template => false
        index => "nginx_access-%{+yyyy.MM}"
      }
    }
    if "error" in [tags] {
      elasticsearch {
        hosts => ["10.0.0.101:9200"]
        manage_template => false
        index => "nginx_error-%{+yyyy.MM}"
      }
    }
}

3. 测试

  • 启动以上服务

    [root@nginx01 ~]# systemctl start nginx
[root@redis01 ~]# /opt/redis/bin/redis-server /opt/redis/conf/redis_6379.conf
[root@es01 ~]# systemctl start elasticsearch
[root@es01 ~]# systemctl start kibana
[root@nginx01 ~]# systemctl start filebeat
[root@es01 ~]# /usr/share/logstash/bin/logstash -f /root/logstash.yml

  • 发送测试请求

    [root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/
[root@nginx01 opt]# ab -c 10 -n 1000 http://10.0.0.109:80/baidu

  • 查看redis

    10.0.0.104:6379> keys *
1) "nginx_error"
2) "nginx_access"

10.0.0.104:6379> lpop nginx_access
"{\"@timestamp\":\"2020-04-27T06:49:45.566Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"source\":\"/var/log/nginx/access.log\",\"time_local\":\"27/Apr/2020:14:49:37 +0800\",\"x_forwarded\":\"-\",\"log\":{\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"up_addr\":\"-\",\"offset\":2775,\"request\":\"GET / HTTP/1.0\",\"request_time\":\"0.000\",\"up_host\":\"-\",\"remote_addr\":\"10.0.0.109\",\"host\":{\"name\":\"nginx01\"},\"status\":200,\"referer\":\"-\",\"tags\":[\"access\"],\"prospector\":{\"type\":\"log\"},\"input\":{\"type\":\"log\"},\"beat\":{\"name\":\"nginx01\",\"hostname\":\"nginx01\",\"version\":\"6.6.0\"},\"agent\":\"ApacheBench/2.3\",\"upstream_time\":\"-\",\"bytes\":612}"
10.0.0.104:6379> lpop nginx_error
"{\"@timestamp\":\"2020-04-27T06:49:55.558Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"log\":{\"file\":{\"path\":\"/var/log/nginx/error.log\"}},\"source\":\"/var/log/nginx/error.log\",\"tags\":[\"error\"],\"prospector\":{\"type\":\"log\"},\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"nginx01\"},\"message\":\"2020/04/27 14:49:47 [error] 3031#3031: *1009 open() \\\"/usr/share/nginx/html/baidu\\\" failed (2: No such file or directory), client: 10.0.0.109, server: localhost, request: \\\"GET /baidu HTTP/1.0\\\", host: \\\"10.0.0.109\\\"\",\"offset\":1040,\"beat\":{\"version\":\"6.6.0\",\"name\":\"nginx01\",\"hostname\":\"nginx01\"}}"

  • logstash控制台输出

    {
             "tags" => [
        [0] "access"
    ],
          "request" => "GET /baidu HTTP/1.0",
           "offset" => 554720,
             "beat" => {
        "hostname" => "nginx01",
         "version" => "6.6.0",
            "name" => "nginx01"
    },
          "referer" => "-",
       "time_local" => "27/Apr/2020:14:49:47 +0800",
            "input" => {
        "type" => "log"
    },
             "host" => {
        "name" => "nginx01"
    },
           "status" => 404,
          "up_addr" => "-",
          "up_host" => "-",
       "prospector" => {
        "type" => "log"
    },
            "bytes" => 153,
         "@version" => "1",
            "agent" => "ApacheBench/2.3",
     "request_time" => 0.0,
    "upstream_time" => 0.0,
       "@timestamp" => 2020-04-27T06:49:48.731Z,
           "source" => "/var/log/nginx/access.log",
              "log" => {
        "file" => {
            "path" => "/var/log/nginx/access.log"
        }
    },
      "x_forwarded" => "-",
      "remote_addr" => "10.0.0.109"
}

  • 查看kibana

    GET _cat/indices

yellow open nginx_access-2020.04 hikPROoJR0OK3YiX1a-ztA 5 1 2000 0 643.2kb 643.2kb
yellow open nginx_error-2020.04  QUpHiZuuQSetl0m04xrMRQ 5 1 1000 0 995.9kb 995.9kb

GET nginx_access-2020.04/_search

{
  "took" : 4,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 1999,
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "nginx_access-2020.04",
        "_type" : "doc",
        "_id" : "avBpunEBINm9vG5xGD9v",
        "_score" : 1.0,
        "_source" : {
          "tags" : [
            "access"
          ],
          "request" : "GET / HTTP/1.0",
          "offset" : 246975,
          "time_local" : "27/Apr/2020:14:49:37 +0800",
          "referer" : "-",
          "beat" : {
            "hostname" : "nginx01",
            "version" : "6.6.0",
            "name" : "nginx01"
          },
          "input" : {
            "type" : "log"
          },
          "host" : {
            "name" : "nginx01"
          },
          "status" : 200,
          "up_addr" : "-",
          "up_host" : "-",
          "prospector" : {
            "type" : "log"
          },
          "bytes" : 612,
          "@version" : "1",
          "agent" : "ApacheBench/2.3",
          "upstream_time" : 0.0,
          "request_time" : 0.0,
          "@timestamp" : "2020-04-27T06:49:45.660Z",
          "source" : "/var/log/nginx/access.log",
          "log" : {
            "file" : {
              "path" : "/var/log/nginx/access.log"
            }
          },
          "x_forwarded" : "-",
          "remote_addr" : "10.0.0.109"
        }
      }
    ]
  }
}

相关文章