Logstash采集Nginx日志写入ES

Logstash采集Nginx日志方式

• RPM安装Logstash采集Nginx日志
• Docker安装Logstash采集Nginx日志

RPM安装Logstash采集Nginx日志

• 安装jdk

  #新建jdk目录 ~/software/java
  cd ~/software/java
  #下载jdk1.8
  wget http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.tar.gz?AuthParam=1534129356_6b3ac55c6a38ba5a54c912855deb6a22
  #解压
  tar -zxvf jdk-8u181-linux-x64.tar.gz
  #配置JAVA环境变量
  vi /etc/profile
  #写入以下内容
  #java
  export JAVA_HOME=/root/software/java/jdk1.8.0_181
  export PATH=$JAVA_HOME/bin:$PATH
  export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib
  #加载环境变量
  source /etc/profile
  

• 验证jdk

image.png

• 安装Logstash

  #下载logstash rpm文件【与ES、Kibana版本一致】
  wget https://artifacts.elastic.co/downloads/logstash/logstash-7.8.0.rpm
  #rpm安装
  rpm -ivh logstash-7.8.0.rpm
  

  etc/logstash/pipelines.yml 为管道配置，添加新的logstash配置文件后要检查管道中是否关联启用

• Nginx配置文件

  #user  nobody;
  worker_processes  2;

  #error_log  logs/error.log;
  #error_log  logs/error.log  notice;
  #error_log  logs/error.log  info;

  #pid        logs/nginx.pid;

  events {
      worker_connections  1024;
  }

  http {
      include       mime.types;
      default_type  application/octet-stream;

      log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                  '$status $body_bytes_sent "$http_referer" '
                  '"$http_user_agent" "$http_x_forwarded_for" "$upstream_response_time" "$request_time"';

      #access log json格式配置
      log_format json '{ "@timestamp": "$time_iso8601", '
                     '"time": "$time_iso8601", '
                     '"remote_addr": "$remote_addr", '
                     '"remote_user": "$remote_user", '
                     '"body_bytes_sent": "$body_bytes_sent", '
                     '"request_time": "$request_time", '
                     '"status": "$status", '
                     '"host": "$host", '
                     '"request": "$request", '
                     '"request_method": "$request_method", '
                     '"uri": "$uri", '
                     '"http_referrer": "$http_referer", '
                     '"body_bytes_sent":"$body_bytes_sent", '
                     '"http_x_forwarded_for": "$http_x_forwarded_for", '
                     '"http_user_agent": "$http_user_agent" '
                '}';

      access_log  /var/log/nginx/access.log  main;
      #设置json格式access log文件路径
      access_log  /var/log/nginx/access_json.log json;

      sendfile        on;
      #tcp_nopush     on;

      #keepalive_timeout  0;
      keepalive_timeout  65;

      include /etc/nginx/vhost/*.conf;

      #gzip  on;
}

• Logstash配置文件nginx日志采集配置

  cp /etc/logstash/logstash-sample.conf  /etc/logstash/conf.d/nginx-log.conf
  vi nginx-log.conf      
  #写入以下内容
  input {
      file{
          path => "/var/log/nginx/access_json.log"  #需要采集的json格式nginx access日志文件路径
          codec => json                             #数据格式
          #start_position => "beginning"            #默认为采集新增内容，begining为从文件开头开始采集
          #type => "nginx_log"                      #类型
      }
  }
  output {
      #es配置
      elasticsearch {
          hosts => "eshost:9200"      #es地址
          index => "logstash-nginx"   #写入的索引名
          user => elastic             #es用户
          password => espassword      #es密码
      }
      #logstash控制台输入采集到的数据，用于调试
      #stdout {
      #    codec => rubydebug
      #}
  }
  

• 后台启动logstash服务

  #logstash服务路径：/usr/share/logstash/bin/logstash
  #后台启动logstash服务，并将内容输出到/var/log/logstash/output_nginx.log文件下
  nohup /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx-log.conf >> /var/log/logstash/output_nginx.log 2>&1 &
  #查看logstash服务
  ps -ef | grep logstash
  #关闭logstash服务
  kill -9 logstash进程号
  

kibana查看配置

• 选择logstash-nginx索引，创建索引模式

• discover面板查看

  
  image.png

Docker安装Logstash采集Nginx日志

• docker安装logstash，logstash服务运行在容器内，采用file类型只能采集容器内存在的文件内容，无法采集宿主机文件，需要使用syslog方式采集宿主机日志

• nginx配置需要开启syslog推送

  access_log syslog:server=接收主机ip:514,facility=local7,tag=nginx,severity=info,nohostname main_json;