一、logstash服务端安装

1、版本：logstash-6.2.3

上传tar包，直接解压到安装目录/opt

2、处理x-pack包

[root@omnis-server bin]# ls
benchmark.sh         logstash               logstash.lib.sh      ruby       system-install
cpdump               logstash.bat           logstash-plugin      setup.bat  x-pack-6.2.3.zip
dependencies-report  logstash-keystore      logstash-plugin.bat  start.sh
ingest-convert.sh    logstash-keystore.bat  pqcheck              stop.sh
[root@omnis-server bin]# pwd
/opt/logstash-6.2.3/bin
[root@omnis-server bin]# ./logstash-plugin install file:///opt/logstash-6.2.3/bin/x-pack-6.2.3.zip

3、添加配置文件

可以在当前目录下面创建一个config目录，创建logstash_index.conf配置文件（名字自定义）。

[root@omnis-server config]# pwd
/opt/logstash-6.2.3/config
[root@omnis-server config]# ls
jvm.options  log4j2.properties  logstash_index.conf  logstash.yml  pipelines.yml  startup.options
[root@omnis-server config]# more logstash_index.conf 
input {
     redis {
            host => "192.168.12.172"
            port => "6378" 
            data_type => "list"
            key => "logstash"
            type => "redis-input"
            }
}
output {
        elasticsearch {
                        hosts => "58.48.177.198:9200"
                        user => "elastic"
                        password => "Elapp211!"
                      }
}

数据来源是redis数据库，去向是elasticsearch，而redis数据库里面的数据来源则是各个jboss服务器上面的jboss日志文件

4、查看logstash.yml文件

[root@omnis-server config]# pwd
/opt/logstash-6.2.3/config
[root@omnis-server config]# ls
jvm.options  log4j2.properties  logstash_index.conf  logstash.yml  pipelines.yml  startup.options
[root@omnis-server config]# cat logstash.yml | grep -Ev "^#|^$"
http.host: "192.168.12.172"
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.url: "http://58.48.177.198:9200"
xpack.monitoring.elasticsearch.username: "logstash_system" 
xpack.monitoring.elasticsearch.password: "Logapp211!"

5、启动程序

使用nohup方式启动

nohup /opt/logstash-6.2.3/bin/logstash  -f /opt/logstash-6.2.3/config/logstash_index.conf > /opt/logstash-6.2.3/log/logstatash.log &

或者

[root@omnis-server bin]# pwd
/opt/logstash-6.2.3/bin
[root@omnis-server bin]# ls
benchmark.sh         logstash               logstash.lib.sh      ruby       system-install
cpdump               logstash.bat           logstash-plugin      setup.bat  x-pack-6.2.3.zip
dependencies-report  logstash-keystore      logstash-plugin.bat  start.sh
ingest-convert.sh    logstash-keystore.bat  pqcheck              stop.sh
[root@omnis-server bin]#  sh start.sh

二、ansible批量安装logstash客户端

客户端与服务端安装的使用的是同一套程序，不同的是配置文件不通。由于客户端的机器比较多，不可能一台台的去部署，使用ansible工具进行批量部署,如下:

1、下载logstash客户端的包

在Gitlab上拉取logstash_clinet_install，然后解压

1645584932943_98E9B9E1-89CC-48aa-AAFF-7C195EA3C98E.png

[root@omnis-server logstash_clinet_install]# ls
logstash-2.3.4.tar.gz    logstashinstall.yml  start.sh     update_logstash_config.yml
logstash_agent_cdr.conf  README.md            stop_cdr.sh  update_logstash_damonfile.yml
logstashinstall_cdr.yml  start_cdr.sh         stop.sh      var.yml
[root@omnis-server logstash_clinet_install]# pwd
/root/logstash_clinet_install

2、编写playbook

[root@omnis-server logstash_clinet_install]# ls
logstash-2.3.4.tar.gz    logstashinstall.yml  start.sh     update_logstash_config.yml
logstash_agent_cdr.conf  README.md            stop_cdr.sh  update_logstash_damonfile.yml
logstashinstall_cdr.yml  start_cdr.sh         stop.sh      var.yml
[root@omnis-server logstash_clinet_install]# pwd
/root/logstash_clinet_install
[root@omnis-server logstash_clinet_install]# more logstashinstall.yml 
---
- hosts: "spbsc"
  gather_facts: True
  vars_files:
    - var.yml
  tasks:
  - name: delete logstash file
    file:
        path: "{ item }"
        state: absent
    with_items:
      - /opt/earth/logstash-2.3.4
      - /tmp/logstash-2.3.4.tar.gz
    ignore_errors: yes
  - name: copy file to clinet
    copy: src=./logstash-2.3.4.tar.gz  dest=/tmp owner=th  mode=755
  - name: tar file
    shell: tar zxf /tmp/logstash-2.3.4.tar.gz  -C /opt/earth
  - name: make conf directory
    shell: mkdir -p /opt/earth/logstash-2.3.4/conf
  - name: copy conf file
    template: src=./logstash_agent.conf dest=/opt/earth/logstash-2.3.4/conf/logstash_agent.conf owner=th
 mode=0644
    notify:
      - Restart Logstash
    ignore_errors: yes
  - name: copy damon script
    copy: 
      src: ./{{ item.src }}
      dest: /opt/earth/logstash-2.3.4/bin/{{ item.dest }}
      owner: th
      mode: 0755
    with_items:
      - { src: 'start.sh', dest: 'start.sh' }
      - { src: 'stop.sh',  dest: 'stop.sh' }
    notify:
      - Restart Logstash

  handlers:
  - name: Restart Logstash
    shell: sh /opt/earth/logstash-2.3.4/bin/{{ item }}
    with_items:
      - stop.sh
      - start.sh

3、编写配置文件logstash_agent.conf

将配置文件logstash_agent.conf放在/etc/ansible/template下面

[root@omnis-server template]# ls logstash_agent.conf
logstash_agent.conf
[root@omnis-server template]# pwd
/etc/ansible/template
[root@omnis-server template]# more logstash_agent.conf
input { 
    file {
            add_field => { "log_project" => "SPBRM" }
            add_field => { "log_host" => "192.168.12.172" }
            add_field => { "log_area" => "SPBRM"}
            type => "{{ansible_hostname}}-log"
            discover_interval => "2"
            path => "/opt/earth/jboss/standalone/log/*.log"
            codec => multiline 
            {
                # Grok pattern names are valid! 
                charset => "UTF-8"
                pattern => "^%{TIMESTAMP_ISO8601}"
                negate => true
                what => previous
            }
         }
     
}

filter {
    mutate {
            rename => [ "type","log_server"]
           } 
}

filter {
  grok {
    match => ["message" , "%{TIMESTAMP_ISO8601:log_time}( -)? %{LOGLEVEL:log_level} \[(?<log_class>(
.*?)|((?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$_]+))\] \((?<log_thread>(.*?))\)" ]
    } 
}
 
output {
    redis {
         host => "192.168.12.172"
         data_type =>"list"
         key => "logstash"
             port => "6378"
       }
}

4、执行playbook脚本

[root@omnis-server logstash_clinet_install]# pwd
/root/logstash_clinet_install
[root@omnis-server logstash_clinet_install]# ansible-playbook logstashinstall.yml  -e "host=spbsc" --start-at-task="copy conf file" -u th -k
[root@omnis-server logstash_clinet_install]# ansible-playbook logstashinstall.yml  -e "host=baksc" --start-at-task="copy conf file" -u th -k

执行完成此脚本后，logstash包会自动上传到客户机并自动修改配置文件，然后启动程序