FileBeat配置输入配置

多个输入配置

#=========================== Filebeat inputs =============================

filebeat.inputs:

# Each - is an input. Most options can be set at the input level, so
# you can use different inputs for various configurations.
# Below are the input specific configurations.

- type: log

  # Change to true to enable this input configuration.
#
  enabled: true

  # Paths that should be crawled and fetched. Glob based paths.
  paths:
  - /home/elk/filebeat-6.6.1-linux-x86_64/nginx-log/access.log
    #- c:\programdata\elasticsearch\logs\*
  tags: ["nginx-access"]

- type: log
  enable: true
  paths:
   - /home/elk/filebeat-6.6.1-linux-x86_64/nginx-log/error.log
  tags: ["nginx-error"]

配置信息详解

###################### Filebeat Configuration Example #########################

# This file is an example configuration file highlighting only the most common

# options. The filebeat.reference.yml file from the same directory contains all the

# supported options with more comments. You can use it as a reference.

#

# You can find the full configuration reference here:

# https://www.elastic.co/guide/en/beats/filebeat/index.html

# For more available modules and options, please see the filebeat.reference.yml sample

# configuration file.

#=========================== Filebeat inputs =============================

#=========================== Filebeat 输入配置 ===========================

filebeat.inputs:

# Each - is an input. Most options can be set at the input level, so

# you can use different inputs for various configurations.

# Below are the input specific configurations.

# 输入filebeat的类型，包括log(具体路径的日志),stdin(键盘输入),redis,udp,docker,tcp,syslog,可以同时配置多个(包括相同类型的)

# 具体的每种类型的配置信息可以通过官网:https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html 了解

- type: log

  # Change to true to enable this input configuration.

  # 配置是否生效

  enabled: true

  # Paths that should be crawled and fetched. Glob based paths.

  # 指定要监控的日志，可以指定具体得文件或者目录

  paths:

    #- /var/log/*.log (这是默认的,自行可以修改）

    - /usr/local/tomcat/logs/catalina.out

  # Exclude lines. A list of regular expressions to match. It drops the lines that are

  # matching any regular expression from the list.

  # 在输入中排除符合正则表达式列表的那些行。

  #exclude_lines: ['^DBG']

  # Include lines. A list of regular expressions to match. It exports the lines that are

  # matching any regular expression from the list.

  # 包含输入中符合正则表达式列表的那些行（默认包含所有行），include_lines执行完毕之后会执行exclude_lines

  #include_lines: ['^ERR', '^WARN']

  # Exclude files. A list of regular expressions to match. Filebeat drops the files that

  # are matching any regular expression from the list. By default, no files are dropped.

  # 忽略掉符合正则表达式列表的文件

  #exclude_files: ['.gz$']

  # Optional additional fields. These fields can be freely picked

  # to add additional information to the crawled log files for filtering

  # 向输出的每一条日志添加额外的信息，比如“level:debug”，方便后续对日志进行分组统计。

  # 默认情况下，会在输出信息的fields子目录下以指定的新增fields建立子目录，例如fields.level

  # 这个得意思就是会在es中多添加一个字段，格式为 "filelds":{"level":"debug"}

  #fields:

  #  level: debug

  #  review: 1

  #  module: mock 

  ### Multiline options

  ### 日志中经常会出现多行日志在逻辑上属于同一条日志的情况，所以需要multiline参数来详细阐述。

  # Multiline can be used for log messages spanning multiple lines. This is common

  # for Java Stack Traces or C-Line Continuation

  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [

  # 多行匹配正则表达式，比如：用空格开头(^[[:space:]]),或者是否以[开头(^\[)。正则表达式是非常复杂的，详细见filebeat的正则表达式官方链接：https://www.elastic.co/guide/en/beats/filebeat/current/regexp-support.html

  multiline.pattern: ^\[

  # Defines if the pattern set under pattern should be negated or not. Default is false.

  # 该参数意思是是否否定多行融入。

  #multiline.negate: false

  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern

  # that was (not) matched before or after or as long as a pattern is not matched based on negate.

  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash

  # 取值为after或before。该值与上面的pattern与negate值配合使用

  # ----------------------------------------------------------------------------------------------------

  #|multiline.pattern|multiline.negate|multiline.match|                      结论                      |

  # ----------------------------------------------------------------------------------------------------

  #|      true      |    true        |    before    |表示匹配行是结尾,和前面不匹配的组成一行完整的日志|

  # ----------------------------------------------------------------------------------------------------

  #|      true      |    true        |    after    |表示匹配行是开头,和后面不匹配的组成一行完整的日志|

  # ----------------------------------------------------------------------------------------------------

  #|      true      |    false      |    before    |表示匹配的行和后面不匹配的一行组成一行完整的日志 |

  # ----------------------------------------------------------------------------------------------------

  #|      true      |    false      |    after    |表示匹配的行和前面不匹配的一行组成一行完整的日志 |

  # ----------------------------------------------------------------------------------------------------

  multiline.match: after

  # Specifies a regular expression, in which the current multiline will be flushed from memory, ending the multiline-message.

  # 表示符合该正则表达式的，将从内存刷入硬盘。

  #multiline.flush_pattern

  # The maximum number of lines that can be combined into one event.

  # If the multiline message contains more than max_lines, any additional lines are discarded. The default is 500.

  # 表示如果多行信息的行数超过该数字，则多余的都会被丢弃。默认值为500行

  #multiline.max_lines: 500

  # After the specified timeout, Filebeat sends the multiline event even if no new pattern is found to start a new event. The default is 5s.

  # 表示超过timeout的时间(秒)还没有新的一行日志产生，则自动结束当前的多行、形成一条日志发出去

  #multiline.timeout: 5

#============================= Filebeat modules ===============================

# 引入filebeat的module配置

filebeat.config.modules:

  # Glob pattern for configuration loading

  path: ${path.config}/modules.d/*.yml

  # Set to true to enable config reloading

  # 是否允许重新加载

  reload.enabled: false

  # Period on which files under path should be checked for changes

  # 重新加载的时间间隔

  #reload.period: 10s

#==================== Elasticsearch template setting ==========================

# Elasticsearch模板配置

setup.template.settings:

  # 数据分片数

  index.number_of_shards: 3

  # 数据分片备份数

  #index.number_of_replicas: 1

  #index.codec: best_compression

  #_source.enabled: false

#================================ General =====================================

# The name of the shipper that publishes the network data. It can be used to group

# all the transactions sent by a single shipper in the web interface.

# 设置filebeat的名字，如果配置为空，则用该服务器的主机名

#name:

# The tags of the shipper are included in their own field with each

# transaction published.

# 额外添加的tag标签

#tags: ["service-X", "web-tier"]

# Optional fields that you can specify to add additional information to the

# output.

# 额外添加的字段和值

#fields:

#  env: staging

#============================== Dashboards =====================================

# dashboards的相关配置

# These settings control loading the sample dashboards to the Kibana index. Loading

# the dashboards is disabled by default and can be enabled either by setting the

# options here, or by using the `-setup` CLI flag or the `setup` command.

# 是否启用仪表盘

#setup.dashboards.enabled: false

# The URL from where to download the dashboards archive. By default this URL

# has a value which is computed based on the Beat name and version. For released

# versions, this URL points to the dashboard archive on the artifacts.elastic.co

# website.

# 仪表盘地址

#setup.dashboards.url:

#============================== Kibana =====================================

# kibana的相关配置

# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

# This requires a Kibana endpoint configuration.

setup.kibana:

  # Kibana Host

  # Scheme and port can be left out and will be set to the default (http and 5601)

  # In case you specify and additional path, the scheme is required: http://localhost:5601/path

  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601

  # kibana地址

  #host: "localhost:5601"

  # Kibana Space ID

  # ID of the Kibana Space into which the dashboards should be loaded. By default,

  # the Default Space will be used.

  # kibana的空间ID

  #space.id:

#============================= Elastic Cloud ==================================

# These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/).

# The cloud.id setting overwrites the `output.elasticsearch.hosts` and

# `setup.kibana.host` options.

# You can find the `cloud.id` in the Elastic Cloud web UI.

#cloud.id:

# The cloud.auth setting overwrites the `output.elasticsearch.username` and

# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.

#cloud.auth:

#================================ Outputs =====================================

# 输出配置

# Configure what output to use when sending the data collected by the beat.

#-------------------------- Elasticsearch output ------------------------------

# 输出到es

#output.elasticsearch:

  # Array of hosts to connect to.

  # ES地址

  # hosts: ["localhost:9200"]

  # ES索引

  # index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"

  # Optional protocol and basic auth credentials.

  # 协议

  #protocol: "https"

  # ES用户名

  #username: "elastic"

  # ES密码

  #password: "changeme"

#----------------------------- Logstash output --------------------------------

# 输出到logstash

output.logstash:

  # The Logstash hosts

  # logstash地址

  hosts: ["localhost:5044"]

  # Optional SSL. By default is off.

  # List of root certificates for HTTPS server verifications

  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  # Certificate for SSL client authentication

  #ssl.certificate: "/etc/pki/client/cert.pem"

  # Client Certificate Key

  #ssl.key: "/etc/pki/client/cert.key"

#================================ Procesors =====================================

# Configure processors to enhance or manipulate events generated by the beat.

processors:

  #主机相关 信息

  - add_host_metadata: ~

# 云服务器的元数据信息,包括阿里云ECS 腾讯云QCloud AWS的EC2的相关信息 

  - add_cloud_metadata: ~

  #k8s元数据采集

  #- add_kubernetes_metadata: ~

  # docker元数据采集

  #- add_docker_metadata: ~

  # 执行进程的相关数据

  #- - add_process_metadata: ~

#================================ Logging =====================================

# Sets log level. The default log level is info.

# Available log levels are: error, warning, info, debug

#logging.level: debug

# At debug level, you can selectively enable logging only for some components.

# To enable all selectors use ["*"]. Examples of other selectors are "beat",

# "publish", "service".

#logging.selectors: ["*"]

#============================== Xpack Monitoring ===============================

# filebeat can export internal metrics to a central Elasticsearch monitoring

# cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

# reporting is disabled by default.

# Set to true to enable the monitoring reporter.

#xpack.monitoring.enabled: false

# Uncomment to send the metrics to Elasticsearch. Most settings from the

# Elasticsearch output are accepted here as well. Any setting that is not set is

# automatically inherited from the Elasticsearch output configuration, so if you

# have the Elasticsearch output configured, you can simply uncomment the

# following line.

#xpack.monitoring.elasticsearch: