curl -L https://toolbelt.treasuredata.com/sh/install-amazon1-td-agent3.sh | sh

/etc/td-agent/

[root@ip td-agent]# cat td-agent.conf

@include /etc/td-agent/conf.d/*.conf

/etc/td-agent/conf.d

[root@ip conf.d]# cat td-gamex.conf

<source>

@type forward

port 24224

</source>

##########################################################################################

<source>

@type tail

format json

time_key time

time_format %N

pos_file /var/log/td-agent/a.log.pos

path /opt/supervisor/log/log*.%d.%m.%Y.log

tag log.106

</source>

##########################################################################################

<match debug.**>

@type stdout

</match>

##########################################################################################

<match log.**>

@type forward # forward模式，转发给其他服务器处理

send_timeout 60s

recover_wait 10s

transport

phi_threshold 16

hard_timeout 60s

<server>

 name log.shard

 host 【IP】

 port 24224

 weight 1

</server>

<secondary>

 @type file

 path /var/log/td-agent/log-forward-failed

</secondary>

</match>

##########################################################################################

例如添加一个 client log

[root@ip conf.d]# cat client.conf

<source>

@type tail

format json

time_key time

time_format %N

pos_file /var/log/td-agent/client_150.log.pos

path /opt/supervisor/log/client*.%d.%m.%Y.log

tag client.150

</source>

##########################################################################################

<match debug.**>

@type stdout

</match>

##########################################################################################

<match client_log.**>

@type forward # forward模式，转发给其他服务器处理

send_timeout 60s

recover_wait 10s

transport

phi_threshold 16

hard_timeout 60s

<server>

 name client_log

 host 【IP】

 port 24224

 weight 1

</server>

<secondary>

 @type file

 path /var/log/td-agent/logics-forward-failed

</secondary>

</match>

##########################################################################################

中转机器上添加配置

[root@ip conf.d]# cat td-client.conf

<match debug.**>

@type stdout

</match>

##################################################################################

<match client_log.**>

@type copy

<store>

@type elasticsearch_dynamic

host  【IP】

port 9200

request_timeout 15s #defaults to 5s

reload_connections false

reload_on_failure true # defaults to false

logstash_format true

logstash_prefix client_logiclog

logstash_dateformat %Y.%m.%d

time_key time

<buffer>

  @type file

  path /var/log/td-agent/buffer/td-client_log-buffer

  chunk_limit_size 512MB #Default: 8MB (memory) / 256MB (file)

  total_limit_size 32GB #Default: 512MB (memory) / 64GB (file)

  chunk_full_threshold 0.9 #output plugin will flush the chunk when actual size reaches chunk_limit_size * 
  chunk_full_threshold

  compress text #The option to specify compression of each chunks, during events are buffered

  flush_mode default

  flush_interval 15s #Default: 60s

  flush_thread_count 1 #Default: 1 The number of threads of output plugins, which is used to write chunks in parallel

  delayed_commit_timeout 60 #The timeout seconds until output plugin decides that async write operation fails

  overflow_action throw_exception

  retry_timeout 10m

</buffer>

</store>

</match>