Flume

1、新建脚本/root/20190623/flume/script.txt，内容如下

a1.sources = source1

a1.sinks = sink1

a1.channels = channel1

# Describe/configure tail -F source1

a1.sources.source1.type = exec

a1.sources.source1.command = tail -F -n 20 /tmp/log.log

a1.sources.source1.channels = channel1

#configure host for source

a1.sources.source1.interceptors = i1

a1.sources.source1.interceptors.i1.type = host

a1.sources.source1.interceptors.i1.hostHeader = hostname

a1.sinks.sink1.type = hdfs

a1.sinks.sink1.hdfs.path =hdfs://xxhostname:8020/tmp/tail/%y-%m-%d/%H

a1.sinks.sink1.hdfs.filePrefix = %{hostname}/events-

a1.sinks.sink1.hdfs.maxOpenFiles = 5000

a1.sinks.sink1.hdfs.batchSize= 500

a1.sinks.sink1.hdfs.fileType = DataStream

a1.sinks.sink1.hdfs.writeFormat =Text

a1.sinks.sink1.hdfs.rollSize = 0

a1.sinks.sink1.hdfs.rollCount = 1000000

a1.sinks.sink1.hdfs.rollInterval = 600

a1.sinks.sink1.hdfs.useLocalTimeStamp = true

# Use a channel which buffers events in memory

a1.channels.channel1.type = memory

a1.channels.channel1.keep-alive = 120

a1.channels.channel1.capacity = 500000

a1.channels.channel1.transactionCapacity = 600

# Bind the source and sink to the channel

a1.sources.source1.channels = channel1

a1.sinks.sink1.channel = channel1

2、进入flume的bin目录执行命令:

flume-ng agent --conf ../conf/ -f /root/20190623/flume/script.txt -Dflume.root.logger=DEBUG,console -n a1

3、查看相应的文件

 hadoop fs -ls

 hadoop fs -text 文件路径

注：查看端口

/opt/hadoop-2.6.5/etc/hadoop