Flume

安装Flume 1.6.0-cdh5.7.0

配置环境变量

export FLUME_HOME=/root/app/apache-flume-1.6.0-cdh5.7.0-bin
export PATH=$FLUME_HOME/bin:$PATH

配置Flume JAVA_HOME

/root/app/apache-flume-1.6.0-cdh5.7.0-bin/conf
/flume-env.sh
export JAVA_HOME=/usr/local/jdk8

配置 conf

A) 配置Source
B)配置 Channnel
C)配置 Sink
D) 把以上三个组件串起来


# Define a memory channel called ch1 on agent1
agent1.channels.ch1.type = memory
 
# Define an Avro source called avro-source1 on agent1 and tell it
# to bind to 0.0.0.0:41414. Connect it to channel ch1.
a1.sources.avro-source1.channels = ch1
a1.sources.avro-source1.type = avro
a1.sources.avro-source1.bind = 0.0.0.0
a1.sources.avro-source1.port = 41414
 
# Define a logger sink that simply logs all events it receives
# and connect it to the other end of the same channel.
a1.sinks.log-sink1.channel = ch1
a1.sinks.log-sink1.type = logger
 
# Finally, now that we've defined all of our components, tell
# agent1 which ones we want to activate.
a1.channels = ch1
a1.sources = avro-source1
a1.sinks = log-sink1

# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
 
# Define source

a1.sources.r1.type = netcat
a1.sources.r1.bind = lixinxin
a1.sources.r1.port = 44444

# Define sink
a1.sinks.k1.type = logger

#Define a memory channel called ch1 on agent1
a1.channels.c1.type = memory

#Bind the source and sink to channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

启动agent

flume-ng agent \
--name a1 \
--conf $FLUME_HOME/conf \
--conf-file $FLUME_HOME/conf/example.conf \
-Dflume.root.logger=INFO,console

flume配置agent.png
启动flume.png

停止 flume

ps -ef|grep flume
 kill -9 pid