大数据之群起脚本

注：启动群起脚本前给与脚本执行权限:chmod +x 脚本文件名称

一、zookeeper群起脚本

#!/bin/bash

case $1 in

"start") {

for i in hadoop102 hadoop103 hadoop104; do

echo ============== zookeeper $i 启动 ===============

ssh $i "/opt/module/zookeeper/bin/zkServer.sh start"

    done

};;

"stop") {

for i in hadoop102 hadoop103 hadoop104; do

echo ============== zookeeper $i 停止 ===============

ssh $i "/opt/module/zookeeper/bin/zkServer.sh stop"

    done

};;

"status") {

for i in hadoop102 hadoop103 hadoop104; do

echo ============== zookeeper $i 状态 ===============

ssh $i "/opt/module/zookeeper/bin/zkServer.sh status"

    done

};;

esac

二、Kafka群起脚本

#!/bin/bash

case $1 in

"start") {

for i in hadoop102 hadoop103 hadoop104; do

echo ============== kafka $i 启动 ===============

ssh $i "/opt/module/kafka/bin/kafka-server-start.sh -daemon /opt/module/kafka/config/server.properties"

    done

};;

"stop") {

for i in hadoop102 hadoop103 hadoop104; do

echo ============== kafka $i 停止 ===============

ssh $i "/opt/module/kafka/bin/kafka-server-stop.sh stop"

    done

};;

esac

三、Flume脚本

#!/bin/bash

case $1 in

start )

for i in hadoop102 hadoop103; do

echo ============== flume $i 启动 ===============

ssh $i "/opt/module/flume-1.9.0/bin/flume-ng agent -n a1 -c /opt/module/flume-1.9.0/conf/ -f /opt/module/flume-1.9.0/datas/taildir_kafka.conf >/dev/null 2>&1 &"

    done

;;

stop )

for i in hadoop102 hadoop103; do

echo ============== flume $i 启动 ===============

ssh $i "ps -ef | awk '/taildir_kafka.conf/ && !/awk{print \$2}' | xargs kill -9"

    done

;;

esac

五、批量显示jps脚本

#!/bin/bash

params=$@

i=2

for((i=2 ; i <= 4 ; i = $i + 1)) ; do

    echo ==============hadoop10$i $params =============

    ssh hadoop10$i "source /etc/profile;$params"

done

六、日志生成脚本

for i in hadoop102 hadoop103; do

echo "=============$i=============="

ssh $i "cd /opt/module/applog ; java -jar gmall2020-mock-log-2020-05-10.jar >/dev/null 2>&1 &"

done

七、sqoop导数据脚本，不完善,后续需完善

APP=gmall

sqoop=/opt/module/sqoop/bin/sqoop

if [-n "$2" ]; then

do_date=$2

else

do_date=`date -d '-1 day' +%F`

fi

import_data(){

$sqoop import \

--connect jdbc:mysql://hadoop102:3306/$APP \

--username root \

--password 123456 \

--query "$2 and \$CONDITIONS" \

--target-dir /origin_data/$APP/db/$1/$do_date \

--delete-target-dir \

--num-mappers 2 \

--fields-terminated-by '\t' \

--compress \

--compression-codec lzop \

--null-string '\\N' \

--null-no-string '\\N'

#建索引

hadoop jar /opt/module/hadoop-3.1.4/share/hadoop/common/hadoop-lzo-0.4.21.jar \

com.hadoop.compression.lzo.DistributedLzoIndexer /origin_data/$APP/db/$1/$do_date

}