Create EMR

?xml version="1.0" encoding="UTF-8"?
project_name=$1
uuid=cat /proc/sys/kernel/random/uuid

create emr cluster

cluster_id_json=`aws emr create-cluster \
--region us-west-2 \
--name "${project_name}" \
--release-label emr-5.13.0 \
--instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m4.xlarge InstanceGroupType=CORE,InstanceCount=1,InstanceType=m4.xlarge \
--ec2-attributes KeyName=dpppkey,InstanceProfile=DataPipelineDefaultResourceRole,SubnetId=subnet-e2f2dfa4 \
--service-role DataPipelineDefaultRole \
--applications Name=Spark Name=hive`

sleep 2m

id_var=$(echo ${cluster_id_json}|grep -o  -E "ClusterId.*")
id_var1=${id_var:13}
cluster_id=${id_var1%%\"*}
echo cluster_id : $cluster_id

get cluster info

sleep 2m

cluster_info=`aws --region us-west-2 emr describe-cluster --cluster-id $cluster_id`
echo cluster_info : $cluster_info
get master id
sleep 2m

master_var=$(echo ${cluster_info}|grep -o -E "MASTER.*" | grep -o  -E "Id(.*?)")
master_var1=${master_var:6}
master_id=${master_var1%%\"*}
echo master_id : $master_id

get instance info

sleep 2m

instance_group_info=`aws emr --region us-west-2 list-instances --cluster-id $cluster_id`

echo instance_group_info : $instance_group_info
get master ip
sleep 2m

ip_var=$(echo ${instance_group_info}|grep -o -E "$master_id.*" | grep -o  -E "PrivateIpAddress(.*?)")
ip_var1=${ip_var:20}
echo $ip_var1
private_ip_address=${ip_var1%%\"*}
echo private_ip_address : $private_ip_address
echo project_name : $project_name

update rundeck xml file

line=`grep -n "name=\"${project_name}\"" /data/01/home/tnuser/rundeck/projects/trips/etc/resources.xml | cut -d : -f 1`

if [ $line -gt "0" ];then

echo ${project_name} is exist in /data/01/home/tnuser/rundeck/projects/trips/etc/resources.xml

sed -i "${line}d" /data/01/home/tnuser/rundeck/projects/trips/etc/resources.xml

fi

sed "2 a<node name=\"${project_name}\" description=\"Rundeck Client Node\" tags=\"\" hostname=\"${private_ip_address}\" osArch=\"amd64\" osFamily=\"unix\" osName=\"Linux\" osVersion=\"3.10.0-693.11.6.el7.x86_64\" username=\"hadoop\"/>" -i /data/01/home/tnuser/rundeck/projects/trips/etc/resources.xml

status_num=$(echo ${instance_group_info}|grep -c "State")

running_num=$(echo ${instance_group_info}|grep -c "RUNNING")

if [ $status_num -eq $running_num ];then

echo emr start success !!

fi