Hadoop-入门篇环境搭建（四）

转载请注明原文出处

hadoop搭建

阿牛的资料下载
Hadoop-入门篇环境搭建（一）
Hadoop-入门篇环境搭建（二）
Hadoop-入门篇环境搭建（三）
Hadoop-入门篇环境搭建（四）

机器配置

node1

+ vi ~/.bash_profile
  export HADOOP_PREFIX=/opt/soft/hadoop
+ source ~/.bash_profile

# cd /opt/soft/hadoop/etc/hadoop

# vi mapred-env.sh
  export JAVA_HOME=/usr/java/jdk1.7.0_79
#vi hadoop-env.sh
  export JAVA_HOME=/usr/java/jdk1.7.0_79

# vi slaves
  node2
  node3
  node4

+ mkdir -p /opt/data/hadoop
+ mkdir  -p /opt/data/journalnode
#vi hdfs-site.xml
#集群名字ID mycluster请替换为下面的value=hadoop
<property>
  <name>dfs.nameservices</name>
  <value>hadoop</value>
</property>
#集群有哪些机器
<property>
  <name>dfs.ha.namenodes.hadoop</name>
  <value>nn1,nn2</value>
#集群的namenode ip
</property>
  <name>dfs.namenode.rpc-address.hadoop.nn1</name>
  <value>node1:8020</value>
</property>
<property>
  <name>dfs.namenode.rpc-address.hadoop.nn2</name>
  <value>node2:8020</value>
</property>

#连接页面
<property>
  <name>dfs.namenode.http-address.hadoop.nn1</name>
  <value>node1:50070</value>
</property>
<property>
  <name>dfs.namenode.http-address.hadoop.nn2</name>
  <value>node2:50070</value>
</property>

<property>
  <name>dfs.namenode.shared.edits.dir</name>
  <value>qjournal://node2:8485;node3:8485;node4:8485/hadoop</value>
</property>
<property>
  <name>dfs.client.failover.proxy.provider.hadoop</name>
</property>

<property>
  <name>dfs.ha.fencing.methods</name>
  <value>sshfence</value>
</property>

<property>
  <name>dfs.ha.fencing.ssh.private-key-files</name>
  <value>/root/.ssh/id_dsa</value>
</property>


<property>
  <name>dfs.ha.fencing.ssh.connect-timeout</name>
  <value>60000</value>
</property>
#mkdir -p /opt/data/journalnode
<property>
  <name>dfs.journalnode.edits.dir</name>
  <value>/opt/data/journalnode</value>
</property>
#自动选举
 <property>
   <name>dfs.ha.automatic-failover.enabled</name>
   <value>true</value>
 </property>
#zk节点的位置
  <property>
   <name>ha.zookeeper.quorum</name>
   <value>node1:2181,node2:2181,node3:2181</value>
 </property>

#in your core-site.xml file:
#<property>
#  <name>fs.defaultFS</name>
#  <value>hdfs://hadoop</value>
#</property>
#


vi core-site.xml

 <property>
   <name>ipc.client.connect.max.retries</name>
    <value>20</value>
    <description>
    </description>
  </property>

  <property>
   <name>ipc.client.connect.retry.interval</name>
    <value>5000</value>
    <description>
      Indicates the number of milliseconds aclient will wait for before retrying to establish a server connection.
    </description>
  </property>


<property>
        <name>ha.zookeeper.quorum</name>
        <value>node1:2181,node2:2181,node3:2181</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/data/hadoop</value>
    </property>

<property>
  <name>fs.defaultFS</name>
  <value>hdfs://hadoop</value>
</property>


hdfs高可用搭建完成
yarn搭建

cd /opt/soft/hadoop/etc/hadoop
#vi yarn-site.xml
<property>
   <name>yarn.resourcemanager.ha.enabled</name>
   <value>true</value>
 </property>
 #this is yarn id not hdfs's id
 <property>
   <name>yarn.resourcemanager.cluster-id</name>
   <value>hadoop-yarn</value>
 </property>
 #this is resourcemanager id how many resourcemanager ,if you want to set RSM please set this
 <property>
   <name>yarn.resourcemanager.ha.rm-ids</name>
   <value>rm1,rm2</value>
 </property>
 <property>
   <name>yarn.resourcemanager.hostname.rm1</name>
   <value>node3</value>
 </property>
 <property>
   <name>yarn.resourcemanager.hostname.rm2</name>
   <value>node4</value>
 </property>
 #there is ZK ,please synchronous your's ZK cluster
 <property>
   <name>yarn.resourcemanager.zk-address</name>
   <value>node1:2181,node2:2181,node3:2181</value>
 </property>
 #this is a defects in yarn ,You have to remember
 #please mapreduce_shuffle
  <property>
   <name>yarn.nodemanager.aux-services</name>
   <value>mapreduce_shuffle</value>
 </property>

16年的学习笔记，留言相互交流。

关注老铁的公众号，从小菜比到老菜比

你假笨 462563010加QQ群一起学习
欢迎关注spring4all