Hadoop2.7.7安装配置

hadoop2.8.5 NameNode高可用HA配置

• /etc/hosts 配置

192.168.56.100   master
192.168.56.101   node1
192.168.56.102   node2
192.168.56.103   node3
192.168.56.104   node4

• etc/hadoop/slaves配置

node1
node2
node3
node4

• etc/hadoop/hadoop-env.sh配置

export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk

• etc/hadoop/core-site.xml配置

<configuration>
    <property>
        <name>dfs.nameservices</name>
        <value>mycluster</value>
    </property>
    <property>
        <name>dfs.ha.namenodes.mycluster</name>
        <value>nn1,nn2</value>
    </property>
    <property>
        <name>dfs.namenode.rpc-address.mycluster.nn1</name>
        <value>master:9000</value>
    </property>
    <property>
        <name>dfs.namenode.rpc-address.mycluster.nn2</name>
        <value>node1:9000</value>
    </property>
    <property>
        <name>dfs.namenode.http-address.mycluster.nn1</name>
        <value>master:50070</value>
    </property>
    <property>
        <name>dfs.namenode.http-address.mycluster.nn2</name>
        <value>node1:50070</value>
    </property>
    <property>
        <name>dfs.namenode.shared.edits.dir</name>
        <value>qjournal://master:8485;node1:8485;node2:8485;node3:8485;node4:8485/mycluster</value>
    </property>
    <property>
        <name>dfs.client.failover.proxy.provider.mycluster</name>
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    </property>
    <property>
        <name>dfs.ha.fencing.ssh.private-key-files</name>
        <value>/root/.ssh/id_rsa</value>
    </property>
    <property>
        <name>dfs.ha.fencing.methods</name>
        <value>sshfence
        shell(/bin/true)
        </value>
    </property>
    <property>
        <name>dfs.ha.fencing.ssh.connect-timeout</name>
        <value>30000</value>
    </property>
    <property>
        <name>dfs.journalnode.edits.dir</name>
        <value>/usr/local/lib/hadoop-2.7.7/journal/node/local/data</value>
    </property>
    <property>
        <name>dfs.ha.automatic-failover.enabled</name>
        <value>true</value>
    </property>
</configuration>

hadoop2.7.7 hdfs启动

• 启动 zookeeper
• 启动 journalnode

#所有机器上操作
sbin/hadoop-daemons.sh start journalnode

• 格式化 namenode 并启动

#master主机器上操作
hdfs namenode -format
bin/hdfs zkfc -formatZK
sbin/start-dfs.sh

• 备用节点启动

#node1机器上操作
hdfs namenode -bootstrapStandby
sbin/hadoop-daemon.sh start namenode

• 切换namenode

hadoop-2.7.7/bin/hdfs haadmin -help

hadoop3.2.1 hdfs高可用HA配置官方参考文档

hadoop2.7.7 yarn 高可用HA配置

• etc/hadoop/yarn-site.xml配置

<configuration>
    <property>
        <name>yarn.resourcemanager.ha.enabled</name>
        <value>true</value>
    </property>
    <property>
        <name>yarn.resourcemanager.cluster-id</name>
        <value>cluster1</value>
    </property>
    <property>
        <name>yarn.resourcemanager.ha.rm-ids</name>
        <value>rm1,rm2</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname.rm1</name>
        <value>master</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname.rm2</name>
        <value>node1</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address.rm1</name>
        <value>master:8088</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address.rm2</name>
        <value>node1:8088</value>
    </property>
    <property>
        <name>yarn.resourcemanager.zk-address</name>
        <value>master:2181,node1:2181,node2:2181,node3:2181,node4:2181</value>
    </property>
</configuration>

yarn相关命令

• 启动yarn

#master中启动
sbin/start-yarn.sh

• 查看 active standby状态

bin/yarn rmadmin -getServiceState rm1
#active 
bin/yarn rmadmin -getServiceState rm2
#standby

• 切换active standby状态

bin/yarn rmadmin -transitionToStandby rm1
bin/yarn rmadmin -transitionToActive rm2 --forcemanual

hadoop3.2.1 yarn HA高可用配置 官方参考文档