HadoopHA高可用集群搭建

• HDFS高可用

hadoop01	hadoop02	hadoop03
DataNode	DataNode	DataNode
NameNode		NameNode
JournalNode		JournalNode
ZKFC		ZKFC

• YARN高可用

hadoop01	hadoop02	hadoop03
NodeManager	NodeManager	NodeManager
ResourceManager	ResourceManager

• 默认端口

服务	端口
DataNode	9864/9866/9867
NameNode	8020/9870
JournalNode	8480/8485
ZKFC	8019
NodeManager	8040/8042/13562
ResourceManager	8030/8031/8032/8033/8088

• 步骤

# 创建目录
mkdir -p /home/justin/hdp/tmp
mkdir -p /home/justin/hdp/journal
mkdir -p /home/justin/hdp/dfs/name
mkdir -p /home/justin/hdp/dfs/data
# 修改权限
chmod -R g-w /home/justin/hdp
# 在规划的NameNode上分别启动JournalNode
# nn1 & nn2
sbin/hadoop-daemon.sh start journalnode
# 格式化namenode
hdfs namenode -format
# 格式化zkfc
hdfs zkfc -formatZK
# 将namenode的name目录从nn1拷贝到nn2
scp -r justin@nn1:/home/justin/hdp/dfs/name /home/justin/hdp
# 启动dfs
sbin/start-dfs.sh
# 启动yarn
sbin/start-yarn.sh
# 停止yarn
sbin/stop-yarn.sh
# 停止dfs
sbin/stop-dfs.sh

• 配置文件参考
• core-site.xml

<configuration>
    <!-- 默认文件系统 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://cluster</value>
    </property>
    <!-- 检查点删除间隔(分钟) -->
    <property>
        <name>fs.trash.interval</name>
        <value>1440</value>
    </property>
    <!-- ZK地址 -->
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>master:2181</value>
    </property>
    <!-- 基础缓存目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/justin/hdp/tmp</value>
    </property>
</configuration>

• mapred-site.xml

<configuration>
  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
  <property>
    <name>mapreduce.jobhistory.address</name>
    <value>hadoop02:10020</value>
  </property>
  <property>
    <name>mapreduce.jobhistory.webapp.address</name>
    <value>hadoop02:19888</value>
  </property>
</configuration>

• hdfs-site.xml

<configuration>
  <!-- namenode存储目录 -->
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>/home/justin/hdp/dfs/name</value>
  </property>
  <!-- datanode存储目录 -->
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>/home/justin/hdp/dfs/data</value>
  </property>
  <!-- 副本数 -->
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
  <!-- ip域名解析 -->
  <property>
    <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
    <value>false</value>
  </property>
  <!-- namenode高可用 -->
  <property>
    <name>dfs.nameservices</name>
    <value>cluster</value>
  </property>
  <property>
    <name>dfs.ha.namenodes.cluster</name>
    <value>nn1,nn2</value>
  </property>
  <property>
    <name>dfs.namenode.rpc-address.cluster.nn1</name>
    <value>hadoop01:8020</value>
  </property>
  <property>
    <name>dfs.namenode.http-address.cluster.nn1</name>
    <value>hadoop01:9870</value>
  </property>
  <property>
    <name>dfs.namenode.rpc-address.cluster.nn2</name>
    <value>hadoop03:8020</value>
  </property>
  <property>
    <name>dfs.namenode.http-address.cluster.nn2</name>
    <value>hadoop03:9870</value>
  </property>
  <!-- namenode同步目录 -->
  <property>
    <name>dfs.namenode.shared.edits.dir</name>
    <value>qjournal://hadoop01:8485;hadoop03:8485/cluster</value>
  </property>
  <!-- 高可用开启 -->
  <property>
    <name>dfs.ha.automatic-failover.enabled</name>
    <value>true</value>
  </property>
  <!--  -->
  <property>
    <name>dfs.client.failover.proxy.provider.cluster</name>
    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
  </property>
  <!-- journalnode文件存储目录 -->
  <property>
    <name>dfs.journalnode.edits.dir</name>
    <value>/home/justin/hdp/journal</value>
  </property>
  <!-- 执行脚本 -->
  <property>
    <name>dfs.ha.fencing.methods</name>
    <value>
      sshfence
      shell(/bin/true)
    </value>
  </property>
  <property>
    <name>dfs.ha.fencing.ssh.private-key-files</name>
    <value>/home/justin/.ssh/id_rsa</value>
  </property>
  <!-- 毫秒 -->
  <property>
    <name>dfs.ha.fencing.ssh.connect-timeout</name>
    <value>30000</value>
  </property>
  <!-- 读优化 -->
  <property>
    <name>dfs.client.read.shortcircuit</name>
    <value>true</value>
  </property>
  <property>
    <name>dfs.domain.socket.path</name>
    <value>/home/justin/hdp/dn_socket</value>
  </property>
</configuration>

• yarn-site.xml

<configuration>
  <!-- 高可用开启 -->
  <property>
    <name>yarn.resourcemanager.ha.enabled</name>
    <value>true</value>
  </property>
  <!-- yarn集群名称 -->
  <property>
    <name>yarn.resourcemanager.cluster-id</name>
    <value>yarn168</value>
  </property>
  <!-- resourceManager节点ID -->
  <property>
    <name>yarn.resourcemanager.ha.rm-ids</name>
    <value>rm1,rm2</value>
  </property>
  <!-- 高可用配置 -->
  <property>
    <name>yarn.resourcemanager.hostname.rm1</name>
    <name>hadoop02</name>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.address.rm1</name>
    <name>hadoop02:8088</name>
  </property>
  <property>
    <name>yarn.resourcemanager.hostname.rm2</name>
    <name>hadoop01</name>
  </property>
  <property>
    <name>yarn.resourcemanager.webapp.address.rm2</name>
    <name>hadoop01:8088</name>
  </property>
  <!-- ZK地址 -->
  <property>
    <name>hadoop.zk.address</name>
    <value>master:2181</value>
  </property>
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <!-- 资源配置 -->
  <property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>4096</value>
  </property>
  <property>
    <name>yarn.nodemanager.resource.cpu-vcores</name>
    <value>2</value>
  </property>
  <property>
    <name>yarn.nodemanager.pmem-check-enabled</name>
    <value>false</value>
  </property>
  <property>
    <name>yarn.nodemanager.vmem-check-enabled</name>
    <value>false</value>
  </property>
  <property>
    <name>yarn.scheduler.minimum-allocation-mb</name>
    <value>256</value>
  </property>
  <property>
    <name>yarn.scheduler.maximum-allocation-mb</name>
    <value>4096</value>
  </property>
  <property>
    <name>yarn.scheduler.minimum-allocation-vcores</name>
    <value>1</value>
  </property>
  <property>
    <name>yarn.scheduler.maximum-allocation-vcores</name>
    <value>2</value>
  </property>
  <!-- 日志保留期限 -->
  <property>
    <name>yarn.nodemanager.log.retain-seconds</name>
    <value>86400</value>
  </property>
  <property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
  </property>
  <property>
    <name>yarn.log-aggregation.retain-seconds</name>
    <value>86400</value>
  </property>
  <property>
    <name>yarn.log.server.url</name>
    <value>http://hadoop02:19888/jobhistory/logs</value>
  </property>
</configuration>