hadoop环境配置

SecureCRT 下载位置

sec.png

/home/program/hadoop-3.1.0/

hadoop-env.sh

cd hadoop-3.1.0/etc/hadoop
vim hadoop-env.sh
export JAVA_HOME=/home/program/java/jdk1.8.0_171/

core-site.xml

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/demo/hadoopdata</value>
    </property>
</configuration>

hdfs-site.xml

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

yarn site.xml

<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.resourcemanager.address</name>
        <value>master:18040</value>
    </property>
    <property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>master:18030</value>
    </property>
    <property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>master:18025</value>
    </property>
    <property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>master:18141</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>master:18088</value>
    </property>
</configuration>

mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>yarn.app.mapreduce.am.env</name>
        <value>HADOOP_MAPRED_HOME=/home/program/hadoop-3.1.0</value>
    </property>
    <property>
        <name>mapreduce.map.env</name>
        <value>HADOOP_MAPRED_HOME=/home/program/hadoop-3.1.0</value>
    </property>
    <property>
        <name>mapreduce.reduce.env</name>
        <value>HADOOP_MAPRED_HOME=/home/program/hadoop-3.1.0</value>
    </property>
</configuration>

hadoop-3.1.0/etc/hadoopworkers

localhost
slave1
slave2

启动hadoop

chown demo hadoop-3.1.0
chgrp -R demo hadoop-3.1.0

cd /home/program/hadoop-3.1.0/sbin
[demo@master sbin]$ start-dfs.sh
Starting namenodes on [master]
Starting datanodes
slave1: WARNING: /home/program/hadoop-3.1.0/logs does not exist. Creating.
slave2: WARNING: /home/program/hadoop-3.1.0/logs does not exist. Creating.
Starting secondary namenodes [master]
[demo@master sbin]$ start-yarn.sh
Starting resourcemanager
Starting nodemanagers
[demo@master sbin]$ jps
78897 ResourceManager
79377 Jps
78448 DataNode
79206 NodeManager
78651 SecondaryNameNode
78284 NameNode

停止hadoop

stop-yarn.sh
stop-dfs.sh