51cto赵强HADOOP学习（一）

安装linux和jdk.上传jdk和hadoop

#mkdir training
#tar -zxvf jdk-8u144-linux-x64.tar.gz
#cd jdk1.8.0_144/
#vi .bash_profile
JAVA_HOME=/root/training/jdk1.8.0_144
export JAVA_HOME

PATH=$JAVA_HOME/bin:$PATH
export PATH
#source .bash_profile
#which java
/root/training/jdk1.8.0_144/bin/java
#java -version
java version "1.8.0_144"
Java(TM) SE Runtime Environment (build 1.8.0_144-b01)
Java HotSpot(TM) 64-Bit Server VM (build 25.144-b01, mixed mode)

安装hadoop（上传hadoop）

#cd training
# tar -zxvf hadoop-2.4.1.tar.gz
#vi ~/.bash_profile
HADOOP_HOME=/root/training/hadoop-2.4.1
export HADOOP_HOME

PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
export PATH
#source ~/.bash_profile 

本地模式

#cd /etc/hadoop
#vi hadoop-env.sh
export JAVA_HOME=/root/training/jdk1.8.0_144
#cd ~/training
#mkdir data
#cd data
#mkdir input
#mkdir output
#cd input/
#vi data.txt
I love Beijing
I love China
Beijing is the capital of China
#more data.txt
#cd ~/training
#cd hadoop-2.4.1/share/hadoop/mapreduce/
#pwd
#hadoop jar hadoop-mapreduce-examples-2.4.1.jar 
# hadoop jar hadoop-mapreduce-examples-2.4.1.jar wordcount
# hadoop jar hadoop-mapreduce-examples-2.4.1.jar wordcount ~/training/data/input/data.txt ~/training/data/output/
# rm -rf /root/training/data/output
# hadoop jar hadoop-mapreduce-examples-2.4.1.jar wordcount ~/training/data/input/data.txt ~/training/data/output/
#vi /etc/hosts
192.168.56.100 master
# hadoop jar hadoop-mapreduce-examples-2.4.1.jar wordcount ~/training/data/input/data.txt ~/training/data/output/
# cd ~/training/data/output/
# more part-r-00000

伪分布模式

image.png

# cd ~/training/hadoop-2.4.1/etc/hadoop/
#vi hdfs-site.xml
<property>
    <name>dfs.replication</name>
    <value>1</value>
</property>

#mkdir /root/training/hadoop-2.4.1/tmm
#vi core-site.xml
<property>
    <name>fs.defaultFS</name>
    <value>hdfs://192.168.56.100:9000</value>
</property>
<property>
    <name>hadoop.tmp.dir</name>
    <value>/root/training/hadoop-2.4.1/tmm</value>
</property>

#cp mapred-site.xml.template mapred-site.xml
#vi mapred-site.xml
<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>

#vi yarn-site.xml
<property>
    <name>yarn.resourcemanager.hostname</name>
    <value>192.168.56.100</value>
</property>
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>

#cd ~/training
#ls hadoop-2.4.1/tmm
#hdfs namenode -format
#ls hadoop-2.4.1/tmm
#start-all.sh
# jps
4576 SecondaryNameNode
5042 Jps
4435 DataNode
4710 ResourceManager
4328 NameNode
4988 NodeManager

#hdfs dfsadmin -report
# hdfs dfs -mkdir /input
#hdfs dfs -put data/input/data.txt /input/data.txt
#hdfs dfs -lsr /
# cd hadoop-2.4.1/share/hadoop/mapreduce/
# hadoop jar hadoop-mapreduce-examples-2.4.1.jar wordcount /input/data.txt /output
#hdfs dfs -lsr /
#hdfs dfs -cat /output/part-r-00000
#stop-all.sh
#jps

全分布模式

image.png

设置免密码登录

#cd
#ls .ssh/
#ssh-keygen -t rsa
#ls .ssh/
#ssh-copy-id -i .ssh/id_rsa.pub root@192.168.56.100
#more .ssh/authorized_keys
#start-all.sh
#jps
#stop-all.sh
#jps

hadoop的WebConsole

# cd training/hadoop-2.4.1/share/hadoop/mapreduce/
# hadoop jar hadoop-mapreduce-examples-2.4.1.jar wordcount /input/data.txt /output1