美文网首页Java面试指南
马士兵Hadoop学习

马士兵Hadoop学习

作者: Java面试指南 | 来源:发表于2018-11-29 12:44 被阅读2次

    软件包位置

    https://pan.baidu.com/s/1pLFVWkr

    文档地址

    http://www.mashibing.com/hadoop_install.html

    http://www.mashibing.com/hadoop_test.html

    http://www.mashibing.com/hdfs_java.html

    http://www.mashibing.com/yarn_test.html

    http://www.mashibing.com/map_reduce.html

    IP 地址设置

    master:  192.168.56.101slave1:  192.168.56.102slave2:  192.168.56.103slave3:  192.168.56.104

    上传文件。

    解压jdk

    #rpm -ivh jdk-8u91-linux-x64.rpm#cd /usr#ls#java#cd

    解压hadoop

    #tar -xvf hadoop-2.7.3.tar.gz#mv hadoop-2.7.3 hadoop #mv hadoop /usr/local

    配置

    #vi /usr/local/hadoop/etc/hadoop/hadoop-env.sh

    把export JAVA_HOME=${JAVA_HOME}改成export JAVA_HOME=/usr/java/default,然后:wq保存退出。

    #vi /etc/profile//在最下面加入:exportPATH=$PATH:/usr/local/hadoop/bin:/usr/local/hadoop/sbin

    source/etc/profile

    关机复制三台。

    关闭防火墙。

    #systemctl stop firewalld#systemctl disable firewalld

    启用

    #vi /usr/local/hadoop/etc/hadoop/core-site.xmlfs.defaultFShdfs://master:9000

    #vi /etc/hosts192.168.56.101 master192.168.56.102 slave1192.168.56.103 slave2192.168.56.104 slave3#hdfs namenode -format

    master:

    #hadoop-daemon.shstartnamenode#jps

    slaves:

    #hadoop-daemon.shstartdatanode#jps

    查看集群情况

    #hdfs dfsadmin -report | more

    用浏览器浏览

    192.168.56.101:50070

    集中式管理集群。

    关闭集群。

    master:

    #hadoop-daemon.shstopnamenode#jps

    slaves:

    #hadoop-daemon.shstopdatanode#jps

    master

    #vi /usr/local/hadoop/etc/hadoop/slavesslave1slave2slave3

    #start-dfs.sh

    slaves:

    #jps

    设置免密登录(master)

    #cd.ssh#ssh-keygen-trsa#ssh-copy-idslave1#ssh-copy-idslave2#ssh-copy-idslave3#ssh-copy-idmaster#stop-dfs.sh#start-dfs.sh

    对文件进行增删改查操作

    #hadoop fs -ls /#cd /usr/local#ll#hadoop fs -put ./hadoop-2.7.3.tar.gz /#hadoop fs -ls /

    修改默认属性

    #cd hadoop/etc/hadoop#vi hdfs-site.xmldfs.replication2

    #cd /usr/local#hadoop fs -put ./jdk-8u91-Linux-x64.rpm /#hadoop fs -ls /

    修改检查时间

    #cd hadoop/etc/hadoop#vi hdfs-site.xmldfs.namenode.heartbeat.recheck-interval10000

    #stop-dfs.sh#start-dfs.sh

    修改配置文件,所有机器都改

    /usr/local/hadoop/etc/hadoop#vi /usr/local/hadoop/etc/hadoop/core-site.xmlhadoop.tmp.dir/var/hadoop#hdfs namenode -format

    #cdvi hello.txt#hadoop fs -put ./hello.txt /#hadoop fs -ls /

    用eclipse打开,file,new ,project,java project,HelloHDFS

    -DHADOOP_USER_NAME=root

    image.png

    导入用到的包

    image.png

    lib下的所有包

    image.png

    publicstaticvoidmain(String[] args)throwsException{

    URL url =newURL("http://www.baidu.com");        InputStreamin= url.openStream();        IOUtils.copyBytes(in,System.out,4096,true);

    URL url =newURL("hdfs://192.168.56.101:9000/hello.txt");        InputStreamin= url.openStream();        IOUtils.copyBytes(in,System.out,4096,true);

    URL.setURLStreamHandlerFactory(newFsUrlStreamHandlerFactory());        URL url =newURL("hdfs://192.168.56.101:9000/hello.txt");        InputStreamin= url.openStream();        IOUtils.copyBytes(in,System.out,4096,true);

    Configuration conf =newConfiguration();        conf.set("fs.defaultFS","hdfs://192.168.56.101:9000");        FileSystem fileSystem = FileSystem.get(conf);

    新建一个目录

    booleansuccess = fileSystem.mkdirs(newPath("/xuehuai"));        System.out.println(success);

    master

    #hadoop fs -ls /

    判断存不存在

    success = fileSystem.exists(newPath("/hello.txt"));        System.out.println(success);

    删除一个文件

    success = fileSystem.delete(newPath("/msb"),true);        System.out.println(success);

    master:

    #cd /usr/local/hadoop/etc/hadoop#vi hdfs-site.xmldfs.permissions.enabledfalse#hadoop-daemon.sh stop namenode#hadoop-daemon.sh start namenode

    FSDataOutputStream out = fileSystem.create(newPath("/test.data"),true);        FileInputStream fis =newFileInputStream("d:/test1/hive-env.sh.template");        IOUtils.copyBytes(fis, out,4096,true);

    master:

    #hadoop fs -ls /#hadoop fs -text /test.data#hadoop fs -rm /test.data

    FSDataOutputStreamout= fileSystem.create(new Path("/test.data"),true);        FileInputStreamin= new FileInputStream("d:/test1/hive-env.sh.template");        byte[] buf = new byte[4096];intlen =in.read(buf);while(len !=-1) {out.write(buf,0,len);            len =in.read(buf);        }in.close();out.close();

    }

    }

    配置计算调度系统yarn和计算引擎mr

    配置yarn(所有)

    #vi /usr/local/hadoop/etc/hadoop/yarn-site.xmlyarn.resourcemanager.hostnamemasteryarn.nodemanager.aux-servicesmapreduce_shuffleyarn.nodemanager.auxservices.mapreduce.shuffle.classorg.apache.hadoop.mapred.ShuffleHandler

    配置mapreduce.(所有)

    #cp /usr/local/hadoop/etc/hadoop/mapred-site.xml.template /usr/local/hadoop/etc/hadoop/mapred-site.xml#vi /usr/local/hadoop/etc/hadoop/mapred-site.xmlmapreduce.framework.nameyarn

    #start-yarn.sh//master

    浏览器:192.168.56.101:8088

    3、上传文件

    #vi input.txt#hadoop fs -mkdir /input#hadoop fs -put input.txt /input#hadoop fs -ls /input#find /usr/local/hadoop -name *example*.jar#hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar word

    相关文章

      网友评论

        本文标题:马士兵Hadoop学习

        本文链接:https://www.haomeiwen.com/subject/wxoxcqtx.html