美文网首页
黑猴子的家:Docker 搭建Hadoop完全分布式

黑猴子的家:Docker 搭建Hadoop完全分布式

作者: 黑猴子的家 | 来源:发表于2021-04-30 13:56 被阅读0次

    1、拉取centos基础镜像

    [root@hadoop108 ~]# docker pull centos:7.5.1804
    

    2、创建centos-jdk:1.0 镜像

    [root@hadoop108 software]# mkdir docker-jdk
    [root@hadoop108 software]# cd docker-jdk
    [root@hadoop108 docker-jdk]# touch Dockerfile
    [root@hadoop108 docker-jdk]# vim Dockerfile
    FROM centos:7.5.1804
    RUN mkdir -p /opt/software
    RUN mkdir -p /opt/module
    COPY jdk-8u144-linux-x64.tar.gz /opt/software/
    RUN tar -xzvf /opt/software/jdk-8u144-linux-x64.tar.gz -C /opt/module
    RUN rm -rf /opt/software/jdk-8u144-linux-x64.tar.gz
    ENV JAVA_HOME=/opt/module/jdk1.8.0_144
    ENV PATH=$PATH:$JAVA_HOME/bin
    [root@hadoop108 docker-jdk]# docker build -t centos-jdk:1.0 ./
    [root@hadoop108 docker-jdk]# docker images
    

    3、创建 centos-jdk-ssh:1.0:1.0 镜像

    [root@hadoop108 software]# mkdir docker-jdk-ssh
    [root@hadoop108 software]# cd docker-jdk-ssh
    [root@hadoop108 docker-jdk-ssh]# touch Dockerfile
    [root@hadoop108 docker-jdk-ssh]# vim Dockerfile
    FROM centos_java8:1.0
    MAINTAINER AlexMK liufengji@aliyun.com
    RUN curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
    RUN sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
    RUN yum makecache
    RUN yum install -y openssh-server openssh-clients vim net-tools
    RUN sed -i '/^HostKey/'d /etc/ssh/sshd_config
    RUN echo 'HostKey /etc/ssh/ssh_host_rsa_key'>>/etc/ssh/sshd_config
    RUN ssh-keygen -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key
    RUN echo 'root:000000' | chpasswd
    EXPOSE 22
    # 开放所有端口
    #EXPOSE 22-65535
    RUN mkdir -p /opt
    RUN echo '#!/bin/bash' >> /opt/run.sh
    RUN echo '/usr/sbin/sshd -D' >> /opt/run.sh
    RUN chmod +x /opt/run.sh
    CMD ["/opt/run.sh"]
    [root@hadoop108 docker-jdk-ssh]# docker build -t centos-jdk-ssh:1.0 ./
    

    4、运行容器

    [root@hadoop108 ~]# docker run --name hadoop102 -h hadoop102 -d centos-jdk-ssh:1.0
    [root@hadoop108 ~]# docker run --name hadoop103 -h hadoop103 -d centos-jdk-ssh:1.0
    [root@hadoop108 ~]# docker run --name hadoop104 -h hadoop104 -d centos-jdk-ssh:1.0
    

    5、查看容器网络

    [root@hadoop108 ~]# docker network inspect bridge
    

    6、通过这种方式是可以(不推荐)

    [root@hadoop108 ~]# docker exec -it hadoop102 /bin/bash
    [root@hadoop108 ~]# docker exec -it hadoop103 /bin/bash
    [root@hadoop108 ~]# docker exec -it hadoop104 /bin/bash
    

    7、通过端口映射的方式(不推荐)

    [root@hadoop108 ~]# docker run --name hadoop105 -h hadoop105 -p 522:22 -d centos-jdk-ssh:1.0
    

    8、安装工具

    [root@hadoop108 ~]# yum install -y bridge-utils git
    [root@hadoop108 ~]# git clone https://github.com/jpetazzo/pipework.git
    [root@hadoop108 ~]# cp pipework/pipework /usr/bin/
    

    9、配置网络

    [root@hadoop108 ~]# cd /usr/local/bin/
    [root@hadoop108 bin]# touch docker_network.sh
    [root@hadoop108 bin]# chmod 755 docker_network.sh
    [root@hadoop108 bin]# vim docker_network.sh
    #!/bin/bash
    
    docker start hadoop102
    docker start hadoop103
    docker start hadoop104
    
    brctl addbr br0; \
    ip link set dev br0 up; \
    ip addr del 192.168.2.108/24 dev ens33; \
    ip addr add 192.168.2.108/24 dev br0; \
    brctl addif br0 ens33; \
    ip route add default via 192.168.2.2 dev br0
    
    sleep 5
    
    pipework br0 hadoop102 192.168.2.102/24@192.168.2.2
    pipework br0 hadoop103 192.168.2.103/24@192.168.2.2
    pipework br0 hadoop104 192.168.2.104/24@192.168.2.2
    [root@hadoop108 ~]# docker_network.sh
    

    10、配置主机和ip的映射

    [root@hadoop102 ~]# vim /etc/hosts
    192.168.2.102 hadoop102
    192.168.2.103 hadoop103
    192.168.2.104 hadoop104
    
    [root@hadoop103 ~]# vim /etc/hosts
    192.168.2.102 hadoop102
    192.168.2.103 hadoop103
    192.168.2.104 hadoop104
    
    [root@hadoop104 ~]# vim /etc/hosts
    192.168.2.102 hadoop102
    192.168.2.103 hadoop103
    192.168.2.104 hadoop104
    

    11、ssh 免密登陆

    [root@hadoop102 ~]# ssh-kengen -t rsa
    [root@hadoop102 ~]# cd ~/.ssh
    [root@hadoop102 .ssh]# ssh-copy-id hadoop102
    [root@hadoop102 .ssh]# ssh-copy-id hadoop103
    [root@hadoop102 .ssh]# ssh-copy-id hadoop104
    
    [root@hadoop103 ~]# ssh-kengen -t rsa
    [root@hadoop103~]# cd ~/.ssh
    [root@hadoop103 .ssh]# ssh-copy-id hadoop102
    [root@hadoop103 .ssh]# ssh-copy-id hadoop103
    [root@hadoop103 .ssh]# ssh-copy-id hadoop104
    
    [root@hadoop104 ~]# ssh-kengen -t rsa
    [root@hadoop104 ~]# cd ~/.ssh
    [root@hadoop104 .ssh]# ssh-copy-id hadoop102
    [root@hadoop104 .ssh]# ssh-copy-id hadoop103
    [root@hadoop104 .ssh]# ssh-copy-id hadoop104
    

    12、配置jdk环境变量

    [root@hadoop102 ~]# vim  /etc/profile
    # JAVA_HOME
    export JAVA_HOME=/opt/module/jdk1.8.0_144
    export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
    export PATH=$PATH:$JAVA_HOME/bin
    [root@hadoop102 ~]# source /etc/profile
    [root@hadoop102 ~]# java -version
    
    [root@hadoop103 ~]# vim  /etc/profile
    # JAVA_HOME
    export JAVA_HOME=/opt/module/jdk1.8.0_144
    export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
    export PATH=$PATH:$JAVA_HOME/bin
    [root@hadoop103 ~]# source /etc/profile
    [root@hadoop103 ~]# java -version
    
    [root@hadoop104 ~]# vim  /etc/profile
    # JAVA_HOME
    export JAVA_HOME=/opt/module/jdk1.8.0_144
    export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
    export PATH=$PATH:$JAVA_HOME/bin
    [root@hadoop104 ~]# source /etc/profile
    [root@hadoop104 ~]# java -version
    

    13、安装部署hadoop

    102 103 104
    hadoop-env.sh
    yarn-env.sh
    mapred-env.sh
    core-site.xml
    hdfs-site.xml
    yarn-site.xml
    mapred-site.xml
    slaves
    

    14、格式化hadoop

    [root@hadoop102 hadoop-2.7.2]# bin/hdfs namenode -format
    

    15、启动hadoop

    [root@hadoop102 hadoop-2.7.2]# sbin/start-dfs.sh
    [root@hadoop103 hadoop-2.7.2]# sbin/start-yarn.sh
    

    16、浏览器访问

    192.168.2.102:50070
    192.168.2.103:8088
    

    相关文章

      网友评论

          本文标题:黑猴子的家:Docker 搭建Hadoop完全分布式

          本文链接:https://www.haomeiwen.com/subject/obunrltx.html