119.向HDFS跨网段写数据

作者: 大勇任卷舒 | 来源:发表于2022-09-07 17:52 被阅读0次

119.向HDFS跨网段写数据
HDFS MapReduce
082-BigData-10HDFS上传与下载机制
python导出insert 数据sql脚本
HDFS读写流程
hdfs文件迁移
Hadoop HDFS 数据流
源码|HDFS之DataNode：写数据块（3）
[Hadoop] HDFS 详解一（原理篇）
4.HDFS的数据流

两种方式具体架构如下图：
注：所有Hadoop节点同时配置有两个网段，用于内网通信的192网段以及用于外网的172网段，而客户端机器只与外网网段172通。
注：所有Hadoop节点同时配置有两个网段，用于内网通信的192网段以及用于外网的172网段，而客户端机器只与外网网段172通。
WebHDFS访问HDFS

package com.cloudera.hdfs.nonekerberos;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import java.io.IOException;
import java.net.URI;

public class WebHDFSTest {
    public static void main(String[] args) {
        Configuration configuration = new Configuration();
        WebHdfsFileSystem webHdfsFileSystem = new WebHdfsFileSystem();
        try {
            webHdfsFileSystem.initialize(new URI("http://13.250.102.77:50070"), configuration);
            System.out.println(webHdfsFileSystem.getUri());
            //向HDFS Put文件
            webHdfsFileSystem.copyFromLocalFile(new Path("/Users/fayson/Desktop/run-kafka"), new Path("/fayson1"));
            //列出HDFS根目录下的所有文件
            FileStatus[] fileStatuses =  webHdfsFileSystem.listStatus(new Path("/"));
            for (FileStatus fileStatus : fileStatuses) {
                System.out.println(fileStatus.getPath().getName());
            }
            webHdfsFileSystem.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

HttpFS访问HDFS

package com.cloudera.hdfs.nonekerberos;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.security.UserGroupInformation;
import java.net.URI;

public class HttpFSDemo {
    public static void main(String[] args) {
        Configuration configuration = new Configuration();
        UserGroupInformation.createRemoteUser("fayson");
        WebHdfsFileSystem webHdfsFileSystem = new WebHdfsFileSystem();
        try {
            webHdfsFileSystem.initialize(new URI("http://52.221.198.3:14000"), configuration);
            System.out.println(webHdfsFileSystem.getUri());
            //向HDFS Put文件
            webHdfsFileSystem.copyFromLocalFile(new Path("/Users/fayson/Desktop/run-kafka/"), new Path("/fayson1-httpfs"));
            //列出HDFS根目录下的所有文件
            FileStatus[] fileStatuses =  webHdfsFileSystem.listStatus(new Path("/"));
            for (FileStatus fileStatus : fileStatuses) {
                System.out.println(fileStatus.getPath().getName());
            }
            webHdfsFileSystem.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

使用webhdfs向HDFS put数据时异常

log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
http://13.250.102.77:50070
java.net.UnknownHostException: cdh04.fayson.com:50075: cdh04.fayson.com
   at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
   at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
   at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
   at java.lang.reflect.Constructor.newInstance(Constructor.java:408)
   at org.apache.hadoop.hdfs.web.WebHdfsFileSystem$AbstractRunner.runWithRetry(WebHdfsFileSystem.java:693)
   at org.apache.hadoop.hdfs.web.WebHdfsFileSystem$AbstractRunner.access$100(WebHdfsFileSystem.java:519)
   at org.apache.hadoop.hdfs.web.WebHdfsFileSystem$AbstractRunner$1.run(WebHdfsFileSystem.java:549)
   at java.security.AccessController.doPrivileged(Native Method)
   at javax.security.auth.Subject.doAs(Subject.java:422)
   at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1920)
   at org.apache.hadoop.hdfs.web.WebHdfsFileSystem$AbstractRunner.run(WebHdfsFileSystem.java:545)
   at org.apache.hadoop.hdfs.web.WebHdfsFileSystem.create(WebHdfsFileSystem.java:1252)
   at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925)
   at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906)
   at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:803)
   at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:368)
   at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:359)
   at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:341)
   at org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:2057)
   at org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:2025)
   at org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1990)
   at com.cloudera.hdfs.nonekerberos.WebHDFSTest.main(WebHDFSTest.java:31)
   at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   at java.lang.reflect.Method.invoke(Method.java:483)
   at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)