1. 在window环境开发HDFS（文件）
1.1. 导入jar包
i. 解压hadoop安装包
ii. 导入/share/hadoop/common/Hadoop-common-2.8.5.jar 以及依赖包 lib下全部jar
iii. 导入/share/haddop/hdfs/hadoop-hdfs-client-2.8.5-tests.jar 以及依赖包lib下全部jar
链接: https://pan.baidu.com/s/1pgyCp17E8IKJ46fM87jZrA 提取码: 3251
1.2. 开始写java代码
先构造Configuration对象
构造时，会先加载jar包中的默认配置xxx-default.xml
再加载用户写的xml 放在src下面 xxx-site.xml 会覆盖默认配置
在代码中可以在此conf.set(name,value);, 会再次覆盖配置
public class HdfsClient {
public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
//设置HDFS的配置
//会从项目的classpath中加载core-default.xml hdfs-default.xml core-site.xml hdfs-site.xml等配置文件
Configuration conf = new Configuration();
//设置副本数量为2
conf.set("dfs.replication","2");
//指定切块文件大小为64M
conf.set("dfs.blocksize","64M");
//构造一个访问指定HDFS系统的客户端对象，参数1.HDFS系统的URL，参数2,客户端指定的参数，参数3.客户端身份
FileSystem fileSystem = FileSystem.get(new URI("hdfs://vm01:9000/"),conf,"root");
}
}
1.2.1. 上传文件，删文件
//上传一个文件到hdfs中
fileSystem.copyFromLocalFile(
new Path("C:\\Users\\81022\\Desktop\\1.txt"),new Path("/javaTest/1.txt"));
//fileSystem.delete(new Path("/video.zip"),true);
fileSystem.close();
1.2.2. 下载文件，（需要在windows中配置hadoop环境）
在Windows下配置环境变量 HADOOP_HOME
Window版本
https://drive.google.com/open?id=1JSVGVmxTtTehgzcUenwZXxjjrSBFL6dZ
fileSystem.copyToLocalFile(new Path("/javaTest/1.txt"),new Path("C:\\Users\\81022\\Desktop\\"));
1.2.3. 移动文件，改名
fileSystem.rename(new Path("/1.txt"),new Path("/javaTest/2.txt"));
1.2.4. 创建文件夹
fileSystem.mkdirs(new Path("/xx/yy/zz"));
1.2.5. 读文件
RemoteIterator<LocatedFileStatus> iter = fileSystem.listFiles(new Path("/"), true);
while (iter.hasNext()){
LocatedFileStatus next = iter.next();
System.out.println("==============================");
System.out.println("文件位置+"+next.getPath());
System.out.println("块大小："+next.getBlockSize());
System.out.println("文件副本数量："+next.getReplication());
}
原始代码
package com.looc.test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
/**
* @author chenPeng
* @version 1.0.0
* @ClassName HdfsClient.java
* @Description TODO
* @createTime 2019年01月18日 16:17:00
*/
public class HdfsClientTest {
FileSystem fileSystem = null;
/**
* 测试查询
* @Author chenpeng
* @Description //TODO
* @Date 19:28
* @Param []
* @return void
**/
@Test
public void fondTest() throws IOException {
RemoteIterator<LocatedFileStatus> iter = fileSystem.listFiles(new Path("/"), true);
while (iter.hasNext()){
LocatedFileStatus next = iter.next();
System.out.println("==============================");
System.out.println("文件位置+"+next.getPath());
System.out.println("块大小："+next.getBlockSize());
System.out.println("文件副本数量："+next.getReplication());
}
fileSystem.close();
}
/**
* 测试目录及文件查询
* @Author chenpeng
* @Description //TODO
* @Date 19:28
* @Param []
* @return void
**/
@Test
public void fondAllTest() throws IOException {
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
for (FileStatus status:fileStatuses) {
System.out.println("==============================");
System.out.println("文件位置+"+status.getPath());
System.out.println(status.isDirectory()? "这是文件夹":"这是文件");
}
fileSystem.close();
}
/**
* 初始化
* @Author chenpeng
* @Description //TODO
* @Date 18:57
* @Param []
* @return void
**/
@Before
public void init() throws URISyntaxException, IOException, InterruptedException {
Configuration conf = new Configuration();
conf.set("dfs.replication","2");
conf.set("dfs.blocksize","64M");
fileSystem = FileSystem.get(new URI("hdfs://vm01:9000/"),conf,"root");
}
/**
* 下载文件
* @Author chenpeng
* @Description //TODO
* @Date 18:57
* @Param []
* @return void
**/
@Test
public void testGet() throws IOException {
fileSystem.copyToLocalFile(
new Path("/javaTest/1.txt"),new Path("C:\\Users\\81022\\Desktop\\"));
fileSystem.close();
}
/**
* 移动文件
* @Author chenpeng
* @Description //TODO
* @Date 19:03
* @Param []
* @return void
**/
@Test
public void renameTest() throws IOException {
fileSystem.rename(new Path("/1.txt"),new Path("/javaTest/1.txt"));
fileSystem.close();
}
/**
* 创建目录
* @Author chenpeng
* @Description //TODO
* @Date 19:24
* @Param []
* @return void
**/
@Test
public void mkdirTest() throws IOException {
fileSystem.mkdirs(new Path("/xx/yy/zz"));
fileSystem.close();
}
/**
* 测试删除文件
* @Author chenpeng
* @Description //TODO
* @Date 18:08
* @Param [fileSystem]
* @return void
**/
@Test
public void testDelete() throws IOException {
fileSystem.delete(new Path("/video.zip"),true);
fileSystem.close();
}
/**
* 上传文件测试
* @Author chenpeng
* @Description //TODO
* @Date 18:07
* @Param [fileSystem]
* @return void
**/
@Test
public void testPut() throws IOException {
fileSystem.copyFromLocalFile(
new Path("C:\\Users\\81022\\Desktop\\1.txt"),new Path("/xx/yy/zz/1.txt"));
fileSystem.close();
}
}
读取文件中的内容，以及向文件中写内容
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
/**
* @author chenPeng
* @version 1.0.0
* @ClassName HdfsReanAndWriteDemo.java
* @Description TODO
* @createTime 2019年01月28日 22:37:00
*/
public class HdfsReanAndWriteDemo {
FileSystem fileSystem;
private String url = "hdfs://vm01:9000/";
@Before
public void init() throws URISyntaxException, IOException, InterruptedException {
fileSystem = FileSystem.get(new URI(url),new Configuration(),"root");
}
/**
* 读取文件
* @Author chenpeng
* @Description //TODO
* @Date 22:54
* @Param []
* @return void
**/
@Test
public void readerHdfsDemo() throws IOException {
FilterInputStream in = fileSystem.open(new Path("/javaTest/2.txt"));
BufferedReader br = new BufferedReader(new InputStreamReader(in,"utf-8"));
String line = null;
while ((line = br.readLine())!=null){
System.out.println(line);
}
br.close();
in.close();
}
/**
* 读取文件 指定位置开始
* @Author chenpeng
* @Description //TODO
* @Date 22:54
* @Param []
* @return void
**/
@Test
public void readerRandomHdfsDemo() throws IOException {
FSDataInputStream in = fileSystem.open(new Path("/javaTest/1.txt"));
//指定从第三个开始读
in.seek(4);
byte[] bs = new byte[4];
int len = 0;
while ((len = in.read(bs))!=-1){
System.out.print(new String(bs,0,len,"gbk"));
}
in.close();
}
@Test
public void writeHDFSDemo() throws IOException {
fileSystem.delete(new Path("/javaTest/2.txt"),true);
FSDataOutputStream out = fileSystem.create(new Path("/javaTest/2.txt"),true);
out.write("测试第二行".getBytes());
out.close();
}
@After
public void last() throws IOException {
fileSystem.close();
}
}
网友评论