java远程操作HDFS

java远程操作hadoop的分布式文件系统hdfs

需要导入的hadoop的jar包
hadoop解压包中有所需的jar包
hadoop-2.5.2/share/hadoop/common/hadoop-common-2.5.2.jar
hadoop-2.5.2/share/hadoop/common/lib/*.jar(lib下的全部jar包)
hadoop-2.5.2/share/hadoop/hdfs/hadoop-hdfs-2.5.2.jar

如果是集群内部用eclipse中的java程序访问hdfs的话，可以直接写代码操作，如果是远程操作的话需要配置hdfs-site.xml文件，关闭权限检查，不配置的话会报错Permission denied

hdfs-site.xml
<name>dfs.permissions.enabled</name>
<value>false</value>

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

/**
 * java远程或本地操作hdfs文件系统（eclipse）
 * @author root
 *
 */
public class HDFSFile {

    public static void main(String [] args) throws IOException {
        /**
         * java读取hdfs文件信息
         */
        /* 方法一 */
        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
        URL url = new URL("hdfs://master:9000/hello.txt");
        InputStream in = url.openStream();
        IOUtils.copyBytes(in,System.out,4096,true);//in是输入流，System.out是标准输出，4096是指定缓存区大小，true是读完后把输入流自动关闭
        
        /* 方法二 */
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS","hdfs://master:9000");
        FileSystem filesystem = FileSystem.get(conf);
    
        boolean success = filesystem.mkdirs(new Path("/test"));
        System.out.println(success);
        
        // 判断文件夹是否存在
        success = filesystem.exists(new Path("/hello.txt"));
        System.out.println(success);
        
        // 删除文件夹或文件
        success = filesystem.delete(new Path("/test"),true);
        System.out.println(success);
        // true --> now   false --> delete home
        
        success = filesystem.exists(new Path("/test"));
        System.out.println(success);
        
        // 向hdfs文件中写入数据
        FSDataOutputStream out = filesystem.create(new Path("/test.data"),true);
        FileInputStream in = new FileInputStream("/root/a.txt");
        
        byte [] buf = new byte[4096];
        int len = in.read(buf);
        while (len != -1) {
            out.write(buf,0,len);
            len = in.read(buf);
        }
        in.close();
        out.close();
        
        // 查看文件hdfs文件目录
        FileStatus[] fs = filesystem.listStatus(new Path("/"));
        for(FileStatus status :fs) {
            System.out.println(status.getPath());
            System.out.println(status.getPermission());
            System.out.println(status.getReplication());//copy how much
        }
    }
}