Hadoop和spring的集成

1.首先添加依赖

<!--添加spring-hadoop的依赖-->
  <dependency>
    <groupId>org.springframework.data</groupId>
    <artifactId>spring-data-hadoop</artifactId>
    <version>RELEASE</version>
  </dependency>

创建资源文件目录 resource

创建 beans.xml 配置文件

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:hdp="http://www.springframework.org/schema/hadoop"
       xmlns:context="http://www.springframework.org/schema/context"
       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
        http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd">


    <hdp:configuration id="hadoopConfiguration">
        fs.defaultFS=${spring.hadoop.fsUri}
    </hdp:configuration>

    <context:property-placeholder location="application.properties"/>

    <hdp:file-system id="fileSystem"
                     configuration-ref="hadoopConfiguration"
                     user="hadoop"/>


</beans>

创建 application.properties

spring.hadoop.fsUri=hdfs://hadoop000:8020

测试代码如下：

package com.imooc.hadoop.spring;
/*
 * Created by  zsh
 */

import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

/**
 * 使用Spring Hadoop来访问HDFS文件系统
 */
public class SpringHadoopHDFSApp {

    private ApplicationContext ctx;
    private FileSystem fileSystem;

    @Test
    public void testMkdir()throws Exception{
        fileSystem.mkdirs(new Path("/springhdfs"));
    }

    @Test
    public void testShow()throws Exception{
        FSDataInputStream in = fileSystem.open(new Path("/springhdfs/hello.txt"));
        IOUtils.copyBytes(in,System.out,1024);
        in.close();
    }

    @Test
    public void testListPath()throws Exception{
        FileStatus[] fileStatus = fileSystem.listStatus(new Path("/springhdfs"));
        for (FileStatus fileStatus1 : fileStatus){
            System.out.println("> "+fileStatus1.getPath());
        }
    }

    @Before
    public void setUp(){
        ctx = new ClassPathXmlApplicationContext("beans.xml");
        fileSystem = (FileSystem) ctx.getBean("fileSystem");
    }

    @After
    public void tearDown(){
        ctx = null;
    }
}