使用MapReduce计算小案例

准备工作

1.首先要确定你本地有hadoop 的环境

2. 新建maven 项目
3. 引入 依赖

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client</artifactId>
      <version>2.6.0-cdh5.7.0</version>
    </dependency>

4. 准备测试数据：

image.png

话不多说，直接上java 代码

package com.zyh.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * 使用MapReduce开发WordCount用用程序
 */
public class WordCountApp {
    /**
     * 读取输入文件
     * <p>
     * Mapper
     * LongWritable 文件的偏移量
     * Text hadoop 里面的text 就是 java里面的String
     */
    public static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable> {
        LongWritable one = new LongWritable(1);
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            // 接收到的 每一行数据
            String line = value.toString();
            // 按照制定分隔符分开
            String[] words = line.split(" ");
            for (String word : words) {
                context.write(new Text(word), one);
            }
        }
    }

    /**
     * 归并操作
     */
    public static class MyReducer extends Reducer<Text, LongWritable, Text, LongWritable> {
        @Override
        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
            long sum = 0;
            for (LongWritable value : values) {
                sum += value.get();
            }
            context.write(key, new LongWritable(sum));
        }
    }

    /**
     * 封装了所有的mapreduce的 作业信息
     * @param args
     */
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        //创建configuration
        Configuration configuration = new Configuration();



        Job job = Job.getInstance(configuration, "wordCount");
        //设置 job的处理类
        job.setJarByClass(WordCountApp.class);
        //设置作业处理的输入路径
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        //设置map相关的参数
        job.setMapperClass(MyMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);


        //设置 reduce 相关参数
        job.setReducerClass(MyReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);

        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

代码写完需要编译运行

运行自己开发的 mapreduce 需要操作如下 ：

1. 首先到Hadoop sbin目录运行 start-all.sh
   2.运行 jar 后面加入我们 的主类名 。 和两个参数 分别为 输入路径地址 和输出

java jar jar包 主类名全路径 hadoop输入文件路径 hadoop 输出结果文件路径