美文网首页
hadoop 3.2.1无法在WIN10上运行

hadoop 3.2.1无法在WIN10上运行

作者: 我的小猫不见了 | 来源:发表于2020-05-15 21:07 被阅读0次

    1.不说了,上代码.先是POM , 我这是 hadoop3.2.1的

    <?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
    
        <groupId>org.example</groupId>
        <artifactId>Hadoop3.2.1</artifactId>
        <version>1.0-SNAPSHOT</version>
        <build>
            <plugins>
                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-compiler-plugin</artifactId>
                    <configuration>
                        <source>7</source>
                        <target>7</target>
                    </configuration>
                </plugin>
            </plugins>
        </build>
    
        <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>commons-logging</groupId>
            <artifactId>commons-logging</artifactId>
            <version>1.2</version>
        </dependency>
        </dependencies>
    </project>
    

    2.这是Map任务 按数字大小分区 ,
    原本是生成的5位以内的正整数 20万个 ,
    做自定义分区实验
    其中1-10000位0区 , 10000-20000为1区 , 其余为2区
    这是代码

    import org.apache.hadoop.conf.Configured;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.NullWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Partitioner;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.util.Tool;
    import org.apache.hadoop.util.ToolRunner;
    
    import java.io.IOException;
    
    public class TotalSortV2 extends Configured implements Tool {
        static class SimpleMapper extends Mapper<LongWritable, Text, IntWritable, IntWritable> {
            @Override
            protected void map(LongWritable key, Text value,
                               Context context) throws IOException, InterruptedException {
                IntWritable intWritable = new IntWritable(Integer.parseInt(value.toString()));
                context.write(intWritable, intWritable);
            }
        }
    
        static class SimpleReducer extends Reducer<IntWritable, IntWritable, IntWritable, NullWritable> {
            @Override
            protected void reduce(IntWritable key, Iterable<IntWritable> values,
                                  Context context) throws IOException, InterruptedException {
                for (IntWritable value : values)
                    context.write(value, NullWritable.get());
            }
        }
    
        public static class IteblogPartitioner extends Partitioner<IntWritable, IntWritable> {
            @Override
            public int getPartition(IntWritable key, IntWritable value, int numPartitions) {
                int keyInt = Integer.parseInt(key.toString());
                if (keyInt < 10000) {
                    return 0;
                } else if (keyInt < 20000) {
                    return 1;
                } else {
                    return 2;
                }
            }
        }
    
        @Override
        public int run(String[] args) throws Exception {
            
            Job job = Job.getInstance(getConf());
            job.setJarByClass(TotalSortV2.class);
            FileInputFormat.addInputPath(job, new Path("hdfs://node2:8020/have"));
            FileOutputFormat.setOutputPath(job, new Path("hdfs://node2:8020/have2"));
    
            job.setMapperClass(SimpleMapper.class);
            job.setReducerClass(SimpleReducer.class);
            job.setPartitionerClass(IteblogPartitioner.class);
            job.setMapOutputKeyClass(IntWritable.class);
            job.setMapOutputValueClass(IntWritable.class);
            job.setOutputKeyClass(IntWritable.class);
            job.setOutputValueClass(NullWritable.class);
            job.setNumReduceTasks(3);
            job.setJobName("dw_subject");
            return job.waitForCompletion(true) ? 0 : 1;
        }
    
        public static void main(String[] args) throws Exception {
            int exitCode = ToolRunner.run(new TotalSortV2(), args);
            System.exit(exitCode);
        }
    }
    
    

    3.我是在idea上调试 ,看能不能运行 ,结果不能. 解决方法如下

    image.png
    把 wintul解压后 , 里面的两个文件放到你电脑JAVA的jdk 的bin目录下 , 就可以了
    https://gitee.com/Alpha365/db

    相关文章

      网友评论

          本文标题:hadoop 3.2.1无法在WIN10上运行

          本文链接:https://www.haomeiwen.com/subject/kfihohtx.html