08-flink之wordCount

流程

wordcount流程图.png

Batch

默认 text

public class WordCountData {

    public static final String[] WORDS = new String[] {
        "To be, or not to be,--that is the question:--",
        "Whether 'tis nobler in the mind to suffer",
        "The slings and arrows of outrageous fortune",
        "Or to take arms against a sea of troubles,",
        "And by opposing end them?--To die,--to sleep,--",
        "No more; and by a sleep to say we end",
        "The heartache, and the thousand natural shocks",
        "That flesh is heir to,--'tis a consummation",
        "Devoutly to be wish'd. To die,--to sleep;--",
        "To sleep! perchance to dream:--ay, there's the rub;",
        "For in that sleep of death what dreams may come,",
        "When we have shuffled off this mortal coil,",
        "Must give us pause: there's the respect",
        "That makes calamity of so long life;",
        "For who would bear the whips and scorns of time,",
        "The oppressor's wrong, the proud man's contumely,",
        "The pangs of despis'd love, the law's delay,",
        "The insolence of office, and the spurns",
        "That patient merit of the unworthy takes,",
        "When he himself might his quietus make",
        "With a bare bodkin? who would these fardels bear,",
        "To grunt and sweat under a weary life,",
        "But that the dread of something after death,--",
        "The undiscover'd country, from whose bourn",
        "No traveller returns,--puzzles the will,",
        "And makes us rather bear those ills we have",
        "Than fly to others that we know not of?",
        "Thus conscience does make cowards of us all;",
        "And thus the native hue of resolution",
        "Is sicklied o'er with the pale cast of thought;",
        "And enterprises of great pith and moment,",
        "With this regard, their currents turn awry,",
        "And lose the name of action.--Soft you now!",
        "The fair Ophelia!--Nymph, in thy orisons",
        "Be all my sins remember'd."
    };

    public static DataSet<String> getDefaultTextLineDataSet(ExecutionEnvironment env) {
        return env.fromElements(WORDS);
    }

批处理 wordCount

public class WordCount {

    public static void main(String[] args) throws Exception {

        final ParameterTool params = ParameterTool.fromArgs(args);

        // 设置执行环境
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

        // web 界面提供该参数
        env.getConfig().setGlobalJobParameters(params);

        // 定义文本变量
        DataSet<String> text;
        if (params.has("input")) {
            // 读取路径下
            text = env.readTextFile(params.get("input"));
        } else {
            // 读取默认文本
            text = WordCountData.getDefaultTextLineDataSet(env);
        }

        DataSet<Tuple2<String, Integer>> counts =
                // split up the lines in pairs (2-tuples) containing: (word,1)
                text.flatMap(new Tokenizer())
                // 按照Tuple2第一个元素分组
                .groupBy(0)
                .sum(1);

        // emit result
        if (params.has("output")) {
            //打印到路径下
            counts.writeAsCsv(params.get("output"), "\n", " ");
            // execute program
            env.execute("WordCount Example");
        } else {
            //控制台打印
            System.out.println("Printing result to stdout. Use --output to specify output path.");
            counts.print();
        }

    }

    /**
     * Implements the string tokenizer that splits sentences into words as a user-defined
     * FlatMapFunction. The function takes a line (String) and splits it into
     * multiple pairs in the form of "(word,1)" ({@code Tuple2<String, Integer>}).
     */
    public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {

        @Override
        public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
            // 单词按照空格划分
            String[] tokens = value.toLowerCase().split("\\W+");

            // 每个单词转为 Tuple2元素
            for (String token : tokens) {
                if (token.length() > 0) {
                    out.collect(new Tuple2<>(token, 1));
                }
            }
        }
    }

Stream

流处理 wordCount

public class WordCount {

   public static void main(String[] args) throws Exception {

      // Checking input parameters
      final ParameterTool params = ParameterTool.fromArgs(args);

      // 获取执行环境
      final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

      // web 界面获取参数
      env.getConfig().setGlobalJobParameters(params);

      // 定义 文本 变量(用于接收文本流)
      DataStream<String> text;
      if (params.has("input")) {
         // 读取提供的流
         text = env.readTextFile(params.get("input"));
      } else {
         // 读物默认的流
         text = env.fromElements(WordCountData.WORDS);
      }

      DataStream<Tuple2<String, Integer>> counts =
         // split up the lines in pairs (2-tuples) containing: (word,1)
         text.flatMap(new Tokenizer())
         // group by the tuple field "0" and sum up tuple field "1"
         .keyBy(0).sum(1);

      // emit result
      if (params.has("output")) {
         counts.writeAsText(params.get("output"));
      } else {
         System.out.println("Printing result to stdout. Use --output to specify output path.");
         counts.print();
      }

      // 执行程序
      env.execute("Streaming WordCount");
   }

   /**
    * Implements the string tokenizer that splits sentences into words as a
    * user-defined FlatMapFunction. The function takes a line (String) and
    * splits it into multiple pairs in the form of "(word,1)" ({@code Tuple2<String,
    * Integer>}).
    */
   public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {

      @Override
      public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
         // normalize and split the line
         String[] tokens = value.toLowerCase().split("\\W+");

         // emit the pairs
         for (String token : tokens) {
            if (token.length() > 0) {
               out.collect(new Tuple2<>(token, 1));
            }
         }
      }
   }

}