Scala和Java分别实现的WordCount

桃扇骨 2023-03-14 05:58 135阅读 0赞

在使用之前，对Maven进行配置：参考：[Java项目管理之Maven下载及安装][Java_Maven] 和项目环境配置 [idea创建项目及配置详解和测试代码][idea]  
之后建立一个data文件夹里面写一个word.txt文件，用于统计。

1.使用Scala实现单词统计

package WordCount
    
    import org.apache.spark.rdd.RDD
    import org.apache.spark.{ SparkConf, SparkContext}
    
    object SparkWC { 
      def main(args: Array[String]): Unit = { 
        val conf = new SparkConf() //设置Spark运行的名称、资源
        //设置名称、
        conf.setAppName("wordcount") //设置运行模式，使用local本地模式
        conf.setMaster("local")
    
        //创建Spark的上下文,通往Spark集群的唯一通道
        val sc = new SparkContext(conf)
    
        //返回类型是一个RDD类型
        val lines: RDD[String] = sc.textFile("./data/word")
        //按空格进行切开
        val words: RDD[String] = lines.flatMap(line => { 
          line.split(" ")
        }) //计数
        val PairWord: RDD[(String, Int)] = words.map(word => { 
          new Tuple2(word, 1)
        })
        //先分组，再对value值进行计算
        val res: RDD[(String, Int)] = PairWord.reduceByKey((v1: Int, v2: Int) => { 
          v1 + v2
        })
    
        //打印结果
        res.foreach(one => { 
          println(one)
        })
      }
    }

使用scala代码的时候，由于Scala的特性，代码还可以继续简化：

package WordCount
    
    import org.apache.spark.rdd.RDD
    import org.apache.spark.{ SparkConf, SparkContext}
    
    object SparkWC1 { 
      def main(args: Array[String]): Unit = { 
        val conf = new SparkConf().setAppName("wordcount").setMaster("local")
        val sc = new SparkContext(conf)
        sc.textFile("./data/word").flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _).foreach(println)
        sc.stop()
      }
    }

运行Scala代码，在控制台查看：  
![在这里插入图片描述][20200516144831285.gif]  
2.使用Java实现单词统计

package WordCount;
    
    import org.apache.spark.SparkConf;
    import org.apache.spark.api.java.JavaPairRDD;
    import org.apache.spark.api.java.JavaRDD;
    import org.apache.spark.api.java.JavaSparkContext;
    import org.apache.spark.api.java.function.FlatMapFunction;
    import org.apache.spark.api.java.function.Function2;
    import org.apache.spark.api.java.function.PairFunction;
    import org.apache.spark.api.java.function.VoidFunction;
    import org.apache.spark.sql.catalyst.plans.logical.Except;
    import scala.Tuple2;
    
    import java.util.Arrays;
    import java.util.Iterator;
    
    public class JavaWC { 
        public static void main(String[] args) { 
            SparkConf sparkConf = new SparkConf();
            sparkConf.setAppName("javaWC");
            sparkConf.setMaster("local");
            JavaSparkContext javaSparkContext = new JavaSparkContext(sparkConf);
            JavaRDD<String> lines = javaSparkContext.textFile("./data/word");
            JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() { 
                @Override
                public Iterator<String> call(String lines) throws Exception { 
                    return Arrays.asList(lines.split(" ")).iterator();
                }
            });
    
            JavaPairRDD<String, Integer> PairWord = words.mapToPair(new PairFunction<String, String, Integer>() { 
                @Override
                public Tuple2<String, Integer> call(String s) throws Exception { 
                    return new Tuple2<String, Integer>(s, 1);
                }
            });
            JavaPairRDD<String,Integer> res=PairWord.reduceByKey(new Function2<Integer, Integer, Integer>() { 
                @Override
                public Integer call(Integer v1, Integer v2) throws Exception { 
                    return v1 + v2;
                }
            });
            res.foreach(new VoidFunction<Tuple2<String, Integer>>() { 
                @Override
                public void call(Tuple2<String, Integer> stringIntegerTuple2) throws Exception { 
                    System.out.println(stringIntegerTuple2);
                }
            });
        }
    }

运行查看：效果是一样的：  
![在这里插入图片描述][20200516145158575.gif]  
但是在这里很显然使用java代码实现代码过于冗余。不像Scala那样简洁：

[Java_Maven]: https://blog.csdn.net/qq_44973159/article/details/105777093
[idea]: https://blog.csdn.net/qq_44973159/article/details/105792078
[20200516144831285.gif]: /images/20230312/8e93c3267e5c4408a03b3ab2e2767e1f.png
[20200516145158575.gif]: /images/20230312/86dd83ddc64f46998c67f506ffd432f7.png