spark wordcount完整工程代码（含pom.xml）

桃扇骨 2023-10-03 09:56 30阅读 0赞

## 工程目录概览 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70]

## 代码 ##

package com.zxl.spark.atguigu
    
    import org.apache.spark.rdd.RDD
    import org.apache.spark.{
        SparkConf, SparkContext}
    
    object L01_WordCount {
        
      def main(args: Array[String]): Unit = {
        
        // 创建 Spark 运行配置对象
        val sparkConf = new SparkConf().setMaster("local").setAppName("L01_WordCount")
        // 创建 Spark 上下文环境对象（连接对象）
        val sparkContext = new SparkContext(sparkConf)
    
        // 读取文件数据
        sparkContext.setLogLevel("ERROR")
        // 将文件中的数据进行分词
        val fileRDD: RDD[String] = sparkContext.textFile("src/main/input/word.txt")
        // 转换数据结构 word => (word, 1)
        val wordRDD: RDD[String] = fileRDD.flatMap(_.split(","))
        // 将转换结构后的数据按照相同的单词进行分组聚合
        val word2OneRDD: RDD[(String, Int)] = wordRDD.map((_, 1))
        // 将数据聚合结果采集到内存中
        val word2CountRDD: RDD[(String, Int)] = word2OneRDD.reduceByKey(_ + _)
        // 打印结果
        val word2Count: Array[(String, Int)] = word2CountRDD.collect()
        word2Count.foreach(println)
        //阻塞以查看日志
        while (true){
        }
        sparkContext.stop()
      }
    
    }

## pom ##

<?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
    
        <groupId>org.zxl</groupId>
        <artifactId>SparkDemo1</artifactId>
        <version>1.0-SNAPSHOT</version>
    
        <properties>
            <maven.compiler.source>8</maven.compiler.source>
            <maven.compiler.target>8</maven.compiler.target>
            <spark.version>3.1.1</spark.version>
            <spark.scala.version>2.12</spark.scala.version>
        </properties>
    
        <dependencies>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-core_${spark.scala.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-sql_${spark.scala.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-streaming_${spark.scala.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>
    
            
    
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-hive_${spark.scala.version}</artifactId>
                <version>${spark.version}</version>
            </dependency>
    
            
            <dependency>
                <groupId>mysql</groupId>
                <artifactId>mysql-connector-java</artifactId>
                <version>5.1.48</version>
            </dependency>
    
            
            <dependency>
                <groupId>org.codehaus.janino</groupId>
                <artifactId>commons-compiler</artifactId>
                <version>3.1.0</version>
            </dependency>
        </dependencies>
    
        <build>
            <plugins>
                
                <plugin>
                    <groupId>net.alchim31.maven</groupId>
                    <artifactId>scala-maven-plugin</artifactId>
                    <version>3.2.2</version>
                    <executions>
                        <execution>
                            
                            <goals>
                                <goal>testCompile</goal>
                            </goals>
                        </execution>
                    </executions>
                </plugin>
                <plugin>
                    <groupId>org.apache.maven.plugins</groupId>
                    <artifactId>maven-assembly-plugin</artifactId>
                    <version>3.1.1</version>
                    <configuration>
                        <descriptorRefs>
                            <descriptorRef>jar-with-dependencies</descriptorRef>
                        </descriptorRefs>
                    </configuration>
                    <executions>
                        <execution>
                            <id>make-assembly</id>
                            <phase>package</phase>
                            <goals>
                                <goal>single</goal>
                            </goals>
                        </execution>
                    </executions>
                </plugin>
            </plugins>
        </build>
    
    </project>

## input/word.txt ##

hello,zxl
    hello,zhangxueliang

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20210702131818297.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk=,size_16,color_FFFFFF,t_70