RDD,DataFrame,DataSet三者之间的转换

喜欢ヅ旅行 2024-02-18 10:30 106阅读 0赞

![在这里插入图片描述][d8023d452f0647cd93026bd2bafbbcd9.png]

import org.apache.spark.rdd.RDD
    import org.apache.spark.sql.{
        DataFrame, Dataset, SparkSession}
    
    case class Person(id:Int,name:String,sex:String,age:Int)
    object DataSetCreate {
        
    
      def main(args: Array[String]): Unit = {
        
        val spark = SparkSession
          .builder()
          .appName("test")
          .master("local[4]")
          .getOrCreate()
        import spark.implicits._
    
        val list = List(Person(1,"zhangsan","man",10),Person(2,"zhang2","woman",66),Person(3,"zhang3","man",70),Person(4,"zhang4","man",22))
    
        //TODO RDD转DataFrame:toDF
        val rdd = spark.sparkContext.parallelize(list)
        val df: DataFrame = rdd.toDF()
    
        //TODO RDD转DataSet:toDS
        val ds: Dataset[Person] = rdd.toDS()
    
        //TODO DataFrame转RDD:df.rdd
        //row类型取值
        val rdd2 = df.rdd
        val rdd3: RDD[(String, Int)] = rdd2.map(row => {
        
          val age = row.getAs[Int](3)
          val name = row.getAs[String](1)
          (name, age)
        })
        //打印结果 List((zhangsan,10), (zhang2,66), (zhang3,70), (zhang4,22))
        println(rdd3.collect().toList)
    
        //TODO DataSet转RDD:ds.rdd
        val rdd4: RDD[Person] = ds.rdd
        val rdd5: RDD[(String, Int)] = rdd4.map(x => (x.name, x.age))
        println(rdd5.collect().toList)
    
        //TODO DataSet转DataFrame:ds.toDF
        val df3: DataFrame = ds.toDF()
        df3.show()
    
        //TODO DataFrame转DataSet:df.as(行类型)
        //TODO 行类型可以写元组，元组的元素个数必须和列的个数一直，元组的元素必须与列的类型对应或者能自动转换
        //TODO 行类型可以写样例类，样例类的属性名必须与列名一致，属性类型必须与列的类型一致或者能够自动转换
        val ds6: Dataset[Person] = df.as[Person]
        ds6.show()
        
      }
    
    
    }

[d8023d452f0647cd93026bd2bafbbcd9.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/02/18/b4d29ba3628b4fc6a3879511af5a4637.png