dataframe 的基本操作

package bl.test.spark

import org.apache.spark.sql.SparkSession

object DataFrameApp {
  def main(args: Array[String]) {

    val spark = SparkSession.builder().appName("DataFrameApp")
      .master("local[2]").getOrCreate()

    val peopleDF = spark.read.format("json").load("file:////home/zy/Desktop/success.json")

    //输出dataframe对应的schema信息
    //peopleDF.printSchema()
    //展示前100条
    //peopleDF.show(100)
    //只显示IMEI字段的前20条
    //peopleDF.select("IMEI").show()
    //查询某几个列的数据 并且计算
    //peopleDF.select(peopleDF.col("IMEI"),(peopleDF.col("TaskID")+10).as("TaskID2")).show()
    //根据某一列的值进行过滤
    peopleDF.filter((peopleDF.col("TaskID")>20)).show()

    //根据某一列进行分组 在进行聚合操作 select TaskID,count(1) from table group by TaskID
    peopleDF.groupBy("TaskID").count().show()
    spark.stop()
  }
}