190、Spark 2.0之Dataset开发详解-typed操

distinct和dropDuplicates，都是用来进行去重的
区别在于，distinct，是根据每一条数据，进行完整内容的比对和去重
dropDuplicates，可以根据指定的字段进行去重

代码

object TypedOperation {

  case class Employee(name: String, age: Long, depId: Long, gender: String, salary: Long)

  def main(args: Array[String]): Unit = {
    val sparkSession = SparkSession
      .builder()
      .appName("BasicOperation")
      .master("local")
      .getOrCreate()

    import sparkSession.implicits._
    import org.apache.spark.sql.functions._

    val employeePath = this.getClass.getClassLoader.getResource("employee.json").getPath

    val employeeDF = sparkSession.read.json(employeePath)

    val employeeDS = employeeDF.as[Employee]

    employeeDS.distinct().show()

    employeeDS.dropDuplicates(Seq("name")).show()

  }
}