189、Spark 2.0之Dataset开发详解-typed操

coalesce和repartition操作,都是用来重新定义分区的
区别在于：coalesce，只能用于减少分区数量，而且可以选择不发生shuffle
repartiton，可以增加分区，也可以减少分区，必须会发生shuffle，相当于是进行了一次重分区操作

代码

object TypedOperation {

  case class Employee(name: String, age: Long, depId: Long, gender: String, salary: Long)

  def main(args: Array[String]): Unit = {
    val sparkSession = SparkSession
      .builder()
      .appName("BasicOperation")
      .master("local")
      .getOrCreate()

    import sparkSession.implicits._

    val employeePath = this.getClass.getClassLoader.getResource("employee.json").getPath

    val employeeDF = sparkSession.read.json(employeePath)

    val employeeDS = employeeDF.as[Employee]
    println(employeeDS.rdd.partitions.size)

    val employeeDSRepartitioned = employeeDS.repartition(5)
    println(employeeDSRepartitioned.rdd.partitions.size)

    val employeeDSCoalesced = employeeDSRepartitioned.coalesce(3)
    println(employeeDSCoalesced.rdd.partitions.size)
  }
}