[TOC]

---

今天我们主要分析下broker被选为controller之后，主要干了什么。门面代码先列出来：

def onControllerFailover() {
    if (isRunning) {
        info("Broker %d starting become controller state transition".format(config.brokerId))
        //read controller epoch from zk
        readControllerEpochFromZookeeper()
        // increment the controller epoch
        incrementControllerEpoch(zkUtils.zkClient)
        // before reading source of truth from zookeeper, register the listeners to get broker/topic callbacks
        registerReassignedPartitionsListener()
        registerIsrChangeNotificationListener()
        registerPreferredReplicaElectionListener()
        partitionStateMachine.registerListeners()
        replicaStateMachine.registerListeners()
        initializeControllerContext()
        replicaStateMachine.startup()
        partitionStateMachine.startup()
        // register the partition change listeners for all existing topics on failover
        controllerContext.allTopics.foreach(topic => partitionStateMachine.registerPartitionChangeListener(topic))
        info("Broker %d is ready to serve as the new controller with epoch %d".format(config.brokerId, epoch))
        brokerState.newState(RunningAsController)
        maybeTriggerPartitionReassignment()
        maybeTriggerPreferredReplicaElection()
        /* send partition leadership info to all live brokers */
        sendUpdateMetadataRequest(controllerContext.liveOrShuttingDownBrokerIds.toSeq)
        if (config.autoLeaderRebalanceEnable) {
            info("starting the partition rebalance scheduler")
            autoRebalanceScheduler.startup()
            autoRebalanceScheduler.schedule("partition-rebalance-thread", checkAndTriggerPartitionRebalance,
                5, config.leaderImbalanceCheckIntervalSeconds.toLong, TimeUnit.SECONDS)
        }
        deleteTopicManager.start()
    }
    else
        info("Controller has been shut down, aborting startup/failover")
}

一个门面，涉及到的监听器和其他内容比较多，我们一一分析。

一、controller epoch

首先从zk的节点/controller_epoch下获取之前的epoch，然后将其+1后持久化到zk中。

二、注册监听器

这块就是订阅zk的节点信息，如果节点信息有变化，会做出一些操作。

2.1 registerReassignedPartitionsListener

private def registerReassignedPartitionsListener() = {
    zkUtils.zkClient.subscribeDataChanges(ZkUtils.ReassignPartitionsPath, partitionReassignedListener)
}

这块订阅的路径是：/admin/reassign_partitions，表示的是分区的重新分配。如果有变化，会有下面的操作：

/**
    * Starts the partition reassignment process unless -
    * 1. Partition previously existed
    * 2. New replicas are the same as existing replicas
    * 3. Any replica in the new set of replicas are dead
    * If any of the above conditions are satisfied, it logs an error and removes the partition from list of reassigned
    * partitions.
    */
class PartitionsReassignedListener(controller: KafkaController) extends IZkDataListener with Logging {
    this.logIdent = "[PartitionsReassignedListener on " + controller.config.brokerId + "]: "
    val zkUtils = controller.controllerContext.zkUtils
    val controllerContext = controller.controllerContext

    /**
    * Invoked when some partitions are reassigned by the admin command
    * @throws Exception On any error.
    */
    @throws(classOf[Exception])
    def handleDataChange(dataPath: String, data: Object) {
        debug("Partitions reassigned listener fired for path %s. Record partitions to be reassigned %s"
            .format(dataPath, data))
        //解析zk节点上的数据
        val partitionsReassignmentData = zkUtils.parsePartitionReassignmentData(data.toString)
        //获取需要重新分配的分区列表
        val partitionsToBeReassigned = inLock(controllerContext.controllerLock) {
            partitionsReassignmentData.filterNot(p => controllerContext.partitionsBeingReassigned.contains(p._1))
        }
        partitionsToBeReassigned.foreach { partitionToBeReassigned =>
            inLock(controllerContext.controllerLock) {
            //首先判断topic是否正在等待被删除，如果是，就把相关的分区从列表中删除
                if (controller.deleteTopicManager.isTopicQueuedUpForDeletion(partitionToBeReassigned._1.topic)) {
                    error("Skipping reassignment of partition %s for topic %s since it is currently being deleted"
                        .format(partitionToBeReassigned._1, partitionToBeReassigned._1.topic))
                    controller.removePartitionFromReassignedPartitions(partitionToBeReassigned._1)
                } else {//进行重分配
                    val context = new ReassignedPartitionsContext(partitionToBeReassigned._2)
                    controller.initiateReassignReplicasForTopicPartition(partitionToBeReassigned._1, context)
                }
            }
        }
    }

    /**
        * Called when the leader information stored in zookeeper has been delete. Try to elect as the leader
        *
        * @throws Exception
        * On any error.
        */
    @throws(classOf[Exception])
    def handleDataDeleted(dataPath: String) {
    }
}

下面我们具体看下重新分配的过程,也就是initiateReassignReplicasForTopicPartition里面做了什么。

2.1.1 initiateReassignReplicasForTopicPartition

def initiateReassignReplicasForTopicPartition(topicAndPartition: TopicAndPartition,reassignedPartitionContext: ReassignedPartitionsContext) {
    val newReplicas = reassignedPartitionContext.newReplicas
    val topic = topicAndPartition.topic
    val partition = topicAndPartition.partition
    //获取存活的replica
    val aliveNewReplicas = newReplicas.filter(r => controllerContext.liveBrokerIds.contains(r))
    try {
        val assignedReplicasOpt = controllerContext.partitionReplicaAssignment.get(topicAndPartition)
        assignedReplicasOpt match {
            case Some(assignedReplicas) =>
                if (assignedReplicas == newReplicas) {
                    throw new KafkaException("Partition %s to be reassigned is already assigned to replicas".format(topicAndPartition) +
                        " %s. Ignoring request for partition reassignment".format(newReplicas.mkString(",")))
                } else {
                    if (aliveNewReplicas == newReplicas) {
                        info("Handling reassignment of partition %s to new replicas %s".format(topicAndPartition, newReplicas.mkString(",")))
                        // first register ISR change listener 监听ISR变化
                        watchIsrChangesForReassignedPartition(topic, partition, reassignedPartitionContext)
                    controllerContext.partitionsBeingReassigned.put(topicAndPartition, reassignedPartitionContext)
                        // mark topic ineligible for deletion for the partitions being reassigned
                        deleteTopicManager.markTopicIneligibleForDeletion(Set(topic))
                        onPartitionReassignment(topicAndPartition, reassignedPartitionContext)
                    } else {
                        // some replica in RAR is not alive. Fail partition reassignment
                        throw new KafkaException("Only %s replicas out of the new set of replicas".format(aliveNewReplicas.mkString(",")) +
                            " %s for partition %s to be reassigned are alive. ".format(newReplicas.mkString(","), topicAndPartition) +
                            "Failing partition reassignment")
                    }
                }
            case None => throw new KafkaException("Attempt to reassign partition %s that doesn't exist"
                .format(topicAndPartition))
        }
    } catch {
        case e: Throwable => error("Error completing reassignment of partition %s".format(topicAndPartition), e)
            // remove the partition from the admin path to unblock the admin client
            removePartitionFromReassignedPartitions(topicAndPartition)
    }
}

我们分析到代码watchIsrChangesForReassignedPartition时，发现里面定义的数据监听之后，其实也是调用了onPartitionReassignment，所以我们之间看下onPartitionReassignment，这是重新分配的重点。

2.1.2 onPartitionReassignment

这个方法由重新分区监听器触发，当admin触发时，它首先创建/admin/reassign_partitions路径，以触发zk监听器。分区重新分配会经历下面几步：

RAR = Reassigned replicas 重新分配的副本

OAR = Original list of replicas for partition，分区最初的副本列表

AR = current assigned replicas：当前分配的副本

• 1、通过OAR + RAR更新zk中的AR
• 2、发送LeaderAndIsr请求给AR中的每个副本，我们这样做的目的是强制更新zk中的controller epoch。
• 3、将RAR-OAR中副本状态变为新副本状态NewReplica，启动新副本
• 4、等待RAR中所有副本与leader同步
• 5、将RAR中所有的副本设置为OnlineReplica状态
• 6、设置AR到RAR的内存中
• 7、如果leader不在RAR中，从RAR中选举一个leader。如果需要选举，需要发送LeaderAndIsr请求。如果不是，那么controller epoch会自增，然后发送LeaderAndIsr请求。在任何情况下，都要保证AR=RAR。防止出现leader把RAR-OAR中的副本加到isr中。
• 8、把OAR-RAR中的副本设为OfflineReplica状态。当OfflineReplica状态变化时，我们会移除zk中ISR的OAR-RAR部分，然后发送LeaderAndIsr给leader，通知他ISR的缩减。然后，我们把OAR-RAR的副本状态改为StopReplica。
• 9、将OAR-RAR中所有的副本状态改为StopReplica。这会物理删除这些副本。
• 10、使用RAR更新ZK中的AR
• 11、更新zk节点/admin/reassign_partitions，删除对应的分区
• 12、选举完成后，副本和isr信息变化了。重新发送更新源数据的请求给每个broker。

整个过程比较绕，需要仔细理解下，下面是一个简单的过程，可以参考。

* For example, if OAR = {1, 2, 3} and RAR = {4,5,6}, the values in the assigned replica (AR) and leader/isr path in ZK
* may go through the following transition.
* AR                 leader/isr
* {1,2,3}            1/{1,2,3}           (initial state)
* {1,2,3,4,5,6}      1/{1,2,3}           (step 2)
* {1,2,3,4,5,6}      1/{1,2,3,4,5,6}     (step 4)
* {1,2,3,4,5,6}      4/{1,2,3,4,5,6}     (step 7)
* {1,2,3,4,5,6}      4/{4,5,6}           (step 8)
* {4,5,6}            4/{4,5,6}           (step 10)
*
* Note that we have to update AR in ZK with RAR last since it's the only place where we store OAR persistently.
* This way, if the controller crashes before that step, we can still recover.

2.2 registerIsrChangeNotificationListener

注册路径/isr_change_notification监听器。

/**
* Called when leader intimates of isr change
*
* @param controller
*/
class IsrChangeNotificationListener(controller: KafkaController) extends IZkChildListener with Logging {

override def handleChildChange(parentPath: String, currentChildren: util.List[String]): Unit = {
    import scala.collection.JavaConverters._

    inLock(controller.controllerContext.controllerLock) {
        debug("[IsrChangeNotificationListener] Fired!!!")
        val childrenAsScala: mutable.Buffer[String] = currentChildren.asScala
        try {
            val topicAndPartitions: immutable.Set[TopicAndPartition] = childrenAsScala.map(x => getTopicAndPartition(x)).flatten.toSet
            if (topicAndPartitions.nonEmpty) {
                controller.updateLeaderAndIsrCache(topicAndPartitions)
                processUpdateNotifications(topicAndPartitions)
            }
        } finally {
            // delete processed children
            childrenAsScala.map(x => controller.controllerContext.zkUtils.deletePath(
                ZkUtils.IsrChangeNotificationPath + "/" + x))
        }
    }
}

主要是更新下leader和isr的缓存，主要是controller的epoch，然后发送更新源数据的请求。

2.3 registerPreferredReplicaElectionListener

监听/admin/preferred_replica_election路径的数据，preferred replica在leader挂掉的情况下，会直接被选为leader，也就是就是assigned replicas列表中的第一个replica。

三、分区和副本状态机

3.1 注册分区状态机监听器

首先是分区状态机，分区的状态有以下几个：

• NonExistentPartition，分区不存在，他的前一个状态只能是OfflinePartition
• NewPartition：新分区，还没有选出leader，前一个状态为NonExistentPartition
• OnlinePartition：分区上线，leader已经选举出来了，前一个状态为NewPartition/OfflinePartition
• OfflinePartition：分区下线，前一个状态为NewPartition/OnlinePartition

// register topic and partition change listeners
def registerListeners() {
  registerTopicChangeListener()
  if(controller.config.deleteTopicEnable)
    registerDeleteTopicListener()
}

监听/brokers/topics路径数据变化，如果允许删除topic的话，监听/admin/delete_topics路径数据变化。

下面我们看下两个监听背后的动作。

3.1.1 registerTopicChangeListener

这块主要处理了/brokers/topics路径下一些topic的变化，包括新增和删除的后续操作。

/**
 * This is the zookeeper listener that triggers all the state transitions for a partition
 */
class TopicChangeListener extends IZkChildListener with Logging {
  this.logIdent = "[TopicChangeListener on Controller " + controller.config.brokerId + "]: "

@throws(classOf[Exception])
def handleChildChange(parentPath : String, children : java.util.List[String]) {
  inLock(controllerContext.controllerLock) {
    if (hasStarted.get) {
      try {
        val currentChildren = {
          import JavaConversions._
          debug("Topic change listener fired for path %s with children %s".format(parentPath, children.mkString(",")))
          (children: Buffer[String]).toSet
        }
        val newTopics = currentChildren -- controllerContext.allTopics
        val deletedTopics = controllerContext.allTopics -- currentChildren
        controllerContext.allTopics = currentChildren

        val addedPartitionReplicaAssignment = zkUtils.getReplicaAssignmentForTopics(newTopics.toSeq)
        controllerContext.partitionReplicaAssignment = controllerContext.partitionReplicaAssignment.filter(p =>
          !deletedTopics.contains(p._1.topic))
        controllerContext.partitionReplicaAssignment.++=(addedPartitionReplicaAssignment)
        info("New topics: [%s], deleted topics: [%s], new partition replica assignment [%s]".format(newTopics,
          deletedTopics, addedPartitionReplicaAssignment))
        if(newTopics.size > 0)
          controller.onNewTopicCreation(newTopics, addedPartitionReplicaAssignment.keySet.toSet)
      } catch {
        case e: Throwable => error("Error while handling new topic", e )
      }
    }
  }
}
}

3.1.2 registerDeleteTopicListener

监听zk节点，把需要删除的topic放到待删除队列中，然后由kafka执行删除，主要删除的是zk下面相关的节点，和日志文件。

3.2 注册副本状态机监听器

副本状态机，有以下几种状态：

• NewReplica：controller在重新分区时会创建新副本，这个状态下，只能收到成为follower的请求，前一个状态是NonExistentReplica。
• OnlineReplica：副本启动后的状态，这个状态下，他可以收到成为leader或follower的请求。前一个状态可以是NewReplica, OnlineReplica or OfflineReplica。
• OfflineReplica：分区挂掉后的状态，前一个状态为NewReplica, OnlineReplica
• ReplicaDeletionStarted：副本删除开始时的状态，前一个状态为OfflineReplica
• ReplicaDeletionSuccessful：副本响应删除请求时没有错误码，这时候的状态，前一个状态为ReplicaDeletionStarted
• ReplicaDeletionIneligible：副本删除失败的状态，前一个状态为ReplicaDeletionStarted
• NonExistentReplica：副本删除成功后的状态，前一个状态为ReplicaDeletionSuccessful。

3.2.1 registerBrokerChangeListener

监听/brokers/ids路径下的节点变化。主要是broker是否有新增或者删除，然后做对应的操作。

/**
 * This is the zookeeper listener that triggers all the state transitions for a replica
 */
class BrokerChangeListener() extends IZkChildListener with Logging {
  this.logIdent = "[BrokerChangeListener on Controller " + controller.config.brokerId + "]: "
  def handleChildChange(parentPath : String, currentBrokerList : java.util.List[String]) {
    info("Broker change listener fired for path %s with children %s".format(parentPath, currentBrokerList.sorted.mkString(",")))
    inLock(controllerContext.controllerLock) {
      if (hasStarted.get) {
        ControllerStats.leaderElectionTimer.time {
          try {
            val curBrokers = currentBrokerList.map(_.toInt).toSet.flatMap(zkUtils.getBrokerInfo)
            val curBrokerIds = curBrokers.map(_.id)
            val liveOrShuttingDownBrokerIds = controllerContext.liveOrShuttingDownBrokerIds
            val newBrokerIds = curBrokerIds -- liveOrShuttingDownBrokerIds
            val deadBrokerIds = liveOrShuttingDownBrokerIds -- curBrokerIds
            val newBrokers = curBrokers.filter(broker => newBrokerIds(broker.id))
            controllerContext.liveBrokers = curBrokers
            val newBrokerIdsSorted = newBrokerIds.toSeq.sorted
            val deadBrokerIdsSorted = deadBrokerIds.toSeq.sorted
            val liveBrokerIdsSorted = curBrokerIds.toSeq.sorted
            info("Newly added brokers: %s, deleted brokers: %s, all live brokers: %s"
            .format(newBrokerIdsSorted.mkString(","), deadBrokerIdsSorted.mkString(","), liveBrokerIdsSorted.mkString(",")))
            newBrokers.foreach(controllerContext.controllerChannelManager.addBroker)
            deadBrokerIds.foreach(controllerContext.controllerChannelManager.removeBroker)
            if(newBrokerIds.size > 0)
              controller.onBrokerStartup(newBrokerIdsSorted)
            if(deadBrokerIds.size > 0)
              controller.onBrokerFailure(deadBrokerIdsSorted)
        } catch {
            case e: Throwable => error("Error while handling broker changes", e)
        }
        }
    }
    }
}
}

3.3 初始化controller上下文

这块主要获取了一些原始数据，包括topic、分区等等，然后启动了一些管理器。

private def initializeControllerContext() {
    // update controller cache with delete topic information
    //存活的brokerId列表
    controllerContext.liveBrokers = zkUtils.getAllBrokersInCluster().toSet
    //所有的topic
    controllerContext.allTopics = zkUtils.getAllTopics().toSet
    //所有topic的分区信息
    controllerContext.partitionReplicaAssignment = zkUtils.getReplicaAssignmentForTopics(controllerContext.allTopics.toSeq)
    //分区的leader信息
    controllerContext.partitionLeadershipInfo = new mutable.HashMap[TopicAndPartition, LeaderIsrAndControllerEpoch]
    //已经挂掉的broker列表，默认为空
    controllerContext.shuttingDownBrokerIds = mutable.Set.empty[Int]
    // update the leader and isr cache for all existing partitions from Zookeeper
    updateLeaderAndIsrCache()
    // start the channel manager
    startChannelManager()
    initializePreferredReplicaElection()
    initializePartitionReassignment()
    initializeTopicDeletion()
    info("Currently active brokers in the cluster: %s".format(controllerContext.liveBrokerIds))
    info("Currently shutting brokers in the cluster: %s".format(controllerContext.shuttingDownBrokerIds))
    info("Current list of topics in the cluster: %s".format(controllerContext.allTopics))
}

前面几行已经有了注释，也比较清楚，下面我们从startChannelManager开始。这个ChannelManager是什么？其实就是用于leader与各个broker通信的通道。这个manager也就是管理这些请求的管理器。

这里主要处理几种请求：

• LEADER_AND_ISR
• STOP_REPLICA
• UPDATE_METADATA_KEY

这个通道启动完成后，就是初始化三个动作：

• initializePreferredReplicaElection
• initializePartitionReassignment
• initializeTopicDeletion

3.4 副本状态机监听器启动

也就是replicaStateMachine.startup()。这个方法通过读取zk中的分区信息，把所有的副本状态改为OnlineReplica。

/**
 * Invoked on successful controller election. First registers a broker change listener since that triggers all
 * state transitions for replicas. Initializes the state of replicas for all partitions by reading from zookeeper.
 * Then triggers the OnlineReplica state change for all replicas.
 */
def startup() {
  // initialize replica state
  initializeReplicaState()
  // set started flag
  hasStarted.set(true)
  // move all Online replicas to Online
  handleStateChanges(controllerContext.allLiveReplicas(), OnlineReplica)
  info("Started replica state machine with initial state -> " + replicaState.toString())
}

3.5 分区状态机监听器启动

类似于副本状态机监听器，这个也是初始化了分区的状态，然后把分区的状态变为OnlineState。

/**
 * Invoked on successful controller election. First registers a topic change listener since that triggers all
 * state transitions for partitions. Initializes the state of partitions by reading from zookeeper. Then triggers
 * the OnlinePartition state change for all new or offline partitions.
 */
def startup() {
  // initialize partition state
  initializePartitionState()
  // set started flag
  hasStarted.set(true)
  // try to move partitions to online state
  triggerOnlinePartitionStateChange()

  info("Started partition state machine with initial state -> " + partitionState.toString())
}

3.6 自动负载定时器

如果开启了auto.leader.rebalance.enable参数，那么就会启动分区负载定时器。配置中可以设置leader.imbalance.check.interval.seconds参数，表示定时检查的时间间隔，单位为秒。

if (config.autoLeaderRebalanceEnable) {
    info("starting the partition rebalance scheduler")
    autoRebalanceScheduler.startup()
    autoRebalanceScheduler.schedule("partition-rebalance-thread", checkAndTriggerPartitionRebalance,
        5, config.leaderImbalanceCheckIntervalSeconds.toLong, TimeUnit.SECONDS)
}

我们可以着重看下checkAndTriggerPartitionRebalance方法。

private def checkAndTriggerPartitionRebalance(): Unit = {
    if (isActive()) {
        trace("checking need to trigger partition rebalance")
        // get all the active brokers
        var preferredReplicasForTopicsByBrokers: Map[Int, Map[TopicAndPartition, Seq[Int]]] = null
        inLock(controllerContext.controllerLock) {
            preferredReplicasForTopicsByBrokers =
                controllerContext.partitionReplicaAssignment.filterNot(p => deleteTopicManager.isTopicQueuedUpForDeletion(p._1.topic)).groupBy {
                    case (topicAndPartition, assignedReplicas) => assignedReplicas.head
                }
        }
        debug("preferred replicas by broker " + preferredReplicasForTopicsByBrokers)
        // for each broker, check if a preferred replica election needs to be triggered
        preferredReplicasForTopicsByBrokers.foreach {
            case (leaderBroker, topicAndPartitionsForBroker) => {
                var imbalanceRatio: Double = 0
                var topicsNotInPreferredReplica: Map[TopicAndPartition, Seq[Int]] = null
                inLock(controllerContext.controllerLock) {
                    topicsNotInPreferredReplica =
                        topicAndPartitionsForBroker.filter {
                            case (topicPartition, replicas) => {
                                controllerContext.partitionLeadershipInfo.contains(topicPartition) &&
                                    controllerContext.partitionLeadershipInfo(topicPartition).leaderAndIsr.leader != leaderBroker
                            }
                        }
                    debug("topics not in preferred replica " + topicsNotInPreferredReplica)
                    val totalTopicPartitionsForBroker = topicAndPartitionsForBroker.size
                    val totalTopicPartitionsNotLedByBroker = topicsNotInPreferredReplica.size
                    imbalanceRatio = totalTopicPartitionsNotLedByBroker.toDouble / totalTopicPartitionsForBroker
                    trace("leader imbalance ratio for broker %d is %f".format(leaderBroker, imbalanceRatio))
                }
                // check ratio and if greater than desired ratio, trigger a rebalance for the topic partitions
                // that need to be on this broker
                if (imbalanceRatio > (config.leaderImbalancePerBrokerPercentage.toDouble / 100)) {
                    topicsNotInPreferredReplica.foreach {
                        case (topicPartition, replicas) => {
                            inLock(controllerContext.controllerLock) {
                                // do this check only if the broker is live and there are no partitions being reassigned currently
                                // and preferred replica election is not in progress
                                if (controllerContext.liveBrokerIds.contains(leaderBroker) &&
                                    controllerContext.partitionsBeingReassigned.size == 0 &&
                                    controllerContext.partitionsUndergoingPreferredReplicaElection.size == 0 &&
                                    !deleteTopicManager.isTopicQueuedUpForDeletion(topicPartition.topic) &&
                                    controllerContext.allTopics.contains(topicPartition.topic)) {
                                    onPreferredReplicaElection(Set(topicPartition), true)
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}
}

3.7 启动删除topic进程

如果允许程序自动删除topic的话（delete.topic.enable=true），那么就会启动这个进程。