ctdbd进程通信流程

ctdb_start_daemon
--> ux_socket_bind(ctdb) // 准备好unix域通信的listen fd
--> ctdb->daemon.sd = socket(AF_UNIX, SOCK_STREAM, 0)
--> bind(ctdb->daemon.sd, (struct sockaddr )&addr, sizeof(addr))
--> listen(ctdb->daemon.sd, 100)
--> tdb_reopen_all(false)
--> ctdb_tcp_init(ctdb); // 初始化tcp通信的相关方法：ctdb_tcp_methods变量中
--> ctdb->methods->initialise(ctdb)即ctdb_tcp_initialise*
--> ctdb_tcp_listen(ctdb)
--> struct ctdb_tcp ctcp = talloc_get_type(ctdb->private_data,struct ctdb_tcp);
--> ctcp->listen_fd = socket(sock.sa.sa_family, SOCK_STREAM, IPPROTO_TCP);
--> bind(ctcp->listen_fd, (struct sockaddr * )&sock, sock_size) != 0);
--> listen(ctcp->listen_fd, 10)
--> fde = event_add_fd(ctdb->ev, ctcp, ctcp->listen_fd, EVENT_FD_READ,ctdb_listen_event, ctdb); // **等待其他ctdbd进程tcp通信建链**
--> 当ctcp->listen_fd变成可读：ctdb_listen_event
--> fd = accept(ctcp->listen_fd, (struct sockaddr )&addr, &len);
--> struct ctdb_incoming in; in = talloc_zero(ctcp, struct ctdb_incoming); in->fd = fd; in->ctdb = ctdb;
--> in->queue = ctdb_queue_setup(ctdb, in, in->fd, CTDB_TCP_ALIGNMENT, ctdb_tcp_read_cb, in, "ctdbd-%s", incoming_node);
--> ctdb_queue_set_fd(queue, fd)
--> queue->fde = event_add_fd(queue->ctdb->ev, queue, fd, EVENT_FD_READ, queue_io_handler, queue); // 等待对端发tcp消息过来
--> ctdb_tcp_add_node(ctdb->nodes[i]) // for循环初始化主动建链需要的相关结构：ctdb->nodes[i]有子成员tnode保存在private_data字段；还有一个子成员queue保存在tnode的out_queue
--> struct ctdb_tcp_node tnode; tnode = talloc_zero(node, struct ctdb_tcp_node);
--> node->private_data = tnode; // 主动建链的链路信息保存在ctdb->nodes[i].private_data中 // tnode的fd是connectfd和queue的fd相同
--> tnode->out_queue = ctdb_queue_setup(node->ctdb, node, tnode->fd, CTDB_TCP_ALIGNMENT, ctdb_tcp_tnode_cb, node, "to-node-%s", node->name); // 处理tcp链路关闭的包
--> queue = talloc_zero(mem_ctx, struct ctdb_queue); // mem_ctx是第二个参数即node
--> queue->ctdb = ctdb; queue->fd = fd; queue->private_data = private_data; queue->callback = callback; // private_data是node
--> ctdb_queue_set_fd(queue, fd) // fd是tnode->fd
--> queue->fde = event_add_fd(queue->ctdb->ev, queue, fd, EVENT_FD_READ,queue_io_handler, queue); // （1）监听读事件：被动tcp链接的数据包
--> ctdb_set_public_addresses(ctdb, true);
--> ctdb_attach_databases(ctdb)
--> fde = event_add_fd(ctdb->ev, ctdb, ctdb->daemon.sd, EVENT_FD_READ, ctdb_accept_client, ctdb); // **等待本节点recover和cmd工具进程unix域通信建链**
--> 当ctdb->daemon.sd变成可读 ctdb_accept_client
--> client->queue = ctdb_queue_setup(ctdb, client, fd, CTDB_DS_ALIGNMENT, ctdb_daemon_read_cb, client, "client-%u", client->pid);
--> queue->fd = fd;
--> queue->callback = callback;
--> ctdb_queue_set_fd(queue, fd)
--> queue->fde = event_add_fd(queue->ctdb->ev, queue, fd, EVENT_FD_READ, queue_io_handler, queue); // （2）监听读事件：recover和cmd工具发来的unix域数据包
--> ctdb->methods->start(ctdb)即ctdb_tcp_start
--> ctdb_tcp_connect_node(ctdb->nodes[i]); // for循环**主动向其他节点的ctdbd进程建链**
--> tnode->connect_te = event_add_timed(ctdb->ev, tnode, timeval_zero(), ctdb_tcp_node_connect, node);
--> ctdb_tcp_node_connect
--> tnode->fd = socket(sock_out.sa.sa_family, SOCK_STREAM, IPPROTO_TCP);
--> bind(tnode->fd, (struct sockaddr )&sock_in, sockin_size)
--> connect(tnode->fd, (struct sockaddr )&sock_out, sockout_size) // tnode->fd是实际读写的fd即connectfd
--> tnode->connect_fde = event_add_fd(node->ctdb->ev, tnode, tnode->fd, EVENT_FD_WRITE|EVENT_FD_READ, ctdb_node_connect_write, node); 等待发送消息包 (3)监听主动建链的写事件
当tnode->fd变成可写即有tcp消息即将发走：ctdb_node_connect_write
--> ctdb_queue_set_fd(tnode->out_queue, tnode->fd);
--> queue->fd = fd; 等待回的tcp应答消息
--> queue->fde = event_add_fd(queue->ctdb->ev, queue, fd, EVENT_FD_READ,queue_io_handler**, queue); （4）监听主动建链的读事件：收到的数据包
--> 当收到对端tcp消息queue->fd变成可读 queue_io_handler
--> queue_io_read(queue)
--> nread = sys_read(queue->fd,queue->buffer.data + queue->buffer.length,num_ready);
--> queue_process(queue);
--> queue->callback(data, pkt_size, queue->private_data);即 ctdb_tcp_read_cb：见前面ctdb_queue_setup
--> in->ctdb->upcalls->recv_pkt(in->ctdb, data, cnt);即ctdb_recv_pkt：main函数中设置的
--> **ctdb_input_pkt**(ctdb, hdr); // **ctdbd进程收到的tcp消息和recover的消息最终都是它来处理！！！！！！**
--> tnode->connect_te = event_add_timed(ctdb->ev, tnode, timeval_current_ofs(1, 0), ctdb_tcp_node_connect, node); 保证对端节点down再up可以快速重连！！！
--> event_loop_wait(ctdb->ev); 阻塞在这里