redis.conf详解之maxclients

本文基于 redis_version:6.2.5

用法

设置一个redis实例支持的最大连接数。

maxclients 10000

 

注意事项：

• 默认情况下，一个实例支持1万条连接。
• 如果你的服务是redis-cluster集群，最大连接数会与内部连接数共享。例：3主3从的cluster集群（6个实例）。每个实例会创建 (6-1)*2=10个内部连接。假设每个实例的maxclients配置为100，那么100条连接中会被占用10条。如果你的cluster集群很庞大，尤其要注意这一点。
• 如果你要设置的maxclients再加上32超过资源限制（例：我要设置maxclient=1000，会用1032去判断是否超过资源限制），redis会根据内部逻辑去设置实际可以支持的maxclients。具体看源码部分。

 

实操

配置一个三主的cluster集群，分别为6370，6380，6390。

$ redis-cli --cluster check 127.0.0.1:6370
127.0.0.1:6370 (3cd5de09...) -> 0 keys | 5461 slots | 0 slaves.
127.0.0.1:6390 (d4ea04d6...) -> 0 keys | 5461 slots | 0 slaves.
127.0.0.1:6380 (fad5592c...) -> 0 keys | 5462 slots | 0 slaves.
[OK] 0 keys in 3 masters.
0.00 keys per slot on average.
>>> Performing Cluster Check (using node 127.0.0.1:6370)
M: 3cd5de0940dbbed9dcf0f3d9541464664dd05c3d 127.0.0.1:6370
   slots:[0-5460] (5461 slots) master
M: d4ea04d67e5e7dc8e1bae6e586c3999edbd99c16 127.0.0.1:6390
   slots:[10923-16383] (5461 slots) master
M: fad5592c85a84e2d97ea3158468594c46720c28d 127.0.0.1:6380
   slots:[5461-10922] (5462 slots) master
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.

---

内部连接的样子见下图。

cluster集群内部连接图示.png

---

以监听在6380的实例为例子，它的内部连接数为（3-1）X 2 = 4条（后4行）

$ lsof -p 2688
COMMAND    PID USER   FD      TYPE DEVICE SIZE/OFF   NODE NAME
redis-ser 2688 root  cwd       DIR  0,173     4096 396203 /root/redis-cluster
...忽略部分...
redis-ser 2688 root    6u     IPv4 571968      0t0    TCP localhost:6380 (LISTEN)
redis-ser 2688 root    7wW     REG  0,173      370 397395 /root/redis-cluster/nodes-6380.conf
redis-ser 2688 root    8u     IPv4 571969      0t0    TCP localhost:16380 (LISTEN)
redis-ser 2688 root   10u     IPv4 571977      0t0    TCP localhost:46051->localhost:16370 (ESTABLISHED)
redis-ser 2688 root   11u     IPv4 572704      0t0    TCP localhost:16380->localhost:44247 (ESTABLISHED)
redis-ser 2688 root   12u     IPv4 569320      0t0    TCP localhost:45051->localhost:16390 (ESTABLISHED)
redis-ser 2688 root   13u     IPv4 569321      0t0    TCP localhost:16380->localhost:40545 (ESTABLISHED)

设置下maxclients=5（设置完毕后不要关闭该窗口和该连接，此时连接数是4个内部连接+当前连接=5条）

$ redis-cli -p 6380
127.0.0.1:6380> config get maxclients
1) "maxclients"
2) "10000"
127.0.0.1:6380> config set maxclients 5
OK
127.0.0.1:6380> config get maxclients
1) "maxclients"
2) "5"
127.0.0.1:6380>

另启一个窗口，建立连接发送请求报连接超限。因为5+1=6超过了maxclients=5。

$ redis-cli -p 6380
127.0.0.1:6380> get c
(error) ERR max number of clients + cluster connections reached

 

redis源码

cluster port = 基础端口+1W（例子：实例监听在6379端口，则cluster port为16379），其他节点与cluster port建立内部连接。

// https://github.com/redis/redis/blob/6.2.6/src/cluster.h
12 #define CLUSTER_PORT_INCR 10000 /* Cluster port = baseport + PORT_INCR */

---

关于连接数的判断（1101～1109行）：

• 如果是cluster集群，则判断外部链接数(server.clients)+内部连接数（clusterConn）的大小是否超过maxclients并输出相应错误。
• 非cluster集群只判断外部链接数的大小是否超过maxclients.并输出相应错误。

// https://github.com/redis/redis/blob/6.2.6/src/networking.c
1082 static void acceptCommonHandler(connection *conn, int flags, char *ip) {
...忽略部分...
1101     if (listLength(server.clients) + getClusterConnectionsCount()
1102         >= server.maxclients)
1103     {
1104         char *err;
1105         if (server.cluster_enabled)
1106             err = "-ERR max number of clients + cluster "
1107                   "connections reached\r\n";
1108         else
1109             err = "-ERR max number of clients reached\r\n";
1110
1111         /* That's a best effort error message, don't check write errors.
1112          * Note that for TLS connections, no handshake was done yet so nothing
1113          * is written and the connection will just drop. */
1114         if (connWrite(conn,err,strlen(err)) == -1) {
1115             /* Nothing to do, Just to avoid the warning... */
1116         }
1117         server.stat_rejected_conn++;
1118         connClose(conn);
1119         return;
1120     }

---

内部连接数的获取：

• 如果是cluster集群模式，返回(节点数-1)*2。节点数包含主节点与从节点。
• 普通模式，返回0。

// https://github.com/redis/redis/blob/6.2.6/src/networking.c
 779 unsigned long getClusterConnectionsCount(void) {
 780     /* We decrement the number of nodes by one, since there is the
 781      * "myself" node too in the list. Each node uses two file descriptors,
 782      * one incoming and one outgoing, thus the multiplication by 2. */
 783     return server.cluster_enabled ?
 784            ((dictSize(server.cluster->nodes)-1)*2) : 0;
 785 }

---

maxclients被设置为实际支持的数值

// https://github.com/redis/redis/blob/6.2.6/src/config.c
// CONFIG_MIN_RESERVED_FDS = 32
// 这个数字是为redis持久化、侦听套接字、日志文件等额外操作保留的文件描述符数
void adjustOpenFilesLimit(void) {
    // maxfiles设置为maxclients+32
    rlim_t maxfiles = server.maxclients+CONFIG_MIN_RESERVED_FDS;
    struct rlimit limit;
    // getrlimit是一个系统调用，返回rlim_cur，rlim_max
    // rlim_cur是进程资源的实际限制
    // rlim_max是rlim_cur限制的上限，rlim_cur不可超过rlim_max
    if (getrlimit(RLIMIT_NOFILE,&limit) == -1) { // 获取失败
        serverLog(LL_WARNING,"Unable to obtain the current NOFILE limit (%s), assuming 1024 and setting the max clients configuration accordingly.",
            strerror(errno));
        // maxclients被设置为 1024-32
        server.maxclients = 1024-CONFIG_MIN_RESERVED_FDS;
    } else {
        // 保存下来当前获取到的限制数
        rlim_t oldlimit = limit.rlim_cur;

        // redis所需要的fd数量超过了获取到的限制数
        if (oldlimit < maxfiles) {
            rlim_t bestlimit;
            int setrlimit_error = 0;

            // 先设置bestlimit为redis所需要的fd数量
            bestlimit = maxfiles;
            // 如果redis所需要的fd数量一直大于获取到的限制数，就一直循环
            while(bestlimit > oldlimit) {
                // 每次循环将redis所需要的fd数量减16
                rlim_t decr_step = 16;

                limit.rlim_cur = bestlimit;
                limit.rlim_max = bestlimit;
                // 通过系统调用，去重新设置限制数，设置成功跳出while循环
                if (setrlimit(RLIMIT_NOFILE,&limit) != -1) break;
                setrlimit_error = errno;

                // 如果redis所需要的fd数量比16还小了
                if (bestlimit < decr_step) {
                    // 将redis所需要的fd数量设置为最初我们拿到的限制数并跳出while循环
                    bestlimit = oldlimit;
                    break;
                }
                // 递减16
                bestlimit -= decr_step;
            }

            // 当前的redis所需要的fd数量比最开始拿到的限制数还小，还是用之前的限制数吧
            if (bestlimit < oldlimit) bestlimit = oldlimit;

            // 已经处理过的redis所需要的fd数量比最开始的redis所需要的fd数量小
            if (bestlimit < maxfiles) {
                unsigned int old_maxclients = server.maxclients;
                // 设置maxclients
                server.maxclients = bestlimit-CONFIG_MIN_RESERVED_FDS;
                // 如果经过一番操作已经处理过的redis所需要的fd数量比17还小，直接挂掉！
                if (bestlimit <= CONFIG_MIN_RESERVED_FDS) {
                    serverLog(LL_WARNING,"Your current 'ulimit -n' "
                        "of %llu is not enough for the server to start. "
                        "Please increase your open file limit to at least "
                        "%llu. Exiting.",
                        (unsigned long long) oldlimit,
                        (unsigned long long) maxfiles);
                    exit(1);
                }
                // 忽略部分，一些日志打印
            } else {
                // 忽略部分，一些日志打印
            }
        }
    }
}

 

原生注释

# Set the max number of connected clients at the same time. By default
# this limit is set to 10000 clients, however if the Redis server is not
# able to configure the process file limit to allow for the specified limit
# the max number of allowed clients is set to the current file limit
# minus 32 (as Redis reserves a few file descriptors for internal uses).
#
# Once the limit is reached Redis will close all the new connections sending
# an error 'max number of clients reached'.
#
# IMPORTANT: When Redis Cluster is used, the max number of connections is also
# shared with the cluster bus: every node in the cluster will use two
# connections, one incoming and another outgoing. It is important to size the
# limit accordingly in case of very large clusters.
#
# maxclients 10000

---

本文属于原创，首发于微信公众号【小易哥学呀学】，如需转载请后台留言。
加微信入交流群。vx：17610015120