image.png

线上发现接口获取缓存变慢，300ms就不太正常了，偶尔连接超时情况

客户端配置：

redis.host=xxxx
redis.port=26899
redis.timeout=2000
redis.maxActive=500
redis.maxIdle=450
redis.minIdle=5
redis.testOnBorrow=false
redis.maxWait=2000

 <dependency>
      <groupId>redis.clients</groupId>
      <artifactId>jedis</artifactId>
      <version>2.1.0</version>
</dependency>

redis.clients.jedis.exceptions.JedisConnectionException: java.net.SocketTimeoutException: Read timed out
        at redis.clients.jedis.Protocol.process(Protocol.java:79)
        at redis.clients.jedis.Protocol.read(Protocol.java:131)
        at redis.clients.jedis.Connection.getBinaryBulkReply(Connection.java:182)
        at redis.clients.jedis.Connection.getBulkReply(Connection.java:171)
        at redis.clients.jedis.Jedis.get(Jedis.java:67)
        at redis.clients.jedis.ShardedJedis.get(ShardedJedis.java:43)
        .....
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:221)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:136)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:104)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandleMethod(RequestMappingHandlerAdapter.java:743)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:672)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:82)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:933)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:867)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:951)
        at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:853)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:650)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:827)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at com.tiefan.fbs.fsp.base.filter.Base64DecodingFilter.doFilter(Base64DecodingFilter.java:51)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at com.tiefan.fbs.fsp.base.filter.FrameWorkFilter.doFilter(FrameWorkFilter.java:113)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:106)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)
        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
        at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
        at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:442)
        at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1083)
        at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:640)
        at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:318)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:170)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at java.net.SocketInputStream.read(SocketInputStream.java:127)
        at redis.clients.util.RedisInputStream.fill(RedisInputStream.java:109)
        at redis.clients.util.RedisInputStream.readByte(RedisInputStream.java:45)
        at redis.clients.jedis.Protocol.process(Protocol.java:64)
        ... 58 common frames omitted

我们这出现这种情况的原因

redis集群中的一台报文件系统错误，导致磁盘IO问题，坑。。。
同时我们的程序重度依赖redis，做了调整：

• batch获取redis数据，以及业务并行查询，减少redis查询次数及节省时间开销
• 上层系统使用hystrix做熔断
• redis响应缓慢非超时，使用fallback机制走db查询
• 重新调整了redis客户端配置
  降低了连接数使用，我们目前不是很大，testOnBorrow设置为true，能够减少无效连接的可能
  把timeout，maxWait设置为100ms，在redis不可用的情况下快速失败，直接查db

redis.host=xxxx
redis.port=26899
redis.timeout=100
redis.maxActive=300
redis.maxIdle=200
redis.minIdle=5
redis.testOnBorrow=true
redis.maxWait=100

其他情况也可能产生这种错误：

出现redis.clients.jedis.exceptions.JedisConnectionException: java.net.SocketTimeoutException: connect timed out问题的大概原因：
1、读写超时设置的过短。
2、命令本身就比较慢。
3、客户端与服务端网络不正常。
4、Redis自身发生阻塞，造成tcp-backlog已满，造成新的连接失败

参考：

redis tcp-backlog配置

JedisPool异常Jedis链接处理

https://www.cnblogs.com/qlong8807/p/5149007.html
源码分析：https://www.jianshu.com/p/dcf1491afbe7