My application immediately started facing socket timeout error after I migrated my elastic cache cluster on aws from cache.t2.micro to cache.t2.small. Which ultimately lead to Could not get a resource from the pool exceptions. This lead to application failure as all server threads were occupied in this.
Below are the details of libraries used
spring-data-redis:2.1.3.RELEASE redis.clients.jedis:2.9.0
This was a clustered redis with 2 shards and 1 replica.
However the application worked fine after restart.
Logs
at org.springframework.data.redis.connection.jedis.JedisExceptionConverter.convert(JedisExceptionConverter.java:67) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.connection.jedis.JedisExceptionConverter.convert(JedisExceptionConverter.java:41) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.PassThroughExceptionTranslationStrategy.translate(PassThroughExceptionTranslationStrategy.java:44) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.FallbackExceptionTranslationStrategy.translate(FallbackExceptionTranslationStrategy.java:42) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.connection.jedis.JedisClusterConnection.convertJedisAccessException(JedisClusterConnection.java:760) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.connection.jedis.JedisClusterStringCommands.convertJedisAccessException(JedisClusterStringCommands.java:556) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.connection.jedis.JedisClusterStringCommands.setEx(JedisClusterStringCommands.java:207) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.connection.DefaultedRedisConnection.setEx(DefaultedRedisConnection.java:295) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.core.DefaultValueOperations$4.potentiallyUsePsetEx(DefaultValueOperations.java:268) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.core.DefaultValueOperations$4.doInRedis(DefaultValueOperations.java:261) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.core.RedisTemplate.execute(RedisTemplate.java:224) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.core.RedisTemplate.execute(RedisTemplate.java:184) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.core.AbstractOperations.execute(AbstractOperations.java:95) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.core.DefaultValueOperations.set(DefaultValueOperations.java:256) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at org.springframework.data.redis.core.DefaultBoundValueOperations.set(DefaultBoundValueOperations.java:131) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
at com.sample.rca.redis.CacheService.set(CacheService.java:76) [classes!/:na]
at com.sample.rca.redis.CacheService.setKey(CacheService.java:154) [classes!/:na]
at com.sample.rca.redis.RedisPaymentStore.store(RedisPaymentStore.java:20) [classes!/:na]
at com.sample.rca.controller.Endpoint.set(Endpoint.java:23) [classes!/:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_342]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_342]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_342]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_342]
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:189) [spring-web-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138) [spring-web-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:102) [spring-webmvc-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895) [spring-webmvc-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:800) [spring-webmvc-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87) [spring-webmvc-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1038) [spring-webmvc-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:942) [spring-webmvc-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1005) [spring-webmvc-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:897) [spring-webmvc-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:634) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:882) [spring-webmvc-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:741) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53) [tomcat-embed-websocket-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:99) [spring-web-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:92) [spring-web-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:93) [spring-web-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:200) [spring-web-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) [spring-web-5.1.3.RELEASE.jar!/:5.1.3.RELEASE]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:199) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:490) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:408) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:791) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1417) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_342]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_342]
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) [tomcat-embed-core-9.0.13.jar!/:9.0.13]
at java.lang.Thread.run(Thread.java:750) [na:1.8.0_342]
Caused by: redis.clients.jedis.exceptions.JedisConnectionException: Could not get a resource from the pool
at redis.clients.util.Pool.getResource(Pool.java:53) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.JedisPool.getResource(JedisPool.java:226) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.JedisSlotBasedConnectionHandler.getConnectionFromSlot(JedisSlotBasedConnectionHandler.java:66) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:116) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:141) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:141) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:141) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.JedisClusterCommand.runWithRetries(JedisClusterCommand.java:141) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.JedisClusterCommand.runBinary(JedisClusterCommand.java:60) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.BinaryJedisCluster.setex(BinaryJedisCluster.java:268) ~[jedis-2.9.0.jar!/:na]
at org.springframework.data.redis.connection.jedis.JedisClusterStringCommands.setEx(JedisClusterStringCommands.java:205) ~[spring-data-redis-2.1.3.RELEASE.jar!/:2.1.3.RELEASE]
... 66 common frames omitted
Caused by: redis.clients.jedis.exceptions.JedisConnectionException: java.net.SocketTimeoutException: connect timed out
at redis.clients.jedis.Connection.connect(Connection.java:207) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.BinaryClient.connect(BinaryClient.java:93) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.BinaryJedis.connect(BinaryJedis.java:1767) ~[jedis-2.9.0.jar!/:na]
at redis.clients.jedis.JedisFactory.makeObject(JedisFactory.java:106) ~[jedis-2.9.0.jar!/:na]
at org.apache.commons.pool2.impl.GenericObjectPool.create(GenericObjectPool.java:883) ~[commons-pool2-2.6.0.jar!/:2.6.0]
at org.apache.commons.pool2.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:436) ~[commons-pool2-2.6.0.jar!/:2.6.0]
at org.apache.commons.pool2.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:365) ~[commons-pool2-2.6.0.jar!/:2.6.0]
at redis.clients.util.Pool.getResource(Pool.java:49) ~[jedis-2.9.0.jar!/:na]
... 76 common frames omitted
Caused by: java.net.SocketTimeoutException: connect timed out
at java.net.PlainSocketImpl.socketConnect(Native Method) ~[na:1.8.0_342]
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ~[na:1.8.0_342]
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ~[na:1.8.0_342]
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ~[na:1.8.0_342]
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[na:1.8.0_342]
at java.net.Socket.connect(Socket.java:607) ~[na:1.8.0_342]
at redis.clients.jedis.Connection.connect(Connection.java:184) ~[jedis-2.9.0.jar!/:na]
... 83 common frames omitted
