java.net.SocketException: Connection reset on External REST Calls

Question

We do have following Configuration to build Spring beans.

Built configuration beans in Spring Beans with Connection Pool manager and ClosableHttpClient.

RestClient class autowires RestTemplate objects to make post calls.

import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.web.client.ResponseErrorHandler;
import org.springframework.web.client.RestTemplate;

import java.time.Duration;
import java.util.List;

@Configuration
@Data
@Slf4j
public class ApplicationConfig {

    @Value("${nexus.security.enabled}")
    private boolean securityEnabled;

    @Value("${nexus.security.ignored}")
    private List<String> securityIgnoredList;

    @Value("${external.card.readTimeoutInMillis}")
    private int readTimeoutInMillis;

    @Value("${external.card.connectionTimeoutInMillis}")
    private int connectionTimeoutInMillis;

    @Value("${external.card.connectionRequestTimeoutInMillis}")
    private int connectionRequestTimeoutInMillis;

    @Value("${external.card.maxTotalConnections}")
    private int maxTotalConnections;

    @Value("${external.card.defaultMaxPerRoute}")
    private int defaultMaxPerRoute;

    @Bean
    @Qualifier(ConfigConstants.CONFIG_NAME_DEFAULT)
    public PoolingHttpClientConnectionManager poolingHttpClientConnectionManager()
    {
        log.info("PoolingHttpClientConnectionManager Build for {}", ConfigConstants.CONFIG_NAME_DEFAULT);
        PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager();
        poolingHttpClientConnectionManager.setMaxTotal(maxTotalConnections);
        poolingHttpClientConnectionManager.setDefaultMaxPerRoute(defaultMaxPerRoute);
        return poolingHttpClientConnectionManager;
    }

    @Bean(destroyMethod = "close")
    @Qualifier(ConfigConstants.CONFIG_NAME_DEFAULT)
    public CloseableHttpClient closeableHttpClient(
            @Qualifier(ConfigConstants.CONFIG_NAME_DEFAULT) PoolingHttpClientConnectionManager poolingHttpClientConnectionManager
    ) {
        log.info("CloseableHttpClient Build for {}", ConfigConstants.CONFIG_NAME_DEFAULT);
        return HttpClientBuilder.create().setConnectionManager(poolingHttpClientConnectionManager).build();
    }

    @Bean
    @Qualifier(ConfigConstants.CONFIG_NAME_DEFAULT)
    public HttpComponentsClientHttpRequestFactory requestFactory(
            @Qualifier(ConfigConstants.CONFIG_NAME_DEFAULT) CloseableHttpClient httpClient
    ) {
        log.info("HttpComponentsClientHttpRequestFactory Build for {}", ConfigConstants.CONFIG_NAME_DEFAULT);
        HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(httpClient);
        requestFactory.setReadTimeout(readTimeoutInMillis);
        requestFactory.setConnectTimeout(connectionTimeoutInMillis);
        requestFactory.setConnectionRequestTimeout(connectionRequestTimeoutInMillis);

        return requestFactory;
    }

    @Bean
    @Qualifier(ConfigConstants.CONFIG_NAME_DEFAULT)
    public RestTemplate restTemplate(@Qualifier(ConfigConstants.CONFIG_NAME_DEFAULT) RestTemplateBuilder builder,
            ResponseErrorHandler errorHandler
    )
    {
        log.info("RestTemplate Build for {}", ConfigConstants.CONFIG_NAME_DEFAULT);
        RestTemplate restTemplate = builder.build();
        restTemplate.getMessageConverters().add(new MappingJackson2HttpMessageConverter());
        restTemplate.getMessageConverters().add(new StringHttpMessageConverter());
        restTemplate.setErrorHandler(errorHandler);
        return restTemplate;
    }


    @Bean
    @Qualifier(ConfigConstants.CONFIG_NAME_DEFAULT)
    public RestTemplateBuilder restTemplateBuilder(
            @Qualifier(ConfigConstants.CONFIG_NAME_DEFAULT) HttpComponentsClientHttpRequestFactory requestFactory
    ) {
        log.info("RestTemplateBuilder Build for {}", ConfigConstants.CONFIG_NAME_DEFAULT);
        return new RestTemplateBuilder()
                .setReadTimeout(Duration.ofMillis(readTimeoutInMillis))
                .setConnectTimeout(Duration.ofMillis(connectionTimeoutInMillis))
                .requestFactory(
                        () -> requestFactory);

    }
}

And RestClient autowired into

public RestClient(@Qualifier(ConfigConstants.CONFIG_NAME_DEFAULT) RestTemplate restTemplate) {
    this.restTemplate = restTemplate;
}

Following code has been used to make call to

ResponseEntity<String> response = restTemplate.exchange(endpointURL, HttpMethod.POST, httpEntity, String.class);

Often service is working as expected.

But, it throws following error sometimes...

java.net.SocketException: Connection reset
        at java.base/java.net.SocketInputStream.read(SocketInputStream.java:186)
        at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)
        at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:478)
        at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
        at java.base/sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.java:70)
        at java.base/sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1364)
        at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:973)
        at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)

After few minutes, the service is working as expected.

Could you please help me to fix this problem?