5

I use AddHttpClient() dependency injection to add a named client to a transient service. At times, when I execute netstat -a on the server, I see many connections open with TIME_WAIT or CLOSE_WAIT status. I believe that these connections take up so much resource that, other TCP connections are unable to operate. Is this possible? Is there a way to stop these, and is it safe?

public class Startup
{
    public Startup(IConfiguration configuration)
    {
        Configuration = configuration;
    }

    public IConfiguration Configuration { get; }

    // This method gets called by the runtime. Use this method to add services to the container.
    public void ConfigureServices(IServiceCollection services)
    {
        ServicePointManager.DefaultConnectionLimit = 200;

        services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_2_2);

        services.AddHttpClient(FirebaseService.FirebaseServiceClient, ConfigureFirebaseClient);

        services.AddTransient<FirebaseService>();
    }

    // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
    public void Configure(IApplicationBuilder app, IHostingEnvironment env)
    {
        if (env.IsDevelopment())
        {
            app.UseDeveloperExceptionPage();
        }
        else
        {
            // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
            app.UseHsts();
        }

        app.UseHttpsRedirection();
        app.UseMvc();
    }

    void ConfigureFirebaseClient(HttpClient client)
    {
        var scopes = new string[] { "https://www.googleapis.com/auth/firebase.messaging" };

        Stream certificateStream = File.OpenRead("firebase-adminsdk.json");

        var serviceCredentials = GoogleCredential.FromStream(certificateStream);
        certificateStream.Close();

        var scopedCredentials = serviceCredentials.CreateScoped(scopes);
        var token = scopedCredentials.UnderlyingCredential.GetAccessTokenForRequestAsync().GetAwaiter().GetResult();
        client.SetBearerToken(token);
    }
}

public class FirebaseService
{
    public static string FirebaseServiceClient = "FirebaseServiceClient";

    private HttpClient _client;

    private readonly ILogger<FirebaseService> _logger;
    private readonly string _messagingUrl; 

    public FirebaseService(
        ILogger<FirebaseService> logger,
        IHttpClientFactory clientFactory)
    {
        _logger = logger;
        _messagingUrl = "https://fcm.googleapis.com/v1/projects/test2/messages:send";
        _client = clientFactory.CreateClient(FirebaseServiceClient);
    }

    public async Task<string> PostToFirebase(Dictionary<string, string> payload)
    {
        HttpResponseMessage result = null;
        string cont = null;
        try
        {
            var content = JsonConvert.SerializeObject(payload, Formatting.None);
            var stringContent = new StringContent(content, Encoding.UTF8, "application/json");

            result = await _client.PostAsync(_messagingUrl, stringContent);
            cont = await result.Content.ReadAsStringAsync();
            return cont;
        }
        finally
        {
            result?.Dispose();
        }
    }

}

public class ValuesController : ControllerBase
{
    private readonly IServiceProvider _serviceProvider;

    public ValuesController(IServiceProvider serviceProvider)
    {
        _serviceProvider = serviceProvider;
    }

    [HttpGet]
    public async Task<IActionResult> Get()
    {
        var payload = new Dictionary<string, string>();
        List<Task> tasks = new List<Task>();
        for (int i = 0; i < 100; i++)
        {
            FirebaseService firebaseService = (FirebaseService)_serviceProvider.GetService(typeof(FirebaseService));
            var task = firebaseService.PostToFirebase(payload);
            tasks.Add(task);
            Console.WriteLine(i);
        }

        await Task.WhenAll(tasks.ToArray());

        //Console.WriteLine(result);

        return Ok();
    }

}
Ahmet
  • 906
  • 1
  • 8
  • 22
  • 1
    https://aspnetmonsters.com/2016/08/2016-08-27-httpclientwrong/ and you should consider using `HttpClientFactory` since you're in Core land: https://learn.microsoft.com/en-us/dotnet/standard/microservices-architecture/implement-resilient-applications/use-httpclientfactory-to-implement-resilient-http-requests – Ian Kemp Mar 01 '19 at 09:47
  • Hello, That's what I'm doing actually. I'm using named clients as in https://learn.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-2.2#named-clients @IanKemp – Ahmet Mar 01 '19 at 13:04
  • 1
    More info about this issue is in https://github.com/dotnet/corefx/issues/35698#issuecomment-468880589 – Ahmet Mar 03 '19 at 23:47
  • @Ahmet the issue you linked to points to problems with your *code*, forgetting to dispose the responses. It has nothing to do with HttpClient. A response contains a network stream from which your application will read the contents. The message can't know that you're done with it if you don't call *dispose*. The network connection won't be closed until the response message gets garbage collected – Panagiotis Kanavos Mar 08 '19 at 13:04
  • @Ahmet please post your actual code so this question can get a correct answer. The accepted one is simply wrong – Panagiotis Kanavos Mar 08 '19 at 13:07
  • Ok. I added to the message. But as you can read from https://github.com/dotnet/corefx/issues/35698#issuecomment-468880589 There is confusion between core developers and document writers, consumers (me) of the component @PanagiotisKanavos – Ahmet Mar 08 '19 at 15:25
  • @Ahmet post your *code*, not a link to a project. The code in the issue is less than 10 lines. There's no confusion either - the responses weren't disposed. The *same* problem occurs with HttpWebRequest if you forget to dispose the response. In fact, I bet you've disabled the domain connection limit, otherwise you'd find that you could only make two connections at a time. That would be because each undisposed response would keep the connection open until it was GCd – Panagiotis Kanavos Mar 08 '19 at 15:31
  • Maybe we are talking about two different things but you are telling me that I have forgotten to do something that is not documented in https://learn.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-2.2#named-clients – Ahmet Mar 08 '19 at 15:37
  • I didn't change domain limit to unlimited. I changed it to 100 as I need more than 2 concurrent requests per domain. As I have said, even when I added the dispose, it didn't fix the problem. I have added the code to the question – Ahmet Mar 08 '19 at 15:38

3 Answers3

1

CLOSE_WAIT - the other side closed the connection.

TIME_WAIT - the local end point (your app) closed the connection.

Both connections are kept for a few more minutes just in case there are some delayed packets from the other side.

"I believe that these connections take up so much resource that, other TCP connections are unable to operate. Is this possible?" - I think not. They are just keepind a port opened. It depends how many there are. If you have a few hundreds you will be ok.

"Is there a way to stop these, and is it safe?" - I dont think so. They all have the same PID so if you try killing one your all app will shut down.

Looking forward for better answers.

Simon
  • 71
  • 2
  • 9
  • 2
    I do thousands of post requests to the same url over a brief amount of time. So when those connections remain open for a few minutes, it accumulates to over 9k open connections After they pass 9k, I start to get socket exceptions. – Ahmet Mar 05 '19 at 15:54
  • Can you please, add some some code and tell us what exceptions do you get? – Simon Mar 05 '19 at 15:58
  • 1
    I have created a test repo for this. https://github.com/ahmettahasakar/HttpClientTest I made it so that there are 100 concurrent posts to Firebase Cloud Messaging. Even though I post blank payloads, the connections still seem to stay open. In order for project to work, you need a Firebase authentication file. You can generate one easily from https://firebase.google.com/docs/admin/setup There has been some correspondence from Microsoft as well on https://github.com/dotnet/corefx/issues/35698#issuecomment-468880589 – Ahmet Mar 05 '19 at 16:32
  • @Ahmet you should have posted that code in the question. It shows you forgot to dispose the response messages – Panagiotis Kanavos Mar 08 '19 at 13:06
  • 1
    I followed the documentation. Documentation was missing it. I had even asked the writers of the documentation, and they were confused as well. Also, even when I added the dispose, it didn't fix the problem. – Ahmet Mar 08 '19 at 15:11
1

I realized that the connections that I had problems with weren't originating from client.PostAsync(). In fact they were from the firebase token authentication requests in Client Configuration action of IHttpClientFactory. So that's why when I switched to a singleton or a static property, CreateClient(clientName) wasn't called more than once, and the problem was gone. Although this was written in the documentation clearly, I had missed it. Each time CreateClient is called, a new instance of HttpClient is created and the configuration action is called.

Ahmet
  • 906
  • 1
  • 8
  • 22
-2

Well it's because you might be using it with the wrong lifetime management methodology. HttpClient has a socket exhaustion problem, thus it should be used as a singleton if possible.

This article will answer your question. Also read this about a workaround for DNS changes.

Guru Stron
  • 102,774
  • 10
  • 95
  • 132
Kristóf Tóth
  • 791
  • 4
  • 19
  • I have read this article. Because of the socket exhaustion problem IHttpClientFactory is introduced. And I'm using that. It's supposed to reuse the same sources, but it does not. – Ahmet Mar 05 '19 at 15:50
  • 1
    Then youre better off implementing your own, it's pretty easy to register one as a singleton if you are using an IoC container. – Kristóf Tóth Mar 05 '19 at 19:25
  • 1
    I can always to that, but what's the point of using a named client from `IHttpClientFactory` then? – Ahmet Mar 06 '19 at 14:05
  • 1
    @Ahmet this answer is simply wrong. HttpClientFactory *does* cache and reuse HttpClient*Handler* instances so there's no need to use a singleton. A singleton won't be able to detect DNS changes. That's why HttpClientFactory *recycles* those handlers after some time, typically 2 minutes – Panagiotis Kanavos Mar 08 '19 at 13:02
  • @KristófTóth yes, which is why HttpClientFactory was created, to cache and *recycle* instances. The issue you linked to says that a singleton should *not* be used because it doesn't respect DNS changes – Panagiotis Kanavos Mar 08 '19 at 13:36
  • @PanagiotisKanavos maybe I undestood something wrong, but as per my understanding the factory has similar issues as per the threads posted here. Thats why I suggested using a singleton in conjunction with ConnectionLeaseTimeout | PooledConnectionLifetime. But I might be wrong. – Kristóf Tóth Mar 08 '19 at 13:41
  • No it doesn't. The issue the OP opened in Github shows that the OP's code didn't dispose the *responses* properly which means the socket connections behind each response remained open until the responses were GC'd – Panagiotis Kanavos Mar 08 '19 at 13:56
  • I selected this as the answer because implementing my own was the only solution that worked. I don't care about the DNS changes for now. My only problem was connections not being reused, and new sockets were being opened and kept open for minutes, which led to socket exhaustion. @KristófTóth said it right. HttpClientFactory also has similar issues. – Ahmet Mar 08 '19 at 15:15
  • As Microsoft has confirmed HttpClientFactory is meant to reuse the same connections, https://github.com/dotnet/corefx/issues/35698#issuecomment-469068526 but it does not. – Ahmet Mar 08 '19 at 15:17