C# UI becomes unresponsive during processing in background thread

Question

I am testing the validity of a large list of proxy servers concurrently. During this testing, many exceptions are being raised and caught. Although I am doing the testing in a background thread, my UI becomes unresponsive unless I use a SemaphoreSlim object to control the concurrency.

I know this is a self imposed bottle neck, and when scaling with an even larger list of proxies to test, I was hoping there might be a better way to solve the problem.

private void ValidateProxiesButton_Click(object sender, EventArgs e)
{
    new Thread(async () =>
    {
        Thread.CurrentThread.IsBackground = true;
        await ValidateProxiesAsync(proxies, judges, tests, 10);
    }).Start();
}


public async Task ValidateProxiesAsync(IEnumerable<Proxy> proxies, IEnumerable<ProxyJudge> judges, IEnumerable<ProxyTest> tests = null, int maxConcurrency = 20)
{
    if (proxies.Count() == 0)
    {
        throw new ArgumentException("Proxy list empty.");
    }

    foreach (var proxy in proxies)
    {
        proxy.Status = ProxyStatus.Queued;
    }

    //Get external IP to check if proxy is anonymous.
    var publicIp = await WebUtility.GetPublicIP();
    foreach (var judge in judges)
    {
        judge.Invalidation = publicIp;
    }

    await ValidateTestsAsync(judges.ToList<IProxyTest>());
    var validJudges = judges.ToList<IProxyTest>().GetValidTests();
    if (validJudges.Count == 0)
    {
        throw new ArgumentException("No valid judges found.");
    }

    if (tests != null)
    {
        await ValidateTestsAsync(tests.ToList<IProxyTest>());
    }

    var semaphore = new SemaphoreSlim(maxConcurrency);
    var tasks = new List<Task>();
    foreach (var proxy in proxies)
    {
        tasks.Add(Task.Run(async () =>
        {
            await semaphore.WaitAsync();
            proxy.Status = ProxyStatus.Testing;
            var isValid = await proxy.TestValidityAsync((IProxyTest)validJudges.GetRandomItem());
            proxy.Status = isValid ? ProxyStatus.Valid : ProxyStatus.Invalid;
            semaphore.Release();
        }));
    }

    await Task.WhenAll(tasks);
}

Inside proxy.TestValidityAsync method

public async Task<bool> TestValidityAsync(IProxyTest test, int timeoutSeconds = 30)
{
    try
    {
        var req = WebRequest.Create(test.URL);
        req.Proxy = new WebProxy(this.ToString());
        var respBody = await WebUtility.GetResponseStringAsync(req).TimeoutAfter(new TimeSpan(0, 0, timeoutSeconds));
        if (respBody.Contains(test.Validation))
        {
            return true;
        }
        else
        {
            return false;
        }
    }
    catch (Exception)
    {
        return false;
    }
}

Answer 1

So I found a working solution, it is to add the TPL Dataflow NuGet package to my project and then use the TransformBlock class. When I do this, my UI stays very responsive even if I am processing tons of concurrent requests that often throw exceptions. The code below is proof of concept, I will update it when I translate it to work with my project.

Source: Throttling asynchronous tasks

private async void button1_Click(object sender, EventArgs e)
{

    var downloader = new TransformBlock<string, WebResponse>(
            url => Download(url),
            new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 200 }
        );

    var buffer = new BufferBlock<WebResponse>();
    downloader.LinkTo(buffer);

    var urls = new List<string>();
    for (int i = 0; i < 100000; i++)
    {
        urls.Add($"http://example{i}.com");
    }

    foreach (var url in urls)
        downloader.Post(url);
    //or await downloader.SendAsync(url);

    downloader.Complete();
    await downloader.Completion;

    IList<WebResponse> responses;
    if (buffer.TryReceiveAll(out responses))
    {
        //process responses        
    }
}

private WebResponse Download(string url)
{
    WebResponse resp = null;
    try
    {
        var req = WebRequest.Create(url);
        resp = req.GetResponse();
    }
    catch (Exception)
    {

    }
    return resp;
}
}