I have very bad perfomance of HttpWebRequest.GetResponse method when it is called from different thread for different URLs. For example if we have one thread and execute url1 it requires 3sec. If we ececute url1 and url2 in parallet it requires 10sec, first request ended after 8sec and second after 10sec.
If we exutet 10 URLs url1, url2, ... url0 it requires 1min 4 sec!!! first request ended after 50 secs!
I use GetResponse method.
I tried te set DefaultConnectionLimit but it doesn't help.
If use BeginGetRequest/EndGetResponse methods it works very fast but only if this methods called from one thread. If from different it is also very slowly.
I need to execute Http requests from many threads at one time.
the same code executed in each thread. If only one thread GetResponse method works very fast. Also IP addreses in URLs are different for each thread. If you compile and run the following code you will see that requests hanled one by one. Fist executed 3 secs, second 8 secs, third 15secs .... i.e. no gains from multithreading.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.Xml;
using System.Reflection;
using System.Threading;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Collections;
namespace HttpRequestExample
{
class HttpPerformer
{
private Thread thread = null;
HttpWebRequest httpRequest = null;
public void start(string url)
{
thread = new Thread(new ThreadStart(WorkerThread));
thread.Name = url;
thread.Start();
}
public void WorkerThread()
{
try
{
HttpWebRequest req = (HttpWebRequest)WebRequest.Create((string)thread.Name);
Console.WriteLine(DateTime.Now + " : before get response " + thread.Name);
HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
Console.WriteLine(DateTime.Now + " : after get response " + thread.Name);
}
catch (Exception e)
{
Console.WriteLine(DateTime.Now + " : Exception : " + e.Message + thread.Name);
}
}
}
class HttpAccessUtils
{
public static bool SetAllowUnsafeHeaderParsing20()
{
//Get the assembly that contains the internal class
Assembly aNetAssembly = Assembly.GetAssembly(typeof(System.Net.Configuration.SettingsSection));
if (aNetAssembly != null)
{
//Use the assembly in order to get the internal type for the internal class
Type aSettingsType = aNetAssembly.GetType("System.Net.Configuration.SettingsSectionInternal");
if (aSettingsType != null)
{
//Use the internal static property to get an instance of the internal settings class.
//If the static instance isn't created allready the property will create it for us.
object anInstance = aSettingsType.InvokeMember("Section",
BindingFlags.Static | BindingFlags.GetProperty | BindingFlags.NonPublic, null, null, new object[] { });
if (anInstance != null)
{
//Locate the private bool field that tells the framework is unsafe header parsing should be allowed or not
FieldInfo aUseUnsafeHeaderParsing = aSettingsType.GetField("useUnsafeHeaderParsing", BindingFlags.NonPublic | BindingFlags.Instance);
if (aUseUnsafeHeaderParsing != null)
{
aUseUnsafeHeaderParsing.SetValue(anInstance, true);
return true;
}
}
}
}
return false;
}
}
class Program
{
static void Main(string[] args)
{
HttpAccessUtils.SetAllowUnsafeHeaderParsing20();
ServicePointManager.UseNagleAlgorithm = true;
ServicePointManager.Expect100Continue = true;
ServicePointManager.CheckCertificateRevocationList = true;
ServicePointManager.DefaultConnectionLimit = 200; //ServicePointManager.DefaultPersistentConnectionLimit;
ServicePointManager.MaxServicePoints = 100;
Console.WriteLine(ServicePointManager.MaxServicePoints);
ArrayList a = new ArrayList(150);
for (int i = 100; i < 220; i++)
{
a.Add("http://207.242.7." + i.ToString());
}
for (int i = 0; i < a.Count; i++)
{
HttpPerformer hp = new HttpPerformer();
hp.start((string)a[i]);
}
}
static void performRequest(object url)
{
try
{
HttpWebRequest req = (HttpWebRequest)WebRequest.Create((string)url);
Console.WriteLine(DateTime.Now + " : before get response " + url);
HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
Console.WriteLine(DateTime.Now + " : after get response " + url);
}
catch (Exception e)
{
Console.WriteLine(DateTime.Now + " : Exception : " + e.Message + (string)url);
}
}
}
}
Сan anyone ever encountered such a problem? Thank you for any suggestions.
