How can I do digest authentication with HttpWebRequest?

Question

Various articles (1, 2) I discovered make this look easy enough:

WebRequest request = HttpWebRequest.Create(url);

var credentialCache = new CredentialCache();
credentialCache.Add(
  new Uri(url), // request url
  "Digest", // authentication type
  new NetworkCredential("user", "password") // credentials
);

request.Credentials = credentialCache;

However, this only works for URLs without URL parameters. For example, I can download http://example.com/test/xyz.html just fine, but when I attempt to download http://example.com/test?page=xyz, the result is a 400 Bad Request message with the following in the server's logs (running Apache 2.2):

Digest: uri mismatch - </test> does not match request-uri </test?page=xyz>

My first idea was that the digest specification requires URL parameters to be removed from the digest hash -- but removing the parameter from the URL passed to credentialCache.Add() didn't change a thing. So it must be the other way around and somewhere in the .NET framework is wrongly removing the parameter from the URL.

Answer 1

Code taken from this post has worked perfectly for me Implement Digest authentication via HttpWebRequest in C#

I had following issue, when ever I browser the feed url in a browser it asked for username and password and worked fine, however any of the above code samples were not working, on inspecting Request/Response Header (in web developer tools in firefox) i could see header having Authorization of type digest.

Step 1 Add:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Security.Cryptography;
using System.Text.RegularExpressions;
using System.Net;
using System.IO;

namespace NUI
{
    public class DigestAuthFixer
    {
        private static string _host;
        private static string _user;
        private static string _password;
        private static string _realm;
        private static string _nonce;
        private static string _qop;
        private static string _cnonce;
        private static DateTime _cnonceDate;
        private static int _nc;

    public DigestAuthFixer(string host, string user, string password)
    {
        // TODO: Complete member initialization
        _host = host;
        _user = user;
        _password = password;
    }

    private string CalculateMd5Hash(
        string input)
    {
        var inputBytes = Encoding.ASCII.GetBytes(input);
        var hash = MD5.Create().ComputeHash(inputBytes);
        var sb = new StringBuilder();
        foreach (var b in hash)
            sb.Append(b.ToString("x2"));
        return sb.ToString();
    }

    private string GrabHeaderVar(
        string varName,
        string header)
    {
        var regHeader = new Regex(string.Format(@"{0}=""([^""]*)""", varName));
        var matchHeader = regHeader.Match(header);
        if (matchHeader.Success)
            return matchHeader.Groups[1].Value;
        throw new ApplicationException(string.Format("Header {0} not found", varName));
    }

    private string GetDigestHeader(
        string dir)
    {
        _nc = _nc + 1;

        var ha1 = CalculateMd5Hash(string.Format("{0}:{1}:{2}", _user, _realm, _password));
        var ha2 = CalculateMd5Hash(string.Format("{0}:{1}", "GET", dir));
        var digestResponse =
            CalculateMd5Hash(string.Format("{0}:{1}:{2:00000000}:{3}:{4}:{5}", ha1, _nonce, _nc, _cnonce, _qop, ha2));

        return string.Format("Digest username=\"{0}\", realm=\"{1}\", nonce=\"{2}\", uri=\"{3}\", " +
            "algorithm=MD5, response=\"{4}\", qop={5}, nc={6:00000000}, cnonce=\"{7}\"",
            _user, _realm, _nonce, dir, digestResponse, _qop, _nc, _cnonce);
    }

    public string GrabResponse(
        string dir)
    {
        var url = _host + dir;
        var uri = new Uri(url);

        var request = (HttpWebRequest)WebRequest.Create(uri);

        // If we've got a recent Auth header, re-use it!
        if (!string.IsNullOrEmpty(_cnonce) &&
            DateTime.Now.Subtract(_cnonceDate).TotalHours < 1.0)
        {
            request.Headers.Add("Authorization", GetDigestHeader(dir));
        }

        HttpWebResponse response;
        try
        {
            response = (HttpWebResponse)request.GetResponse();
        }
        catch (WebException ex)
        {
            // Try to fix a 401 exception by adding a Authorization header
            if (ex.Response == null || ((HttpWebResponse)ex.Response).StatusCode != HttpStatusCode.Unauthorized)
                throw;

            var wwwAuthenticateHeader = ex.Response.Headers["WWW-Authenticate"];
            _realm = GrabHeaderVar("realm", wwwAuthenticateHeader);
            _nonce = GrabHeaderVar("nonce", wwwAuthenticateHeader);
            _qop = GrabHeaderVar("qop", wwwAuthenticateHeader);

            _nc = 0;
            _cnonce = new Random().Next(123400, 9999999).ToString();
            _cnonceDate = DateTime.Now;

            var request2 = (HttpWebRequest)WebRequest.Create(uri);
            request2.Headers.Add("Authorization", GetDigestHeader(dir));
            response = (HttpWebResponse)request2.GetResponse();
        }
        var reader = new StreamReader(response.GetResponseStream());
        return reader.ReadToEnd();
    }
}

}

Step 2: Call new method

DigestAuthFixer digest = new DigestAuthFixer(domain, username, password);
string strReturn = digest.GrabResponse(dir);

if Url is: http://xyz.rss.com/folder/rss then domain: http://xyz.rss.com (domain part) dir: /folder/rss (rest of the url)

you could also return it as stream and use XmlDocument Load() method.

Answer 2

You said you removed the querystring paramters, but did you try going all the way back to just the host? Every single example of CredentialsCache.Add() I've seen seems to use only the host, and the docs for CredentialsCache.Add() list the Uri parameter as "uriPrefix", which seems telling.

In other words, try this out:

Uri uri = new Uri(url);
WebRequest request = WebRequest.Create(uri);

var credentialCache = new CredentialCache(); 
credentialCache.Add( 
  new Uri(uri.GetLeftPart(UriPartial.Authority)), // request url's host
  "Digest",  // authentication type 
  new NetworkCredential("user", "password") // credentials 
); 

request.Credentials = credentialCache;

If this works, you will also have to make sure that you don't add the same "authority" to the cache more than once... all requests to the same host should be able to make use of the same credential cache entry.

Answer 3

In earlier answers everybody use the obsolete WEbREquest.Create method. So here is my async solution what up to date for recently trending's:

public async Task<string> LoadHttpPageWithDigestAuthentication(string url, string username, string password)
    {
        Uri myUri = new Uri(url);
        NetworkCredential myNetworkCredential = new NetworkCredential(username, password);
        CredentialCache myCredentialCache = new CredentialCache { { myUri, "Digest", myNetworkCredential } };
        var request = new HttpClient(new HttpClientHandler() { Credentials = myCredentialCache, PreAuthenticate = true});

        var response = await request.GetAsync(url);

        var responseStream = await response.Content.ReadAsStreamAsync();

        StreamReader responseStreamReader = new StreamReader(responseStream, Encoding.Default);

        string answer = await responseStreamReader.ReadToEndAsync();

        return answer;
    }

Answer 4

The solution is to activate this parameter in apache:

    BrowserMatch "MSIE" AuthDigestEnableQueryStringHack=On 

More info : http://httpd.apache.org/docs/2.0/mod/mod_auth_digest.html#msie

Then add this property in your code for the webrequest object:

    request.UserAgent = "MSIE"

it work very well for me

Answer 5

I think the second URL points to dynamic page and you should first call it using GET to get the HTML and then to download it. No experience in this field though.