Nothing is wrong with hyphens in atrribute names or values thats valid html, the problem with your source is that they use javascript on the client to render the the html, to verify that you can download the html page and you will notice that the elements you are looking for do not exist.
To parse such pages where javascript need to be executed first, for that you could use a web browser control and then pass the html to HAP.
Here is a simple example on how to use the WinForms web browser control:
private void ParseSomeHtmlThatRenderedJavascript(){
var browser = new System.Windows.Forms.WebBrowser() { ScriptErrorsSuppressed = true };
string link = "yourLinkHere";
//This will be called when the web page loads, it better be a class member since this is just a simple demonstration
WebBrowserDocumentCompletedEventHandler onDocumentCompleted = new WebBrowserDocumentCompletedEventHandler((s, evt) => {
//Do your HtmlParsingHere
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(browser.DocumentText);
var someNode = doc.DocumentNode.SelectNodes("yourxpathHere");
});
//subscribe to the DocumentCompleted event using our above handler before navigating
browser.DocumentCompleted += onDocumentCompleted;
browser.Navigate(link);
}
Also you could take look at Awesomium and some other embedded WebBrowser controls.
Also if you want to run the WebBrowser in console app here is a sample, if you don't get it the use windows forms, this sample is with the help of this SO answer WebBrowser Control in a new thread
using System;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using HtmlAgilityPack;
namespace ConsoleApplication276
{
// a container for a url and a parser Action
public class Link
{
public string link{get;set;}
public Action<string> parser { get; set; }
}
public class Program
{
// Entry Point of the console app
public static void Main(string[] args)
{
try
{
// download each page and dump the content
// you can add more links here, associate each link with a parser action, as for what data should the parser generate create a property for that in the Link container
var task = MessageLoopWorker.Run(DoWorkAsync, new Link() {
link = "google.com",
parser = (string html) => {
//do what ever you need with hap here
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
var someNodes = doc.DocumentNode.SelectSingleNode("//div");
} });
task.Wait();
Console.WriteLine("DoWorkAsync completed.");
}
catch (Exception ex)
{
Console.WriteLine("DoWorkAsync failed: " + ex.Message);
}
Console.WriteLine("Press Enter to exit.");
Console.ReadLine();
}
// navigate WebBrowser to the list of urls in a loop
public static async Task<Link> DoWorkAsync(Link[] args)
{
Console.WriteLine("Start working.");
using (var wb = new WebBrowser())
{
wb.ScriptErrorsSuppressed = true;
TaskCompletionSource<bool> tcs = null;
WebBrowserDocumentCompletedEventHandler documentCompletedHandler = (s, e) =>
tcs.TrySetResult(true);
// navigate to each URL in the list
foreach (var arg in args)
{
tcs = new TaskCompletionSource<bool>();
wb.DocumentCompleted += documentCompletedHandler;
try
{
wb.Navigate(arg.link.ToString());
// await for DocumentCompleted
await tcs.Task;
// after the page loads pass the html to the parser
arg.parser(wb.DocumentText);
}
finally
{
wb.DocumentCompleted -= documentCompletedHandler;
}
// the DOM is ready
Console.WriteLine(arg.link.ToString());
Console.WriteLine(wb.Document.Body.OuterHtml);
}
}
Console.WriteLine("End working.");
return null;
}
}
// a helper class to start the message loop and execute an asynchronous task
public static class MessageLoopWorker
{
public static async Task<Object> Run(Func<Link[], Task<Link>> worker, params Link[] args)
{
var tcs = new TaskCompletionSource<object>();
var thread = new Thread(() =>
{
EventHandler idleHandler = null;
idleHandler = async (s, e) =>
{
// handle Application.Idle just once
Application.Idle -= idleHandler;
// return to the message loop
await Task.Yield();
// and continue asynchronously
// propogate the result or exception
try
{
var result = await worker(args);
tcs.SetResult(result);
}
catch (Exception ex)
{
tcs.SetException(ex);
}
// signal to exit the message loop
// Application.Run will exit at this point
Application.ExitThread();
};
// handle Application.Idle just once
// to make sure we're inside the message loop
// and SynchronizationContext has been correctly installed
Application.Idle += idleHandler;
Application.Run();
});
// set STA model for the new thread
thread.SetApartmentState(ApartmentState.STA);
// start the thread and await for the task
thread.Start();
try
{
return await tcs.Task;
}
finally
{
thread.Join();
}
}
}
}
