I'm trying to crawl and parse following RSS feed: http://english.alarabiya.net/.mrss/en/sports.xml
When I open it in browser it gives me the normal RSS feed that I want to parse. But when I download it inside Java it shows me the following:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
<meta http-equiv="Content-Script-Type" content="text/javascript">
<script type="text/javascript">
function getCookie(c_name) { // Local function for getting a cookie value
if (document.cookie.length > 0) {
c_start = document.cookie.indexOf(c_name + "=");
if (c_start!=-1) {
c_start=c_start + c_name.length + 1;
c_end=document.cookie.indexOf(";", c_start);
if (c_end==-1)
c_end = document.cookie.length;
return unescape(document.cookie.substring(c_start,c_end));
}
}
return "";
}
function setCookie(c_name, value, expiredays) { // Local function for setting a value of a cookie
var exdate = new Date();
exdate.setDate(exdate.getDate()+expiredays);
document.cookie = c_name + "=" + escape(value) + ((expiredays==null) ? "" : ";expires=" + exdate.toGMTString()) + ";path=/";
}
function getHostUri() {
var loc = document.location;
return loc.toString();
}
setCookie('YPF8827340282Jdskjhfiw_928937459182JAX666', '105.183.123.12', 10);
try {
location.reload(true);
} catch (err1) {
try {
location.reload();
} catch (err2) {
location.href = getHostUri();
}
}
</script>
</head>
<body>
<noscript>This site requires JavaScript and Cookies to be enabled. Please change your browser settings or upgrade your browser.</noscript>
</body>
</html>
I'm using simple stream reading, here is my code:
try {
URL url = new URL("http://english.alarabiya.net/.mrss/en/sports.xml");
BufferedReader in = new BufferedReader(
new InputStreamReader(url.openStream()));
String inputLine;
while ((inputLine = in.readLine()) != null)
System.out.println(inputLine);
in.close();
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
Is there anyone know how to parse the main RSS content and bypass the Javascript part with cookies? Or any ideas?
P.S.: I'm using Rome library to crawl the RSS feed, but I think the problem out of its scope.
