You may have seen my desperate frustrations over the past few weeks on here. I've been scraping some wait time data and am still unable to grab data from these two sites
At first I tried BS4 for Python. Sample code below for HCA Virgina
from BeautifulSoup import BeautifulSoup
import requests
url = 'http://hcavirginia.com/home/'
r = requests.get(url)
soup = BeautifulSoup(r.text)
wait_times = [span.text for span in soup.findAll('span', attrs={'class': 'ehc-er-digits'})]
fd = open('HCA_Virginia.csv', 'a')
for w in wait_times:
fd.write(w + '\n')
fd.close()
All this does is print blanks to the console or the CSV. So I tried it with PhantomJS since someone told me it may be loading with JS. Yet, same result! Prints blanks to console or CSV. Sample code below.
var page = require('webpage').create(),
url = 'http://hcavirginia.com/home/';
page.open(url, function(status) {
if (status !== "success") {
console.log("Can't access network");
} else {
var result = page.evaluate(function() {
var list = document.querySelectorAll('span.ehc-er-digits'), time = [], i;
for (i = 0; i < list.length; i++) {
time.push(list[i].innerText);
}
return time;
});
console.log (result.join('\n'));
var fs = require('fs');
try
{
fs.write("HCA_Virginia.csv", '\n' + result.join('\n'), 'a');
}
catch(e)
{
console.log(e);
}
}
phantom.exit();
});
Same issues with Centura Health :(
What am I doing wrong?