I've been trying to scrape the follow site (http://epl.squawka.com/english-premier-league/21-05-2017/watford-vs-man-city/matches). What I want to do is click 'shots' then let the javascript load and then hover over each of the 'shot elements' and pull the data which appears (the time and player info).
My issue is that I keep on getting an 'Unable to locate element' error when trying to hover over the 'shot elements'. When I print the page_source at the point I try to do this I can clearly see the elements ARE in fact there as well as visibly when I save a screenshot.
I think the elements may be hidden somehow. I tried 'switch to' (perhaps incorrectly) to no avail.
driver = webdriver.Firefox(executable_path='/usr/bin/geckodriver')
driver.get("http://epl.squawka.com/english-premier-league/21-05-2017/watford-vs-man-city/matches")
try:
element = WebDriverWait(driver, 1000).until(EC.element_to_be_clickable((By.XPATH,"//*[@id='mc-content-wrap']/div[2]/div[1]")))
finally:
driver.find_element_by_xpath("//*[@id='mc-content-wrap']/div[2]/div[1]").click()
time.sleep(1)
driver.find_element_by_id("mc-stat-shot").click()
time.sleep(5)
shotsVar = -1
html = driver.page_source
bsObj = BeautifulSoup(html, "html.parser")
for circle in bsObj.find("svg",{'height':'224.6'}).findAll('circle'):
shotsVar += 1
if circle['r'] == '6.5':
shotsXpathCode = ("//*[@id='mc-pitch-view']/div[2]/div[1]/svg/g[%s]/circle" % shotsVar)
print(shotsXpathCode)
try:
element = WebDriverWait(driver, 100).until(EC.presence_of_element_located((By.XPATH,"%s" % shotsXpathCode)))
finally:
element_to_hover_over = driver.find_element_by_xpath("%s" % shotsXpathCode)
hover = ActionChains(driver).move_to_element(element_to_hover_over)
hover.perform()
