0

I am scraping gas and electricity prices at uswitch.com and have got to the end of the process where the website outputs prices. I want to print the cost of the cheapest deal, after including plans that require switching directly through the supplier. The code below does this, after copying the xpath of the cheapest deal 

//*[@id='my-results']/div[3]/div[2]/div[1]/div/div[2]/div/div[2]/a/div[1]/span[2]/span[1]

, but the result printed to screen is not the cheapest deal, but two frames down, the cheapest deal not including plans that requires switching directly through the supplier, which I enabled. For me this is just selenium picking the wrong div, but maybe something dynamic is causing the code problems.

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support import ui
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import TimeoutException

driver = webdriver.Chrome(executable_path=ChromeDriverManager().install())

# Open webpage
driver.get('https://www.uswitch.com/gas-electricity/')

# Locates 'input postcode' box
search_form = driver.find_element_by_id('input-postcode')

# eneters a postcode in the 'input postcode' box
search_form.send_keys('SW7 2BX')

# Click on the 'Compare energy deals now' to submit the form and continue
driver.find_element_by_xpath("//*[contains(text(),'Compare energy deals     now')]").click()

# Once the page loads, wait for 2 seconds
driver.implicitly_wait(2)

# Click on the 'Skip' button to skip selecting an address
driver.find_element_by_css_selector('.us-link.us-margin-top.us-float--    right').click()

#Once the page loads, wait for 2 seconds THIS ISNT WORKING????? (NOR ARE     THE OTHERS)
driver.implicitly_wait(2)

# Click on the 'Continue' button under 'Your property'
driver.find_element_by_xpath("//*[contains(text(),'Continue')]").click()

# Click on the 'Continue' button under 'Your supply'
driver.find_element_by_xpath("//*[contains(text(),'Continue')]").click()

# Click on the 'Continue' button under 'Your gas and electricity'
driver.find_element_by_xpath("//*[contains(text(),'Continue')]").click()

# Click on the 'Continue' button under 'Your usage'
driver.find_element_by_xpath("//*[contains(text(),'Continue')]").click()

# Under 'your gas usage', select the 'I use ...' option
driver.find_element_by_xpath("//input[@name='usage-gas' and     @value='usage']").click()

# Creating a variable that represents the gas usage box 
gas_usage_box=driver.find_element_by_xpath("//input[@name='usage-for-gas'     and @class='us-form-input']")

# Locates the 'I use ...' box under 'Your gas usage'
gas_usage_box.click()

# Enters AQ into 'I use ...' box
gas_usage_box.send_keys('1001')

# Click on 'Continue' button under 'Your gas usage box'
driver.find_element_by_xpath("//*[contains(text(),'Continue')]").click()

# Under 'your electricity usage', select the 'I use ...' option
driver.find_element_by_xpath("//input[@name='usage-electricity' and     @value='usage']").click()

# Creating a variable that represents the electricity usage box 
electricity_usage_box=driver.find_element_by_xpath("//input[@name='usage-    for-electricity' and @class='us-form-input']")

# Locates the 'I use ...' box under 'Your gas usage'
electricity_usage_box.click()

# Enters EAC into 'I use ...' box
electricity_usage_box.send_keys('1001')

# Wait 5 seconds after enetering EAC
wait = ui.WebDriverWait(driver, 5)

# Click 'Find cheaper deals' button under 'Your electricity usage'
driver.find_element_by_xpath("//*[contains(text(),'Find cheaper     deals')]").click()

# Wait 5 seconds after enetering EAC
wait = ui.WebDriverWait(driver, 5)

# Choose to view plans that 'require switching directly through the     supplier'
#driver.find_element_by_xpath("//input[@name='show-switchable-plans-    filters' and @class='fulfilability-toggle__input us-form-input js-    fulfilability-toggle' and @type='radio' and @value='false']")

driver.find_element_by_xpath("//input[@name='show-switchable-plans-filters' and @value='false']").click()

wait = ui.WebDriverWait(driver, 20)

mytext = driver.find_element_by_xpath("//*[@id='my-    results']/div[3]/div[2]/div[1]/div/div[2]/div/div[2]/a/div[1]/span[2]/span[1]")

print mytext.get_attribute('innerHTML')

The only difference I can see between the two different types of prices is the one which requires switching directly through the supplier is inside a class of type 'unfufillable', which may aid an xpath search, which I am new to.

<div class="new-result-mobile-row__wrapper new-result-mobile-row__wrapper-- 
unfulfillable">
<span>
Estimated cost
</span><span class="new-result-mobile-row__wrapper__link__cost__figure">
<span class="new-result-mobile-row__wrapper__link__cost__figure__pounds">
£257</span>
<span class="new-result-mobile-row__wrapper__link__cost__figure__pence">
.19</span></span>
<span>per year</span>
<span>
(£21.43 pm)

EDIT:

SOLUTION: I don't know what the problem was but inserting driver.refresh() instead of the webdriver.wait, solved the problem.

magd
  • 99
  • 7
  • what is your expected output? – KunduK Apr 03 '19 at 10:23
  • @KajalKundu Output should be £257, I think it may be to do with the page not loading correctly after enabling direct deals, but I thought wait = ui.WebDriverWait(driver, 20), would deal with that. – magd Apr 03 '19 at 10:29
  • Try the below code with following imports it should work. – KunduK Apr 03 '19 at 10:34

2 Answers2

0

Use WebDriverWait to handle Dynamic Element.

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions   

print(WebDriverWait(driver, 10).until(expected_conditions.element_to_be_clickable((By.XPATH, '//span[@class="new-result-mobile-row__wrapper__link__cost__figure__pounds"]'))).text)

Edited.

print(driver.execute_script("return arguments[0].innerHTML;",driver.find_element_by_xpath('//span[@class="new-result-mobile-row__wrapper__link__cost__figure__pounds"]')))
KunduK
  • 32,888
  • 5
  • 17
  • 41
  • hmm, that results in a timeout, is the pounds figure clickable? Maybe because it's hidden text, that's why I had to return innerHTML – magd Apr 03 '19 at 10:37
  • Based on your snippet html it is retrieving the value.can you check if there any iframe or can you share full page html link. – KunduK Apr 03 '19 at 10:41
  • The full html is too much to paste here, but the code in the question goes to uswitch.com and follows several steps to get the results page, running it will show the end result html. – magd Apr 03 '19 at 10:44
  • @magd : can you try the edited code.See if this work as you mentioned hidden text – KunduK Apr 03 '19 at 10:49
  • OK this is weird, but inserting driver.refresh() before getting the figure solved the problem. – magd Apr 03 '19 at 10:49
0

Fixed locators you use, please check best practice or official locator strategies for the webdriver.
Do not use explicit and implicit wait in the same time, check when to use explicit wait vs. implicit wait.

Code is a basic example, you can modify, create methods.. to make it more readable:

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By


driver = webdriver.Chrome()
wait = WebDriverWait(driver, 15)

# Open webpage
driver.get('https://www.uswitch.com/gas-electricity/')

driver.find_element_by_id('input-postcode').send_keys('SW7 2BX')

# Click on the 'Compare energy deals now' to submit the form and continue
driver.find_element_by_xpath("//button[.='Compare energy deals now']").click()

# Click on the 'Skip' button to skip selecting an address
wait.until(EC.element_to_be_clickable((By.LINK_TEXT, "Skip"))).click()

# Click on the 'Continue' button under 'Your property'
driver.find_element_by_css_selector("#your-property .continue-button button").click()

# Click on the 'Continue' button under 'Your supply'
driver.find_element_by_css_selector("#your-supply .continue-button button").click()

# Click on the 'Continue' button under 'Your gas and electricity'
driver.find_element_by_css_selector("#dual-fuel .continue-button button").click()

# Click on the 'Continue' button under 'Your usage'
driver.find_element_by_css_selector("#your-usage .continue-button button").click()

# Under 'your gas usage', select the 'I use ...' option
driver.find_element_by_css_selector("input[name=usage-gas][value=usage]").click()
# Creating a variable that represents the gas usage box
gas_usage_input = driver.find_element_by_css_selector("input[name=usage-for-gas]")
# Locates the 'I use ...' box under 'Your gas usage'
gas_usage_input.click()
# Enters AQ into 'I use ...' box
gas_usage_input.send_keys('1001')
# Click on 'Continue' button under 'Your gas usage box'
driver.find_element_by_css_selector("#gas-usage .continue-button button").click()

# Under 'your electricity usage', select the 'I use ...' option
driver.find_element_by_css_selector("input[name=usage-electricity][value=usage]").click()
# Creating a variable that represents the electricity usage box
electricity_usage_input = driver.find_element_by_css_selector("input[name=usage-for-electricity]")
# Locates the 'I use ...' box under 'Your gas usage'
electricity_usage_input.click()
# Enters EAC into 'I use ...' box
electricity_usage_input.send_keys('1001')

# Click 'Find cheaper deals' button under 'Your electricity usage'
driver.find_element_by_css_selector(".cheaper-deals button").click()

# Choose to view plans that 'require switching directly through the     supplier'
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input.fulfilability-toggle__input[value=false]"))).click()

lowest_price = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "span.new-result-row__cost__figure"))).text
lowest_price_pound = driver.find_element_by_css_selector("span.new-result-row__cost__figure__pounds").text
lowest_price_pence = driver.find_element_by_css_selector("span.new-result-row__cost__figure__pence").text.replace(".", "")

print(lowest_price)
Sers
  • 12,047
  • 2
  • 12
  • 31