I'm having an issue with Selenium. The code below in question is working and can log everything I need, but it can only log up until 10 - coincidently 10 is the number of products I see on the page without scrolling down so my suspicion is it cannot click on these since they're technically not in view and a scrolling action would be needed or something that makes that element visible. I have tried several ways to do this but I was unable to find an elegant up to date way.
most pages have about 25 items and the code recognizes that but again, cannot locate the element to click on and will throw this
On Item:10 of 25
list index out of range
On Item:11 of 25
list index out of range
On Item:12 of 25
list index out of range
On Item:13 of 25
list index out of range
On Item:14 of 25
....
class InspectProducts:
def __init__(self):
phantom_js_path = '/usr/bin/phantomjs'
self.driver = webdriver.PhantomJS(executable_path=phantom_js_path)
self.driver.maximize_window()
def test(self, url_path: str, category: str):
self.driver.get(url_path)
products = self.driver.find_elements_by_class_name('catProd-image-wrapper')
for x in range(0, len(products)):
try:
print(f'On Item:{x} of {len(products)}')
option = self.driver.find_elements_by_class_name('catProd-image-wrapper')[x]
option.click()
# on product page now
product_name = self.driver.find_element_by_class_name('prod-title').text
label = self.driver.find_elements_by_tag_name('img')
product_image_label = label[4].get_attribute('src')
if product_image_label.endswith('uploadB.jpg') or product_image_label.endswith('uploadA.jpg'):
nutrition_label = product_image_label
print(nutrition_label)
elif not product_image_label.endswith('uploadB.jpg') or not product_image_label.endswith('uploadA.jpg'):
nutrition_label = 'no label'
product_description = self.driver.find_element_by_class_name('productSectionContent').text
product_rating = self.driver.find_elements_by_xpath('//*[@id="content"]/div[2]/form/div[1]/div/div[1]/div[3]/div[2]/div/span/span/span[1]')[0].text
product_price = self.driver.find_elements_by_xpath('/html/body/div[3]/div[3]/div/div[2]/form/div/div/div[2]/div[2]/div[1]/span[2]/span')[0].text
# you can print stuff here ..
# back to product listing page
self.utils.go_back_to_previous_page()
except IndexError as e:
print(e)
print(f'ALL DONE {category}')
if __name__ == "__main__":
inspect_products = InspectProducts()
inspect_products.test('https://www.svncanada.com/list-amino-acid-blends.php', 'amino acids')
