Selenium web scraping using python - anydrop down without name

Question

I am trying to figure out how to go through all dropdown and get the price and the variation name. I can do this via name attribute but different product pages have different attributes so it wouldnt be best practise. I completed this code using select by element id but received errors below. I am also trying to figure out how to copy to csv for this but it doesnt seem to work.

import csv
browser = webdriver.Chrome(executable_path='C:\Users\userman\PycharmProjects\seleniumTest\drivers\chromedriver.exe')
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from selenium.webdriver.support.select import Select



from selenium import webdriver
import csv
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select

from selenium.webdriver.support.select import Select
from selenium.webdriver.support.select import Select
from selenium.webdriver import Chrome

import itertools
from pprint import pformat

url = "https://www.ebay.co.uk/itm/Pashmina-Scarf-100-Viscose-Plain-Wrap-Shawl-Stole-Scarf-Many-Colours-Available/252342060680"

browser.get(url)
optionones = len(browser.find_elements_by_xpath("//select[@id='msku-sel-1']/option"))
for colNum in range(optionones):
    #select color
    optiononeEle = browser.find_element_by_xpath("(//select[id='msku-sel-1']/option)[" + str(colNum+1) + "]")
    optionone = optiononeEle.text
    optiononeEle.click()

    # get the sizes
    sizes = len(browser.find_elements_by_xpath("//select[id='msku-sel-2']/option"))
    for sizeNum in range(sizes):
        # select optionone
        optiontwoEle = browser.find_element_by_xpath("(//select[id='msku-sel-2']/option)[" + str(sizeNum + 1) + "]")
        size = optiontwoEle.text
        optiontwoEle.click()
        price = browser.find_element_by_xpath("//span[@id='prcIsum']").text
        print ("optionone:" + optionone)
        print( "optiontwoEle:" + optiontwoEle)
        print("Price: "+ price)
        print ("----------------------------------------")

Traceback (most recent call last):
  File "C:/Users/userman/PycharmProjects/seleniumTest/test/test310.py", line 30, in <module>
    optiononeEle = browser.find_element_by_xpath("(//select[id='msku-sel-1']/option)[" + str(colNum+1) + "]")
  File "C:\Python27\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 394, in find_element_by_xpath
    return self.find_element(by=By.XPATH, value=xpath)
  File "C:\Python27\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 978, in find_element
    'value': value})['value']
  File "C:\Python27\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
    self.error_handler.check_response(response)
  File "C:\Python27\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"(//select[id='msku-sel-1']/option)[1]"}

Process finished with exit code 1

Expected output is name of option 1 (e.g size but could be anything) + variation type name (e.g small) and then price.

Answer 1

Why not find the number of those dropdowns then loop setting each dropdown into a variable and use usual syntax for selecting options within?

number_of_dropdowns =  len(WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "[id^='msku-sel-']"))))

for i in range(1, number_of_dropdowns + 1):
    current_dropdown = driver.find_element_by_css_selector("[id^='msku-sel-" + str(i) + "']")
    #then use usual syntax to work with select 

See this extensive set of answers for working with option selection:

Selenium - Python - drop-down menu option value

---

Imports for wait condition:

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

---

Example to loop them :

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.select import Select

urls = ['https://www.ebay.co.uk/itm/Pashmina-Scarf-100-Viscose-Plain-Wrap-Shawl-Stole-Scarf-Many-Colours-Available/252342060680']
d = webdriver.Chrome()

for url in urls:
    d.get(url)
    number_of_dropdowns =  len(WebDriverWait(d,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "[id^='msku-sel-']"))))

    for i in range(1, number_of_dropdowns + 1):
        #then use usual syntax to work with select 
        select = Select(d.find_element_by_css_selector("[id^='msku-sel-" + str(i) + "']"))
        options = select.options
        for index in range(1, len(options) - 1): #avoid first index
            select.select_by_index(index)
            #print(select.first_selected_option.text)
            #your code to extract info per option goes here