It's my first time working with Selenium and web scraping. I have been trying to get the menu item and prices for a certain restaurant in California from the following website (https://www.fastfoodmenuprices.com/baskin-robbins-prices/). I have been able to use Selenium to get to make it select California from the dropdown menu but I keep running into the problem of not being able to scrape the menu items and prices and coming up with a blank data frame. How do I scrape the menu item and prices from the following website and store them into a data frame? The code is below:
from selenium import webdriver
import time
from selenium.webdriver.support.ui import Select
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
import pandas as pd
from bs4 import BeautifulSoup
path = "/path/to/chromedriver"
driver = webdriver.Chrome(executable_path = path)
url = "https://www.fastfoodmenuprices.com/baskin-robbins-prices/"
driver.get(url)
Select(WebDriverWait(driver,20).until(EC.visibility_of_element_located((By.XPATH, "//select[@class='tp-variation']")))).select_by_value("MS4yOA==")
print(driver.page_source)
driver.quit
menu = []
prices = []
content = driver.page_source
soup = BeautifulSoup (content, features = "html.parser")
for element in soup.findAll('div', attrs = {'tbody': 'row-hover'}):
menu = element.find ('td', attrs = {'class': "column-1"})
prices = element.find('td', attrs = {'class':'column-3'})
menu.append(menu.text)
prices.append(prices.text)
df = pd.DataFrame({'Menu Item':menu, 'Prices':prices})
df
