I'm using selenium to scrape data on a dynamic website. after picking the elements I set a for loop to append to a list instead of iterating the whole set of data. it iterates only 5x and stops. what could be the cause of this?
import time
from selenium import webdriver
from selenium.webdriver.chrome import service
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
ser= Service("C:\Program Files (x86)\chromedriver.exe")
options = webdriver.ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-logging'])
driver = webdriver.Chrome(options=options,service=ser)
driver.get('https://soundcloud.com/jujubucks')
print(driver.title)
wait = WebDriverWait(driver,30)
wait.until(EC.element_to_be_clickable((By.ID,"onetrust-accept-btn-handler"))).click()
song_contents = driver.find_elements(By.CLASS_NAME, 'soundList__item')
song_list = []
for option in song_contents:
search = option.find_element(By.XPATH, ".//a[contains(@class,'soundTitle__username')]/span").text
search_song = option.find_element(By.XPATH, ".//a[contains(@class,'soundTitle__title')]/span").text
search_date = option.find_element(By.XPATH, ".//time[contains(@class,'relativeTime')]/span").text
search_plays = option.find_element(By.XPATH, ".//span[contains(@class,'sc-ministats-small')]/span").text
option ={
'Artist': search,
'Song_title': search_song,
'Date': search_date,
'Streams': search_plays
}
song_list.append(option)
df = pd.DataFrame(song_list)
print(df)
driver.quit()
Output
Stream Juju Bucks music | Listen to songs, albums, playlists for free on SoundCloud
Artist Song_title Date Streams
0 Juju Bucks Squad Too Deep Ft. Cool Prince (Outro) Posted 1 year ago 31
1 Juju Bucks Tropikana ft. P-Dogg Amazing Posted 1 year ago 31
2 Juju Bucks Party Ka Mngani Ft. X-Poll Posted 1 year ago 31
3 Juju Bucks Joy Ft. Black Sushi & Gavin Bowden Posted 1 year ago 31
4 Juju Bucks Amazing ft. X-Poll Posted 1 year ago 31