0

I'm using selenium to scrape data on a dynamic website. after picking the elements I set a for loop to append to a list instead of iterating the whole set of data. it iterates only 5x and stops. what could be the cause of this?

import time
from selenium import webdriver
from selenium.webdriver.chrome import service
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd

ser= Service("C:\Program Files (x86)\chromedriver.exe")
options = webdriver.ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-logging'])
driver = webdriver.Chrome(options=options,service=ser)
driver.get('https://soundcloud.com/jujubucks')
print(driver.title)

wait = WebDriverWait(driver,30)

wait.until(EC.element_to_be_clickable((By.ID,"onetrust-accept-btn-handler"))).click()

song_contents = driver.find_elements(By.CLASS_NAME, 'soundList__item')

song_list = []

for option in song_contents:
    search = option.find_element(By.XPATH, ".//a[contains(@class,'soundTitle__username')]/span").text
    search_song = option.find_element(By.XPATH, ".//a[contains(@class,'soundTitle__title')]/span").text
    search_date = option.find_element(By.XPATH, ".//time[contains(@class,'relativeTime')]/span").text
    search_plays = option.find_element(By.XPATH, ".//span[contains(@class,'sc-ministats-small')]/span").text
   
    option ={
        'Artist': search, 
        'Song_title': search_song, 
        'Date': search_date,
        'Streams': search_plays
    }

    song_list.append(option)

df = pd.DataFrame(song_list)
print(df)


driver.quit()

Output

Stream Juju Bucks music | Listen to songs, albums, playlists for free on SoundCloud
       Artist                              Song_title               Date Streams
0  Juju Bucks  Squad Too Deep Ft. Cool Prince (Outro)  Posted 1 year ago      31
1  Juju Bucks            Tropikana ft. P-Dogg Amazing  Posted 1 year ago      31
2  Juju Bucks              Party Ka Mngani Ft. X-Poll  Posted 1 year ago      31
3  Juju Bucks      Joy Ft. Black Sushi & Gavin Bowden  Posted 1 year ago      31
4  Juju Bucks                      Amazing ft. X-Poll  Posted 1 year ago      31
  • show us the value of `song_contents`? – Riley Oct 28 '21 at 10:11
  • https://stackoverflow.com/questions/22702277/crawl-site-that-has-infinite-scrolling-using-python Your page has only 5 loaded at the start you could do this to get them all or requests. – Arundeep Chohan Oct 28 '21 at 21:44

0 Answers0