Html Parser pulling from previous webpage

Question

I have a script that loads a page and saves a bunch of data ids from multiple containers. I then want to open up new urls appending those said data ids onto the end of the urls. For each url I want to locate all the hrefs and compare them to a list of specific links and if any of them match I want to save that link and a few other details to a table.

I have managed to get it to open the url with the appended data id but when I try to search for elements in the new page it either pulls them from the first url that was parsed if I try to findAll from soup again or I constantly get this error when I try to run another html.parser.

ResultSet object has no attribute 'findAll'. You're probably treating a list of items like a single item. Did you call find_all() when you meant to call find()?

Is it not possible to run another parser or am I just doing something wrong?

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup as soup
from selenium.webdriver.common.action_chains import ActionChains

url = "http://csgo.exchange/id/76561197999004010#x"

driver = webdriver.Firefox()

driver.get(url)
import time
time.sleep(15)
html = driver.page_source
soup = soup(html, "html.parser")

containers = soup.findAll("div",{"class":"vItem"})

print(len(containers))
data_ids = [] # Make a list to hold the data-id's

for container in containers:
    test = container.attrs["data-id"]
    data_ids.append(test) # add data-id's to the list
    print(str(test))

for id in data_ids:
    url2 = "http://csgo.exchange/item/" + id
    driver.get(url2)
    import time
    time.sleep(2)   
    soup2 = soup(html, "html.parser")
    containers2 = soup2.findAll("div",{"class":"bar"})
    print(str(containers2))

with open('scraped.txt', 'w', encoding="utf-8") as file:
    for id in data_ids:
        file.write(str(id)+'\n') # write every data-id to a new line

Answer 1

Not sure exactly what you want from each page. You should add waits. I add waits looking for hrefs in the flow history section of each page (if present). It should illustrate the idea.

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

url = 'http://csgo.exchange/id/76561197999004010'
driver = webdriver.Chrome()
driver.get(url)
ids = [item.get_attribute('data-id') for item in WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "[data-id]")))]
results = []
baseURL = 'http://csgo.exchange/item/'
for id in ids:
    url = baseURL + id
    driver.get(url)
    try:
        flowHistory = [item.get_attribute('href') for item in WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#tab-history-flow [href]")))]
        results.append([id, flowHistory])
    except:
        print(url)

Answer 2

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

url = 'http://csgo.exchange/id/76561197999004010'
profile = webdriver.FirefoxProfile()
profile.set_preference("permissions.default.image", 2) # Block all images to load websites faster.
driver = webdriver.Firefox(firefox_profile=profile)
driver.get(url)
ids = [item.get_attribute('data-id') for item in WebDriverWait(driver,30).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "[data-id]")))]
results = []
baseURL = 'http://csgo.exchange/item/'
for id in ids:
    url = baseURL + id
    driver.get(url)
    try:
        pros = ['http://csgo.exchange/profiles/76561198149324950']
        flowHistory = [item.get_attribute('href') for item in WebDriverWait(driver,3).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#tab-history-flow [href]")))]
        if flowHistory in pros:
            results.append([url,flowHistory])
            print(results)
    except:
        print()

Answer 3

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

urls = ['http://csgo.exchange/id/76561197999004010']
profile = webdriver.FirefoxProfile()
profile.set_preference("permissions.default.image", 2) # Block all images to load websites faster.
driver = webdriver.Firefox(firefox_profile=profile)
for url in urls:
    driver.get(url)
ids = [item.get_attribute('data-id') for item in WebDriverWait(driver,30).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "[data-id]")))]
results = []
pros = ['http://csgo.exchange/profiles/76561198149324950', 'http://csgo.exchange/profiles/76561198152970370']
baseURL = 'http://csgo.exchange/item/'
for id in ids:
    url = baseURL + id
    driver.get(url)
try:

    flowHistory = [item.get_attribute('href') for item in WebDriverWait(driver,2).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#tab-history-flow [href]")))]
    match = []
    for string in pros:
        if string in flowHistory:
            match = string
            break

    if match:
        pass 

        results.append([url,match])
        print(results)
except:
    print()