My python script only pulls the first result then moves on to the next url

Question

I am creating a python scraper for a website to pull price, product number, cat number, description. When I run this script it only pulls the first item of the page then moves on to the next url. New to python just wondering how I can modify to pull all of the products from the page. Thanks to clarify the first url only has one product on it but the second the third all have many products that are not being pulled.

 import requests
from bs4 import BeautifulSoup
import random
import time

product_urls = [
'https://www.qiagen.com/us/shop/pcr/primer-sets/miscript-precursor-assays/#orderinginformation',
'https://www.qiagen.com/us/shop/pcr/primer-sets/miscript-primer-assay-plate/#orderinginformation',
'https://www.qiagen.com/us/shop/pcr/primer-sets/miscript-primer-assays/#orderinginformation', 
]

for URL in product_urls:
    page = requests.get(URL)
    soup = BeautifulSoup(page.text,"lxml")
    timeDelay = random.randrange(5, 25)

    for item in soup.select('.content'):
        cat_name = item.select_one('.title').text.strip()
        cat_discription = item.select_one('.copy').text.strip()
        product_name = (item.find('div',{'class':'headline'}).text.strip())
        product_discription = (item.find('div',{'class': 'copy'}).text.strip())
        product_number = (item.find('td',{'class': 'textLeft paddingTopLess'}).text.strip())
        cat_number = (item.find('td',{'class': 'textRight paddingTopLess2'}).text.strip())
        product_price = (item.find('span',{'class': 'prc'}).text.strip())
        print("Catagory Name: {}\n\nCatagory Discription:  {}\n\nProduct Name:  {}\n\nProduct Discription:  {}\n\nProduct Number:  {}\n\nCat No:  {}\n\nPrice:  {}\n\n".format(cat_name,cat_discription,product_name,product_discription,product_number,cat_number,product_price))
        time.sleep(timeDelay)

Answer 1

You can get the tables elements from the pane class div. The 4th table is the main product & the 5th (when existing) is the additionnal products.

In the following example, I use list comprehension to output list of tuples with title, description, product n°, category n° & price :

from bs4 import BeautifulSoup
import requests

product_urls = [
    'https://www.qiagen.com/us/shop/pcr/primer-sets/miscript-precursor-assays/#orderinginformation',
    'https://www.qiagen.com/us/shop/pcr/primer-sets/miscript-primer-assay-plate/#orderinginformation',
    'https://www.qiagen.com/us/shop/pcr/primer-sets/miscript-primer-assays/#orderinginformation', 
    'https://www.qiagen.com/us/shop/pcr/real-time-pcr-enzymes-and-kits/miscript-target-protectors/#orderinginformation',
    'https://www.qiagen.com/us/shop/pcr/real-time-pcr-enzymes-and-kits/two-step-qrt-pcr/miscript-sybr-green-pcr-kit/#orderinginformation'
]

session = requests.Session()

for URL in product_urls:

    response = session.get(URL)
    soup = BeautifulSoup(response.content, "html.parser")

    tables = soup.find_all("div", {"class":"pane"})[0].find_all("table")

    if (len(tables) > 4):
        product_list = [
            (
                t[0].findAll("div", {"class":"headline"})[0].text.strip(), #title
                t[0].findAll("div", {"class":"copy"})[0].text.strip(),     #description
                t[1].text.strip(),                                         #product number
                t[2].text.strip(),                                         #category number
                t[3].text.strip()                                          #price
            )
            for t in (t.find_all('td') for t in tables[4].find_all('tr'))
            if t
        ]
    elif (len(tables) == 1):
        product_list = [
            (
                t[0].findAll("div", {"class":"catNo"})[0].text.strip(),    #catNo
                t[0].findAll("div", {"class":"headline"})[0].text.strip(), #headline
                t[0].findAll("div", {"class":"price"})[0].text.strip(),    #price
                t[0].findAll("div", {"class":"copy"})[0].text.strip()      #description
            )
            for t in (t.find_all('td') for t in tables[0].find_all('tr'))
            if t
        ]
    else:
        print("could not parse main product")

    print(product_list)

    if len(tables) > 5:
        add_product_list = [
            (
                t[0].findAll("div", {"class":"title"})[0].text.strip(), #title
                t[0].findAll("div", {"class":"copy"})[0].text.strip(),  #description
                t[1].text.strip(),                                      #product number
                t[2].text.strip(),                                      #category number
                t[3].text.strip()                                       #price
            )
            for t in (t.find_all('td') for t in tables[5].find_all('tr'))
            if t
        ]
        print(add_product_list)

Check this answer if you want to convert the tuples index into single list for each field