3

I am trying to scrape a website with Scrapy but i am having problem in scraping the lazy loading products. This is my code.

def parse(self, response):
    hxs = HtmlXPathSelector(response)
    sites = hxs.select('//ul[@id="productsCatalog"]/li')
    items = []
    for site in sites:
        item = DmozItem()
        item['link'] = site.select('a/@href').extract()
        item['title'] = site.select('a/span[4]/text()').extract()[0].strip()
        item['price'] = site.select('a/span[5]/span/text()').extract()
        item['brand'] = site.select('a/span[3]/text()').extract()
        items.append(item)

    return items
user2747776
  • 109
  • 1
  • 15

0 Answers0