I am trying to scrape a website with Scrapy but i am having problem in scraping the lazy loading products. This is my code.
def parse(self, response):
hxs = HtmlXPathSelector(response)
sites = hxs.select('//ul[@id="productsCatalog"]/li')
items = []
for site in sites:
item = DmozItem()
item['link'] = site.select('a/@href').extract()
item['title'] = site.select('a/span[4]/text()').extract()[0].strip()
item['price'] = site.select('a/span[5]/span/text()').extract()
item['brand'] = site.select('a/span[3]/text()').extract()
items.append(item)
return items