I try to scrape a website using Scrapy. Pagination is quite difficult because it's using AJAX and I don't know how to do. Besides, when I click on the next page the URL doesn't chaange at all.
When I go in the network tab of the developper tool, I see this:
And this is my item code:
import json
import scrapy
import re
import pkgutil
from scrapy.loader import ItemLoader
from annonces_spiders.items import AnnonceItem
class GlenMarchSpider(scrapy.Spider):
name = 'annonces_results'
def __init__(self, *args, **kwargs):
data_file = pkgutil.get_data("annonces_spiders", "json/input/db_scrap_url_lp_js_10000_reduced.json")
self.data = json.loads(data_file)
def start_requests(self):
for item in self.data:
request = scrapy.Request(item['url_lp'], callback=self.parse)
request.meta['item'] = item
yield request
def parse(self, response):
item = response.meta['item']
item['results'] = []
for caritem in response.css("li.li-result"):
data = AnnonceItem()
data["marque"] = caritem.css("span.brand::text").extract_first().capitalize().replace(" ", "").strip()
data["model"] = caritem.css("span.sub-title::text").extract_first().capitalize().replace(" ", "").strip()
data["model_year"] = caritem.css("li:nth-child(3) > div.upper::text").extract_first().replace(" ", "").strip()
data["price_str"] = caritem.css("p.prix::text").extract_first().strip()
if "NC" not in data["price_str"]:
data["price_int"] = int(caritem.css("p.prix::text").extract_first().replace("€", "").replace(" ", "").strip())
else:
data["price_int"] = None
data["annonce_date"] = caritem.css("p.btn-publication::text").extract_first().strip().replace("/", "-")
data["country"] = caritem.css("div.location > img::attr(alt)").extract_first().capitalize().strip()
item['results'].append(data)
yield item
I tried to use Splash and a Lua script but the pagination seems impossible.