In general I understand how to work with Scrapy and x-path to parse the html. However, I don't know how to grab the HAR data.
mport scrapy
from scrapy_splash import SplashRequest
class QuotesSpider(scrapy.Spider):
name = 'quotes'
allowed_domains = ['quotes.toscrape.com']
start_urls = ['http://quotes.toscrape.com/js']
def start_requests(self):
for url in self.start_urls:
yield SplashRequest(url=url,
callback=self.parse,
endpoint='render.html')
def parse(self, response):
quotes = response.xpath('//*[@class="quote"]')
for quote in quotes:
yield { 'author': quote.xpath('.//*[@class="author"]/text()').extract_first(),
'quote': quote.xpath('.//*[@class="text"]/text()').extract_first()
}
script = """function main(splash)
assert(splash:go(splash.args.url))
splash:wait(0.3)
button = splash:select("li[class=next] a")
splash:set_viewport_full()
splash:wait(0.1)
button:mouse_click()
splash:wait(1)
return {url = splash:url(),
html = splash:html(),
har = splash:har()}
end """
yield SplashRequest(url=response.url,
callback=self.parse,
endpoint='execute',
args={'lua_source':script})
What would be the next statement in the script to export the HAR data to a file? How do I yield all the network data to a file? Any insights would be appreciated.
