I started using PyQt5 for dynamic javascript scraping and already ran into trouble. When i have multiple urls to scrape, python crashes after first or second url, no matter what the domain. I can get data from the first, but not from the second page. Error logs from windows show Qt5WebEngineCore.dll as the cause of error, but I got no clue what to do. I really didnt find anything useful on it from elswhere on the web. Here's the code:
import sys
import requests
from PyQt5.QtWidgets import QApplication
from PyQt5.QtCore import QUrl
from PyQt5.QtWebEngineWidgets import QWebEnginePage
from bs4 import BeautifulSoup
import re
class Client(QWebEnginePage):
def __init__(self, url):
self.app = QApplication(sys.argv)
QWebEnginePage.__init__(self)
self.html = ''
self.loadFinished.connect(self._on_load_finished)
self.load(QUrl(url)) #Ignote mainFrame from PyQt4
self.app.exec_()
def _on_load_finished(self):
self.html = self.toHtml(self.Callable)
print('Load finished')
def Callable(self, html_str):
self.html = html_str
self.app.quit()
def scrape_pyqt5():
lists = ['example1.com/a', 'example1.com/b', 'example1.com/c']
for url in lists:
r = Client(url)
bs = BeautifulSoup(r.html, 'html.parser')
for link in bs.find_all('div', {'id': 'media-player'}):
for directlink in link.find_all('iframe'):
print(directlink)
