I'm using playwright in my Render server. Locally, I don't have any error. But on the Render server, sometimes it works with the goto function but sometimes I get an error. I have a link list(I added example links in the source code) and give the functions. I couldn't find any solution for this. This is the error log:
: GOTO: page.goto: Navigation failed because page was closed!
May 16 06:28:40 PM =========================== logs ===========================
May 16 06:28:40 PM navigating to "https://in.investing.com/equities/housing-development-finance-technical?timeFrame=60", waiting until "load"
May 16 06:28:40 PM ============================================================
May 16 06:28:40 PM at getResults (/usr/src/app/index.js:151:10)
May 16 06:28:40 PM at async scrape (/usr/src/app/index.js:223:23) {
May 16 06:28:40 PM name: 'Error'
May 16 06:28:40 PM }
May 16 06:28:40 PM ERROR => page.$eval: Target page, context or browser has been closed
May 16 06:28:40 PM at getResults (/usr/src/app/index.js:160:33)
May 16 06:28:40 PM at async scrape (/usr/src/app/index.js:223:23) {
May 16 06:28:40 PM name: 'Error'
May 16 06:28:40 PM }
May 16 06:28:40 PM ERROR => browserContext.newPage: Target page, context or browser has been closed
May 16 06:28:40 PM at getResults (/usr/src/app/index.js:133:34)
May 16 06:28:40 PM at async scrape (/usr/src/app/index.js:223:23) {
May 16 06:28:40 PM name: 'Error'
May 16 06:28:40 PM }
Code:
Link_Lst = [
"https://in.investing.com/equities/axis-bank-technical",
"https://in.investing.com/equities/tata-motors-ltd-technical",
"https://in.investing.com/equities/icici-bank-ltd-technical",
..}
async function getResults(lnk) {
const results = [];
const timeFrames = [1, 5, 15];
for (const i of timeFrames) {
try {
const browser = await chromium.launch({
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--single-process',
"--no-zygote",
'--ignore-certificate-errors',
"--disable-features=AudioServiceOutOfProcess"
// "--remote-debugging-port"
],
defaultViewport: chromium.defaultViewport,
executablePath: await chromium.executablePath(),
headless: chromium.headless,
});
const context = await browser.newContext({
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
});
const page = await context.newPage();
const url = `${lnk}?timeFrame=${i * 60}`;
console.log(`Getting: ${url}`);
await page.evaluate(() => navigator.userAgent)
await page.setViewportSize({ width: 1920, height: 1080 });
await page.setDefaultNavigationTimeout(0);
await page
.goto(url, { waitUntil: "load", timeout: 0 })
.then(() => console.log("Goto Success!!"))
.catch((err) => console.log("GOTO:", err));
await page
.waitForSelector("section.forecast-box-graph", {
visible: true,
})
.then(() => console.log("Success"))
.catch(async (err) => console.log("ERR", err));
const status = await page.$eval(
"section.forecast-box-graph .title",
(el) => el.textContent
);
const bank_name = await page.$eval("h1.main-title.js-main-title", (el) =>
el.textContent.trim()
);
results.push({
bankName: bank_name,
status: status,
lnk: lnk.replace(/-/g, " ").split("/").pop(),
url: url,
});
// await page.close();
await browser.close();
} catch (e) {
console.log("ERROR =>", e);
}
}
sendRequest();
return results;
}
....
async function scrape() {
while (true) {
for (const lnk of Link_Lst) {
let Buy_ = true;
let Sell_ = true;
let focusCount = 0
const results = await getResults(lnk);
}}