I'm abling to fetch some data but not all with the function describe in similar question and copied to this question.
The function does exactly what it is meant to do. But my problem is with this method the desired request not appear on to screen.
The desired request should respond with products JSON information. and it didn't. when I surf to the same URL without coding I can see that request with full responed and I can see in UI all products arrived from this request. But when doing it in this method and (```headless: false'') the UI shows all excluding the products data. What am I missing here?
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch();
const [page] = await browser.pages();
const results = []; // collects all results
let paused = false;
let pausedRequests = [];
const nextRequest = () => { // continue the next request or "unpause"
if (pausedRequests.length === 0) {
paused = false;
} else {
// continue first request in "queue"
(pausedRequests.shift())(); // calls the request.continue function
}
};
await page.setRequestInterception(true);
page.on('request', request => {
if (paused) {
pausedRequests.push(() => request.continue());
} else {
paused = true; // pause, as we are processing a request now
request.continue();
}
});
page.on('requestfinished', async (request) => {
const response = await request.response();
const responseHeaders = response.headers();
let responseBody;
if (request.redirectChain().length === 0) {
// body can only be access for non-redirect responses
responseBody = await response.buffer();
}
const information = {
url: request.url(),
requestHeaders: request.headers(),
requestPostData: request.postData(),
responseHeaders: responseHeaders,
responseSize: responseHeaders['content-length'],
responseBody,
};
results.push(information);
nextRequest(); // continue with next request
});
page.on('requestfailed', (request) => {
// handle failed request
nextRequest();
});
await page.goto('http://example.com', { timeout: 0 }); // otherwise i got Navigation timeout of 30000 ms exceeded
console.log(results);
await browser.close();
})();