Get elements from page.evaluate in Puppeteer?

Question

I'm working with Node.js and Puppeteer for the first time and can't find a way to output values from page.evaluate to the outer scope.

My algorithm:

1. Login
2. Open URL
3. Get ul
4. Loop over each li and click on it
5. Wait for innetHTML to be set and add it's src content to an array.

How can I return data from page.evaluate()?

const puppeteer = require('puppeteer');

const CREDENTIALS = require(`./env.js`).credentials;
const SELECTORS = require(`./env.js`).selectors;
const URLS = require(`./env.js`).urls;

async function run() {
    try {
        const urls = [];
        const browser = await puppeteer.launch({headless: false});
        const page = await browser.newPage();

        await page.goto(URLS.login, {waitUntil: 'networkidle0'});
        await page.type(SELECTORS.username, CREDENTIALS.username);
        await page.type(SELECTORS.password, CREDENTIALS.password);
        await page.click(SELECTORS.submit);
        await page.waitForNavigation({waitUntil: 'networkidle0'});
        await page.goto(URLS.course, {waitUntil: 'networkidle0'});

        const nodes = await page.evaluate(selector => {
            let elements = document.querySelector(selector).childNodes;
            console.log('elements', elements);
            return Promise.resolve(elements ? elements  : null);
        }, SELECTORS.list);

        const links = await page.evaluate((urls, nodes, VIDEO) => {
            return Array.from(nodes).forEach((node) => {
                node.click();
                return Promise.resolve(urls.push(document.querySelector(VIDEO).getAttribute('src')));
            })
        }, urls, nodes, SELECTORS.video);
        const output = await links;
    } catch (err) {
        console.error('err:', err);
    }
}

run();

Answer 1

The function page.evaluate() can only return a serializable value, so it is not possible to return an element or NodeList back from the page environment using this method.

You can use page.$$() instead to obtain an ElementHandle array:

const nodes = await page.$$(`${selector} > *`); // selector children

If the length of the constant nodes is 0, then make sure you are waiting for the element specified by the selector to be added to the DOM with page.waitForSelector():

await page.waitForSelector(selector);

Answer 2

Use page.evaluateHandle() to return a DOM node as a Puppeteer ElementHandle that you can manipulate in Node.

Answer 3

let elementsHendles = await page.evaluateHandle(() => document.querySelectorAll('a'));
let elements = await elementsHendles.getProperties();
let elements_arr = Array.from(elements.values());