Log that bottom scroll has been reached

Question

I am using the code below to scroll all the way to the bottom of a YouTube page and it works. My question is after the site is scrolled down to the bottom how can I console.log that the bottom have been reached?

note: the solution should work with youtube.com. I have already tried getting the document height and compared it with the scroll height but that didn't work!

const puppeteer = require('puppeteer');

let thumbArr = []
const scrapeInfiniteScrollItems = async(page) => {
  while (true) {
    const previousHeight = await page.evaluate(
      "document.querySelector('ytd-app').scrollHeight"
    );
    await page.evaluate(() => {
      const youtubeScrollHeight =
        document.querySelector("ytd-app").scrollHeight;
      window.scrollTo(0, youtubeScrollHeight);
    });
    await page.waitForFunction(
      `document.querySelector('ytd-app').scrollHeight > ${previousHeight}`, {
        timeout: 0
      }
    );

    const thumbnailLength = (await page.$$('ytd-grid-video-renderer')).length
    //this logs the amount of thumbnails every loop but once bottom scroll has        been reached it stops logging (obviously) but the question is how am I supposed to compare the last amount of thumbnail's found with total thumbnails once the loop has stopped running. Take a look below to better understand my question.
    thumbArr.push(thumbnailLength)

    if (thumbnailLength == thumbArr.at(-1)) {
      console.log('bottom has been reached')
    }

    await page.waitForTimeout(1000)
  }
};

(async() => {
  const browser = await puppeteer.launch({
    headless: false
  });
  const page = await browser.newPage();
  await page.goto('https://www.youtube.com', {
    waitUntil: 'networkidle2',
  });

  await scrapeInfiniteScrollItems(page)
})();

UPDATE:

let clientHeightArr = []
let clientHeightArrTracker = []
const scrapeInfiniteScrollItems = async(browser, page) => {
  var infiniteScrollTrackerInterval = setInterval(async() => {
    clientHeightArrTracker.push(clientHeightArr.length)
    if (clientHeightArrTracker.some((e, i, arr) => arr.indexOf(e) !== i) == true) {
      clearInterval(infiniteScrollTrackerInterval)
      console.log('Bottom is reached')
      //causes error "ProtocolError: Protocol error (Runtime.callFunctionOn): Target closed."
      await browser.close()
    }
  }, 2000)
  while (true) {
    const previousHeight = await page.evaluate(
      "document.querySelector('ytd-app').scrollHeight"
    );

    await page.evaluate(() => {
      const youtubeScrollHeight =
        document.querySelector("ytd-app").scrollHeight;
      window.scrollTo(0, youtubeScrollHeight);
    });

    await page.waitForFunction(
      `document.querySelector('ytd-app').scrollHeight > ${previousHeight}`, {
        timeout: 0
      },
    );

    const clientHeight = await page.$$eval("ytd-app", el => el.map(x => x.clientHeight));
    clientHeightArr.push(clientHeight[0])
    await page.waitForTimeout(1000)
  }
};

(async() => {
  const browser = await puppeteer.launch({
    headless: false
  });
  const page = await browser.newPage();
  await page.goto('https://www.youtube.com/c/mkbhd/videos', {
    waitUntil: 'networkidle2',
  });

  await scrapeInfiniteScrollItems(browser, page)
})();

Answer 1

There are a ton of possible approaches here. After messing with it for a bit I arbitrarily landed on the following strategy and decided it was sufficient for starters:

const puppeteer = require("puppeteer"); // ^19.0.0

let browser;
(async () => {
  browser = await puppeteer.launch();
  const [page] = await browser.pages();
  const ua =
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36";
  await page.setUserAgent(ua);
  await page.setRequestInterception(true);
  page.on("request", req => {
    if (req.resourceType() === "image") {
      req.abort();
    }
    else {
      req.continue();
    }
  });
  const url = "https://www.youtube.com/c/mkbhd/videos";
  await page.goto(url, {waitUntil: "domcontentloaded"});
  await page.waitForSelector("#video-title");
  let thumbs = [];

  for (;;) {
    try {
      await page.waitForFunction(
        `${thumbs.length} !==
          document.querySelectorAll("#video-title").length`,
        {timeout: 10000}
      );
    }
    catch (err) {
      break;
    }

    thumbs = await page.$$eval("#video-title", els => {
      els.at(-1).scrollIntoView();
      return els.map(e => e.textContent);
    });
  }

  thumbs = thumbs.filter(Boolean);
  console.log("total length", thumbs.length);
  console.log("first 10 thumbs:", thumbs.slice(0, 10));
  console.log("last 10 thumbs:", thumbs.slice(-10));
})()
  .catch(err => console.error(err))
  .finally(() => browser?.close());

Output:

total length 1460
first 10 thumbs: [
  'iPhone 14/Pro Impressions: Welcome to Dynamic Island!',
  'Google Pixel Buds Pro Review: Just Get These!',
  'The Hyundai IONIQ 5: I Get It Now!',
  'Android 13 Hands-On: Top 5 Features!',
  'Dope Tech: The Most Extreme Gaming Monitor!',
  'Samsung Z Fold 4/ Flip 4 Impressions + Watch 5 Pro!',
  'Best Back to School Tech 2022!',
  'OnePlus 10T Impressions: Somebody That You Used to Know',
  'Asus Zenfone 9: The New Compact King!',
  "The iPad's Odd New Feature"
]
last 10 thumbs: [
  'HQ Tutorial: Rocket Dock Application',
  'Camstudio Clarity',
  'Tutorial: Camstudio HQ',
  'HQ Tutorial: Ccleaner',
  '15 Year old Golf Swing Analysis',
  'Fraps HD Test in 1080p (18 WOS)',
  'HP Pavilion dv7t Media Center Remote Overview',
  'High fps LG Voyager footage',
  '14 Year knock-down shot (11 Handicap)',
  '13-Year-Old Golf Swing Analysis'
]

This code keeps an array of all thumbs collected so far. In the loop, I first check to see if there are new thumbs--if no new thumbs show up in 10 seconds or so, break and assume we've collected all of the thumbs. Otherwise, scroll the last thumb into view and collect all of the thumbs, extending the result list.

The "wait 10 seconds for new thumbs, then quit" is technically a race condition. If there's some lag, it could trigger a false positive, but it seems good enough for starters. Extending the timeout too generously keeps the script from exiting when the scrape is over, so it's not great for scraping lots of channels with few videos. Maybe better is to wait for the /browse API response after triggering each scroll. This response object has all sorts of additional data in it.

See also Puppeteer - scroll down until you can't anymore