0

OK so I have been banging my head against a wall trying to figure this out.

I am using node.js, selenium webdriver, and firefox.

I want to loop thru a list of URLs and save the resulting HTML to my file system.

The way I have it set up now it will loop thru all of the list but doesn't give me any output until the entire routine is completed and I only get data from the last element in the array, but I get the appropriate number of files.

An example of the code logic that I am using is: 

var urlList = [link1,link2,link3];
var driver = new Builder().forBrowser('firefox').build();

function loopURL(){
    var loopPromises = [];
    urlList.forEach(url=>{
        loopPromises.push(new Promise((resolve,reject)=>{
        resolve(loadAndSaveResults(url))
    })
    return Promise.all(loopPromises);       
}

function loadAndSaveResults(url){
    return driver.wait(until.elementLocated(By.linkText(url)))
    .then(_=> driver.get(url))
    .then(_=> driver.getPageSource())
    .then(pageString => driver.wait(writeToFile(pageString))
}

async function writeToFile(pageString){
    await fs.writeFileSync(filepath,pageString);
{ 

driver.get('http://homepage/index.html')
.then(_=> loopURL())
.then(_=> driver.quit());

What am I missing?

Joshua Harrison
  • 1
  • 1
  • Tip: `urlList.forEach(url=>{ loopPromises.push(new Promise((resolve,reject)=>{ resolve(loadAndSaveResults(url)) }) return Promise.all(loopPromises);` is just a long way to write `return Promise.all(urlList.map(loadAndSaveResults)` :) – Benjamin Gruenbaum Aug 01 '18 at 16:26
  • See https://stackoverflow.com/questions/23803743/what-is-the-explicit-promise-construction-antipattern-and-how-do-i-avoid-it – Benjamin Gruenbaum Aug 01 '18 at 16:26
  • I tried that and unfortunately it kills my webdriver too soon, before it has the chance to get at any data – Joshua Harrison Aug 01 '18 at 17:02

0 Answers0