How to use Promises to send an array of request results?

Question

I want to set an express server that returns the results of the queries of an array of items. I have read in this question that I can use Promise.each for this...

What I intend my code to do is:

Web Scrape a page with a list of movies and return the titles as an array.
Use the title of the movies to do a request of an API for each
Return a response to the client with the results of the API response to every movie

Here's my code:

var express = require('express');
var app = express();

var Xray = require('x-ray');
var x = Xray();

app.get('/', function (req,res){
    var username = 'medicengonzo';
    var pages = 3;

    x('https://letterboxd.com/'+username+'/watchlist/', 'li.poster-container', [{
            movie: 'img@alt'
            }])(function (err,results){
        console.log('Entered x-ray callback fn');
        results.forEach(function(result, index){
            console.log('Iteration');
            console.log(results.length);
            request('http://netflixroulette.net/api/api.php?title='+encodeURIComponent(result.movie),
                    function(err, response, body){
                        movies = [];
                        console.log(count);
                        obj = JSON.parse(body);
                        if(obj.errorcode != 404){
                            movies.push('Movie found: '+obj.show_title, 'ID: '+obj.show_id);
                        }
                        else{
                            movies.push('No movie found');
                        }
                            res.send(movies);
                    }
            );
        });
    })
    .paginate('.paginate-current+li a@href')
    .limit(pages);
});

app.listen(3000, function(){
    console.log('Listening on port 3000!');
});

It's not clear exactly what your code intends to do. Right now, you are doing a `res.send(movies)` inside your `results.forEach()` loop so you can be doing that multiple times for the same request which is definitely a coding error and only the first one will actually be sent as a response. So, what result are you supposed to send as the response? — jfriend00, Sep 24 '17 at 16:32
@BorisCharpentier I'm new to promises, so I don't quite understand how `Promise.all` works... — medicengonzo, Sep 24 '17 at 17:02
@jfriend00 What I'm trying to do is request an API for each of the results I get from a web scrapping. And when all api requests are done send the results to the express server — medicengonzo, Sep 24 '17 at 17:04
But you re-initialize `movies = []` inside your loop so you aren't accumulating movies. Why? Do you mean to be initializing that only once before the loop and accumulating results in the loop? — jfriend00, Sep 24 '17 at 17:26

score 0 · Answer 1 · answered Sep 25 '17 at 22:57

So... After a bit of Googling and a lot of learning about Promises and Chaining Promises I got the code to work:

var express = require('express');
var app = express();

var Xray = require('x-ray');
var x = Xray();

var Promise  = require('bluebird');
var request = require('request');

var debug = true;

app.get('/', function (req,res){

    if(debug) console.log('Page requested!');

    var username = 'medicengonzo';
    var pages = 3;

    var scrapeMovies = function(){
        if(debug) console.log('scrapeMovies()');
        return new Promise((resolve,reject) => {
            x('https://letterboxd.com/'+username+'/watchlist/', 'li.poster-container', [{
                    title: 'img@alt'
                    }])((err, results) => {
                    console.log('x()');
                if(err) reject(err);
                resolve(results);
            })
            .paginate('.paginate-current+li a@href')
            .limit(pages);

        });
    };

    scrapeMovies().then(
            movies => {
        if(debug) console.log('scrapeMovies() resolved');
        return new Promise.all(movies.map((movie, index) => {
            return new Promise((resolve, reject)=> {
                if(debug) console.log('Requesting: ', movie.title);
                request('http://netflixroulette.net/api/api.php?title='+encodeURIComponent(movie.title), (err, response, body) => {
                        if(debug) console.log('Flixroulette requested');
                        var flixMovie = JSON.parse(body);
                        if(flixMovie.errorcode != 404){
                            resolve('Movie found: '+flixMovie.show_title, 'ID: '+flixMovie.show_id);
                        }
                       else{
                            resolve('No movie found...');
                        }
                });
            });
        }));
    }).then((movies) => res.send(movies)).catch((err) => console.log(err));

});

app.listen(3000, function(){
    console.log('Listening on port 3000!');
});

First, I define a function that is going to scrape the movie titles I need and return a promise with an array of said titles: scrapeMovies().

After that I chain that promise to another promise that contains the function that is going to make the requests for me. Instead of array.forEach I used array.map so that it returned an array of promises each one resolving with the data from the request and at the end I res.send() the resolve data from that Promise.all() which is an array with all the data.

How to use Promises to send an array of request results?

1 Answers1