Concatenating JSON objects with callbacks and exporting result

Question

Full disclosure, I am new to all things NodeJS and Express. However, the below code works and is tested, so that should be a good place to start.

I am trying to scrape various pages on the same website, pulling data into a big JSON object from each page. Then I would like to amalgamate all those JSON objects into one, even bigger JSON object, then I would like to export it so I can call it in another file.

Here is my dataFetch.js:

var request = require('request'),
jsdom = require('jsdom');

var PlayerData = [];

var QBFunction = jsdom.env({
    url: 'http://www.fantasypros.com/nfl/projections/qb.php',
    scripts: ["http://code.jquery.com/jquery.js"],
    done: function (errors, window) {
        var $ = window.$;
        var table = $("tbody:gt(0)").hide();
        var QBObject = [];
        table.find("tr").each(function (i, el) {
            var $tds = $(this).find('td');
            QBObject.push({
                'position': 'QB',
                'name': $tds.eq(0).text(),
                'passing_attempts': $tds.eq(1).text(),
                'completions': $tds.eq(2).text(),
                'passing_yards': $tds.eq(3).text(),
                'passing_tds': $tds.eq(4).text(),
                'interceptions': $tds.eq(5).text(),
                'rushing_attempts': $tds.eq(6).text(),
                'rushing_yards': $tds.eq(7).text(),
                'rushing_tds': $tds.eq(8).text(),
                'fumbles_lost': $tds.eq(9).text(),
                'standard_projection': $tds.eq(10).text()
            });
        });
    }
})

exports.PlayerData = PlayerData;

Above, the QBObject is an example of the initial JSON object I create when scraping the webpage. The idea is that I do this for each position so imagine that I have this same code for 5 different positions (I left it out for brevity). The QBObject is getting created perfectly.

My question, when and how do I combine the QBObject with the PlayerObject, keeping in mind that I'll have to also combine it with a RBObject, etc...? I am confused because the QBObject is being created in a callback, and I'm new to asynchronous programming. I tried concatenating the JSON objects in the callback and printing it out to console, but it was empty (I'm assuming because the console log was executed before the JSON objects were created).

Finally, if all goes as planned and the PlayerObject JSON is created as intended, will a simple

var dataFetch = require('./dataFetch');
var PlayerData = dataFetch.PlayerObject;

suffice to bring that data into another javascript file? Thanks for anyone's help, anything is appreciated.

well, for one, exports.Playerdata isn't going to work. you're going to want to export a function that accepts a callback, or a promise that gets resolved when the data is available. — Kevin B, Jul 29 '14 at 18:31

score 0 · Accepted Answer · edited May 23 '17 at 12:11

You may find these discussions useful:
How do JavaScript closures work?
Idiomatic way to wait for multiple callbacks in Node.js

An initial attempt to solve this problem (that I'd likely soon refactor) might look something like:

var request = require('request');
var jsdom = require('jsdom');

function fetchPlayerData(done) {
    var playerObject;

    function gotData() {
       if ( playerObject.RBObject && playerObject.QBObject ) {
          //if all went well, we should have data for all positions
          done(playerObject);
       }
    }

    function qbReady(errors, window) {
        //existing code for scraping except...
        playerObject.QBObject = {
            position: 'qb',
            name: $tds.eq(0).text()
        }; 
        //we write to the player object instead of a local
        gotData();
    } 

    function rbReady(errors, window) {
        //same for other positions
        playerObject.RBObject = {position: 'rb'}; 
        gotData();   
    }

    jsdom.env({
        url: 'http://www.fantasypros.com/nfl/projections/qb.php',
        scripts: ["http://code.jquery.com/jquery.js"],
        done: qbReady 
    });

    jsdom.env({
        url: 'http://www.fantasypros.com/nfl/projections/rb.php',
        scripts: ["http://code.jquery.com/jquery.js"],
        done: rbReady 
    });
}

module.exports = {
    fecthPlayerData: fetchPlayerData
};

And then the usage would be something like:

var fetch = require('./dataFetch');

fetch.fetchPlayerData(function(data){
    console.log(data.QBObject);
});

Wow, thanks for both the code and the information so that I can catch my own fish. Works like a charm, the only difference between my working code and yours is that I populated the QBObject like I have in my question. Then after that loop, I did `playerObject.QBObject = QBObject`, basically the same thing, but just wanted to write it down in case someone else looks at this question. Thanks again — , Jul 29 '14 at 20:15

Concatenating JSON objects with callbacks and exporting result

1 Answers1