Why isnt the final console.log showing the whole data even after waiting for all promises to resolve?

Question

Edit

So if I set a setTimeout on my final console.log, it will log fine, but I guess my new question is although, even with the then statements, why isnt the it triggering after everything is resolved?

I am having an odd bug that I cant figure out why it is not printing the whole structure that I am expecting. I am not getting any errors, and I did try putting in a few console.log, but just get seem to get the anyOf property to log. It always shows empty.

My main issue is that hold is not logging correctly. It is showing anyOf as empty.

const $ = require('cheerio');
const Axios = require('axios').default;
let name = 'sqs';
let basePath = '/AWS_SQS.html'


let hold = {
    '$schema': 'http://json-schema.org/draft-07/schema#',
    '$id': `cf${name}`,
    'description': `CF ${name}`,
    'properties': {
        'Type': {
            'type': 'string',
            'enum': [],
        },
    },
    'anyOf': [] // this keeps coming up empty
};

let typeProperties = (t, p) => {
    return {
        'if': { 'properties': { 'Type': { 'const': t } } },
        'then': {
            'properties': {
                'Properties': {
                    'type': 'object',
                    'properties': p,
                },
            },
        },
    }
}

const makeRequest = (bpath) => {
    let url = 'https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/' + bpath
    return Axios.get(url)
        .then((res) => {
            return res.data;
        })
        .catch(console.log);
};

let getData = makeRequest(basePath).then((data) => {
    let match = $('.listitem>p>a', data);
    match.map((i, e) => {
        let t = $(e);
        hold.properties.Type.enum.push(t.text());
        makeRequest(t.attr('href')).then((newdata) => {
            let holdProperties = {}
            let pType = $($('.variablelist', newdata)[0])
            $('.term>.code', pType).map((i, elem) => {
                let propertyName = $(elem).text()
                holdProperties[propertyName] = {}
            })
            hold.anyOf.push(typeProperties(t.text(), holdProperties))
        }).catch(console.log)
    });
}).catch(console.log)

getData.then(() => {
    // Why doesnt this log fully once everything is resolved? If i put a setTimeout here, it will work fine
    console.log(JSON.stringify(hold));
}).catch(console.log)

Answer 1

Inside of the code associated with getData(), you are doing makeRequest().then() inside a .map(), but completely ignoring the promises that makeRequest() is returning. You're not waiting for them. You need to use Promise.all() to wait on all of them so you can return a promise that is linked to all those other promises finishing.

You can do so like this:

let getData = makeRequest(basePath).then((data) => {
    let match = $('.listitem>p>a', data).get();
    return Promise.all(match.map(item => {
        let t = $(item);
        hold.properties.Type.enum.push(t.text());
        return makeRequest(t.attr('href')).then((newdata) => {
            let holdProperties = {}
            let pType = $($('.variablelist', newdata)[0])
            $('.term>.code', pType).map((i, elem) => {
                let propertyName = $(elem).text()
                holdProperties[propertyName] = {}
            })
            hold.anyOf.push(typeProperties(t.text(), holdProperties))
        }).catch(err => {
            console.log(err);
            throw err;              // propagate error back to caller
        });
    }));
}).catch(console.log)

getData.then(() => {
    // Why doesnt this log fully once everything is resolved? If i put a setTimeout here, it will work fine
    console.log(JSON.stringify(hold));
}).catch(console.log)

Changes in this code:

1. Inside the .map() add return so you have return makeRequest(...) so your .map() will return an array of promises.
2. With that array of promises from .map() use return Promise.all(match.map(...)) so you're waiting for all the promises from the .map() and so you're returning that summary promise.
3. Get a real array from jQuery/cheerio using .get() so you can use real ES6 iteration operations with it.
4. Then switch the arguments to .map() to match what an array.map() uses.

Note, your hold.anyOf array will have results in no guaranteed order since all your parallel makeRequest() calls could finish in any order. If you want them in order, you should return the value from the .then() handler and then Promise.all() will accumulate all the values in order and you can then use them from the array that is the resolved value of Promise.all() where they will be in the order you requested them.