How to achieve recursive Promise calls in Node.js

Question

I am calling an API where I can only fetch 1000 records per request, I was able to achieve this using recursion.

I am now trying to achieve the same using promises, I am fairly new to Node.js and JavaScript too.

I tried adding the recursion code in an if else block but failed

var requestP = require('request-promise');

const option = {
    url: 'rest/api/2/search',
    json: true,
    qs: {
        //jql: "project in (FLAGPS)",
    }
}

const callback = (body) => {

    // some code
    .
    .
    .//saving records to file
    .
    //some code
    if (totlExtractedRecords < total) {  

        requestP(option, callback).auth('api-reader', token, true)
     .then(callback)
    .catch((err) => {
        console.log('Error Observed ' + err)
    })
    }
}

requestP(option).auth('api-reader', token, true)
    .then(callback)
    .catch((err) => {
        console.log('Error Observed ' + err)
    })

I want to execute the method using promise and in a synchronous way, i.e. I want to wait until the records are all exported to a file and continue with my code

can you try making the `callback` as `async` function and `await` on the `requestP` function promise — nivendha, Sep 05 '19 at 11:00
You will find [this Q&A](https://stackoverflow.com/a/50121218/633183) relevant. If you have trouble adapting the techniques there to your particular case, lmk and I'll do my best to help you later today. — Mulan, Sep 05 '19 at 16:13

Amir Popovich · Accepted Answer · 2019-09-05T11:07:55.190

2

I think its better to create your own promise and simply resolve it when your done with your recursion. Here's a simply example just for you to understand the approach

async function myRecursiveLogic(resolveMethod, ctr = 0) {
      // This is where you do the logic
      await new Promise((res) => setTimeout(res, 1000)); // wait - just for example
      ctr++;
      console.log('counter:', ctr);

      if (ctr === 5) {
        resolveMethod(); // Work done, resolve the promise
      } else {
        await myRecursiveLogic(resolveMethod, ctr); // recursion - continue work
      }
    }

// Run the method with a single promise
new Promise((res) => myRecursiveLogic(res)).then(r => console.log('done'));

edited Sep 05 '19 at 11:07

answered Sep 05 '19 at 11:02

Amir Popovich

29,350
9
53
99

I have couple of questions on the above answer. 1. how resolveMethod works? 2. Can u explain me last line? usually we write res() in promise. – Prashant Tapase Sep 05 '19 at 12:33
Do you know what a promise is? If not, read here: https://medium.com/javascript-scene/master-the-javascript-interview-what-is-a-promise-27fc71e77261 .If you do, the resolveMethod is the method you call when you want to resolve the promise. The last line creates a new promise and passes the resolve method as a parameter to your recursive logic function. You now can write code with recursions and when you are done, you simply call the resolve method that was passed and the promise outside gets resolved. – Amir Popovich Sep 05 '19 at 13:27

Scaccoman · Answer 2 · 2019-09-05T13:50:49.647

Here's a clean and nice solution using the latest NodeJS features. The recursive function will continue executing until a specific condition is met (in this example asynchronously getting some data).

const sleep = require('util').promisify(setTimeout)

const recursive = async () => {
  await sleep(1000)
  const data = await getDataViaPromise() // you can replace this with request-promise

  if (!data) {
    return recursive() // call the function again
  }

  return data // job done, return the data
}

The recursive function can be used as follows:

const main = async () => {
  const data = await recursive()
  // do something here with the data
}

score 0 · Answer 3 · answered Sep 05 '19 at 19:03

Using your code, I'd refactored it as shown below. I hope it helps.

const requestP = require('request-promise');
const option = {
    url: 'rest/api/2/search',
    json: true,
    qs: {
        //jql: "project in (FLAGPS)",
    }
};

/* 
    NOTE: Add async to the function so you can udse await inside the function 
*/

const callback = async (body) => {

    // some code

    //saving records to file

    //some code

    try {
        const result = await requestP(option, callback).auth('api-reader', token, true);
        if (totlExtractedRecords < total) {
            return callback(result);
        }
        return result;
    } catch (error) {
        console.log('Error Observed ' + err);
        return error;
    }
}

score 0 · Answer 4 · answered Sep 09 '19 at 14:54

Created this code using feed back from Amir Popovich

const rp = require('Request-Promise')
const fs = require('fs')

const pageSize = 200

const options = {
    url: 'https://jira.xyz.com/rest/api/2/search',
    json: true,
    qs: {
        jql: "project in (PROJECT_XYZ)",            
        maxResults: pageSize,
        startAt: 0,
        fields: '*all'
    },
    auth: {
        user: 'api-reader',
        pass: '<token>',
        sendImmediately: true
    }
}



const updateCSV = (elment) => {
    //fs.writeFileSync('issuedata.json', JSON.stringify(elment.body, undefined, 4))
}


async function getPageinatedData(resolve, reject, ctr = 0) {
    var total = 0

    await rp(options).then((body) => {
        let a = body.issues
        console.log(a)
        a.forEach(element => {
            console.log(element)
            //updateCSV(element)
        });

        total = body.total

    }).catch((error) => {
        reject(error)
        return
    })

    ctr = ctr + pageSize
    options.qs.startAt = ctr

    if (ctr >= total) {
        resolve();
    } else {
        await getPageinatedData(resolve, reject, ctr);
    }
}

new Promise((resolve, reject) => getPageinatedData(resolve, reject))
    .then(() => console.log('DONE'))
    .catch((error) => console.log('Error observed - ' + error.name + '\n' + 'Error Code - ' + error.statusCode));

How to achieve recursive Promise calls in Node.js

4 Answers4