How to slow down this Node looping http request?

Question

I'm trying to call this API with the request module about 200-300 times with a Lambda function. I need to add second between each call so I don't get a 429 response. I've tried a few different ways to make this happen, but it seems to ignore the code to slow it down.

How do people normally slow down these requests in AWS lambda? It would be great if I could insert something like utilities.sleep(1000) in the loop to make it wait a second before continuing. I'm sure there is a simple solution to this issue, but all the examples I've seen seem to make it complex.

function findProjects(items){

    var toggleData = [];

    for( var i = 0; i < items.length; i++ ){
        setTimeout( callToggle( items[i] ), 1000 );
    }

    function callToggle( data ){
        request({
            'url': 'https://www.toggl.com/api/v8/projects/' + data.toggle.data.id,
            'method': 'GET',
            'headers': {
                'Content-Type': 'application/json',
                'Accept': 'application/json'
            },
            'auth': {
                'user': 'xxxxxxx',
                'pass': 'api_token'
        }}, function( error, response, body ){
            if( error ) {

                console.log( error );
                context.done( null, error );
            } else {

                console.log(response.statusCode, "toggle projects were listed");
                var info = JSON.parse(body);
                toggleData.push(info);
            }
        });
    }

    findDocument( toggleData );   
}

Answer 1

You can do something like this:

for(var i = 0; i<items.length; i++){
    setTimeout(callToggl, 1000 + (( i * X ) % Y), items[i]);
}

Where Y is the max delay (1000 + Y) then you want (es 5 sec) and X is the timing for each call (es X=10 : 1000,1010,1020,1030,...

if you want 1s each call:

for(var i = 0; i<items.length; i++){
    setTimeout(callToggl(items[i]), 1000 + ( i * 1000 ));
}

EDIT

for(var i = 0; i<items.length; i++){
    setTimeout(callToggl, 1000 + ( i * 1000 ), items[i]);
}

Answer 2

You can chain the requests together:

function findProjects(items){

    var toggleData = [];

    // chain requests with delay
    items.reduce(function (requestChain, item) {
      return requestChain
        .then(callToggle.bind(null, item))
        .then(wait.bind(null, 1000));
    }, Promise.resolve());

    function wait (ms) {
      return new Promise(function (resolve, reject) {
        setTimeout(resolve, ms);
      });
    }

    function callToggle(data) {
        request({
            'url': 'https://www.toggl.com/api/v8/projects/' + data.toggle.data.id,
            'method': 'GET',
            'headers': {
                'Content-Type': 'application/json',
                'Accept': 'application/json'
            },
            'auth': {
                'user': 'xxxxxxx',
                'pass': 'api_token'
        }}, function( error, response, body ){
            if( error ) {

                console.log( error );
                context.done( null, error );
            } else {

                console.log(response.statusCode, "toggle projects were listed");
                var info = JSON.parse(body);
                toggleData.push(info);
            }
        });
    }

    findDocument( toggleData );   
}

Answer 3

While Node.js is single-threaded, setTimeout does not create a single stack of synchronous calls. When you use your for loop you immediately set each subsequent call to 1000ms ahead, so they all activate at roughly the same time. Instead, you likely want to use a third-party promise lib to wait.

You could do something as simple as:

const Bluebird = require('bluebird');
const $http = require('http-as-promised');
const _ = require('lodash');

const timeout = 1000;
const items = []; 
const headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' };
const auth = { 'user': 'xxxxxxx', 'pass': 'api_token' };

const makeRequest = (id) => {
  const url = 'https://www.toggl.com/api/v8/projects/' + id;
  return $http.get({ url, headers, auth }); 
};

const waitCall = (data) => {
    return Bluebird
    .resolve(makeRequest(data.toggl.data.id))
    .wait(timeout);
};

Bluebird.mapSeries(items, waitCall);

http://bluebirdjs.com/docs/api/promise.mapseries.html

Answer 4

As you probably know, javascript code does not block on io (unless using specific sync api's which block your entire code which is a bad practice that should be avoided unless you have a really good reason to block the entire code(loading config files on startup... etc...))

so what you need to do, is simply wait for the response

it used to be a bit complicated in the past to orchestrate it all, but currently, using --harmony flag (to activate latest js features in node) you can use the shiny new async functions syntax. (await/async)

the function you are running inside must be declared async, then after each iteration, you need to wait for the response of that http call using "await" keyword. this keyword makes the code seem as if its blocking and waiting for the resolved answer, although its not.

i used "fetch" instead of "request" because it plays nice with async functions (being promise based) but you can use any other method as long as you return a promise. (you can even promisify your existing callback based api with Promise object, but it will make everything look uglier, so please don't :) )

here is the modified code. I'm not really sure if it works as is. But the idea is pretty clear i think.

On any case, its a great opportunity for you to learn how to work with async functions in case you didn't use them already, they really make life easier.

//making enture function async, so you can use 'await' inside it
async function findProjects(items){

    var toggleData = [];

    for( var i = 0; i < items.length; i++ ){
        //setTimeout( callToggle( items[i] ), 1000 );
        //instead of using a timeout, you need to wait for response before continuing to next iteration
        await response = callToggle(items[i]);
        toggleData.push(JSON.parse(response.body));
    }

    async function callToggle( data ){
        /*request({
            'url': 'https://www.toggl.com/api/v8/projects/' + data.toggle.data.id,
            'method': 'GET',
            'headers': {
                'Content-Type': 'application/json',
                'Accept': 'application/json'
            },
            'auth': {
                'user': 'xxxxxxx',
                'pass': 'api_token'
        }}, function( error, response, body ){
            if( error ) {

                console.log( error );
                context.done( null, error );
            } else {

                console.log(response.statusCode, "toggle projects were listed");
                var info = JSON.parse(body);
                toggleData.push(info);
            }
        });*/
        // to make things simpler, use fetch instead of request which is promise based
        var myInit = {  'method': 'GET',
                        'headers': {
                            'Content-Type': 'application/json',
                            'Accept': 'application/json'
                        },
                        'auth': {
                            'user': 'xxxxxxx',
                            'pass': 'api_token'
                        }
                    };
        return fetch("https://www.toggl.com/api/v8/projects/",myInit);
    }

    findDocument( toggleData );   
}