0

I can't seem to find any documentation or Stack Overflow examples on how to rate limit bulk requests using axios.all to make concurrent requests. I've come across axiom-rate-limit on GitHub and I've found quite a few examples of how to implement that but nothing that shows how to implement it when using axios.all/axios.spread.

Here is my main function with axios.all

async function bulkAxios(url, idArray) {
    var token = "BEARER-TOKEN"
    const AuthStr = "Bearer ".concat(token);
    for (var i in idArray) {
        var config = {
            method: "get",
            url: url + idArray[i],
            headers: { Authorization: AuthStr },
        };
        let newPromise = axios(config)
        axiosArray.push(newPromise)
    }
    console.log("Initiating bulk promises for bulkAxios api calls.")
    const p = Promise
        .all(axiosArray)
        .then(axios.spread((...responses) => {
        responses = responses.map(item => item.data)
        console.log('All Axios API calls succeeded.');
        return(responses)
        }))
        .catch(function (error) {
            console.log(error);
        });
    return(p);
}

I've tried adding

const apiRequest = rateLimit(axios.create(), { maxRequests: 2, perMilliseconds: 1000, maxRPS: 2 })

Then modifying my function to something like this?

async function bulkAxios(url, idArray) {
    var token = "BEARER-TOKEN"
    const AuthStr = "Bearer ".concat(token);
    for (var i in idArray) {
        var config = {
            method: "get",
            url: url + idArray[i],
            headers: { Authorization: AuthStr },
        };
        let newPromise = http(config)
        axiosArray.push(newPromise)
    }
    console.log("Initiating bulk promises for bulkAxios api calls.")
    const p = Promise
        .all(axiosArray)
        .then(http.spread((...responses) => {
        responses = responses.map(item => item.data)
        console.log('All Axios API calls succeeded.');
        return(responses)
        }))
        .catch(function (error) {
            console.log(error);
        });
    return(p);
}

I get this error since it can't seem to access the .spread behind the rateLimit

.then(apiRequest.spread((...responses) => {
                 ^
TypeError: apiRequest.spread is not a function
at bulkAxios (/Users/user/Desktop/app/api/index.js:160:26)
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async routeHandler (/Users/user/Desktop/app/api/index.js:209:25)

If I change

.then(http.spread((...responses)

Back to

.then(axios.spread((...responses)

It works but I can't tell if its actually be rate limited or not.

UPDATE 8/24

// limits axios api requests to 50 per second
const apiRequest = rateLimit(axios.create(), { maxRequests: 50, perMilliseconds: 1000 });

async function bulkAxios(url, idArray) {
    var token = "BEARER-TOKEN"
    const AuthStr = "Bearer ".concat(token);
    for (var i in idArray) {
        var config = {
            method: "get",
            url: url + idArray[i],
            headers: { Authorization: AuthStr },
        };
        let newPromise = apiRequest(config);
        axiosArray.push(newPromise)
    }
    console.log("Initiating bulk promises for bulkAxios api calls.")
    const p = Promise
    .all(axiosArray)
    .then(axios.spread((...responses) => {
        responses = responses.map(item => item.data)
        console.log('All Axios API calls succeeded.');
        return(responses)
    }))
    .catch(function (error) {
        console.log(error);
    });
    return(p);
}
kiddslopp
  • 95
  • 1
  • 8
  • I'm not sure what the right answer to your question is, but I just wanted to mention that the express-rate-limit npm package is not designed for this scenario, it's designed to be used with the express web server. – Nathan Friedly Jul 23 '23 at 15:33
  • 1
    I was actually able to resolve this by simply changing http.spread to axios.spread. Updated original post with working code. – kiddslopp Aug 24 '23 at 20:55

0 Answers0