I can't seem to find any documentation or Stack Overflow examples on how to rate limit bulk requests using axios.all to make concurrent requests. I've come across axiom-rate-limit on GitHub and I've found quite a few examples of how to implement that but nothing that shows how to implement it when using axios.all/axios.spread.
Here is my main function with axios.all
async function bulkAxios(url, idArray) {
var token = "BEARER-TOKEN"
const AuthStr = "Bearer ".concat(token);
for (var i in idArray) {
var config = {
method: "get",
url: url + idArray[i],
headers: { Authorization: AuthStr },
};
let newPromise = axios(config)
axiosArray.push(newPromise)
}
console.log("Initiating bulk promises for bulkAxios api calls.")
const p = Promise
.all(axiosArray)
.then(axios.spread((...responses) => {
responses = responses.map(item => item.data)
console.log('All Axios API calls succeeded.');
return(responses)
}))
.catch(function (error) {
console.log(error);
});
return(p);
}
I've tried adding
const apiRequest = rateLimit(axios.create(), { maxRequests: 2, perMilliseconds: 1000, maxRPS: 2 })
Then modifying my function to something like this?
async function bulkAxios(url, idArray) {
var token = "BEARER-TOKEN"
const AuthStr = "Bearer ".concat(token);
for (var i in idArray) {
var config = {
method: "get",
url: url + idArray[i],
headers: { Authorization: AuthStr },
};
let newPromise = http(config)
axiosArray.push(newPromise)
}
console.log("Initiating bulk promises for bulkAxios api calls.")
const p = Promise
.all(axiosArray)
.then(http.spread((...responses) => {
responses = responses.map(item => item.data)
console.log('All Axios API calls succeeded.');
return(responses)
}))
.catch(function (error) {
console.log(error);
});
return(p);
}
I get this error since it can't seem to access the .spread behind the rateLimit
.then(apiRequest.spread((...responses) => {
^
TypeError: apiRequest.spread is not a function
at bulkAxios (/Users/user/Desktop/app/api/index.js:160:26)
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async routeHandler (/Users/user/Desktop/app/api/index.js:209:25)
If I change
.then(http.spread((...responses)
Back to
.then(axios.spread((...responses)
It works but I can't tell if its actually be rate limited or not.
UPDATE 8/24
// limits axios api requests to 50 per second
const apiRequest = rateLimit(axios.create(), { maxRequests: 50, perMilliseconds: 1000 });
async function bulkAxios(url, idArray) {
var token = "BEARER-TOKEN"
const AuthStr = "Bearer ".concat(token);
for (var i in idArray) {
var config = {
method: "get",
url: url + idArray[i],
headers: { Authorization: AuthStr },
};
let newPromise = apiRequest(config);
axiosArray.push(newPromise)
}
console.log("Initiating bulk promises for bulkAxios api calls.")
const p = Promise
.all(axiosArray)
.then(axios.spread((...responses) => {
responses = responses.map(item => item.data)
console.log('All Axios API calls succeeded.');
return(responses)
}))
.catch(function (error) {
console.log(error);
});
return(p);
}