I have been working on this data gathering module, that is supposed to get data from different bitcoin markets and standardise all the received data so it can be inserted into a mongodb database for later use. (The module is written in node 4.3.x)
The problem I have is that I need consistency in how the data is represented in the database. So whenever a request times out, instead of catching the get request and log an error, I want to resolve a '0'. Additionally the received data contains trades that need to be cut in order. This needs to happen so that the trades can be cut properly, so data is not written twice. For that I have implemented two queues:
1: TimestampQueue - holds timestamps. The timestamp in [0] is the next expected response
2: objectQueue - holds received responses
=> as soon as the object in objectQueue[0] equals the timestamp in timestampQueue[0] => do data manipulation and insert into database.
The problem lies that the axios.get request that should catch a timeout doesn't do that consistently.
It happens after random timeframes, but on average the queue gets stuck after 2hrs.
To make things clearer here some important code snippets: httpclient making the axios request:
get(url) {
return this.instance.get(url) //instance just defined a timeout. Nothing special
.then(response => {
return response.data;
})
.catch(error => {
throw error; //THIS SEEMINGLY DOESN'T GET EXECUTED IN THE DESCRIBED CASE
});
}
Now the marketHandler that resolves the request:
getMethodFromMarket(method, market, timestamp){
if(this.markets[market]){
if(this.markets[market].methods[method]) {
var url = this.markets[market].methods[method];
let result = {};
result[method] = {};
result[method][market] = {};
return this.client.get(url)
.then(data => {
result[method][market] = data;
log.debug("Successfully received " + method + " for " + market + " : " + timestamp);
return result;
})
.catch(err => {
result[method][market] = 0;
log.error(new Error("Failed to get " + method + " for " + market + ": " + timestamp));
log.error(err);
return result;
});
} else{
return Promise.reject(new Error(method + " not available for " + market));
}
} else {
return Promise.reject(new Error("Market not specified in config"));
}
}
The code that makes the requests for all defined markets (for one method) and joins them in one object:
//returns promise to get *method* from all markets specified in
//config.json
getAllMarkets(method, timestamp){
let getMarketsPromises = [];
let result = {};
result[method] = {};
Object.keys(this.markets).forEach(market => {
result[method][market] = {};
getMarketsPromises.push(this.getMethodFromMarket(method, market, timestamp));
});
return Promise.all(getMarketsPromises)
.then(results => {
for(let i = 0; i < results.length; i++){
let market = Object.keys(results[i][method])[0];
result[method][market] = results[i][method][market];
}
log.debug("Got all markets for " + method + " for " + timestamp);
return result;
})
}
The code that makes the requests for all methods and markets and joins them in the final object that gets manipulated from a different module and inserted into the database:
//returns promise to get trades and depths from markets specified in
//config.json
getEverything(timestamp){
let getMethodPromises = [];
let result = {timestamp};
this.methods.forEach(method => {
result[method] = {};
getMethodPromises.push(this.getAllMarkets(method, timestamp))
});
return Promise.all(getMethodPromises)
.then(results =>{
for(let i = 0; i < results.length; i++){
let method = Object.keys(results[i])[0];
result[method] = results[i][method];
}
log.debug("Got everything for " + timestamp);
return result;
})
}
I have tested the whole process without any data manipulation. Only those functions and inserting it into the database.
The implementation of the 2 queues:
//handles the incoming responses from markets and sorts
//them according to their timestamp
queueResponse(marketInfo){
this.marketInfoQueue.push(marketInfo);
this.marketInfoQueue.sort(function(a, b){
return a.timestamp - b.timestamp;
})
}
//returns queued Responses in order of timestamps.
getQueuedResponses(){
var i = 0;
var results = [];
log.debug("TimestampQueue: "+ this.timestampQueue[0] + " | objectQueue: " + this.marketInfoQueue[0].timestamp);
while(this.marketInfoQueue[i] && this.timestampQueue[i] == this.marketInfoQueue[i].timestamp){
results.push(this.marketInfoQueue.shift());
this.timestampQueue.shift();
i++;
}
return results;
}
//pushes new timestamp into timestampQueue to keep
//incoming responses in order
queueTimestamp(timestamp){
this.timestampQueue.push(timestamp);
}
I have been trying to fix this problem for more than 3 weeks now, and I am absolutely clueless.
TLDR: Axios get request does not resolve or reject. Even though a timeout of 5000ms is defined in the instance used in the httpClient module.