Syncano Codebox - Call API - parse JSON - get Reference - Save new Objects

Question

I am using Syncano as a baas, where I am trying to call an external API to receive a JSON array. This JSON needs to be parsed and afterwards stored in syncano. Before that I need to receive the reference object from the DB to link it to the new team object.

I receive the team (json) array & reference object successfully. But I am unable to get the new data stored, as only 12-14 teams (has to be 18) get saved.

I tried this & that with promises but it didn´t work out. Anyone a good advise how to rewrite the code to store all data? Thank you - here is what I have so far...

//TODO: get from ARGS when executing this codebox
var teamKey = 394;
var requestURL = 'http://api.football-data.org/v1/soccerseasons/' + teamKey + "/teams";

var request = require("request");
var Syncano = require('syncano');
var Promise = require('bluebird');
var account = new Syncano({
  accountKey: "abc"
});
var promises = [];

//from: http://docs.syncano.io/v1.0/docs/data-objects-filtering
//"_eq" means equals to
var filter = {
  "query": {
    "apikey": {
      "_eq": apiKey
    }
  }
};

request({
  headers: {
    'X-Auth-Token': 'abc'
  },
  url: requestURL,
  'Content-Type': 'application/json;charset=utf-8;',
  method: 'GET',
}, function(error, response, body) {
  if (error) {
    console.log(error);
  } else {
    var json = JSON.parse(body);
    var teamArray = json.teams;
    var newObject;

    account.instance('instance').class('competition').dataobject().list(filter)
    .then(function(compRes) {

        var competitionID = compRes.objects[0].id;
        for (var i = 0; i < teamArray.length; i++) {
          newObject = {
            "name": teamArray[i].name,
            "nameshort": teamArray[i].code,
            "logo": teamArray[i].crestUrl,
            "competition": competitionID
          };

          (account.instance('instance').class('teams').dataobject().add(newObject).then(function(res) {
              console.log(res);
            }).catch(function(err) {
              console.log("Error eq: " + err);
            })
          );
        }
      }).catch(function(err) {
        console.log(err);
      });
  }
});

Do you have the execution time of the current version of code? — Kelly J Andrews, Feb 22 '16 at 20:46
Your code looks basically ok - i think it may be the 15 req/second you are bumping into here. — Kelly J Andrews, Feb 22 '16 at 20:55
@Burkart Yeah, it seems like you're hitting the rate limit of a free account, for Builder (free) it's 15 req/s. I tried your example and I'm getting `Error: Error: "{\"detail\":\"Request was throttled. Expected available in 1.0 second.\"}"` for some requests. — Mariusz Wiśniewski, Feb 23 '16 at 01:35

mido · Accepted Answer · 2016-02-23T13:09:37.757

The issue might be you are finishing the request process before all the save calls are made, you could try Promise.all():

    account.instance('instance').class('competition').dataobject().list(filter)
    .then(function(compRes) {

        var competitionID = compRes.objects[0].id, promises=[];
        for (var i = 0; i < teamArray.length; i++) {
          newObject = {
            "name": teamArray[i].name,
            "nameshort": teamArray[i].code,
            "logo": teamArray[i].crestUrl,
            "competition": competitionID
          };
          promises.push(account.instance('instance').class('teams').dataobject().add(newObject).then(function(res) {
              console.log(res);
            }).catch(function(err) {
              console.log("Error eq: " + err);
            })
          );
        }
        return Promise.all(promises);
      }).catch(function(err) {
        console.log(err);
      });

if too many parallel calls at a time is the issue then chain one call after other:

    account.instance('instance').class('competition').dataobject().list(filter)
    .then(function(compRes) {

        var competitionID = compRes.objects[0].id, promise = Promise.resolve();
        function chainToPromise(promise, teamObj, waitTime){
          waitTime = waitTime || 500;
          return promise.then(function(){
            return new Promise(function(resolve, reject){
              setTimeout(resolve, waitTime);
            });
          }).then(function(){
            return account.instance('instance').class('teams').dataobject().add(teamObj);
          }).then(function(res) {
            console.log(res);
          }).catch(function(err) {
            console.log("Error eq: " + err);
          });
        }
        for (var i = 0; i < teamArray.length; i++) {
          newObject = {
            "name": teamArray[i].name,
            "nameshort": teamArray[i].code,
            "logo": teamArray[i].crestUrl,
            "competition": competitionID
          };
           promise = chainToPromise(promise, newObject);
        }
        return promise;
      }).catch(function(err) {
        console.log(err);
      });

For the first one I get: "[TypeError: Cannot read property 'push' of undefined]" - currently I can´t figure out where this comes from. For the second one, one team gets stored multiple times, It seems that the loop runs 18 times and afterwards the add() method runs only for 1 object multiple times. However, I still get the "Request was throttled" message. — Burkart, Feb 23 '16 at 12:54
@Burkart my bad, updated code, first one I did not initialize `promises`, second code had a closure issue, also I added a gap of 500ms between two save calls in second method... — mido, Feb 23 '16 at 13:11

Syncano Codebox - Call API - parse JSON - get Reference - Save new Objects

1 Answers1