Getting records from DynamoDB recursively using Q.Promises

Question

I am having trouble implementing Q promises with recursive dynamodb call, new to nodejs and q, considering the limitations of the dynamodb to retrieve results, we need to run recursive query to get the required results.

normally we use the query with Q implementation something like this as

    function getDBResults(){

    var q = Q.defer();

    var params = {
        TableName: mytable,
        IndexName: 'mytable-index',

        KeyConditionExpression: 'id = :id',
        FilterExpression: 'deliveryTime between :startTime and :endTime',

        ExpressionAttributeValues: {
            ':startTime': {
                N: startTime.toString()
            },

            ":endTime": {
                N: endTime.toString()
            },
            ":id": {
                S: id.toString()
            }
        },
        Select: 'ALL_ATTRIBUTES',
        ScanIndexForward: false,
    };


     dynamodb.query(params, function(err, data) {
           if (err) {
               console.log('Dynamo fail ' + err);
               q.reject(err);
           } else {
               console.log('DATA'+ data);
               var results = data.Items;
               q.resolve(results);
           }
       });
      return q.promise;

}



getDBResults.then(
  function(data) {
    // handle data
  },
 function(err) {
        //handle error
  }
);

Using recursive query I can get the results but I need those results to be used in another function, but because of nodejs async nature,the next function calls happens already before the recursive query function finishes its job, now I want that I get all the results from the recursive query function and then get as a promise to a new function and finally handle all the data.

recursive query for dynamodb looks like this.

function getDBResults(){

    //var q = Q.defer();

     params = {
        TableName: mytable,
        IndexName: 'mytable-index',

        KeyConditionExpression: 'id = :id',
        FilterExpression: 'deliveryTime between :startTime and :endTime',

        ExpressionAttributeValues: {
            ':startTime': {
                N: startTime.toString()
            },

            ":endTime": {
                N: endTime.toString()
            },
            ":id": {
                S: id.toString()
            }
        },
        Select: 'ALL_ATTRIBUTES',
        ScanIndexForward: false,
    };



dynamodb.query(params, onQueryCallBack);


}


function onQueryCallBack(err, data) {
if (err) {
    console.log('Dynamo fail ' + err);
    console.error("Could not query db" + err);
} else {


if (typeof data.LastEvaluatedKey != "undefined") {
    console.log("query for more...");
    params.ExclusiveStartKey = data.LastEvaluatedKey;
    dynamodb.query(params, onQueryCallBack);
}
data.Items.forEach(function(item) {
    allResults.push(item);
});
//console.log('NO:OF Results:' + allResults.length);

//q.resolve(tickets);

//});

}

Now I want that I can get the results as promise finally so I can handle them in the next function like this.

getDBResults.then(
  function(data) {
    // handle data
  },
 function(err) {
        //handle error
  }
);

Please help me on this, sorry if its a stupid question but recursive calls with promises have made a hurdle for me.

Thanks

How would `onQueryCallBack` be able to resolve `q` when it's not in the scope to see the deferred? — Bergi, May 17 '17 at 07:54
possible duplicate of [How do I convert an existing callback API to promises?](http://stackoverflow.com/q/22519784/1048572) — Bergi, May 17 '17 at 07:55
Its not duplicate Bergi, problem is not converting callback API to promises but its more to recursive callback to promises, yes onQueryCallBack is not in the scope, I posted questions thats why to solve the problem when its a recursive calls — nsgulliver, May 17 '17 at 07:58

score 1 · Answer 1 · answered May 17 '17 at 09:15

First of all, keep the promisified function you already have. Use it as a building block for the recursive solution, instead of trying to alter it!

It might need two small adjustments though:

function getDBResults(startKey){
//                    ^^^^^^^^
    var q = Q.defer();
    var params = {
        ExclusiveStartKey: startKey,
//      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        … // rest as before
    };
    dynamodb.query(params, function(err, data) {
        if (err) {
           q.reject(err);
        } else {
           q.resolve(data);
//                   ^^^^ Not `data.Items`
        }
    });
    return q.promise;
}

Now we can use that to trivially implement the recursive solution:

function getRecursiveDBResults(key) {
    return getDBResults(key).then(function(data) {
        if (typeof data.LastEvaluatedKey != "undefined") {
            return getRecursiveDBResults(data.LastEvaluatedKey).then(items) {
                return data.Items.concat(items);
            });
        } else {
            return data.Items
        }
    });
}

thanks, for the solution, i figured out a solution already but your solution is also effective :), so i accept it — nsgulliver, May 17 '17 at 10:03

score 1 · Accepted Answer · answered May 17 '17 at 10:28

Here is how i solve the problem, Thanks Bergi for your solution as well

function getDBResults() {
    var q = Q.defer();
    var dynamodb = core.getDynamoDB();



    params = {
        TableName: mytable,
        IndexName: 'mytable-index',

        KeyConditionExpression: 'id = :id',
        FilterExpression: 'deliveryTime between :startTime and :endTime',

        ExpressionAttributeValues: {
            ':startTime': {
                N: startTime.toString()
            },

            ":endTime": {
                N: endTime.toString()
            },
            ":id": {
                S: id.toString()
            }
        },
        Select: 'ALL_ATTRIBUTES',
        ScanIndexForward: false,
    };


    var results = [];
    var callback = function(err, data) {
        if (err) {
            console.log('Dynamo fail ' + err);
            q.reject(err);
        } else if (data.LastEvaluatedKey) {
            params.ExclusiveStartKey = data.LastEvaluatedKey;
            dynamodb.query(params, callback);
        } else {
            q.resolve(results);
        }
        data.Items.forEach(function(item) {
            results.push(item);
        });

    }


    dynamodb.query(params, callback);

    return q.promise;
}

Getting records from DynamoDB recursively using Q.Promises

2 Answers2