2

I was just exploring Google Cloud Functions for Firebase (Have the Blaze pay as you go pricing plan enabled).

wanted to test by creating a project that uses TMDB API and copies all the pages and stores the JSON data to firebase database

here is my code

// Array to hold async tasks
var asyncTasks = [];

var page = 0;
var total_pages = 630;

console.log('Started fetching Movies');
console.log('Total Pages ' + total_pages);

// Loop total no of pages times
for (var i = 0; i < total_pages; i++) {
    // We don't actually execute the async action here
    // We add a function containing it to an array of "tasks"
    asyncTasks.push(function (callback) {
        page++;
        console.log("Procesing Page " + page);

        var url = base_url + page;
        client.get(url, function (data, response) {

            // Get rateLimit from Raw Header
            var ratelimit = response.rawHeaders[27];

            // console.log(response.rawHeaders[26] + ' ' + response.rawHeaders[27]);
            if (ratelimit == 2) {
                // Rate Limit Exceeded wait 10 seconds
               console.log("Rate Limited Exceeded");

               // Sleep 10 Seconds to reset Rate Limit
               sleep.sleep(10);
            }

            global.all_data = global.all_data.concat(data.results);
            callback();
        }).on('error', function (err) {
            console.log('Something went wrong on the request', err.request.options);
            problem_pages.push(err.request.options.href);
            // Retrying once more before failure
            var eurl = err.request.options.href;
            client.get(eurl, function (data, response) {
                console.log("Retrying " + eurl);
                global.all_data = global.all_data.concat(data.results);
            });
        });
    });
}
const limit = functions.config().exec.limit;
console.log("Starting || execution [limit" + limit + "]");
async.parallelLimit(asyncTasks, limit, movie_results);
});

Here is the movie_results functions that executes at the end

function movie_results () {
var moviesRef = defaultDatabase.ref("v1/movies");
var statusRef = defaultDatabase.ref("v1/status");

//moviesRef.remove();
var data_json = {};

console.log("Total Movies fetched " + global.all_data.length);

async.each(global.all_data,
    function(mov, cb) {
        data_json[mov.id] = mov;
        cb();
        },
    function() {
        console.log("Result post-processing done");
        console.log("Total =" + data_json.length + " | Expected =" + data_info.total_resu`enter code here`lts);
        console.log("Uploading to Firebase Datastore");
        moviesRef.update(data_json);

        console.log("Finished fetching movies");

        // Update status for this job on status DB
        var datim = new Date();
        statusRef.update({ datim: {"records_fetched": data_json.length, "records_expected": data_info.total_results,
                                   "total_pages": data_info.total_pages, "timestamp": datim}
                        });

        // Closing all db references
        moviesRef.off();
        statusRef.off();
    }
);
}

The issue is that this executes ok most of the times...

As you can see i have added logging to let me know when a function in asynTask is executed

console.log("Procesing Page " + page);

in the Google Cloud logs i see

gcloud beta functions logs read --limit 20 --execution-id 118384524692947

I gets stuck at random pages (whenever it gets stuck.) I understand These features are still in BETA. So can somebody comment that they see the same issue as me. Is there any way to check if there is any Cloud Function Process Executing currently or not and know if is has been killed. I don't think its timing out. Thanks

I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:22.286  Procesing Page 223
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:22.786  Procesing Page 224
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:23.384  Procesing Page 225
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:23.984  Procesing Page 226
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:25.084  Procesing Page 227
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:25.484  Procesing Page 228
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:26.184  Procesing Page 229
I      tmdbUpdatePubSub  118384524692947  2017-04-26 09:47:27.284  Procesing Page 230
Hani Q
  • 135
  • 2
  • 15
  • Did you ever solve? I am experiencing similar issue: https://stackoverflow.com/questions/47137368/google-cloud-function-finishes-execution-but-never-enters-callbacks – Charlton Provatas Nov 07 '17 at 02:10

0 Answers0