I was just exploring Google Cloud Functions for Firebase (Have the Blaze pay as you go pricing plan enabled).
wanted to test by creating a project that uses TMDB API and copies all the pages and stores the JSON data to firebase database
here is my code
// Array to hold async tasks
var asyncTasks = [];
var page = 0;
var total_pages = 630;
console.log('Started fetching Movies');
console.log('Total Pages ' + total_pages);
// Loop total no of pages times
for (var i = 0; i < total_pages; i++) {
// We don't actually execute the async action here
// We add a function containing it to an array of "tasks"
asyncTasks.push(function (callback) {
page++;
console.log("Procesing Page " + page);
var url = base_url + page;
client.get(url, function (data, response) {
// Get rateLimit from Raw Header
var ratelimit = response.rawHeaders[27];
// console.log(response.rawHeaders[26] + ' ' + response.rawHeaders[27]);
if (ratelimit == 2) {
// Rate Limit Exceeded wait 10 seconds
console.log("Rate Limited Exceeded");
// Sleep 10 Seconds to reset Rate Limit
sleep.sleep(10);
}
global.all_data = global.all_data.concat(data.results);
callback();
}).on('error', function (err) {
console.log('Something went wrong on the request', err.request.options);
problem_pages.push(err.request.options.href);
// Retrying once more before failure
var eurl = err.request.options.href;
client.get(eurl, function (data, response) {
console.log("Retrying " + eurl);
global.all_data = global.all_data.concat(data.results);
});
});
});
}
const limit = functions.config().exec.limit;
console.log("Starting || execution [limit" + limit + "]");
async.parallelLimit(asyncTasks, limit, movie_results);
});
Here is the movie_results functions that executes at the end
function movie_results () {
var moviesRef = defaultDatabase.ref("v1/movies");
var statusRef = defaultDatabase.ref("v1/status");
//moviesRef.remove();
var data_json = {};
console.log("Total Movies fetched " + global.all_data.length);
async.each(global.all_data,
function(mov, cb) {
data_json[mov.id] = mov;
cb();
},
function() {
console.log("Result post-processing done");
console.log("Total =" + data_json.length + " | Expected =" + data_info.total_resu`enter code here`lts);
console.log("Uploading to Firebase Datastore");
moviesRef.update(data_json);
console.log("Finished fetching movies");
// Update status for this job on status DB
var datim = new Date();
statusRef.update({ datim: {"records_fetched": data_json.length, "records_expected": data_info.total_results,
"total_pages": data_info.total_pages, "timestamp": datim}
});
// Closing all db references
moviesRef.off();
statusRef.off();
}
);
}
The issue is that this executes ok most of the times...
As you can see i have added logging to let me know when a function in asynTask is executed
console.log("Procesing Page " + page);
in the Google Cloud logs i see
gcloud beta functions logs read --limit 20 --execution-id 118384524692947
I gets stuck at random pages (whenever it gets stuck.) I understand These features are still in BETA. So can somebody comment that they see the same issue as me. Is there any way to check if there is any Cloud Function Process Executing currently or not and know if is has been killed. I don't think its timing out. Thanks
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:22.286 Procesing Page 223
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:22.786 Procesing Page 224
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:23.384 Procesing Page 225
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:23.984 Procesing Page 226
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:25.084 Procesing Page 227
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:25.484 Procesing Page 228
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:26.184 Procesing Page 229
I tmdbUpdatePubSub 118384524692947 2017-04-26 09:47:27.284 Procesing Page 230