I have an API with HTTP endpoint which is on Firebase Functions. It takes care of some processing of received data which is then stored into Firebase Realtime Database, but I'm having a constant problem where a part of object overwrites entire object, making the data invalid and not useful. The problem is that when Google Geocoding completes, the object is already in database and it gets overwritten by location object that was created, despite I used the .update() and have set full object paths. I have confirmed that the data is already there by adding a few console.log() functions and then viewing the Functions logs.
Function that receives the data and process it (some stuff removed like image processing, otherwise the function is too long to be pasted here):
exports = module.exports = (req, res, admin) => {
// allowed domains for this API endpoint
let domains = [
'http://localhost:4200',
'http://localhost:4300',
'http://localhost.local',
'http://www.localhost.local'
];
// make sure that only above domains are accessing it, otherwise return error
if (typeof req.headers.origin !== 'undefined') {
if (domains.indexOf(req.headers.origin) === -1) {
return res.status(403).send({
'status': 'error',
'type': 'not-authorized',
'message': 'You\'re not authorized to use this method!'
});
}
} else {
return res.status(403).send({
'status': 'error',
'type': 'malformed-request',
'message': 'Your request is missing CORS origin header!'
});
}
// errors
let errors = [];
// process uploaded form
const busboy = new Busboy({headers: req.headers});
// process attached fields
busboy.on('field', function(fieldname, val) {
req.body[fieldname] = val;
});
// now process the results
busboy.on('finish', () => {
let report_object = {},
report_object_location = {},
report_key = admin.database().ref().child('/reports/').push().key;
// check "location" and process if
if (typeof req.body.report_address !== 'undefined') {
if (req.body.report_address !== '') {
report_object['reports/' + report_key + '/location/address'] = req.body.report_address;
report_object['reports/' + report_key + '/location/unknown'] = false;
if (typeof req.body.report_latitude !== 'undefined') {
report_object['reports/' + report_key + '/location/latitude'] = parseFloat(req.body.report_latitude);
}
if (typeof req.body.report_longitude !== 'undefined') {
report_object['reports/' + report_key + '/location/longitude'] = parseFloat(req.body.report_longitude);
}
// Google Maps API for geocoding and reverse geocoding
const googleMapsClient = require('@google/maps').createClient({
key: 'xxx'
});
console.log('address: ', req.body.report_address);
if ((typeof req.body.report_latitude === 'undefined' || req.body.report_latitude === '0' || req.body.report_latitude === '' || req.body.report_latitude === 0) && typeof req.body.report_address !== 'undefined') {
console.log('geocoding executed');
googleMapsClient.geocode({'address': req.body.report_address, 'components': {'country':'XX'}}, function(error, response) {
if (!error) {
console.log('formatted: ' + response.json.results[0].formatted_address);
report_object_location['address'] = response.json.results[0].formatted_address.replace(', <country name>', '');
report_object_location['latitude'] = response.json.results[0].geometry.location.lat;
report_object_location['longitude'] = response.json.results[0].geometry.location.lng;
// added so that the data is saved directly and was hoping it won't be overwritten
report_object['reports/' + report_key + '/location/address'] = response.json.results[0].formatted_address.replace(', <country>', '');
report_object['reports/' + report_key + '/location/latitude'] = response.json.results[0].geometry.location.lat;
report_object['reports/' + report_key + '/location/longitude'] = response.json.results[0].geometry.location.lng;
response.json.results[0].address_components.forEach(result => {
if (typeof result.types !== 'undefined') {
if (result.types[0] === 'locality') {
report_object_location['city'] = result.long_name;
report_object['reports/' + report_key + '/location/city'] = result.long_name;
}
}
});
console.log('geocoding complete', new Date().getTime());
admin.database().ref('/reports/' + report_key + '/location').update(report_object_location);
} else {
console.log(error);
}
});
}
} else {
errors.push({
'field': 'report_address',
'type': 'missing',
'message': 'Please enter address'
});
}
} else {
errors.push({
'field': 'report_address',
'type': 'missing',
'message': 'Please enter address'
});
}
// check category and process it
if (typeof req.body.process !== 'undefined') {
if (req.body.process !== '') {
report_object['reports/' + report_key + '/category'] = utils.firebaseKeyEncode(req.body.process);
} else {
errors.push({
'field': 'process',
'type': 'missing',
'message': 'Please select process'
});
}
} else {
errors.push({
'field': 'process',
'type': 'missing',
'message': 'Please select process'
});
}
// check "subject" and process it
if (typeof req.body.subject !== 'undefined') {
if (req.body.subject !== '') {
report_object['reports/' + report_key + '/subject'] = utils.firebaseKeyEncode(req.body.subject);
}
}
// check "reporter" and process if
if (typeof req.body.reporter_name !== 'undefined' && req.body.reporter_name !== '') {
report_object['reports/' + report_key + '/reporter_name'] = req.body.reporter_name;
}
if (typeof req.body.reporter_address !== 'undefined' && req.body.reporter_address !== '') {
report_object['reports/' + report_key + '/reporter_address'] = req.body.reporter_address;
}
if (typeof req.body.reporter_phone !== 'undefined' && req.body.reporter_phone !== '') {
report_object['reports/' + report_key + '/reporter_phone'] = req.body.reporter_phone;
}
if (typeof req.body.reporter_notify !== 'undefined' && req.body.reporter_notify !== '') {
report_object['reports/' + report_key + '/reporter_notify'] = true;
}
if (typeof req.body.reporter_email !== 'undefined' && req.body.reporter_email !== '') {
const emailValidator = require('email-validator');
if (emailValidator.validate(req.body.reporter_email)) {
report_object['reports/' + report_key + '/reporter_email'] = req.body.reporter_email;
} else {
errors.push({
'field': 'reporter_email',
'type': 'invalid',
'message': 'Entered email is not valid!'
});
}
}
// check "note" and copy it
if (typeof req.body.notes !== 'undefined') {
if (req.body.notes !== '') {
report_object['reports/' + report_key + '/notes'] = req.body.notes;
}
}
// add current user
report_object['reports/' + report_key + '/created_user_display_name'] = 'Website';
// add created date & statuses
report_object['reports/' + report_key + '/datetime'] = admin.database.ServerValue.TIMESTAMP;
report_object['reports/' + report_key + '/status'] = 'open';
report_object['reports/' + report_key + '/status_updates/open'] = admin.database.ServerValue.TIMESTAMP;
// any errors?
if (errors.length > 0) {
return res.status(400).send({
'status': 'error',
'type': 'invalid-data',
'message': 'Please fix the data you provided data and re-submit.',
'errors': errors
});
}
// trigger function that saves the data
return exports.saveReportDetails(report_object, report_key, res, admin);
});
// required, otherwise the upload hangs
busboy.end(req.rawBody);
req.pipe(busboy);
};
Function that saves the data into database:
exports.saveReportDetails = function(report_object, report_key, res, admin) {
// add icon marker
admin.database().ref('/settings').once('value', settingsData => {
admin.database().ref('/categories/' + report_object['reports/' + report_key + '/category']).once('value', categoryData => {
let settings = settingsData.val(),
category = categoryData.val(),
description = (typeof category.subjects[report_object['reports/' + report_key + '/subject']] !== 'undefined' && typeof category.subjects[report_object['reports/' + report_key + '/subject']].description !== 'undefined' ? category.subjects[report_object['reports/' + report_key + '/subject']].description : '');
report_object['reports/' + report_key + '/marker_icon'] = {
url: category.icon,
color: category.marker_color,
scaledSize: {
width: settings.map_marker_icon_size,
height: settings.map_marker_icon_size
}
};
let report_history_key = admin.database().ref().child('/reports_history/' + report_key + '/history').push().key;
report_object['reports_history/' + report_key + '/' + report_history_key + '/action'] = 'created';
report_object['reports_history/' + report_key + '/' + report_history_key + '/datetime'] = parseInt(moment().format('x'), 10);
report_object['reports_history/' + report_key + '/' + report_history_key + '/user_display_name'] = 'Website';
report_object['categories_reports/' + report_object['reports/' + report_key + '/category'] + '/' + report_key] = true;
if (report_object['reports/' + report_key + '/subject'] !== 'undefined') {
if (typeof category.subjects !== 'undefined') {
if (typeof category.subjects[report_object['reports/' + report_key + '/subject']] !== 'undefined') {
let subject = category.subjects[report_object['reports/' + report_key + '/subject']];
if (typeof subject.handling_days !== 'undefined') {
report_object['reports/' + report_key + '/handling_days'] = subject.handling_days;
}
if (typeof subject.user_key !== 'undefined' && typeof subject.user_display_name !== 'undefined') {
// report should be assigned to user
let report_history_key2 = admin.database().ref().child('/reports_history/' + report_key + '/history').push().key;
report_object['reports/' + report_key + '/assigned_user_key'] = subject.user_key;
report_object['reports/' + report_key + '/assigned_user_display_name'] = subject.user_display_name;
report_object['reports/' + report_key + '/assigned_user_type'] = subject.user_type;
report_object['reports/' + report_key + '/assigned_datetime'] = admin.database.ServerValue.TIMESTAMP;
report_object['reports/' + report_key + '/status'] = 'assigned';
report_object['reports/' + report_key + '/status_updates/assigned'] = admin.database.ServerValue.TIMESTAMP;
report_object['reports_history/' + report_key + '/' + report_history_key2 + '/action'] = 'assigned';
report_object['reports_history/' + report_key + '/' + report_history_key2 + '/assigned_user_key'] = subject.user_key;
report_object['reports_history/' + report_key + '/' + report_history_key2 + '/assigned_user_display_name'] = subject.user_display_name;
report_object['reports_history/' + report_key + '/' + report_history_key2 + '/datetime'] = admin.database.ServerValue.TIMESTAMP;
report_object['reports_history/' + report_key + '/' + report_history_key2 + '/user_key'] = false;
report_object['reports_history/' + report_key + '/' + report_history_key2 + '/user_display_name'] = 'auto-assigned';
report_object['users_assigned_reports/' + subject.user_key + '/' + report_key] = true;
}
}
}
}
if (typeof report_object['reports/' + report_key + '/subject'] !== 'undefined') {
report_object['subjects_reports/' + report_object['reports/' + report_key + '/subject'] + '/' + report_key] = true;
}
let year = moment().format('Y');
admin.database().ref('/reports_count/' + year).once('value', data => {
let value = data.val();
let number = 0;
if (value !== null) {
number = parseInt(value, 10) + 1;
} else {
number = 1;
}
report_object['reports/' + report_key + '/unique_number'] = year + '#' + number;
// assume all files have uploaded and push data into firebase
admin.database().ref('/').update(report_object);
console.log('save report', new Date().getTime());
});
// send confirmation email?
console.log(report_object['reports/' + report_key + '/reporter_email']);
if (typeof report_object['reports/' + report_key + '/reporter_email'] !== 'undefined') {
if (report_object['reports/' + report_key + '/reporter_email'] !== '' && report_object['reports/' + report_key + '/reporter_email'] !== false) {
const emails = require('./../utils/mailTransportModule');
emails.mailTransport.verify(error => {
if (error) {
console.log(error);
} else {
admin.database().ref('/settings').once('value', function(settings_data) {
let settings = settings_data.val(),
webapp_name = (typeof settings.webapp_name !== 'undefined' ? (settings.webapp_name !== '' ? settings.webapp_name : '<webapp name>') : '<webapp name>'),
webapp_url = (typeof settings.webapp_url !== 'undefined' ? (settings.webapp_url !== '' ? settings.webapp_url : '<webapp url>') : '<webapp url>'),
support_email = (typeof settings.support_email !== 'undefined' ? (settings.support_email !== '' ? settings.support_email : '<webapp email>') : '<webapp email>'),
support_phone = (typeof settings.support_phone !== 'undefined' ? (settings.support_phone !== '' ? settings.support_phone : '-') : '-'),
message = {
from: `"${webapp_name}" <${support_email}>`,
to: report_object['reports/' + report_key + '/reporter_email'],
replyTo: '<replyTo email>',
subject: `<subject>`,
text: emails.emails.newReportConfirmationText(webapp_name, report_object['reports/' + report_key + '/datetime'], utils.firebaseKeyDecode(report_object['reports/' + report_key + '/category']), (report_object['reports/' + report_key + '/subject'] !== false && typeof report_object['reports/' + report_key + '/subject'] !== 'undefined' ? utils.firebaseKeyDecode(report_object['reports/' + report_key + '/subject']) : ''), (description !== 'undefined' ? description : ''), (report_object['reports/' + report_key + '/location/address'] !== false ? report_object['reports/' + report_key + '/location/address'] : '-'), support_email, support_phone),
html: emails.emails.newReportConfirmationHTML(webapp_name, webapp_url, report_object['reports/' + report_key + '/datetime'], utils.firebaseKeyDecode(report_object['reports/' + report_key + '/category']), (report_object['reports/' + report_key + '/subject'] !== false && typeof report_object['reports/' + report_key + '/subject'] !== 'undefined' ? utils.firebaseKeyDecode(report_object['reports/' + report_key + '/subject']) : ''), (description !== 'undefined' ? description : ''), (report_object['reports/' + report_key + '/location/address'] !== false ? report_object['reports/' + report_key + '/location/address'] : '-'), support_email, support_phone),
attachments: []
};
let images = _.filter(report_object, function(v, k){
return _.includes(k, '/images');
});
// check if any image or audio is available and attach them to the message
if (images.length) {
images.forEach((image, index) => {
if (image.startsWith('https://')) {
message.attachments.push({
filename: 'image_' + index + '.jpg',
href: image
});
}
});
}
emails.mailTransport.sendMail(message).then(() => {
}).catch(error => {
return Promise.reject('sendMail error: ' + error, message);
});
});
}
});
}
}
return res.status(200).send({
'status': 'success',
'type': 'report-saved',
'message': ' Report was successfully saved.'
});
});
});
};
I am hoping that I have missed something basic and someone can shed some light onto what, since I am lost with what else to do.
Package JSON for Functions:
{
"name": "<name>",
"version": "0.0.1",
"description": "<description>",
"dependencies": {
"@google-cloud/storage": "^1.5.2",
"@google/maps": "^0.4.5",
"busboy": "^0.2.14",
"connect-busboy": "0.0.2",
"email-validator": "^1.1.1",
"express": "^4.16.2",
"firebase-admin": "^5.8.1",
"firebase-functions": "^0.8.1",
"lodash": "^4.17.4",
"moment": "^2.20.1",
"nodemailer": "^4.4.1",
"sharp": "^0.19.0",
"uuid-v4": "^0.1.0"
},
"scripts": {
"start": "node index.js",
"build": ""
},
"private": true
}
UPDATE: Example object before geocoding:
{
"assigned_datetime" : 1536661321150,
"assigned_user_display_name" : "<name>",
"assigned_user_key" : "<key>",
"assigned_user_type" : "<type>",
"category" : "<category>",
"created_user_display_name" : "<name>",
"created_user_key" : "<key>",
"datetime" : 1536661321150,
"location" : {
"address" : "<full address>",
"city" : "<city>",
"latitude" : <decimal>,
"longitude" : <decimal>,
"unknown" : false
},
"marker_icon" : {
"color" : "#2962ff",
"scaledSize" : {
"height" : 38,
"width" : 38
},
"url" : "assets/img/icons/blue.png"
},
"notes" : "<notes>",
"printed" : true,
"reporter_address" : "<address>",
"reporter_email" : "<email>",
"reporter_name" : "<name>",
"reporter_notified" : 1537282713509,
"reporter_phone" : "<phone>",
"send_email" : true,
"status" : "resolved",
"status_updates" : {
"assigned" : 1536667369830,
"open" : 1536661321150,
"resolved" : 1537282713367
},
"subject" : "<subject>",
"unique_number" : "<number>"
}
Example object after geocoding and saving into reports/<report_key>/location:
{
"location" : {
"address" : "<full address>",
"city" : "<city>",
"latitude" : <decimal>,
"longitude" : <decimal>,
"unknown" : false
}
}
