The Lambda Functions success VS. Errors
Hey Y'all I've scrubbed the internet to find the issues with my node.js function. The Cloudwatch error is the following. Any insight or suggestions would be massively helpful! The code runs but there are some issues. I do not know where the /var/runtime/CallbackContext.js code is which seems to be the source of the problem.
2019-12-11T16:34:24.328Z 0211ab2b-8fea-4391-9cc6-01f5c5244c63 ERROR Invoke Error
"errorType": "Error", " at _homogeneousError (/var/runtime/CallbackContext.js:13:12)",
" at postError (/var/runtime/CallbackContext.js:30:51)",
" at done (/var/runtime/CallbackContext.js:57:7)",
" at fail (/var/runtime/CallbackContext.js:69:7)",
" at Object.fail (/var/runtime/CallbackContext.js:105:16)",
" at /var/task/index.js:42:25",
" at IncomingMessage.<anonymous> (/var/task/index.js:444:13)",
" at IncomingMessage.emit (events.js:203:15)",
" at endReadableNT (_stream_readable.js:1143:12)",
" at process._tickCallback (internal/process/next_tick.js:63:19)"
]
Code: Says the issues is line 42.25 which is the context.fail line.
if (error) {
logFailure(error, failedItems);
context.fail(JSON.stringify(error));
} else {
console.log('Success: ' + JSON.stringify(success));
context.succeed('Success');
}
// v1.1.2
var https = require('https');
var zlib = require('zlib');
var crypto = require('crypto');
var endpoint = 'vpc-esdxl-hx2nubxsqn3wh4hqwmywyy2g6y.us-east-1.es.amazonaws.com';
// Set this to true if you want to debug why data isn't making it to
// your Elasticsearch cluster. This will enable logging of failed items
// to CloudWatch Logs.
var logFailedResponses = false;
exports.handler = function(input, context) {
// decode input from base64
var zippedInput = Buffer.from(input.awslogs.data, 'base64');
// decompress the input
zlib.gunzip(zippedInput, function(error, buffer) {
if (error) { context.fail(error); return; }
// parse the input from JSON
var awslogsData = JSON.parse(buffer.toString('utf8'));
// transform the input to Elasticsearch documents
var elasticsearchBulkData = transform(awslogsData);
// skip control messages
if (!elasticsearchBulkData) {
console.log('Received a control message');
context.succeed('Control message handled successfully');
return;
}
// post documents to the Amazon Elasticsearch Service
post(elasticsearchBulkData, function(error, success, statusCode, failedItems) {
console.log('Response: ' + JSON.stringify({
"statusCode": statusCode
}));
if (error) {
logFailure(error, failedItems);
context.fail(JSON.stringify(error));
} else {
console.log('Success: ' + JSON.stringify(success));
context.succeed('Success');
}
});
});
};
function transform(payload) {
if (payload.messageType === 'CONTROL_MESSAGE') {
return null;
}
var bulkRequestBody = '';
payload.logEvents.forEach(function(logEvent) {
var timestamp = new Date(1 * logEvent.timestamp);
// index name format: atg-YYYY.MM.DD
var indexName = [
'atg-' + timestamp.getUTCFullYear(), // year
('0' + (timestamp.getUTCMonth() + 1)).slice(-2), // month
('0' + timestamp.getUTCDate()).slice(-2) // day
].join('.');
var source = buildSource(logEvent.message, logEvent.extractedFields);
var DXLRegex = /\#DXLORDER\#/;
// var LogDatetimeRegex = /(info|Error|Warning|debug)\t(Mon\s|Sun\s|Sat\s|Fri\s|Thu\s|Wed\s|Tue\s)(.*)\t([0-9])/;
var logtyperegex = /(info|Error|Warning|debug)/;
var WOrderRegex = /\|((W)[0-9][0-9][0-9][0-9][0-9][0-9][0-9])\|/;
var DXLOrderRegex = /(DXL([0-9][0-9][0-9][0-9][0-9][0-9][0-9]|[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]|[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]|[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]))\|/;
var CustEMailRegex = /(DXL([0-9][0-9][0-9][0-9][0-9][0-9][0-9]|[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]|[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]|[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]))\|\s*((.*)@(.*)(\.com|\.net|\.live|\.gov)\s*)\|/;
var EMailReferralRegex = /(#DXLORDER#)\|(SFMC),(\s*)(.*?)(\s*)\|(W[0-9]|\|[0-9])/;
var CouponRegex = /(DXL([0-9][0-9][0-9][0-9][0-9][0-9][0-9]|[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]|[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]|[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])\|(((.*)@(.*(.com|.net|.live|)))\|)(.*)\|)/;
var atgpathregex = /(\/)(.*)(\t)/;
var carttotalregex = /(W[0-9][0-9][0-9][0-9][0-9][0-9][0-9]|null)\|(.*)\|([0-9]|[0-9][0-9]|[0-9][0-9][0-9])\.([0-9]|[0-9][0-9]|[0-9][0-9][0-9])\.([0-9]|[0-9][0-9]|[0-9][0-9][0-9])\.([0-9]|[0-9][0-9]|[0-9][0-9][0-9])\|(DXL)/;
var referralregex = /(#DXLORDER#)\s*\|\s*(.*)\s*,\s*(.*)\s*\|(W[0-9]|\|[0-9])/;
var ClientIPregex = /(([0-9][0-9]|[0-9][0-9][0-9]|[0-9])\.([0-9][0-9]|[0-9][0-9][0-9]|[0-9])\.([0-9][0-9]|[0-9][0-9][0-9]|[0-9])\.([0-9][0-9]|[0-9][0-9][0-9]|[0-9]))\|/;
var ATGIDRegex = /(20[0-9][0-9])(\t)(.*)(\t)(\/)/;
var EmailSegmentRegex = /([a-z]w)([0-9][0-9][0-9][0-9][0-9][0-9])_([0-9][0-9][0-9][0-9])_([A-Z]|[A-Z][A-Z])\|/;
var usernamesucregex = /(findUser: user )(.*)( found.)/;
var incorrectpassregex = /(Attempt to log into account )(.*)(with incorrect password)/;
var nonexistentregex = /(Attempt to log into nonexistent account )(.*)/;
var badcouponregex = /(noCouponFound: No coupon could be found for the claim code:)( )(.*)(.)/;
var ccauthamtregex = /(<R20_CC_AUTH_AMT>)(.*)(<\/R20_CC_AUTH_AMT>)/;
var couponremovedregex = /(in removeCoupon method : Coupon )(.*)( has been expired so removing couon from order )(.*)/;
var giftdollaramtregex = /(<GIFT_DOL>)( )(.*)(<\/GIFT_DOL.)/;
var giftredeemdollarregex = /(<GIFTCERT_REDEEM_DOL>)( )(.*)(<\/GIFTCERT_REDEEM_DOL>)/;
var itemnumregex = /(ITEM_NUM>)(([0-9][0-9][0-9][0-9][0-9] |[A-Z][0-9][0-9][0-9][0-9] )(.*)|HEM)((<\/[A-Z][0-9][0-9]_([0-9]|[0-9][0-9])_ITEM_NUM>))/;
var giftcardpinregex = /(<GIFT_CARD_PIN>)(.*)(<\/GIFT_CARD_PIN>)/;
var linkshareregex = /(#DXLORDER#)\s*\|\s*(linkshare)\s*,\s*(.*)\s*\|(W[0-9]|\|[0-9])/;
var giftcardnumregex = /<GIFT_CARD_NUM>(.*)<\/GIFT_CARD_NUM>/;
var giftcardprocesseddollarregex = /Payment Status in Payment Group,\sso using it.(.*):(.*)/;
var outofstockregex = /(We're sorry,\s*Item :\s*)(.*)\s([A-Z][0-9][0-9][0-9][0-9])(.*)/;
if(outofstockregex.exec(logEvent.message) != null){
var outofstockdesc = outofstockregex.exec(logEvent.message)[2];
var outofstockid = outofstockregex.exec(logEvent.message)[3];
} else{
outofstockdesc = "-";
outofstockid = "-";
}
if(giftcardprocesseddollarregex.exec(logEvent.message) != null){
var giftcardprocesseddollar = Number(giftcardprocesseddollarregex.exec(logEvent.message)[2]);
} else{
giftcardprocesseddollar = 0;
}
if(giftcardnumregex.exec(logEvent.message) != null){
var giftcardnum = giftcardnumregex.exec(logEvent.message)[1];
} else{
giftcardnum = "-";
}
// if(LogDatetimeRegex.exec(logEvent.message) != null){
// var datetimelog = Date(LogDatetimeRegex.exec(logEvent.message)[3]);
// } else {
// datetimelog = "-";
// }
if(DXLRegex.exec(logEvent.message) != null){
var DXLORDER = "DXLORDER";
} else {
DXLORDER = "False";
}
if(giftcardpinregex.exec(logEvent.message) != null){
var giftcardpin = giftcardpinregex.exec(logEvent.message)[2];
} else{
giftcardpin = "-";
}
if(itemnumregex.exec(logEvent.message) != null ){
var itemnum = itemnumregex.exec(logEvent.message)[3];
var itemdescr = itemnumregex.exec(logEvent.message)[4];
if(itemdescr == null && itemnumregex.exec(logEvent.message)[2] == "HEM"){
itemnum = "-";
itemdescr = "HEM";
}
} else {
itemnum = "-";
itemdescr ="-";
}
if(giftredeemdollarregex.exec(logEvent.message) != null){
var giftredeemdollaramt = giftredeemdollarregex.exec(logEvent.message)[3];
} else{
giftredeemdollaramt = "0";
}
if(giftdollaramtregex.exec(logEvent.message) != null){
var giftdollaramt = giftdollaramtregex.exec(logEvent.message)[3];
} else {
giftdollaramt = "0";
}
if(couponremovedregex.exec(logEvent.message) != null){
var removedcoupon = couponremovedregex.exec(logEvent.message)[2];
} else{
removedcoupon = "N/A";
}
if(couponremovedregex.exec(logEvent.message) != null){
var removedcouponorder = couponremovedregex.exec(logEvent.message)[4];
} else{
removedcouponorder = "N/A";
}
if(ccauthamtregex.exec(logEvent.message) != null){
var ccauthamt = ccauthamtregex.exec(logEvent.message)[2];
} else{
ccauthamt = "0";
}
if(badcouponregex.exec(logEvent.message) != null){
var badcoupon = badcouponregex.exec(logEvent.message)[3];
} else{
badcoupon = "-";
}
if(nonexistentregex.exec(logEvent.message) != null){
var nonexistent = nonexistentregex.exec(logEvent.message)[2];
} else{
nonexistent = "-";
}
if(incorrectpassregex.exec(logEvent.message) != null){
var incorrectpass = incorrectpassregex.exec(logEvent.message)[2];
} else{
incorrectpass = "-";
}
if(usernamesucregex.exec(logEvent.message) != null){
var username = usernamesucregex.exec(logEvent.message)[2];
} else{
username = "-";
}
if (EmailSegmentRegex.exec(logEvent.message)!=null){
var EmailMailing = EmailSegmentRegex.exec(logEvent.message)[1];
var EmailDate = EmailSegmentRegex.exec(logEvent.message)[2];
var EmailTime = EmailSegmentRegex.exec(logEvent.message)[3];
var EmailSegment = EmailSegmentRegex.exec(logEvent.message)[4];
} else{
EmailMailing = "-";
EmailDate = "-";
EmailTime = "-";
EmailSegment = "-";
}
if (ATGIDRegex.exec(logEvent.message)!=null && DXLRegex.exec(logEvent.message) != null){
var ATGID = ATGIDRegex.exec(logEvent.message)[3];
} else{
ATGID = "-";
}
if (ClientIPregex.exec(logEvent.message)!= null && DXLRegex.exec(logEvent.message) != null){
var ClientIP = ClientIPregex.exec(logEvent.message)[1];
} else{
ClientIP = "-";
}
if (referralregex.exec(logEvent.message) != null){
var referralsource = referralregex.exec(logEvent.message)[2];
var referral = referralregex.exec(logEvent.message)[3];
if (referralregex.exec(logEvent.message)[2].includes("linkshare") == true) {
referralsource = linkshareregex.exec(logEvent.message)[2];
referral = linkshareregex.exec(logEvent.message)[3];
}
} else{
referralsource = "-";
referral = "-";
}
if (carttotalregex.exec(logEvent.message) != null && DXLRegex.exec(logEvent.message) != null){
var carttotal = Number(carttotalregex.exec(logEvent.message)[2]);
} else {
carttotal = 0;
}
if (atgpathregex.exec(logEvent.message) != null && DXLRegex.exec(logEvent.message) != null){
var atgpath = atgpathregex.exec(logEvent.message)[0];
} else {
atgpath = "-";
}
if (CouponRegex.exec(logEvent.message) != null && DXLRegex.exec(logEvent.message) != null){
var Coupon = CouponRegex.exec(logEvent.message)[8];
var Coupon1 = Coupon.toUpperCase();
if(CouponRegex.exec(logEvent.message)[8] == "null"){
Coupon = "-";
}
} else {
Coupon = "-";
}
if (logtyperegex.exec(logEvent.message) != null){
var logtype = logtyperegex.exec(logEvent.message)[0];
} else{
logtype = "-";
}
if (EMailReferralRegex.exec(logEvent.message)!= null){
var EmailReferral = EMailReferralRegex.exec(logEvent.message)[4];
} else{
EmailReferral = "-";
}
if (CustEMailRegex.exec(logEvent.message)!= null && DXLRegex.exec(logEvent.message) != null){
var CustEMail = CustEMailRegex.exec(logEvent.message)[3];
} else{
CustEMail = "-";
}
if(DXLOrderRegex.exec(logEvent.message) != null && DXLRegex.exec(logEvent.message) != null){
var ATGOrderID = DXLOrderRegex.exec(logEvent.message)[1];
} else {
ATGOrderID = "-";
}
if(WOrderRegex.exec(logEvent.message)!= null && DXLRegex.exec(logEvent.message) != null){
var WOrder = WOrderRegex.exec(logEvent.message)[1];
} else{
WOrder = "-";
}
source['outofstockdesc'] = outofstockdesc;
source['outofstockid'] = outofstockid;
source['giftcardprocesseddollar'] = Number(1*giftcardprocesseddollar);
source['giftcardnum'] = giftcardnum;
source['DXLORDER'] = DXLORDER;
source['giftcardpin'] = giftcardpin;
source['itemnum'] = itemnum ;
source['itemdescr'] = itemdescr;
source['GiftcertRedeemDol'] = "$"+giftredeemdollaramt;
source['giftdollaramt']=giftdollaramt;
source['removedcoupon'] = removedcoupon;
source['removedcouponorder'] = removedcouponorder;
source['ccauthamt'] = ccauthamt;
source['badcoupon'] = badcoupon;
source['nonexistent'] = nonexistent;
source['incorrectpass'] = incorrectpass;
source['username'] = username;
source['EmailMailing'] = EmailMailing;
source['EmailDate']=EmailDate;
source['EmailTime'] = EmailTime;
source['EmailSegment'] = EmailSegment;
source['ATGID'] = ATGID;
source['ClientIP'] = ClientIP;
source['referral'] = referral;
source['referralsource'] = referralsource;
//source['carttotal'] = Number(1* carttotal);
source['carttotaldollar'] = Number(1* carttotal);
source['atgpath'] = atgpath;
source['Coupon'] = Coupon1;
source['logtype'] = logtype;
source['EMailReferral'] = EmailReferral;
source['CustomerEMail'] = CustEMail;
source['ATGOrderID'] = ATGOrderID;
source['WOrder'] = WOrder;
// source['datetimelog'] = new Date(datetimelog).toISOString();
source['@id'] = logEvent.id;
source['@timestamp'] = new Date(1 * logEvent.timestamp).toISOString();
source['@message'] = logEvent.message;
source['@owner'] = payload.owner;
source['@log_group'] = payload.logGroup;
source['@log_stream'] = payload.logStream;
var action = { "index": {} };
action.index._index = indexName;
action.index._type = payload.logGroup;
action.index._id = logEvent.id;
bulkRequestBody += [
JSON.stringify(action),
JSON.stringify(source),
].join('\n') + '\n';
});
return bulkRequestBody;
}
function buildSource(message, extractedFields) {
if (extractedFields) {
var source = {};
for (var key in extractedFields) {
if (extractedFields.hasOwnProperty(key) && extractedFields[key]) {
var value = extractedFields[key];
if (isNumeric(value)) {
source[key] = 1 * value;
continue;
}
jsonSubString = extractJson(value);
if (jsonSubString !== null) {
source['$' + key] = JSON.parse(jsonSubString);
}
source[key] = value;
}
}
return source;
}
jsonSubString = extractJson(message);
if (jsonSubString !== null) {
return JSON.parse(jsonSubString);
}
return {};
}
function extractJson(message) {
var jsonStart = message.indexOf('{');
if (jsonStart < 0) return null;
var jsonSubString = message.substring(jsonStart);
return isValidJson(jsonSubString) ? jsonSubString : null;
}
function isValidJson(message) {
try {
JSON.parse(message);
} catch (e) { return false; }
return true;
}
function isNumeric(n) {
return !isNaN(parseFloat(n)) && isFinite(n);
}
function post(body, callback) {
var requestParams = buildRequest(endpoint, body);
var request = https.request(requestParams, function(response) {
var responseBody = '';
response.on('data', function(chunk) {
responseBody += chunk;
});
response.on('end', function() {
var info = JSON.parse(responseBody);
var failedItems;
var success;
if (response.statusCode >= 200 && response.statusCode < 299) {
failedItems = info.items.filter(function(x) {
return x.index.status >= 300;
});
success = {
"attemptedItems": info.items.length,
"successfulItems": info.items.length - failedItems.length,
"failedItems": failedItems.length
};
}
var error = response.statusCode !== 200 || info.errors === true ? {
"statusCode": response.statusCode,
"responseBody": responseBody
} : null;
callback(error, success, response.statusCode, failedItems);
});
}).on('error', function(e) {
callback(e);
});
request.end(requestParams.body);
}
function buildRequest(endpoint, body) {
var endpointParts = endpoint.match(/^([^\.]+)\.?([^\.]*)\.?([^\.]*)\.amazonaws\.com$/);
var region = endpointParts[2];
var service = endpointParts[3];
var datetime = (new Date()).toISOString().replace(/[:\-]|\.\d{3}/g, '');
var date = datetime.substr(0, 8);
var kDate = hmac('AWS4' + process.env.AWS_SECRET_ACCESS_KEY, date);
var kRegion = hmac(kDate, region);
var kService = hmac(kRegion, service);
var kSigning = hmac(kService, 'aws4_request');
var request = {
host: endpoint,
method: 'POST',
path: '/_bulk',
body: body,
headers: {
'Content-Type': 'application/json',
'Host': endpoint,
'Content-Length': Buffer.byteLength(body),
'X-Amz-Security-Token': process.env.AWS_SESSION_TOKEN,
'X-Amz-Date': datetime
}
};
var canonicalHeaders = Object.keys(request.headers)
.sort(function(a, b) { return a.toLowerCase() < b.toLowerCase() ? -1 : 1; })
.map(function(k) { return k.toLowerCase() + ':' + request.headers[k]; })
.join('\n');
var signedHeaders = Object.keys(request.headers)
.map(function(k) { return k.toLowerCase(); })
.sort()
.join(';');
var canonicalString = [
request.method,
request.path, '',
canonicalHeaders, '',
signedHeaders,
hash(request.body, 'hex'),
].join('\n');
var credentialString = [ date, region, service, 'aws4_request' ].join('/');
var stringToSign = [
'AWS4-HMAC-SHA256',
datetime,
credentialString,
hash(canonicalString, 'hex')
] .join('\n');
request.headers.Authorization = [
'AWS4-HMAC-SHA256 Credential=' + process.env.AWS_ACCESS_KEY_ID + '/' + credentialString,
'SignedHeaders=' + signedHeaders,
'Signature=' + hmac(kSigning, stringToSign, 'hex')
].join(', ');
return request;
}
function hmac(key, str, encoding) {
return crypto.createHmac('sha256', key).update(str, 'utf8').digest(encoding);
}
function hash(str, encoding) {
return crypto.createHash('sha256').update(str, 'utf8').digest(encoding);
}
function logFailure(error, failedItems) {
if (logFailedResponses) {
console.log('Error: ' + JSON.stringify(error, null, 2));
if (failedItems && failedItems.length > 0) {
console.log("Failed Items: " +
JSON.stringify(failedItems, null, 2));
}
}
}
