We have couple of jobs which are set up with retry attempt of 10. Due to nature or external service sometimes during 1st or 2nd time process will fail and we get failure email. But on 3rd or 4th time job will succeeded. Now I want to send email in coordinated manner to earlier failure emails. So we can safely ignore earlier failure emails if we receive successive email with success. I want to send header of email in following pattern; so it gets easier to correlate.
Attempt 1 -> Email Header : JobID - JobName - Attempt 1 - Failed
Attempt 2 -> Email Header : JobID - JobName - Attempt 2 - Failed
Attempt 3 -> Email Header : JobID - JobName - Attempt 3 - Succeeded
How to achieve this in hangfire ? Also I want to retrieve retry attempt number of given Job.
Thanks
My approach so far. Is there any other better approach ?
namespace Forte.Application.Hangfire.Filter
{
public class HangFireEmailerFilter : JobFilterAttribute, IElectStateFilter
{
private readonly IEmailer _emailer;
private readonly IHangfireEmailSettings _hangfireSettings;
public HangFireEmailerFilter(IHangfireEmailSettings hangfireSettings)
{
_hangfireSettings = hangfireSettings;
_emailer = new Emailer(hangfireSettings);
}
public void OnStateElection(ElectStateContext context)
{
if(context?.CandidateState is FailedState || context?.CandidateState is SucceededState)
{
if(_hangfireSettings.IsEnabled)
{
var emailHeader = string.Empty;
var body = string.Empty;
int? totalAttemptsAllowed = null;
var highPriority = false;
var sendEmail = false;
var currentRetryCount = context.GetJobParameter<int>("RetryCount");
var retryAttribute = (AutomaticRetryAttribute) GetCustomAttribute(context.BackgroundJob.Job.Type, typeof(AutomaticRetryAttribute));
var jobId = context.BackgroundJob.Id;
if(retryAttribute != null)
{
totalAttemptsAllowed = retryAttribute.Attempts;
}
if (context.CandidateState is FailedState failedState)
{
//FATAL
if(totalAttemptsAllowed.HasValue && currentRetryCount == totalAttemptsAllowed)
{
emailHeader = $"{EmailLevel.FATAL.ToString()}:{GetEmailSubject(context, currentRetryCount, totalAttemptsAllowed, jobId)}";
highPriority = true;
}
//ERROR
else
{
emailHeader = $"{EmailLevel.FAIL.ToString()}:{GetEmailSubject(context, currentRetryCount, totalAttemptsAllowed, jobId)}";
}
body = _hangfireSettings.JobDashboardUrl + jobId + "\n\n" + failedState.Exception.Message;
sendEmail = true;
}
if (context.CandidateState is SucceededState)
{
if(currentRetryCount > 1) //OK
{
emailHeader = $"{EmailLevel.OK.ToString()}:{GetEmailSubject(context, currentRetryCount, totalAttemptsAllowed, jobId)}";
body = $"{_hangfireSettings.JobDashboardUrl + jobId} {Environment.NewLine} Job Completed.";
sendEmail = true;
}
}
if(sendEmail)
{
_emailer.SendEmail(_hangfireSettings.FromEmail, _hangfireSettings.ToEmail, emailHeader, body, highPriority);
}
}
}
}
private string GetEmailSubject(ElectStateContext context, int retryCount, int? totalAttemptsAllowed, string jobId)
{
if(totalAttemptsAllowed.HasValue)
{
return $"{context.BackgroundJob.Job.Type.Name}.{context.BackgroundJob.Job.Method.Name} " +
$"[{jobId}:{retryCount}/{totalAttemptsAllowed.Value}][{_hangfireSettings.ProcessName}][{_hangfireSettings.MachineName}]";
}
else
{
return $"{context.BackgroundJob.Job.Type.Name}.{context.BackgroundJob.Job.Method.Name} " +
$"[{jobId}:{retryCount}][{_hangfireSettings.ProcessName}][{_hangfireSettings.MachineName}]";
}
}
}
}