i am having a hard time to figure this one out, maybe someone can help me out here.
i have an Asp.net Core 2.0 web application, i am using Hangfire to process jobs in the background, usually its to sync products or orders between different platforms, but there is one job which always fail with the 'Insufficient memory to continue the execution of the program.' and also 'Object reference not set to an instance of an object' i log the trace and it takes me to a Parallel.ForEach where the error is being thrown, now when i do the same thing manually (meaning without hangfire) i get no exception and everything works just fine.
so let me share some code here.
so first we setup hangfire like this. inside the startup class, at the method Configure, we do this.
app.UseHangfireDashboard("/tasks", new DashboardOptions() {
Authorization = new[] { new HangFireAuthorizationFilter() }
});
var schedules = new ScheduleTasks(app.ApplicationServices);
schedules.RegisterRecurentJobs();
now the ScheduleTasks looks like this.
public class ScheduleTasks
{
private readonly IApiService _apiService;
private readonly ISettingService _settingService;
public ScheduleTasks(IServiceProvider serviceProvider)
{
var scope = serviceProvider.CreateScope();
_apiService = scope.ServiceProvider.GetService<IApiService>();
_settingService = scope.ServiceProvider.GetService<ISettingService>();
}
public void RegisterRecurentJobs()
{
var settings = _settingService.LoadSetting<ApiSettings>();
if(!settings.SilentMode)
{
RecurringJob.AddOrUpdate<IApiService>("Config", x => x.SyncConfig(), "* 1 * * *");
RecurringJob.AddOrUpdate<IApiService>("Manufacturers", x => x.SyncManufacturers(), "* 2 * * *");
RecurringJob.AddOrUpdate<IApiService>("Products", x => x.SyncProducts(30, false), "* 3 * * *");
}
}
}
and now here is my api service method that executed the SyncProducts.
public async Task SyncProducts(int days, bool hasSalesOnly = true)
{
var s = new StringBuilder();
try
{
var search = new SearchModel()
{
ModifiedDate = new DateRange()
{
FromDate = DateTime.Today.AddDays(-days)
},
PageSize = 300000,
HaveSalesOnly = hasSalesOnly
};
var ids = await _productService.GetProductFeed(search);
var hasNext = ids?.Result?.AdditionalResultsId;
var failedIds = new List<string>();
var retry = true;
while (ids.Result.Ids != null && ids.Result.Ids.Count > 0)
{
var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = 20 };
s.Append("[");
var start = true;
Parallel.ForEach(ids.Result.Ids, parallelOptions, id =>
{
var itemResult = _productService.GetProductByProductId(id).Result;
if (itemResult.Success)
{
lock (s)
{
if (start)
{
s.Append(itemResult.ResponseString);
}
else
{
var r = "," + itemResult.ResponseString;
s.Append(r);
}
start = false;
}
}
else
{
failedIds.Add(id);
}
itemResult = null;
});
s.Append("]");
var newFileName = "result.json";
var newFileDir = GetJsonFileDirectoryPath() + newFileName;
var baseDir = Environment.CurrentDirectory + "/wwwroot";
var dataPath = baseDir + newFileDir;
File.WriteAllText(dataPath, s.ToString());
s = null;
ids.Result.Ids = null;
var param = new SqlParameter
{
ParameterName = "path",
SqlDbType = SqlDbType.VarChar,
Direction = ParameterDirection.Input,
Value = dataPath
};
var imagesToProcess = await _itemRepository.ExecuteStoredProcedureList<SyncItems>("[dbo].[SyncItems]", param);
foreach (var image in imagesToProcess)
{
if (image.ProductId == null)
continue;
//for now delete all images, and save the new ones.
var imagesToDelete = _pictureService.GetProductPicturesByProductId(image.ProductId.Value);
foreach (var i in imagesToDelete)
{
await _pictureService.DeleteProductPicture(i);
}
if (image.ImageUrls != null)
{
var imageList = image.ImageUrls.Split(',');
foreach (var i in imageList)
{
await _pictureService.InsertProductPictureFromPictureUrl(i, image.ProductId.Value);
}
}
}
if (!string.IsNullOrEmpty(hasNext))
{
ids = await _productService.GetNextPage(ids.Result.AdditionalResultsId);
hasNext = ids.Result.AdditionalResultsId;
}
else
{
ids.Result.Ids = null;
}
if (string.IsNullOrEmpty(hasNext) && ids.Result.Ids == null && retry ||
string.IsNullOrEmpty(hasNext) && ids.Result.Ids.Count == 0 && retry)
{
ids = new BaseSearchResult();
ids.Result.Ids = failedIds;
failedIds = new List<string>();
retry = false;
}
}
ids = null;
}
catch (Exception ex)
{
s = null;
aEvent = new ApiEvent()
{
Level = LogLevel.Critical,
EventDate = DateTime.Now,
User = "Api",
EventType = EventTypeEnum.Product,
EntityTypeId = "",
Event = ex.Message,
Details = ex.StackTrace
};
await _eventService.PublishEvent(aEvent);
}
}
if i execute this method manually, either by writing a functional test to execute the method, or by having a button on my page which triggers this method to execute, everything works fine, but when Hangfire is trying to execute the exact same method, i get this error.
One or more errors occurred. (Insufficient memory to continue the execution of the program.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.) (Object reference not set to an instance of an object.)
and the trace is this.
at System.Threading.Tasks.TaskReplicator.Run[TState](ReplicatableUserAction
1 action, ParallelOptions options, Boolean stopOnFirstFailure) at System.Threading.Tasks.Parallel.ForWorker[TLocal](Int32 fromInclusive, Int32 toExclusive, ParallelOptions parallelOptions, Action1 body, Action2 bodyWithState, Func4 bodyWithLocal, Func1 localInit, Action1 localFinally) --- End of stack trace from previous location where exception was thrown --- at System.Threading.Tasks.Parallel.ThrowSingleCancellationExceptionOrOtherException(ICollection exceptions, CancellationToken cancelToken, Exception otherException) at System.Threading.Tasks.Parallel.ForWorker[TLocal](Int32 fromInclusive, Int32 toExclusive, ParallelOptions parallelOptions, Action1 body, Action2 bodyWithState, Func4 bodyWithLocal, Func1 localInit, Action1 localFinally) at System.Threading.Tasks.Parallel.ForEachWorker[TSource,TLocal](IList1 list, ParallelOptions parallelOptions, Action1 body, Action2 bodyWithState, Action3 bodyWithStateAndIndex, Func4 bodyWithStateAndLocal, Func5 bodyWithEverything, Func1 localInit, Action1 localFinally) at System.Threading.Tasks.Parallel.ForEachWorker[TSource,TLocal](IEnumerable1 source, ParallelOptions parallelOptions, Action1 body, Action2 bodyWithState, Action3 bodyWithStateAndIndex, Func4 bodyWithStateAndLocal, Func5 bodyWithEverything, Func1 localInit, Action1 localFinally) at System.Threading.Tasks.Parallel.ForEach[TSource](IEnumerable1 source, ParallelOptions parallelOptions, Action`1 body) at *****.*****.Services.ApiService.SyncProducts(Int32 days, Boolean hasSalesOnly) in C:***************.****\Services\ApiService.cs:line 493
can anyone help me figure why it fails when hangfire executes the method?
thanks in advance for everyone looking into this.
