I am fetching 700.000 rows from a web service and my app is crashing after a while being terminated due to memory issue, it consumes about 1 GB of ram before it crashes, the code is fairly simple, I fetch the JSON, I put into an array, I loop the array and insert into core data and once done I save the context
the code is shown below
+ (void)fetchTillDataAll:(int)tillId :(int)startAtRow :(int)takeNoOfRows {
if ([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchTillDataAll:tillId = %d, startAtRow = %d, takeNoOfRows = %d", tillId, startAtRow, takeNoOfRows);
}
NSString *finalURL = [NSString stringWithFormat:@"https://host.domain.com:5443/api/till/tilldatav2/%d?StartAtRow=%d&TakeNoOfRows=%d",tillId, startAtRow, takeNoOfRows];
[[[NSURLSession sharedSession] dataTaskWithURL:[NSURL URLWithString:finalURL]
completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
if (error != nil) {
if ([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchTillDataAll:Transport error %@", error);
}
} else {
NSHTTPURLResponse *responseHTTP;
responseHTTP = (NSHTTPURLResponse *) response;
if(responseHTTP.statusCode != 200) {
if ([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchTillDataAll:Server Error %d", (int) responseHTTP.statusCode);
}
} else {
NSArray *tillBasicDataArray = [NSJSONSerialization JSONObjectWithData:data
options:0
error:NULL];
if ([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchTillDataAll:tillBasicDataArray count = %lu", (unsigned long)[tillBasicDataArray count]);
NSLog(@"WebServices:fetchTillDataAll:tillBasicDataArray looks like %@",tillBasicDataArray);
}
AppDelegate *appDelegate = (AppDelegate *)[[UIApplication sharedApplication]delegate];
//NSManagedObjectContext *context =
//appDelegate.persistentContainer.viewContext;
//context.mergePolicy = NSMergeByPropertyStoreTrumpMergePolicy;
NSPersistentContainer *container = appDelegate.persistentContainer;
[container performBackgroundTask:^(NSManagedObjectContext *context ) {
context.mergePolicy = NSMergeByPropertyStoreTrumpMergePolicy;
NSDictionary *tillBasicDataDict = Nil;
//Loop through the array and for each dictionary insert into local DB
// lets work on concurrency here
for (id element in tillBasicDataArray){
tillBasicDataDict = element;
NSString *itemId = [tillBasicDataDict objectForKey:@"itemId"];
NSString *brandId = [tillBasicDataDict objectForKey:@"companyId"];
NSString *languageId = [tillBasicDataDict objectForKey:@"languageCode"];
NSString *colorCode = [NSString stringWithFormat:@"%@", [tillBasicDataDict objectForKey:@"colorCode"]];
NSString *discountable = [tillBasicDataDict objectForKey:@"discountable"];
NSString *exchangeable = [tillBasicDataDict objectForKey:@"exchangeable"];
NSString *noos14 = [tillBasicDataDict objectForKey:@"noos14"];
NSString *sizeCode = [NSString stringWithFormat:@"%@", [tillBasicDataDict objectForKey:@"sizeCode"]];
NSString *taxGroup = [tillBasicDataDict objectForKey:@"taxGroupId"];
NSString *taxRegion = [tillBasicDataDict objectForKey:@"taxRegion"];
NSString *tradeItemDesc = [tillBasicDataDict objectForKey:@"tradeItemDesc"];
NSString *withTax = [tillBasicDataDict objectForKey:@"withTax"];
NSString *status = [tillBasicDataDict objectForKey:@"status"];
// Use Core Data FMD
NSManagedObject *newPimItem = Nil;
newPimItem = [NSEntityDescription
insertNewObjectForEntityForName:@"TillData"
inManagedObjectContext:context];
[newPimItem setValue:itemId forKey:@"itemId"];
[newPimItem setValue:brandId forKey:@"brandId"];
[newPimItem setValue:languageId forKey:@"languageCode"];
[newPimItem setValue:colorCode forKey:@"colorCode"];
[newPimItem setValue:discountable forKey:@"discountable"];
[newPimItem setValue:exchangeable forKey:@"exchangeable"];
[newPimItem setValue:noos14 forKey:@"noos14"];
[newPimItem setValue:sizeCode forKey:@"sizeCode"];
[newPimItem setValue:[NSNumber numberWithInt:[taxGroup intValue]] forKey:@"taxGroup"];
[newPimItem setValue:taxRegion forKey:@"taxRegion"];
[newPimItem setValue:tradeItemDesc forKey:@"tradeItemDesc"];
[newPimItem setValue:[NSNumber numberWithInt:[withTax intValue]] forKey:@"withTax"];
[newPimItem setValue:[NSNumber numberWithInt:[status intValue]] forKey:@"status"];
if ([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchTillDataAll:ItemId in loop = %@", itemId);
NSLog(@"WebServices:fetchTillDataAll:newPimItem = %@", newPimItem);
NSLog(@"WebServices:fetchTillDataAll:CoreData error = %@", error);
}
}
NSError *error = nil;
if (![context save:&error]) {
NSLog(@"Failure to save context: %@\n%@", [error localizedDescription], [error userInfo]);
abort();
} else {
NSUserDefaults *tillUserDefaults = [NSUserDefaults standardUserDefaults];
[tillUserDefaults setInteger:1 forKey:@"hasTillData"];
[tillUserDefaults synchronize];
}
}];
}
}
}] resume];
}
What can I do minimize the foot print so that I am able to download the data? I absolutely must have the data locally in order to allow offline capabilities in the app
----- EDIT -----
After implementing splitting the NSArray into an array of array I still get the same problem, below if the new code as suggested:
split method
+ (NSArray *) splitIntoArraysOfBatchSize:(NSArray *)originalArray :(int)batchSize {
NSMutableArray *arrayOfArrays = [NSMutableArray array];
for(int j = 0; j < [originalArray count]; j += batchSize) {
NSArray *subarray = [originalArray subarrayWithRange:NSMakeRange(j, MIN(batchSize, [originalArray count] - j))];
[arrayOfArrays addObject:subarray];
}
return arrayOfArrays;
}
Looping through the array as follows
+(void)fetchPricelistAll:(int)pricelistId :(int)startAtRow :(int)takeNoOfRows;
{
if ([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchPriceList:priceListId = %d", pricelistId);
}
NSString *finalURL = [NSString stringWithFormat:@"https://host.domain.com:5443/api/till/tillpricelistv2/%d?StartAtRow=%d&TakeNoOfRows=%d",pricelistId, startAtRow, takeNoOfRows];
[[[NSURLSession sharedSession] dataTaskWithURL:[NSURL URLWithString:finalURL]
completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
if (error != nil) {
if ([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchPriceList:Transport error %@", error);
}
} else {
NSHTTPURLResponse *responseHTTP;
responseHTTP = (NSHTTPURLResponse *) response;
if(responseHTTP.statusCode != 200) {
if ([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchPriceList:Server Error %d", (int) responseHTTP.statusCode);
}
} else {
NSArray *priceListObjectArray = [NSJSONSerialization JSONObjectWithData:data
options:0
error:NULL];
if ([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchPriceList:count = %lu", (unsigned long)[priceListObjectArray count]);
NSLog(@"WebServices:fetchPriceList:PricelistObjectArray looks like %@",priceListObjectArray);
}
AppDelegate *appDelegate = (AppDelegate *)[[UIApplication sharedApplication]delegate];
NSPersistentContainer *container = appDelegate.persistentContainer;
NSArray *arrayOfArrays = [NWTillHelper splitIntoArraysOfBatchSize:priceListObjectArray :1000];
for(NSArray *batch in arrayOfArrays) {
[container performBackgroundTask:^(NSManagedObjectContext *context ) {
context.mergePolicy = NSMergeByPropertyStoreTrumpMergePolicy;
NSDictionary *priceListObjectDict;
//Loop through the array and for each dictionary insert into local DB
for (id element in batch) {
priceListObjectDict = element;
NSString *currencyName = [priceListObjectDict objectForKey:@"currencyName"];
NSString *price = [priceListObjectDict objectForKey:@"price"];
NSString *priceIncTax = [priceListObjectDict objectForKey:@"priceIncTAX"];
NSString *validFrom = [priceListObjectDict objectForKey:@"validFromDate"];
NSString *validTo = [priceListObjectDict objectForKey:@"validToDate"];
NSString *itemId = [priceListObjectDict objectForKey:@"itemID"];
NSDateFormatter *dateFormat = [[NSDateFormatter alloc] init];
[dateFormat setDateFormat:@"YYYY-MM-dd'T'HH:mm:ss"];
NSDate *validToDate = [dateFormat dateFromString:validTo];
NSDate *validFromDate = [dateFormat dateFromString:validFrom];
if([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchPriceList:validToDate: >>>> %@ <<<<", validToDate);
NSLog(@"WebServices:fetchPriceList:validFromDate: >>>> %@ <<<<", validFromDate);
}
if([NWTillHelper isDebug] == 1) {
NSLog(@"PimItemListView:tableView:context = %@", context);
}
NSManagedObject *newPrlItem = Nil;
newPrlItem = [NSEntityDescription
insertNewObjectForEntityForName:@"PriceList"
inManagedObjectContext:context];
[newPrlItem setValue:itemId forKey:@"itemId"];
[newPrlItem setValue:validToDate forKey:@"validTo"];
[newPrlItem setValue:validFromDate forKey:@"validFrom"];
[newPrlItem setValue:price forKey:@"price"];
[newPrlItem setValue:priceIncTax forKey:@"priceIncTax"];
[newPrlItem setValue:currencyName forKey:@"currencyName"];
if ([NWTillHelper isDebug] == 1) {
NSLog(@"WebServices:fetchTillData:ItemId in loop = %@", itemId);
NSLog(@"WebServices:fetchTillData:newPrlItem = %@", newPrlItem);
NSLog(@"WebServices:fetchTillData:CoreData error = %@", error);
}
}
NSError *error = nil;
if (![context save:&error]) {
NSLog(@"Failure to save context: %@\n%@", [error localizedDescription], [error userInfo]);
abort();
} else {
NSUserDefaults *tillUserDefaults = [NSUserDefaults standardUserDefaults];
[tillUserDefaults setInteger:1 forKey:@"hasPriceList"];
[tillUserDefaults synchronize];
}
}];
}
}
}
}] resume];
}
It still raises to 2 GB and gets terminated, when the NSURLSession completion block hits memory usage is about 250, I assume that is after it downloads the entire data set into the NSArray, but after that when I want to write to core data all goes terribly wrong and it goes to 2 GB and gets terminated
Why is that?
