I'm implementing the MailChimp.NET wrapper in both synchronous and asynchronous ways and calls are going through without a problem, BUT results tend to get lost in the synchronous methods. In other words, if I send 100 members to be added (by batches of 10 due to the simultaneous connections limit of the MailChimp API), all 100 will indeed be visible in my MC audience but I'll loose from 5 to 25% of the results on code side. Here's the concerned bit of my implementation :
public class MailChimpClient : IDisposable
{
private MailChimpManager _mcm;
private string _apiKey;
private bool _isDisposed;
private ConcurrentQueue<MailChimpMember> _updatedMembersQueue;
private ConcurrentQueue<MailChimpBaseException> _exceptionsQueue;
private const int BatchSize = 10;
private const int TaskDelay = 100;
private ConcurrentQueue<MailChimpMember> UpdatedMembersQueue
{
get { return _updatedMembersQueue = _updatedMembersQueue ?? new ConcurrentQueue<MailChimpMember>(); }
set { _updatedMembersQueue = value; }
}
private ConcurrentQueue<MailChimpBaseException> ExceptionsQueue
{
get { return _exceptionsQueue = _exceptionsQueue ?? new ConcurrentQueue<MailChimpBaseException>(); }
set { _exceptionsQueue = value; }
}
public MailChimpClient(string apiKey)
{
_apiKey = apiKey;
_mcm = new MailChimpManager(apiKey);
}
private async Task AddOrUpdateMember(MailChimpMember member, string listId)
{
try
{
var model = member.ToApiMember();
model = await _mcm.Members.AddOrUpdateAsync(listId, model);
UpdatedMembersQueue.Enqueue(new MailChimpMember(model));
await Task.Delay(TaskDelay);
}
catch (Exception ex)
{
var mccex = new MailChimpClientException($"Error adding/updating member \"{(member != null ? member.MailAddress.ToString() : "NULL")}\" to list with ID \"{listId}\".", ex);
ExceptionsQueue.Enqueue(mccex);
}
}
private MailChimpClientResult AddOrUpdateMemberRange(IEnumerable<MailChimpMember> members, string listId)
{
var batches = members.GetBatches(BatchSize);
var result = new MailChimpClientResult();
var i = 0;
foreach (var batch in batches)
{
AddOrUpdateMemberBatch(batch, listId);
i++;
FlushQueues(ref result);
}
return result;
}
private void AddOrUpdateMemberBatch(MailChimpMember[] batch, string listId)
{
Task.WaitAll(batch.Select(async b => await AddOrUpdateMember(b, listId)).ToArray(), -1);
}
private void FlushQueues(ref MailChimpClientResult result)
{
result.UpdatedMembers.FlushQueue(UpdatedMembersQueue);
result.Exceptions.FlushQueue(ExceptionsQueue);
}
public MailChimpClientResult AddOrUpdate(MailChimpMember member, string listId)
{
return AddOrUpdateMemberRange(new MailChimpMember[] { member }, listId);
}
public MailChimpClientResult AddOrUpdate(IEnumerable<MailChimpMember> members, string listId)
{
return AddOrUpdateMemberRange(members, listId);
}
}
public static class CollectionExtensions
{
public static T[][] GetBatches<T>(this IEnumerable<T> items, int batchSize)
{
var result = new List<T[]>();
var batch = new List<T>();
foreach (var t in items)
{
if (batch.Count == batchSize)
{
result.Add(batch.ToArray());
batch.Clear();
}
batch.Add(t);
}
result.Add(batch.ToArray());
batch.Clear();
return result.ToArray();
}
public static void FlushQueue<T>(this IList<T> list, ConcurrentQueue<T> queue)
{
T item;
while (queue.TryDequeue(out item))
list.Add(item);
}
}
MailChimpMember being a public copy of the MailChimp.NET Member. The problem seems to happen in the batch processing method, the Task.WaitAll(...) instruction firing its completion event before all calls are complete, therefore not all results are queued. I tried delaying the execution of each individual treatment with Task.Delay() but with little to no result.
Does anyone have an idea what is failing in my implementation ?