How to load 30 csv files in parallel using queuejs with d3js?

Question

Two main questions here:

1. We want to load files in parallel

2. Because these files are loaded in parallel, we don't know the exact order they are loaded. I want to parse a key from each file name, with the array of file content parsed by d3.csv as their corresponding values

That is said, the returned results by

   q.awaitAll(function(error, results) { console.log("all done!"); });

should be

results={
  "key1":array_read_from_myData_key1.csv,
  "key2":array_read_from_myData_key2.csv,
  ...
  "key30":array_read_from_myData_key30.csv
}

How to add each file name as a key to each array (results[i]) below?

var q = queue(), // create the queue
    dataSources = [ // the data sources
        'myData_key1.csv',
        'myData_key2.csv',
        'myData_key3.csv',
        ...,
        'myData_key30.csv'
    ];


// Go through each data source and add it to the queue:
dataSources.forEach(function (source) {
    q.defer(function (callback) {
        d3.csv(source, callback);
    });
});

// Wait for all requests to be completed:
q.awaitAll(function (error, results) {
    console.log(results);
})

Answer 1

You could expand your call back inside here:

dataSources.forEach(function (source, index) {
  q.defer(function (callback) {
    d3.csv(source, function(data){
       var fileData = {
           name: dataSources[index],
           data: data
       };
       callback(fileData);
   });
  });
});

Based on your comment you could try doing the following but not 100% sure that it will work :

 var finalData = {};

 dataSources.forEach(function (source, index) {
  q.defer(function (callback) {
    d3.csv(source, function(data){
       if (data) {
         //regex the key from file name
         var matcher = /^(myData)_([a-z][a-z][a-z][0-9])\.csv$/;
         var key = matcher.exec(dataSources[index])[1];
         //assign value to external object
         finalData[key] = data;
         //callback success
         callback(true);
       }
       else {
        //log error if required
        callback(false);
       }
   });
  });
});

// Wait for all requests to be completed:
q.awaitAll(function (error, results) {
   console.log(finalData);
})

Hope this solves your issue.