I am new using the Promise Bluebird in nodeJS.
I would like to read all the CSV files in a specific directory into 1 JSON object array and display the results.
The steps are:
- read the directory to get the file names (function readDir called)
- use a loop to read each CSV file (lodash forOwn loop) and store the content in the JSON object array (function ReadFile called)
- merge the JSON object array (using Lodash assign function) with the main JSON object array (named data_Warehouse) so all data are stored into 1 JSON object array.
- After finished saving all the CSV file data into JSON object array data_Warehouse, display data_Warehouse in console.log (.then section).
The check if all goes to according plan, I put in console.log between the codes. The problem is that the processes are not done by the correct order. I tried to use Bluebird but it still doesn't go according to the plan.
Could you please assist me with this issue?
This is what I have as output:
output:load\Book1.csv
output:load\Book2.csv
output:load\Book3.csv
Results:{}
load\Book1.csv 101
load\Book2.csv 102
load\Books.csv 103
As you can see, the line output 'results' should be as last but it is earlier in the process where the JSON object array data_Warehouse is empty.
var Converter = require("csvtojson").Converter;
var converter = new Converter({});
var _ = require('lodash');
var r = require('rethinkdbdash')(config.rethinkdb);
var fs = Promise.promisifyAll(require('fs'));
readDir()
.then(function(v){
var data_Warehouse={};
_.forOwn(v, function(index){
console.log('output:' + index);
_.assign(data_Warehouse, readFile(index));
});
return mine;
}).then(function(w){
console.log('Results:' + w.length);
})
function readDir() {
return fs.readdirAsync(dir).map(function (fileName) {
var path = Path.join(dir, fileName);
return fs.statAsync(path).then(function(stat) {
return stat.isDirectory() ? readDir(path) : path;
});
}).reduce(function (a, b) {
return a.concat(b);
}, []);
}
function readFile(fileName){
var converter = new Converter({});
fs.createReadStream(fileName).pipe(converter);
converter.on("end_parsed", function (jsonArray) {
console.log(fileName + ' ' + jsonArray[0].Order_ID);
return jsonArray;
})
}
I changed my code (promisifyAll is used) that now is able to read all the files in the specified directory and display the results in the right order, but using the csvtojson to convert the read data into json on each file is now the problem. It will display 'done....' before the console.log('results: ' + result[Ticker].Order_ID); (see below the output).
var join = Promise.join;
var fs = Promise.promisifyAll(require("fs"));
var Converter = require("csvtojson").Converter;
var Promise = require('bluebird');
var dir='./load/';
fs.readdirAsync(dir).map(function(fileName) {
var contents = fs.readFileAsync(fileName).catch(function ignore() {});
return join(fileName, contents, function(fileName, contents) {
return {
fileName: fileName,
contents: contents
}
});
}).each(function(file) {
console.log('File Name: ' + file.fileName + '\n');
console.log('output: ' + file.contents + '\n');
var converter = new Converter({});
var temp = file.contents.toString();
var tmp;
converter.on("end_parsed", function() {});
converter.fromString(temp, function(err, result){
tmp = result;
for(var Ticker in result){
console.log('results: ' + result[Ticker].Order_ID);
}
});
console.log('___________________');
return tmp;
}).then(function(contents) {
console.log('Done.............');
})
My output is now:
File Name: Book1.csv
output: Order_ID,Quantity,Price,Date,Remark
123, 1000, 12.95,10/8/2015, test 786, 100, 7.95,11/6/2015, test2
File Name: Book2.csv
output: Order_ID,Quantity,Price,Date,Remark
4526, 800, 2.95,4/1/2015, test3 129, 10, 3.66,1/23/2014, test4
Done.............
results: 123
results: 786
results: 4526
results: 129
How do I bring the converter csvtojson back in the right order so 'done...' is at the end of the process?
EDITED:
Now I have promisify the csvtojson but for some reason it is not working.
var join = Promise.join;
var fs = Promise.promisifyAll(require("fs"));
var Converter = require("csvtojson").Converter;
var Promise = require('bluebird');
var dir='./load/';
jsons= function(data){
var converter = new Converter({});
var output;
converter.on("end_parsed", function() {});
converter.fromString(data, function(err, result){
for(var Ticker in result){
console.log('results: ' + result[Ticker].Ticker);
}
output = result;
});
return output;
};
var conversionJsons = Promise.promisifyAll(jsons);
fs.readdirAsync(dir).map(function(fileName) {
var contents = fs.readFileAsync(fileName).catch(function ignore() {});
return join(fileName, contents, function(fileName, contents) {
return {
fileName: fileName,
contents: contents
}
});
}).each(function(file) {
console.log('File Name: ' + file.fileName + '\n');
console.log('output: ' + file.contents + '\n');
console.log('___________________');
var temp = file.contents.toString();
conversionJsons(temp).then(function(result){ // <----Error here for then
console.log("results are: "+ result);
return result;
});
}).then(function(contents) {
console.log('Done.............');
})
I get the following error:
"Cannot read property 'then' of undefined"
Could you assist me with this issue?
