0

I have a very large array (10K) and i want to split it (i did according to this: https://stackoverflow.com/a/8495740/2183053 and it worked) but i need to pass the tempArray to request and wait for response which will be passed to savetodb.

Can someone help me out please. To be clear, I want to split the large Array and pass the separated arrays to request function then pass to save to db and continue this process until all the arrays are cleared.

the following is the code i did:

//i used waterfall because it worked in waiting to finish the first task before starting another 
async.waterfall([
  async.apply(handleFile, './jsonFile.json')
  runRequest1,
  savetoDb
], function(err) {
  console.log('waterfall1 complete')
})

function handleFile(data, callback) {
  var name
  var authorNames = []
  require('fs').readFile(data, 'utf8', function(err, data) {
    if (err) throw err
    var content = _.get(JSON.parse(data), [2, 'data'])
    for (var x = 0; x < content.length; x++) {
      authorNames.push(JSON.stringify(content[x]['author']))
    }
    //the large array is authorNames and im splitting it below:
    for (i = 0, j = authorNames.length; i < j; i += chunk) {
      temparray = authorNames.slice(i, i + chunk);
      setTimeout(function() {
        callback(null, temparray)
      }, 2000);
    }
  })
}
CertainPerformance
  • 356,069
  • 52
  • 309
  • 320
A.K.C.F.L
  • 45
  • 3
  • 11

3 Answers3

1

You need promise to handle async things in nodejs

let findAllReceipts = function (accountArray) {
    const a = [];

    for (let i = 0; i < accountArray.length; i++) {
        a.push(new Promise((resolve, reject) => {
            receiptTable.find({accountNo: accountArray[i].accountNo}, function (err, data) {
                if (!err) {
                    resolve(data);
                } else {
                    reject(new Error('findPrice ERROR : ' + err));
                }
            });
        }));
    }

    return Promise.all(a);
};
Biswajit
  • 978
  • 3
  • 11
  • 30
  • hi, thanks for the answer. but how does promise help in the process? – A.K.C.F.L Jun 02 '18 at 04:37
  • promise work like until one service is complete next service waiting for first service result will wait. it works sync – Biswajit Jun 02 '18 at 10:28
  • alright. let me try to learn bout promise on how to implement it in my code. hope it works. thank you so much. i'm new in node, thats y didnt know bout promise. lol – A.K.C.F.L Jun 02 '18 at 10:37
  • 1
    to be true when i was new to nodejs i was not knowing about promise i just stuck in between async. All my code are ruining here and there. So u not the new one. Keep learning from error. – Biswajit Jun 02 '18 at 10:49
1

I added some promise. 

const data = await handleFile('./jsonFile.json');
// save to db 


async function handleFile(filePath) {
    let arrayWillReturn = []; 

    var name
    var authorNames = []
    let data = await getFileData(filePath)
    var content = _.get(JSON.parse(data), [2, 'data'])
    for (var x = 0; x < content.length; x++) {
        authorNames.push(JSON.stringify(content[x]['author']))
    }
    //the large array is authorNames and im splitting it below:
    for (i = 0, j = authorNames.length; i < j; i += chunk) {
        arrayWillReturn.push(authorNames.slice(i, i + chunk));
    }
    return arrayWillReturn;
}

async function getFileData(fileName) {
    return new Promise(function (resolve, reject) {
        fs.readFile(fileName, type, (err, data) => {
            err ? reject(err) : resolve(data);
        });
    });
}
EmreSURK
  • 419
  • 3
  • 13
  • hi, thanks for the answer. I tried running this code, but it says 'await is only valid in async function'. i googled, and it seems there's no solution for it yet. lol. – A.K.C.F.L Jun 02 '18 at 04:39
  • hey, i found out how to deal with this. thank you so much. but arrayWillReturn doesn't return the sliced arrays but it returns the whole bunch 10K array.. any idea why? – A.K.C.F.L Jun 02 '18 at 12:42
0

I'm answering my own question for future references. the only thing that I added to make my code work is Promise. It sounds easy but it took me a while to grasp the function and implementing it but it worked and it was worth it. Thank you so much for the responses.

A.K.C.F.L
  • 45
  • 3
  • 11