0

I'm trying to write to a file then send that file. I need to wait for the file to write before sending then wait for it to send before looping again. I have the file creation function and file sending function returning promises and the start function is async. However I'm getting await only vaild in async function.

const fs = require('fs');
const csv = require('csv-parser')
const { exec } = require("child_process")
var i = 74243;

async function start() {
  fs.createReadStream('agency_summary_test.csv')
    .pipe(csv())
    .on('data', function (row) {
      var cmd = "curl -XPOST -u \'User:Pass\' ";
      var url = "\'https://vpc-test-ovlb6af5uhpnaoqb57oqg5xore.us-gov-east-1.es.amazonaws.com/_bulk\' --data-binary @agency_temp.json";
      var obj = {}
      Object.entries(row).forEach(([key, value]) => {
        if (!isNaN(value)) {
          value = parseInt(value)
        }
        obj[key] = value
      })

      await writeFile(obj, i);

      var cmdString = cmd + url + " -H \'Content-Type: application\/json\'";

      await sendFile(cmdString);

      i++;
    })
}

function writeFile(obj, i) {
  return new Promise((resolve) => {
    fs.writeFileSync('agency_temp.json', JSON.stringify({
      index: {
        _index: "agency_summary",
        _id: i
      }
    }) + "\n" + JSON.stringify(obj) + "\n", (err) => {
      if (err)
        console.log(err);
      else { }
    })
    resolve();
  })
}

function sendFile(cmd) {
  return new Promise((resolve) => {
    exec(cmd, (error, stdout, stderr) => {
      console.log(`stdout: ${stdout}`)
    });
    resolve();
  })
}

start();
pilchard
  • 12,414
  • 5
  • 11
  • 23
coffy43
  • 3
  • 7
  • synchronous functions shouldn't be used in asynchronous logic. – Kevin B Jun 09 '22 at 18:00
  • Since you're using streams why aren't you piping right into a writable? preferably with a [pipeline](https://stackoverflow.com/questions/58875655/whats-the-difference-between-pipe-and-pipeline-on-streams) – pilchard Jun 09 '22 at 18:01
  • This will be way easier if you don't `pipe` but instead read lines from the csv parser in a loop. – Evert Jun 09 '22 at 18:04
  • The error is very clear: `.on('data', function (row) {` needs to be `.on('data', async function (row) {` because you use `await` inside that function. – Aluan Haddad Jun 09 '22 at 18:04
  • @KevinB If i try to run it just in order the flies are wrtten before they are sent causing some to be skipped. I've got a large dataset I'm trying to get into Openserach. – coffy43 Jun 09 '22 at 18:06
  • Ok? that's not a reason to use sync code in async logic. – Kevin B Jun 09 '22 at 18:07
  • 1
    @AluanHaddad Oh my, how did I not see that? That's exactly the error. Thank you! – coffy43 Jun 09 '22 at 18:08

0 Answers0