Javascript FileSystem function not working as intended

Question

I have a simple code to write a js list to a js file.

// Requiring fs module in which
// writeFile function is defined.
var fs = require('fs');

// first overwrite any pre-existing file with the start of the list
fs.writeFile('Output.js', 'export const addressPoints = [', function (err) {
  if (err) throw err;
  console.log('Create file Output.js');
});

for (let i = 0; i < 100; i++) {
    // var string_row = "[" + latlon_datum1_row[i] + "],";
    var string_row = "HI"
    console.log(string_row);
    fs.appendFile('Output.js', string_row, function (err) {
        if (err) throw err;
        console.log('Writing file contents line', i);
    });
}

fs.appendFile('Output.js', '];', function (err) {
if (err) throw err;
    console.log('Append file Output.js');
});

It can be thought of in three parts, one intial component creates the new js script and adds 'export const addressPoints = [' to it. After that, a for-loop runs to write the items of the list, and lastly, '];' is appended to the end of the file.

Very strangely, the code does not work as intended. One would expect the console log to look something like

Create file Output.js
Writing file contents line 1
Writing file contents line 2
Writing file contents line 3
...
Writing file contents line 99
Append file Output.js

However, the output is scrambled and written in the wrong order (showing last few lines):

...
Writing file contents line 96
Writing file contents line 97
Writing file contents line 98
Append file Output.js
Writing file contents line 99
Writing file contents line 91
Writing file contents line 79
Writing file contents line 68

Is there a method that can do this more quickly? If not, why is this error occurring? I assume it has something to do with fs closure but I am not too familiar with js.

You'd be much, much, much better off opening the file *once* and writing what you need to write while it's open, and then closing it at the end. Each time you do an "append" operation, the operating system has to find and reopen the file. — Pointy, Sep 12 '21 at 16:34

score 2 · Answer 1 · edited Sep 13 '21 at 08:56

Extra points for your console.log detail.

As you see, the log has a wrong order. That is due to asynchronous nature of nodejs and javascript.

You need to use callbacks or promises to "wait" for the result of previous operation.

Sample: read a file and print its content

incorrect

fs.readFile('/tmp/foo.txt',function (err, data) {
});

console.log('data is...');

correct

fs.readFile('/tmp/foo.txt',function (err, data) {
  console.log('data is:'+data);
});

As you can see, readFile and almost all nodejs functions are asynchronous. It means that we cannot think like java, c# or python in a sequential or synchronous mode. In this case, readFile needs a function to be executed when file is readed:

function (err, data) {
  console.log('data is:'+data);
}

callbacks

Work passing function which will be executed after the async operation.

var fs = require('fs');

fs.writeFile('Output.js', 'export const addressPoints = [', function (err) {
  if (err) throw err;
  console.log('Create file Output.js');

  //execute after file was updated
  var newContent = "";
  for (let i = 0; i < 100; i++) {
    var string_row = "HI"
    console.log(string_row);
    newContent += string_row+"\n";    
  }
  newContent += "];\n";      
  
  fs.appendFile('Output.js', newContent, function (err) {
    if (err) throw err;
    console.log('Output.js was updated');
  });
  
});

Or a little more readable

var fs = require('fs');

fs.writeFile('Output.js', 'export const addressPoints = [', createCustomFile);

function createCustomFile(err, data){
  //logic here
}

Disadvantage

If you need to execute more operations, you will have a nightmare

promises & await

A way to write more readable (which continues async) code vs nested callbacks

var fs = require('fs');

async function main(){
  await fs.promises.writeFile('Output.js', 'export const addressPoints = [');
  console.log('Create file Output.js');

  for (let i = 0; i < 100; i++) {
      var string_row = "HI"
      console.log(string_row);
      await fs.promises.appendFile('Output.js', string_row);
      console.log('Writing file contents line', i);
  }
  
  await fs.promises.appendFile('Output.js', '];');
  console.log('Append file Output.js');
}

//launch
//don't forget to catch async exceptions here!
main().catch(e => { console.error(e); process.exit(1) })

Javascript FileSystem function not working as intended

1 Answers1

callbacks

promises & await