How to push an array to a file without keeping old data when looping in setInterval?

Question

sorry for the title, I was trying to be specific but it's a bit confusing. Actually the piece of code should be the better way to show you what my problem is:

var i = 0
var jsonArray = []
var startTime = new Date().getTime()

var writeFile = setInterval(function () {
    var currentTime = new Date().getTime()
    var stopTime = 31 * 1000
    var executeScript = setInterval(function () {
        // the script will stop after some time
        if (currentTime - startTime > stopTime) { // time in ms: 60 * 60 * 1000 = 1 hour
            process.exit()
        }
        // making request and fetching data
        http.get(request, function (response) {
            response.pipe(bl(function (err, data) {
                if (err) return console.error(err)
                response.setEncoding('utf8')
                parseString(data, function (err, result) {
                    result = result.SoccerFeed.SoccerDocument[0].MatchData[0].TeamData
                    // creating object and filling it with received data
                    result = createJsObject(result)
                    // converting object in json format
                    var jsonObject = JSON.stringify(result, null, 4)
                    jsonArray.push(jsonObject)
                    // copying json into a new file without overwriting previous written data
                })
            }))
        }).on('error', console.error)
    }, 1 * 1000) // interval in ms: 90 * 1000 = 90 seconds / 1 min 30
    fs.writeFile('API - Opta - ' + moment().format('YYYY - DD.MM (HH:mm:ss)') + '.txt', jsonArray, function(err) {
        if (err) throw err
    })
    jsonArray = []
    i++
}, 10 * 1000)

The problem is that the array I'm pushing in the file is keeping data from the old one, even when I clean it with jsonArray = []. I know I'm doing something wrong but I don't know what.

Thanks in advance for your help

Answer 1

Not sure if it helps, but place this line

var jsonArray = []

after this

var writeFile = setInterval(function () {

and you won't need to clear jsonArray.

so it will be

var writeFile = setInterval(function () {
  var jsonArray = []
  // ...

Answer 2

So, at first I thought it was an asynchronous problem with fs.writeFile getting a blank copy of jsonArray. This is not the case due to the way that javascript handles jsonArray = []. When you make the call to fs.writeFile, you pass it jsonArray which is passed by "reference." When you set it to jsonArray = [] you actually create an entire new object. The object that was passed to fs.writeFile is no longer referenced by jsonArray and it's only scope is in the call to fs.writeFile thus it maintains the state that you passed in. If you had instead called jsonArray.length = 0 it would have always output an empty array to the file because that would have overwritten the actual array fs.writeFile. Check out this post to see that: How do I empty an array in JavaScript?

In short, that's not the problem.

But the actual problem is that http.get is receiving data many times after the file was written. Those things won't get written until your next loop. Your described sequence that things are always one behind makes perfect sense.

Your sequence is this inside your interval executeScript:

1. Check to make sure it's not quitting time
2. Start http.get
   • sometime in the future the response is received and added to jsonArray
3. Start fs.writeFile, contents of jsonArray locked in at this point in time as to what is going to be written to the file
   • sometime in future the file is written
4. Set jsonArray = []

I would propose you have two separate intervals that aren't nested like so:

var i = 0
var jsonArray = []
var startTime = new Date().getTime()

var writeFile = setInterval(function () {
    fs.writeFile('API - Opta - ' + moment().format('YYYY - DD.MM (HH:mm:ss)') + '.txt', jsonArray, function(err) {
        if (err) throw err
    })
    jsonArray = []
    i++
}, 10 * 1000)

var executeScript = setInterval(function () {
    var currentTime = new Date().getTime()
    var stopTime = 31 * 1000

    // the script will stop after some time
    if (currentTime - startTime > stopTime) { // time in ms: 60 * 60 * 1000 = 1 hour
        process.exit()
    }
    // making request and fetching data
    http.get(request, function (response) {
        response.pipe(bl(function (err, data) {
            if (err) return console.error(err)
            response.setEncoding('utf8')
            parseString(data, function (err, result) {
                result = result.SoccerFeed.SoccerDocument[0].MatchData[0].TeamData
                // creating object and filling it with received data
                result = createJsObject(result)
                // converting object in json format
                var jsonObject = JSON.stringify(result, null, 4)
                jsonArray.push(jsonObject)
                // copying json into a new file without overwriting previous written data
            })
        }))
    }).on('error', console.error)
}, 1 * 1000) // interval in ms: 90 * 1000 = 90 seconds / 1 min 30

Now you have two intervals going at the same time. The 1 second loop on executing the script now has lots of data queued up for when the 10 second loop on writing the file kicks off. It writes the file and then immediately clears jsonArray for it to keep adding in subsequent 1 second loops of the executeScript interval.