Pause a program for a while

Question

I'm writing a program that basically does the below tasks.

1. Read a json file.
2. Insert the item from the file to DynamoDB

Below is my code.

const AWS = require('aws-sdk');
var fs = require('fs');
var docClient = new AWS.DynamoDB.DocumentClient();

var i = 0;

var allMovies = JSON.parse(fs.readFileSync('JsonFiles/Cars.json', 'utf8'));
allMovies.forEach(function (user) {
    var params = {
        TableName: "CarsData",

        Item: {
            "id": user.id,
            "make": user.Make.toLowerCase(),
            "model": user.Model.toLowerCase(),
            "year": user.Year,
        }
    };
    if (i % 100 == 0) {
        setTimeout(function () {
            console.log('Blah blah blah blah extra-blah');
        }, 3000);
    }
    docClient.put(params, function (err, data) {
        if (err) {
            console.error("Unable to add user", user.Region, ". Error JSON:", JSON.stringify(err, null, 2));
        } else {
            console.log(`PutItem succeeded: `, i);
        }
    });
    i++;
});

Basically, I'm trying to, run the program and increment i by 1. Once i reaches 100, I want the entire program to pause for 3 seconds and then continue till next time 100 records and so on... the output should be like

put item succeeded: 3825
...99 more records....
Blah blah blah blah extra-blah
----------wait for 3 seconds-----
put item succeeded: 3825
...99 more records....
Blah blah blah blah extra-blah
----------wait for 3 seconds-----
.........and so on till the entire data is inserted

When I run the above code the output that I get is as below.

Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
Blah blah blah blah extra-blah
PutItem succeeded:  3825
PutItem succeeded:  3825

This is quite confusing, please let me know where am I going wrong and how can I fix this.

Thanks

Answer 1

You can try this way. forEach gives index in callback function

const AWS = require('aws-sdk');
var fs = require('fs');
var docClient = new AWS.DynamoDB.DocumentClient();

var allMovies = JSON.parse(fs.readFileSync('JsonFiles/Cars.json', 'utf8'));
allMovies.forEach(function (user, index) {
    var params = {
        TableName: "CarsData",

        Item: {
            "id": user.id,
            "make": user.Make.toLowerCase(),
            "model": user.Model.toLowerCase(),
            "year": user.Year,
        }
    };
    if ((index + 1) % 100 == 0) { // since index starts with 0;
        setTimeout(function () {
            console.log('Blah blah blah blah extra-blah');
        }, 3000);
    }
    docClient.put(params, function (err, data) {
        if (err) {
            console.error("Unable to add user", user.Region, ". Error JSON:", JSON.stringify(err, null, 2));
        } else {
            console.log(`PutItem succeeded: `, i);
        }
    });
});

Very simple example using javascript

var arr = [1,2,3,4];
arr.forEach(function(x, index) {
console.log(x);
if(index == 3) {
    window.setTimeout(function(){ console.log('Im timeout blah blan'); }, 1000);
 }
});

Answer 2

What i might do is ditch using forEach and instead use a self invoking function to iterate through the list and apply a timeout when necessary.

var cooldownIteration = function(array, index) {
  if (index >= array.length) {
    return; //Make sure we dont try to access non-existing indexes
  }
  var user = array[index];

  var params = {
    TableName: "CarsData",

    Item: {
      "id": user.id,
      "make": user.Make.toLowerCase(),
      "model": user.Model.toLowerCase(),
      "year": user.Year,
    }
  };

  index++ // Use the next index for the next time the function is ran

  docClient.put(params, function(err, data) {
    if (err) {
      console.error("Unable to add user", user.Region, ". Error JSON:", JSON.stringify(err, null, 2));
    } else {
      console.log(`PutItem succeeded: `, i);
    }

    //Assuming docClient.put is asynchronous then you should call the next cycle inside of the put as you dont want another cycle to start before one ends
    if (index % 100 == 0) {
      setTimeout(cooldownIteration(array, index), 3000) //Once the operations are done and the index is a multiple of 100 then call the same function again with the increased index 3 seconds later
    } else {
      cooldownIteration(array, index) // If the index is not a multiple of 100 then call the function without delay.
    }
  });
}

var allMovies = JSON.parse(fs.readFileSync('JsonFiles/Cars.json', 'utf8'))
cooldownIteration(allMovies, 0)

Answer 3

My understanding has always been that you cannot simply just "pause" Javascript. When the setTimeout is being triggered the rest of your function will continue to run immediately since it's asynchronous... it'll take 3 seconds for the "blah blah blah" to appear in your console - it won't hold your application up.

---

Edit #1

The only way to accomplish a "pause" in JavaScript is to create a function that takes 3 seconds (or however long you want it to take) to complete. While this isn't the best practice to do since it's effectively locking up your page... the following code will do what it is you want - a three second delay when a certain condition is met.

<script>
var arr = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
arr.forEach(function (x) {
    console.log(x)
    if (x % 3 == 0) {
        console.log("blah blah blah");
        sleep(3000)
    }
});

function sleep(delay) {
    var start = new Date().getTime();
    while (new Date().getTime() < start + delay);
}
</script>

So, whenever a number divisible by 3 appears... it will lock up execution for 3 seconds before continuing on to the additional numbers in the array.

1, 2, 3 ... (3 seconds) ... 4, 5, 6 ... (3 seconds) ... 7, 8, 9 (3 seconds)...

From what I know and have read, this is really the only way to achieve this in JavaScript. I'll provide an additional resource for you read over as well.

Resource: Is There a Sleep Function in JavaScript?

---

Edit #2

Here is your work with the proposed function I mentioned above being used to pause your function for 3 seconds when your condition is met...

const AWS = require('aws-sdk');
var fs = require('fs');
var docClient = new AWS.DynamoDB.DocumentClient();

var i = 0;

var allMovies = JSON.parse(fs.readFileSync('JsonFiles/Cars.json', 'utf8'));
allMovies.forEach(function (user) {
    var params = {
        TableName: "CarsData",

        Item: {
            "id": user.id,
            "make": user.Make.toLowerCase(),
            "model": user.Model.toLowerCase(),
            "year": user.Year,
        }
    };
    if (i % 100 == 0) {
         console.log('Blah blah blah blah extra-blah');
         sleep(3000);
    }
    docClient.put(params, function (err, data) {
        if (err) {
            console.error("Unable to add user", user.Region, ". Error JSON:", JSON.stringify(err, null, 2));
        } else {
            console.log(`PutItem succeeded: `, i);
        }
    });
    i++;
});

function sleep(delay) {
    var start = new Date().getTime();
    while (new Date().getTime() < start + delay);
}