0

I have the following code that finds any documents missing DateUTC and loop through and set the date using a datestring value contained in the document. The data is imported via JSON files so dates need to be cast to UTC date. However i am getting 'Process Out of Memory' when there are >1000 records. The documents are quite large JSON weather readings. I am new to MongoDb and Node but this is not a lot of data to update so I figure i am doing something wrong.

'use strict';
var mongoose = require('mongoose');
var Schema = mongoose.Schema;    
var db = mongoose.connection;
var Grib = require('./models/grib.js');
var fs = require('fs');
var moment = require('moment');

//register call back events

db.on('error', console.error);
db.once('open', function () {

    //do our DB work here
    var today = moment().startOf('day');
    var tomorrow = moment(today).add(1, 'day');
    var yesterday = moment(today).add(-1, 'day');

    Grib.find({ 'header.dateUTC': null }, {}, {}, function (err, array) {
        for (var i = 0, len = array.length; i < len; i++) {
            array[i].header.dateUTC = moment.utc(array[i].header.refTime).toDate();
            array[i].save(function (err) {
                if (err) console.log('save error:' + err);
            });

        };
        console.log('Number of grib dates updated:' + array.length);
    });
DrewMann
  • 3
  • 3
  • What's the type of `array[i].header.refTime`? If it's already a date, you don't need to fetch all the docs to update them one by one. – Shanoor Feb 24 '16 at 14:08

1 Answers1

1

You can always paginate. For example fetch in batches by a 1000, just keep in mind you have to sort objects for it to work. 

var index = 0;
Grib.find({ 'header.dateUTC': null }).sort('created_at').skip(index).limit(1000).exec(function (err, array) {
        index += 1000;
.
.
.

be aware of nodejs async nature. I would suggest to take a look at async each function.

J.D.
  • 1,145
  • 2
  • 15
  • 29