0

I made some code to parse things from some file.log it got 2 different lines that one is line of command and finishing line. So there are 1 000 000 lines of log which is 500 000 entries into DB. Problem inserting and updating database is to slow.

function getDate(){

var date = new Date().getTime() + (2 * 60 * 60 * 1000);
return new Date(date).toISOString().replace(/T/, ' ').replace(/\..+/, ''); // DATETIME in format yyyy-mm-dd hh:mm:ss
}
var detailLog = false;
var express = require("express");
var mysql = require('mysql');
var connection = mysql.createConnection({
host : 'localhost',
user : 'root',
password : '',
database : 'logger'
});
var app = express();
connection.connect(function(err) {
if (!err) {
    console.log(getDate(), "Database is connected ... \n");
} else {
    console.log(getDate(), "Error connecting database ... \n");
    }
});
var file = 'file.log';
var fs = require('fs');
var rl = require('readline').createInterface({

input : require('fs').createReadStream(file)
});
rl.on('line', function(line) {
//searching 'command' since it is in every odd line
if (line.search('command') >= 0) { 

    //parsing every odd line of log

    // insert in logs table
    var queryINS = ""; 
    if (detailLog)
        console.log(getDate(),'shooting INSERT ');
    connection.query(queryINS,
            function(err, rows, fields) {
                if (err) {
                    console.log(getDate(), 'Error while performing
insert into logs query: ', queryINS,err);
                } 
    });
    if (detailLog) 
        console.log(getDate(),'INSERT done');
//searching 'Request finished' since it is in every even line
} else if (line.search('Request finished') >= 0) {

    //parsing every even line of log

    // update logs table
    var queryUPD = "";
    if (detailLog)
        console.log(getDate(),'shooting UPDATE ');
    connection.query(queryUPD,
        function(err, rows, fields) {
            if (err) {
                console.log(getDate(), 'Error while performing update 
logs query: ',queryUPD,err);
            }   
    });
    if (detailLog)
        console.log(getDate(),'UPDATE done');
}   
}).on('close', function() {
    console.log(getDate(), 'Inserted logs into table.');
});
  • 1
    Instead of running one query per line read, you should map all of your inserts into one array and [execute the insert as a batch](http://stackoverflow.com/questions/8899802/how-do-i-do-a-bulk-insert-in-mysql-using-node-js). Probably same process for update – CodingIntrigue Dec 07 '15 at 12:31
  • You may also want to look into pooling. https://github.com/felixge/node-mysql#pooling-connections – Gary Dec 07 '15 at 17:08

0 Answers0