0

I am using node.js v0.10.33 with node-mysql 2.5.4 and express.js 4.10.2 and fast-csv 0.6.0

Any following requests made to the server slows down after a request to "query 9000000 rows and download it as csv". Is there a better way to avoid the slow response?

/**
 * problem is why the server response slows down on a large request mande to mysqlrows_to_csv which queries 9000000 rows and saves as csv
 */

(function () {
    'use strict';
    var async = require('async');
    var csv = require("fast-csv");
    var fs = require('fs');
    var mysql = require('mysql');
    //var express = require('express');
    //var expRouter = express.Router();   // Express Router

    expRouter.get('/mysqlrows_to_csv/', function(req, res, next) {
        mysqlrows_to_csv(function(err) {
            console.log('use socket to emit a notification to client saying csvfile is ready');
        });
        res.json({'mysqlrows_to_csv': true});
        return res.end();
    });

    expRouter.get('/some_other_queries/', function(req, res, next) {
        res.json({'some_other_queries': true});
        return res.end();
    });


    function mysqlrows_to_csv(callback) {
        var csvStream;
        var connProp = {
                host     : 'host',
                user     : 'user',
                password : 'password',
                database : 'database'
        };
        var I = 0;
        async.waterfall([
            function(cb) {
                var connection = mysql.createConnection(connProp);
                connection.connect(function(err) {
                    if (err) return callback(err);
                    csvStream = csv.createWriteStream({headers: true});
                    var writableStream = fs.createWriteStream("dump_mysql_to_csv.csv");
                    writableStream.on("error", function(err){
                        return callback(err);
                    });
                    writableStream.on("finish", function(){
                        console.log("DONE!");
                        return callback(null);
                    });
                    if (csvStream) csvStream.pipe(writableStream);
                    return cb(err, connection);
                });
            },
            function(connection, cb) {
                var qry = 'select * from table';    // this has aleast 9000000 rows
                var query = connection.query(qry);
                query
                    .on('error', function(err){
                        return cb(err);
                    })
                    .on('result', function(row) {
                        connection.pause();
                        I = I + 1;
                        console.log('row no.', I);
                        if (csvStream) csvStream.write(row);
                        connection.resume();
                    })
                    .on('end', function() {
                        console.log('stream-ended')
                        if (csvStream) csvStream.end();
                        return cb(null, connection);
                    });
            },
            function(connection, cb) {
                connection.end(function(err) {
                    if (err) {
                        console.log('Error while disconnecting mysql', err);
                    }
                    console.log('The connection is terminated now');
                    console.log('final conn state:', connection.state);
                });
            }
        ]);
    }
}());
amulllb
  • 3,036
  • 7
  • 50
  • 87
  • 1
    This is a bulk operation, yet you are processing in an iterative mode. A much more performant method would be to use `SELECT ... INTO OUTFILE ... FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' LINES TERMINATED BY '\n'` to dump the data as csv and serve that file. – JRD Oct 28 '15 at 21:25
  • I cannot do that as I my mysql server is remote, so the `SELECT INTO OUTFILE` will only work if i had a local server http://stackoverflow.com/a/2805186/558397 – amulllb Oct 28 '15 at 21:29
  • Yes, run it on the server. Then make the file available to your web process. – JRD Oct 28 '15 at 22:24
  • you mean from the remote database server, make it available using ftp/tftp to my web server? I don't database server's ftp access as i don't maintain the database – amulllb Oct 28 '15 at 22:29
  • 1
    Perfect opportunity to take your database guys out for coffee/tea and tell them about your dilemma. If each of your rows was just a single byte (1 character), you are looking at 8.5 MB of data for 9 million rows. Depending on your total data size, may not be suitable for a web request. – JRD Oct 28 '15 at 22:41

0 Answers0