Heap memory gets full and throw : java.lang.OutOfMemoryError: Java Heap Space

Question

The following code of servlet receives huge JSON string every minute and near about after 2 hours I always got the OutOfMemoryError: Java Heap Space

public class GetScanAlertServlet extends HttpServlet {

private String scanType = "";
private static final String path = "D:\\Mobile_scan_alerts8180";
private static final String stockFileName = "stock.txt";
private static final String foFileName = "fo.txt";
private static Logger logger = null;
private String currDate = "";
private DateFormat dateFormat;
private StringBuffer stockData;
private StringBuffer foData;
 StringBuffer data = new StringBuffer("");
// For average time of received data
private static float sum = 0;
private static float count = 0;
private static float s_sum = 0;
private static float s_count = 0;
private static float fo_sum = 0;
private static float fo_count = 0;

private static final File dir = new File(path);
private static final File stockFile = new File(path + "\\" + stockFileName);
private static final File foFile = new File(path + "\\" + foFileName);

public void init() {

    logger = MyLogger.getScanAlertLogger();

    if(logger == null) {
        MyLogger.createLog();
        logger = MyLogger.getScanAlertLogger();
    }

}

/**
 * Processes requests for both HTTP <code>GET</code> and <code>POST</code>
 * methods.
 *
 * @param request servlet request
 * @param response servlet response
 * @throws ServletException if a servlet-specific error occurs
 * @throws IOException if an I/O error occurs
 */
protected void processRequest(HttpServletRequest request, HttpServletResponse response)
        throws ServletException, IOException {

    PrintWriter out = response.getWriter();
    response.setContentType("text/plain");
    String strScan = "";

    try {

        String asof = null;

        scanType = request.getParameter("type");
        scanType = scanType == null ? "" : scanType;


        if(scanType.length() > 0){

            if(scanType.equalsIgnoreCase("s")) {
                stockData = null;
                stockData = new StringBuffer(request.getParameter("scanData"));
                stockData = stockData == null ? new StringBuffer("") : stockData;
            } else {
                foData = null;
                foData = new StringBuffer(request.getParameter("scanData"));
                foData = foData == null ? new StringBuffer("") : foData;
            }

        }

        asof = request.getParameter("asof");
        asof = asof == null ? "" : asof.trim();

        // Date format without seconds
        DateFormat formatWithoutSec = new SimpleDateFormat("yyyy/MM/dd HH:mm");

        dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
        Date tmp = new Date();

        // format: yyyy/MM/dd HH:mm:ss
        currDate = dateFormat.format(tmp);

        //format: yyyy/MM/dd HH:mm
        Date asofDate = formatWithoutSec.parse(asof);
        Date cDate = formatWithoutSec.parse(currDate);
        cDate.setSeconds(0);


        System.out.println(asofDate.toString()+" || "+cDate.toString());

        int isDataExpired = asofDate.toString().compareTo(cDate.toString());

        if(isDataExpired > 0 || isDataExpired == 0) {

            if(scanType != null && scanType.length() > 0) {
                checkAndCreateDir();
                strScan = scanType.equalsIgnoreCase("s") ? "Stock Data Recieved at "+currDate
                        : "FO Data Recieved at "+currDate;
                //System.out.println(strScan);
            } else {
                strScan = "JSON of scan data not received properly at "+currDate;
                //System.out.println("GSAS: received null or empty");
            }

        } else {
            strScan = "GSAS: " + scanType + ": Received Expired Data of "+asofDate.toString()+" at "+cDate.toString();
            System.out.println(strScan);
        }
        scanType = null;


    } catch (Exception ex) {
        strScan = "Mobile server issue for receiving scan data";
        System.out.println("GSAS: Exception-1: "+ex);
        logger.error("GetScanAlertServlet: processRequest(): Exception: "+ex.toString());
    } finally {
        logger.info("GetScanAlertServlet: "+strScan);
        out.println(strScan);
    }

}

private void checkAndCreateDir() {

    try {
            boolean isStock = false;
            Date ddate = new Date();
            currDate = dateFormat.format(ddate);
            sum += ddate.getSeconds();
            count++;
            logger.info("Total Average Time: "+(sum/count));

            if(scanType.equalsIgnoreCase("s")){ //For Stock
                setStockData(stockData);

                Date date1 = new Date();
                currDate = dateFormat.format(date1);
                s_sum += date1.getSeconds();
                s_count++;
                logger.info("Stock Average Time: "+(s_sum/s_count));

                //file = new File(path + "\\" + stockFileName);
                isStock = true;

            } else if (scanType.equalsIgnoreCase("fo")) { //For FO
                setFOData(foData);

                Date date2 = new Date();
                currDate = dateFormat.format(date2);
                fo_sum += date2.getSeconds();
                fo_count++;
                logger.info("FO Average Time: "+(fo_sum/fo_count));

                //file = new File(path + "\\" +foFileName);
                isStock = false;
            }


            if(!dir.exists()) { // Directory not exists
                if(dir.mkdir()) {

                    if(isStock)
                        checkAndCreateFile(stockFile);
                    else
                        checkAndCreateFile(foFile);

                }
            } else { // Directory already exists

                    if(isStock)
                        checkAndCreateFile(stockFile);
                    else
                        checkAndCreateFile(foFile);
            }

    } catch (Exception e) {
        System.out.println("GSAS: Exception-2: "+e);
        logger.error("GetScanAlertServlet: checkAndCreateDir(): Exception: "+e);
    }

}

private void checkAndCreateFile(File file) {

    try{
        if(!file.exists()){ // File not exists

            if(file.createNewFile()){
                writeToFile(file);
            }

        } else { // File already exists
            writeToFile(file);
        }
    } catch (Exception e) {
        System.out.println("GSAS: Exception-3: "+e);
        logger.error("GetScanAlertServlet: checkAndCreateFile(): Exception: "+e.toString());
    }
}


private void writeToFile(File file) {

    FileOutputStream fop = null;

    try{

        if(scanType.equalsIgnoreCase("s")){ //For Stock
            data = getStockData();
        } else if (scanType.equalsIgnoreCase("fo")) { //For FO
            data = getFOData();
        }

        if(data != null && data.length() > 0 && file != null){

            fop = new FileOutputStream(file);

            byte[] contentBytes = data.toString().getBytes();
            for(byte b : contentBytes){
                fop.write(b);
            }
            //fop.write(contentBytes);

            fop.flush();


        } else {

            System.out.println("GSAS: Data is null/empty string");
            logger.info("GSAS: Data is null or empty string");

        }
        data = null;
    } catch (Exception  e) {
        System.out.println("GSAS: Exception-4: "+e);
        logger.info("GetScanAlertServlet: writeToFile(): Exception: "+e.toString());

    } finally {

        try {

            if(fop != null)
                fop.close();

        } catch (IOException ex) {
                java.util.logging.Logger.getLogger(GetScanAlertServlet.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
}

private String readFromFile(String fileName){

    String fileContent = "";
    try{

        String temp = "";
        File file = new File(fileName);
        if(file.exists()){

            FileReader fr = new FileReader(file);
            BufferedReader br = new BufferedReader(fr);

            while((temp = br.readLine()) != null)
            {
                fileContent += temp;
            }
            br.close();
        } else {
            System.out.println("GSAS: File not exists to read");
            logger.info("GetScanAlertServlet: File not exists to read");
        }

        temp = null;
        file = null;

    } catch (Exception e) {
        System.out.println("GSAS: Exception-5: "+e);
        logger.error("GetScanAlertServlet: readFromFile(): Exception: "+e.toString());
    }
    return fileContent;
}

public StringBuffer getStockData() {

    //String temp="";
    //StringBuffer temp = (StringBuffer)scanDataSession.getAttribute("stock");
    //if(temp != null && temp.length() > 0) {
    //    return temp;
    //}
    if(stockData != null && stockData.length() > 0){
        return stockData;
    } else {
        stockData = null;
        stockData = new StringBuffer(readFromFile(path + "\\"+ stockFileName));
        return stockData;
    }
}


public StringBuffer getFOData(){

    //String temp="";
    //StringBuffer temp = (StringBuffer)scanDataSession.getAttribute("fo");
    //if(temp != null && temp.length() > 0) {
    //    return temp;
    //}
    if(foData != null && foData.length() > 0) {
        return foData; 
    } else {
        foData = null;
        foData = new StringBuffer(readFromFile(path + "\\" + foFileName));
        return foData;
    }   
}

}

I always get the following exception after every 2 hours when I restart my jboss server and as solution I've also increased Heap size but same problem still exists

ERROR [[GetScanAlertServlet]] Servlet.service() for servlet GetScan
AlertServlet threw exception
java.lang.OutOfMemoryError: Java heap space
        at sun.nio.cs.StreamEncoder.write(Unknown Source)
        at java.io.OutputStreamWriter.write(Unknown Source)
        at java.io.Writer.write(Unknown Source)
        at GetScanAlertServlet.writeToFile(GetScanAlertServlet.java:256)
        at GetScanAlertServlet.checkAndCreateFile(GetScanAlertServlet.java:236)
        at GetScanAlertServlet.checkAndCreateDir(GetScanAlertServlet.java:202)
        at GetScanAlertServlet.processRequest(GetScanAlertServlet.java:135)
        at GetScanAlertServlet.doPost(GetScanAlertServlet.java:377)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Appl
icationFilterChain.java:252)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationF
ilterChain.java:173)
        at org.jboss.web.tomcat.filters.ReplyHeaderFilter.doFilter(ReplyHeaderFi
lter.java:81)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Appl
icationFilterChain.java:202)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationF
ilterChain.java:173)
        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperV
alve.java:213)
        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextV
alve.java:178)
        at org.jboss.web.tomcat.security.CustomPrincipalValve.invoke(CustomPrinc
ipalValve.java:39)
        at org.jboss.web.tomcat.security.SecurityAssociationValve.invoke(Securit
yAssociationValve.java:153)
        at org.jboss.web.tomcat.security.JaccContextValve.invoke(JaccContextValv
e.java:59)
        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.j
ava:126)
        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.j
ava:105)
        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineVal
ve.java:107)
        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.jav
a:148)
        at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java
:856)
        at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.proce
ssConnection(Http11Protocol.java:744)
        at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpo
int.java:527)
        at org.apache.tomcat.util.net.MasterSlaveWorkerThread.run(MasterSlaveWor
kerThread.java:112)
        at java.lang.Thread.run(Unknown Source)

Answer 1

Though I do not see the problem right away, your code does not properly handle resource allocation/release. The problems do not have to be caused by manipulating large JSON blob.

I just observed you do not free your resources (you open files, but do not close them in finally blocks -- any reason not to?), you would probably do better with StringBuilder for string manipulation or just use some kind of existing library (apache commons (io, string)) to do it for you.

Scheduled executor service should be properly shutted down (maybe use something your container provides: Jboss thread pool).

Answer 2

To start I had to rewrite most of the code. I know it's bad practice on SO to do that, We are here to teach and help. Not do other peoples work. But I could really stand reading the code and made it hard to follow.

Here are the bullet points of issues I found

• No finally clauses, So your FileWriter, 'FileReader, andBufferedReader` were never closed if an exception occurred.
• Not using static where it could, path and file names never changed. Also your DateFormat never changed so moved that to static
• Not sure why you were setting strings to null when the next line was getting it from the request parameters and if it was null changed it to an empty string anyway.
• Not sure why you were converting dates to strings to compare them. Dates are comparable.

anyway here is the code hope it helps

public class GetScanAlertServlet extends HttpServlet
{
    private static final String PATH            = "D:\\Mobile_scan_alerts";
    private static final String STOCK_FILE_NAME = "stock.txt";
    private static final String FO_FILE_NAME    = "fo.txt";
    private static final String EMPTY           = "";

    private static final DateFormat FORMAT_WITHOUT_SEC = new SimpleDateFormat("yyyy/MM/dd HH:mm");

    // For average time of received data
    private static float SUM      = 0;
    private static float S_SUM    = 0;
    private static float FO_SUM   = 0;
    private static float COUNT    = 0;
    private static float S_COUNT  = 0;
    private static float FO_COUNT = 0;

    private static Logger LOGGER = null;

    private String scanType;
    private String stockData;
    private String foData;

    @Override
    public void init()
    {
        LOGGER = MyLogger.getScanAlertLogger();
        if (LOGGER == null)
        {
            MyLogger.createLog();
            LOGGER = MyLogger.getScanAlertLogger();
        }
    }

    /**
     * Returns a short description of the servlet.
     *
     * @return a String containing servlet description
     */
    @Override
    public String getServletInfo()
    {
        return "Short description";
    }

    /**
     * Handles the HTTP <code>GET</code> method.
     *
     * @param request  servlet request
     * @param response servlet response
     * @throws ServletException if a servlet-specific error occurs
     * @throws IOException      if an I/O error occurs
     */
    @Override
    protected void doGet(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException
    {
        processRequest(request, response);
    }

    /**
     * Handles the HTTP <code>POST</code> method.
     *
     * @param request  servlet request
     * @param response servlet response
     * @throws ServletException if a servlet-specific error occurs
     * @throws IOException      if an I/O error occurs
     */
    @Override
    protected void doPost(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException
    {
        processRequest(request, response);
    }

    /**
     * Processes requests for both HTTP <code>GET</code> and <code>POST</code>
     * methods.
     *
     * @param request  servlet request
     * @param response servlet response
     * @throws ServletException if a servlet-specific error occurs
     * @throws IOException      if an I/O error occurs
     */
    protected void processRequest(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException
    {
        PrintWriter out = response.getWriter();
        response.setContentType("text/plain");
        String strScan = EMPTY;

        try
        {
            scanType = getRequestParameter(request, "type");
            if (scanType.length() > 0)
            {
                if (scanType.equalsIgnoreCase("s"))
                {
                    stockData = getRequestParameter(request, "scanData");
                }
                else
                {
                    foData = getRequestParameter(request, "scanData");
                }
            }

            //format: yyyy/MM/dd HH:mm
            Date asofDate = FORMAT_WITHOUT_SEC.parse(getRequestParameter(request, "asof"));
            Date currDate = new Date();
            currDate.setSeconds(0);
            System.out.println(asofDate.toString() + " || " + currDate.toString());
            if (asofDate.compareTo(currDate) >= 0)
            {
                if (scanType != null && scanType.length() > 0)
                {
                    checkAndCreateDir();
                    strScan =
                            scanType.equalsIgnoreCase("s") ? "Stock Data Recieved at " + currDate :
                                    "FO Data Recieved at " + currDate;
                }
                else
                {
                    strScan = "JSON of scan data not received properly at " + currDate;
                }
            }
            else
            {
                strScan = "GSAS: " + scanType + ": Received Expired Data of " + asofDate.toString()
                        + " at " + currDate.toString();
                System.out.println(strScan);
            }
        }
        catch (Exception ex)
        {
            strScan = "Mobile server issue for receiving scan data";
            LOGGER.error("GetScanAlertServlet: processRequest(): Exception: " + ex.toString());
        }
        finally
        {
            LOGGER.info("GetScanAlertServlet: " + strScan);
            out.println(strScan);
            out.close();
        }
    }

    private void checkAndCreateDir()
    {
        try
        {
            File dir = new File(PATH);
            if (!dir.exists())
            {
                dir.mkdir();
            }
            File file = null;

            SUM += new Date().getSeconds();
            COUNT++;
            LOGGER.info("Total Average Time: " + (SUM / COUNT));

            if (scanType.equalsIgnoreCase("s"))
            { //For Stock
                S_SUM += new Date().getSeconds();
                S_COUNT++;
                LOGGER.info("Stock Average Time: " + (S_SUM / S_COUNT));

                file = new File(PATH + System.lineSeparator() + STOCK_FILE_NAME);
            }
            else if (scanType.equalsIgnoreCase("fo"))
            { //For FO
                FO_SUM += new Date().getSeconds();
                FO_COUNT++;
                LOGGER.info("FO Average Time: " + (FO_SUM / FO_COUNT));

                file = new File(PATH + System.lineSeparator() + FO_FILE_NAME);
            }
            checkAndCreateFile(file);
        }
        catch (Exception e)
        {
            //System.out.println("GSAS: Exception-2: "+e);
            LOGGER.error("GetScanAlertServlet: checkAndCreateDir(): Exception: " + e.toString());
        }
    }

    private void checkAndCreateFile(File file)
    {
        try
        {
            if(!file.exists())
            {
                file.createNewFile();
            }
            writeToFile(file);
        }
        catch (Exception e)
        {
            LOGGER.error("GetScanAlertServlet: checkAndCreateFile(): Exception: " + e.toString());
        }
    }

    private void writeToFile(File file) throws IOException
    {
        String data = EMPTY;
        if (scanType.equalsIgnoreCase("s"))
        { //For Stock
            if (stockData == null)
            {
                stockData = readFromFile(PATH + System.lineSeparator() + STOCK_FILE_NAME);
            }
            data = stockData;
        }
        else if (scanType.equalsIgnoreCase("fo"))
        { //For FO
            if (foData == null)
            {
                foData = readFromFile(PATH + System.lineSeparator() + FO_FILE_NAME);
            }
            data = foData;
        }
        FileWriter fileWriter = null;
        try
        {
            if (data != null && data.length() > 0)
            {
                fileWriter  = new FileWriter(file);
                fileWriter.write(data.toString());
            }
            else
            {
                System.out.println("GSAS: Data is null/empty string");
                LOGGER.info("GSAS: Data is null or empty string");
            }
        }
        catch (Exception e)
        {
            LOGGER.info("GetScanAlertServlet: writeToFile(): Exception: " + e.toString());
        }
        finally
        {
            if (fileWriter != null)
            {
                fileWriter.flush();
                fileWriter.close();
            }
        }
    }

    private String readFromFile(String fileName) throws IOException
    {
        String fileContent = EMPTY;
        FileReader fr = null;
        BufferedReader br = null;
        try
        {
            File file = new File(fileName);
            if (file.exists())
            {
                fr = new FileReader(file);
                br = new BufferedReader(fr);

                String temp;
                while ((temp = br.readLine()) != null)
                {
                    fileContent += temp;
                }
            }
            else
            {
                System.out.println("GSAS: File not exists to read");
                LOGGER.info("GetScanAlertServlet: File not exists to read");
            }
        }
        catch (Exception e)
        {
            LOGGER.error("GetScanAlertServlet: readFromFile(): Exception: " + e.toString());
        }
        finally
        {
            if (fr != null)
            {
                fr.close();
            }
            if (br != null)
            {
                br.close();
            }
        }
        return fileContent;
    }

    private String getRequestParameter(HttpServletRequest request, String parameter)
    {
        String str = request.getParameter(parameter);
        return str == null ? EMPTY : str.trim();
    }
}