JPA Entity Manager does not throw exception the second time when database is down for the same entity

Question

I am using Spring CRUD Repository for handling my database level transactions. My web application is deployed on tomcat and interacts with MySQL database. The scenario I am testing involves the MySQL database going down with the web application deployed on tomcat. The JPA Entity Manager does throw the exception for the 1st time for the same entity(mapped as java class) but, for the second time with the same entity, the entity manager just hangs and it only throws an exception when the tomcat is shutdown.

I have the logs with me and if required I can upload them.

web.xml

    <web-app version="3.0" xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd">
    <!-- Processes application requests -->
    <servlet>

        <servlet-name>appServlet</servlet-name>
        <servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>

        <init-param>
            <param-name>contextClass</param-name>
            <param-value>
                 org.springframework.web.context.support.AnnotationConfigWebApplicationContext
            </param-value>
        </init-param>
        <init-param>
            <param-name>contextConfigLocation</param-name>
            <param-value>
               com.test.WebConfig
            </param-value>
         </init-param>
            <load-on-startup>1</load-on-startup>
            <async-supported>true</async-supported>
    </servlet>

    <servlet-mapping>
    <servlet-name>appServlet</servlet-name>
    <url-pattern>/</url-pattern>
    </servlet-mapping>

    <filter>
        <filter-name>cors</filter-name>
        <filter-class>com.test.filter.CorsFilter</filter-class>
        <async-supported>true</async-supported>
    </filter>
    <filter-mapping>
    <filter-name>cors</filter-name>
    <url-pattern>/*</url-pattern>
    </filter-mapping>
    <error-page>
    <error-code>404</error-code>
    <location>/WEB-INF/pages/404.jsp</location>
  </error-page>

  <error-page>
    <exception-type>java.lang.Exception</exception-type>
    <location>/WEB-INF/pages/404.jsp</location>
  </error-page>
  <error-page>
    <exception-type>com.test.exceptions.AcsException</exception-type>
    <location>/WEB-INF/pages/404.jsp</location>
  </error-page>


   </web-app>

WebConfig.Java

package com.test;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.web.servlet.config.annotation.DefaultServletHandlerConfigurer;
import org.springframework.web.servlet.config.annotation.EnableWebMvc;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter;
import org.springframework.web.servlet.view.InternalResourceViewResolver;
import org.springframework.web.socket.WebSocketHandler;
import org.springframework.web.socket.config.annotation.EnableWebSocket;
import org.springframework.web.socket.config.annotation.SockJsServiceRegistration;
import org.springframework.web.socket.config.annotation.WebSocketConfigurer;
import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry;
import org.springframework.web.socket.handler.PerConnectionWebSocketHandler;
import org.springframework.web.socket.server.standard.ServletServerContainerFactoryBean;

import com.test.service.handler.AgentConnectionHandlerImpl;
import com.test.service.handler.CustomerConnectionHandlerImpl;
import com.test.jsonrpc.JsonRpcCache;

@Configuration
@EnableWebMvc
@EnableWebSocket
@ComponentScan(basePackages={"com.test"})
//Do not enable CGLIB , i.e. (proxyTargetClass=true), default is false. 
@EnableAspectJAutoProxy
public class WebConfig
  extends WebMvcConfigurerAdapter
  implements WebSocketConfigurer
{
  private static final Logger logger = LoggerFactory.getLogger(WebConfig.class);

    @Bean
    public ServletServerContainerFactoryBean createServletServerContainerFactoryBean() {
        ServletServerContainerFactoryBean container = new ServletServerContainerFactoryBean();
        container.setMaxTextMessageBufferSize(32768);
        container.setMaxBinaryMessageBufferSize(32768);
         logger.info("Websocket factory returned");
        return container;
    }


  public void registerWebSocketHandlers(WebSocketHandlerRegistry registry)
  {
    SockJsServiceRegistration custSr=registry.addHandler(coBrowseSockJsCustomerHandler(), new String[] { "/sockjs/test/customer/" }).withSockJS();
    SockJsServiceRegistration agentSr=registry.addHandler(coBrowseSockJsAgentHandler(), new String[] { "/sockjs/test/agent/" }).withSockJS();
    if(custSr!=null){
        custSr.setSessionCookieNeeded(false);
    }else{
        logger.info("Cust SR null");

    }
    if(agentSr!=null){
        agentSr.setSessionCookieNeeded(false);
    }else{
        logger.info("Agent SR null");

    }
    JsonRpcCache.getInstance().init("com.avaya.labs");
    logger.info("Registered Sock JS");
  }

  @Bean
  public InternalResourceViewResolver getInternalResourceViewResolver()
  {
    InternalResourceViewResolver resolver = new InternalResourceViewResolver();
    resolver.setPrefix("/WEB-INF/views/");
    resolver.setSuffix(".jsp");
    return resolver;
  }

  @Bean
  public WebSocketHandler coBrowseSockJsCustomerHandler()
  {
    return new PerConnectionWebSocketHandler(CustomerConnectionHandlerImpl.class);
  }

  @Bean
  public WebSocketHandler coBrowseSockJsAgentHandler()
  {
    return new PerConnectionWebSocketHandler(AgentConnectionHandlerImpl.class);
  }

  @Bean
  public ThreadPoolTaskScheduler sockJsTaskScheduler()
  {
    ThreadPoolTaskScheduler taskScheduler = new ThreadPoolTaskScheduler();
    taskScheduler.setThreadNamePrefix("SockJS-");
    return taskScheduler;
  }

  public void configureDefaultServletHandling(DefaultServletHandlerConfigurer configurer)
  {
    configurer.enable();
  }
}

RepoConfig.java

package com.test;

import javax.persistence.EntityManagerFactory;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;
import org.springframework.core.env.Environment;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.hibernate4.HibernateExceptionTranslator;
import org.springframework.orm.jpa.JpaDialect;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.orm.jpa.vendor.Database;
import org.springframework.orm.jpa.vendor.HibernateJpaDialect;
import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
import org.springframework.transaction.PlatformTransactionManager;

import com.jolbox.bonecp.BoneCPDataSource;

@PropertySource(value = "classpath:db.properties")
@EnableJpaRepositories("com.test.dao")
@Configuration
public class RepoConfig {
    private static final Logger logger = LoggerFactory.getLogger(RepoConfig.class);
    @Autowired
    Environment env;

    @Bean
    public BoneCPDataSource dataSource() {
    BoneCPDataSource boneCPDataSource = null;
    try {
        logger.info("Creating BoneCPDataSource");
        boneCPDataSource = new BoneCPDataSource();
        boneCPDataSource.setDriverClass(env.getProperty("jdbc.driverclass").trim());
        logger.info("JDBS PARAMS " + env.getProperty("jdbc.url").trim());
        boneCPDataSource.setJdbcUrl(env.getProperty("jdbc.url").trim());
        boneCPDataSource.setUsername(env.getProperty("jdbc.username").trim());
        boneCPDataSource.setPassword(env.getProperty("jdbc.password").trim());
        boneCPDataSource.setIdleConnectionTestPeriodInMinutes(Integer.parseInt(env.getProperty(
            "idleconnectiontestperiodinminute").trim()));
        boneCPDataSource.setIdleMaxAgeInMinutes(Integer.parseInt(env.getProperty("idlemaxageinminutes").trim()));
        boneCPDataSource.setMaxConnectionsPerPartition(Integer.parseInt(env
            .getProperty("maxconnectionperpartition").trim()));
        boneCPDataSource.setMinConnectionsPerPartition(Integer.parseInt(env
            .getProperty("minconnectionperpartition").trim()));
        boneCPDataSource.setPartitionCount(Integer.parseInt(env.getProperty("partitioncount").trim()));
        boneCPDataSource.setAcquireIncrement(Integer.parseInt(env.getProperty("acquireincrement").trim()));
        boneCPDataSource.setStatementsCacheSize(Integer.parseInt(env.getProperty("statementcachesize").trim()));
        boneCPDataSource.setReleaseHelperThreads(Integer.parseInt(env.getProperty("releasehelperthread").trim()));
        logger.info("BoneCPDataSource created");
    } catch (Exception e) {
        logger.error("Exception BoneCPDataSource", e);
    }
    return boneCPDataSource;

    }

    @Bean
    public HibernateExceptionTranslator hibernateExceptionTranslator() {
    logger.info("HibernateExceptionTranslator");
    return new HibernateExceptionTranslator();
    }

    @Bean
    @Autowired
    public LocalContainerEntityManagerFactoryBean entityManagerFactory() {
    LocalContainerEntityManagerFactoryBean factory = null;
    try {
        HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter();
        // Boolean.getBoolean(name)
        Boolean generateDdl = false;
        Boolean showSql = false;
        if (env.getProperty("generateddl").trim().equalsIgnoreCase("true")) {
        generateDdl = true;
        }
        if (env.getProperty("showsql").trim().equalsIgnoreCase("true")) {
        showSql = true;
        }
        vendorAdapter.setGenerateDdl(generateDdl);
        vendorAdapter.setShowSql(showSql);
        vendorAdapter.setDatabasePlatform(env.getProperty("jdbc.databaseplatform").trim());

        String databaseType = env.getProperty("databasetype").trim();

        // Default MySql
        Database database = Database.MYSQL;
        // H2, HSQL, INFORMIX, MYSQL, ORACLE, POSTGRESQL, SQL_SERVER,
        if (databaseType.equalsIgnoreCase("postgres")) {
        database = Database.POSTGRESQL;
        } else if (databaseType.equalsIgnoreCase("oracle")) {
        database = Database.ORACLE;
        } else if (databaseType.equalsIgnoreCase("hsql")) {
        database = Database.HSQL;
        } else if (databaseType.equalsIgnoreCase("sqlserver")) {
        database = Database.SQL_SERVER;
        }
        vendorAdapter.setDatabase(database);
        logger.info("EntityManagerFactory LocalContainerEntityManagerFactoryBean");
        factory = new LocalContainerEntityManagerFactoryBean();
        factory.setJpaVendorAdapter(vendorAdapter);
        factory.setPackagesToScan("com.test.dao.model");
        logger.info("EntityManagerFactory com.test.dao.model scanned");
        BoneCPDataSource ds = dataSource();
        factory.setDataSource(ds);
        if (ds != null) {
        logger.info("dataSource is NOT NULL");
        } else {
        logger.info("dataSource is ## NULL");
        }
        logger.info("EntityManagerFactory dataSource");
        // Properties properties = new Properties();
        // properties.setProperty("hibernate.cache.use_second_level_cache",
        // "true");
        // properties.setProperty("hibernate.cache.region.factory_class",
        // "org.hibernate.cache.ehcache.EhCacheRegionFactory");
        // properties.setProperty("hibernate.cache.use_query_cache",
        // "true");
        // properties.setProperty("hibernate.generate_statistics", "true");
        //
        // factory.setJpaProperties(properties);
        //
        // factory.afterPropertiesSet();

        logger.info("Returning EntityManagerFactory");
        return factory;
    } catch (Exception e) {
        logger.error("Exception EntityManagerFactory", e);
    }
    return null;
    }

    @Bean
    @Autowired
    public PlatformTransactionManager transactionManager() {
    JpaTransactionManager txManager = null;
    try {
        EntityManagerFactory emf = entityManagerFactory().getObject();
        if (emf != null) {
        logger.info("EntityManagerFactory is NOT NULL");
        } else {
        logger.info("EntityManagerFactory is ## NULL");
        }
        txManager = new JpaTransactionManager();
        JpaDialect jpaDialect = new HibernateJpaDialect();
        txManager.setEntityManagerFactory(emf);
        txManager.setJpaDialect(jpaDialect);
    } catch (Exception e) {
        logger.error("Exception JpaTransactionManager", e);
    }
    return txManager;
    }

}

db.properties

enabledb=false
jdbc.url=jdbc:mysql://localhost:3306/acsdb
jdbc.username=root
jdbc.password=Avaya123!
jdbc.driverclass=com.mysql.jdbc.Driver
jdbc.databaseplatform=org.hibernate.dialect.MySQL5InnoDBDialect
#Connections older than this are sent a keep-alive statement.
idleconnectiontestperiodinminute=60 
# Maximum age of an unused connection before it is closed off.  
idlemaxageinminutes=120
maxconnectionperpartition=30
minconnectionperpartition=10
#connections are grouped in partitions as this decreases contention when opening a new connection.
partitioncount=1
acquireincrement=5
#Min no of prepared statements to cache
statementcachesize=100
#Number of release-connection helper threads to create per partition
releasehelperthread=3
generateddl=true
showsql=false
databasetype=mysql

ClientConfigDao - Dao class for which the issue has been reported

package com.test.dao;

import org.springframework.data.repository.CrudRepository;

import com.test.dao.model.ClientConfig;

public interface ClientConfigDao extends CrudRepository<ClientConfig, String> {
    ClientConfig findByClientIdAndProductVersion(String clientId,String productVersion);
}

ClientConfig - Entity class

package com.test.dao.model;


import java.io.Serializable;
import java.util.Date;

import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;

@Entity
@Table (name="clientconfig")
public class ClientConfig implements Serializable{
    private static final long serialVersionUID = 1L;
    @Id
    @Column(name = "clientid",nullable=false)
    private String clientId;
    @Column(name = "productversion",nullable=false)
    private String productVersion="1.0";
    @Column(name = "maxlicensesagent")
    private Integer numOfLicensesAgent=50;
    @Column(name = "maxlicensesadmin")
    private Integer numOfLicensesAdmin=50;

    //**All values in minute
    @Column(name = "agenttimeout")
    private Integer agentTimeout=30;
    @Column(name = "agenttimeoutwarnintv")
    private Integer agentTimeoutWarnInterval=5;
    @Column(name = "clienttimeout")
    private Integer clientTimeout=30;//Minute
    @Column(name = "clienttimeoutwarnintv")
    private Integer clientTimeoutWarnInterval=5;

    @Column(name = "macid")
    private String macid;
    @Column(name = "hashedvalue")
    private String hashedValue;
    @Column(name = "allowedorigins")
    private String allowedOrigins;
    @Column(name = "validitydate")
    private Date validityDate;
    //These two columns are temporary arrangement
    //Till the time we do not have a pass phrase generator module
    @Column(name = "startpassphrase")
    private Integer startPassPhrase=5000;
    @Column(name = "lastpassphrase")
    private Integer lastPassPhrase=5000;

    public ClientConfig(){

    }

    public Integer getStartPassPhrase() {
        return startPassPhrase;
    }
    public void setStartPassPhrase(Integer startPassPhrase) {
        this.startPassPhrase = startPassPhrase;
    }

    public Integer getLastPassPhrase() {
        return lastPassPhrase;
    }
    public void setLastPassPhrase(Integer lastPassPhrase) {
        this.lastPassPhrase = lastPassPhrase;
    }
    public Integer getClientTimeout() {
        return clientTimeout;
    }
    public void setClientTimeout(Integer clientTimeout) {
        this.clientTimeout = clientTimeout;
    }
    public Integer getNumOfLicensesAgent() {
        return numOfLicensesAgent;
    }
    public void setNumOfLicensesAgent(Integer numOfLicensesAgent) {
        this.numOfLicensesAgent = numOfLicensesAgent;
    }
    public Integer getNumOfLicensesAdmin() {
        return numOfLicensesAdmin;
    }
    public void setNumOfLicensesAdmin(Integer numOfLicensesAdmin) {
        this.numOfLicensesAdmin = numOfLicensesAdmin;
    }
    public String getAllowedOrigins() {
        return allowedOrigins;
    }
    public void setAllowedOrigins(String allowedOrigins) {
        this.allowedOrigins = allowedOrigins;
    }
    public Date getValidityDate() {
        return validityDate;
    }
    public void setValidityDate(Date validityDate) {
        this.validityDate = validityDate;
    }
    public String getProductVersion() {
        return productVersion;
    }
    public void setProductVersion(String productVersion) {
        this.productVersion = productVersion;
    }
    public String getHashedValue() {
        return hashedValue;
    }
    public void setHashedValue(String hashedValue) {
        this.hashedValue = hashedValue;
    }
    public String getMacid() {
        return macid;
    }
    public void setMacid(String macid) {
        this.macid = macid;
    }
    public String getClientId() {
        return clientId;
    }
    public void setClientId(String clientId) {
        this.clientId = clientId;
    }
    public Integer getAgentTimeout() {
        return agentTimeout;
    }
    public void setAgentTimeout(Integer agentTimeout) {
        this.agentTimeout = agentTimeout;
    }
    public Integer getAgentTimeoutWarnInterval() {
        return agentTimeoutWarnInterval;
    }
    public void setAgentTimeoutWarnInterval(Integer agentTimeoutWarnInterval) {
        this.agentTimeoutWarnInterval = agentTimeoutWarnInterval;
    }
    public Integer getClientTimeoutWarnInterval() {
        return clientTimeoutWarnInterval;
    }
    public void setClientTimeoutWarnInterval(Integer clientTimeoutWarnInterval) {
        this.clientTimeoutWarnInterval = clientTimeoutWarnInterval;
    }

}

BoneCP does not seem to return the exception back to the underlying Spring layer for error handling. Instead it keeps on trying the connection to the database until the connection is established or the tomcat process is killed.