We are getting a lot of stuck thread in our production environment. We have to restart the server, once the queries are stuck. Basically we are using EclipseLink 2.4.1 in a J2EE stack with WebLogic 10.3.6. We often have a lot of "stuck threads", caused by lines in EclipseLink that are getting releaseDeferred locks in concurrency manager. using JPA


11-Jan-2017 15:42:09.408 INFO [main] org.apache.catalina.core.StandardServer.await A valid shutdown command was received via the shutdown port. Stopping the Server instance.
11-Jan-2017 15:42:09.408 INFO [main] org.apache.coyote.AbstractProtocol.pause Pausing ProtocolHandler ["http-nio-8080"]
11-Jan-2017 15:42:09.459 INFO [main] org.apache.coyote.AbstractProtocol.pause Pausing ProtocolHandler ["ajp-nio-8009"]
11-Jan-2017 15:42:09.510 INFO [main] org.apache.catalina.core.StandardService.stopInternal Stopping service Catalina
11-Jan-2017 15:42:09.553 INFO [localhost-startStop-2] org.apache.catalina.core.StandardWrapper.unload Waiting for 12 instance(s) to be deallocated for Servlet [Faces Servlet]
11-Jan-2017 15:42:10.555 INFO [localhost-startStop-2] org.apache.catalina.core.StandardWrapper.unload Waiting for 12 instance(s) to be deallocated for Servlet [Faces Servlet]
11-Jan-2017 15:42:11.556 INFO [localhost-startStop-2] org.apache.catalina.core.StandardWrapper.unload Waiting for 12 instance(s) to be deallocated for Servlet [Faces Servlet]
11-Jan-2017 15:42:11.709 WARNING [localhost-startStop-2] org.apache.catalina.loader.WebappClassLoaderBase.clearReferencesJdbc The web application [ROOT] registered the JDBC driver [com.mysql.jdbc.Driver] but failed to unregister it when the web application was stopped. To prevent a memory leak, the JDBC Driver has been forcibly unregistered.
11-Jan-2017 15:42:11.710 WARNING [localhost-startStop-2] org.apache.catalina.loader.WebappClassLoaderBase.clearReferencesThreads The web application [ROOT] is still processing a request that has yet to finish. This is very likely to create a memory leak. You can control the time allowed for requests to finish by using the unloadDelay attribute of the standard Context implementation. Stack trace of request processing thread:
java.lang.Thread.sleep(Native Method)
org.eclipse.persistence.internal.helper.ConcurrencyManager.releaseDeferredLock(ConcurrencyManager.java:445)
org.eclipse.persistence.internal.identitymaps.CacheKey.releaseDeferredLock(CacheKey.java:465)
org.eclipse.persistence.internal.descriptors.ObjectBuilder.buildObject(ObjectBuilder.java:1070)
org.eclipse.persistence.internal.descriptors.ObjectBuilder.buildWorkingCopyCloneNormally(ObjectBuilder.java:899)
org.eclipse.persistence.internal.descriptors.ObjectBuilder.buildObjectInUnitOfWork(ObjectBuilder.java:852)
org.eclipse.persistence.internal.descriptors.ObjectBuilder.buildObject(ObjectBuilder.java:735)
org.eclipse.persistence.internal.descriptors.ObjectBuilder.buildObject(ObjectBuilder.java:689)
org.eclipse.persistence.queries.ObjectLevelReadQuery.buildObject(ObjectLevelReadQuery.java:805)
org.eclipse.persistence.queries.ReadAllQuery.registerResultInUnitOfWork(ReadAllQuery.java:962)
org.eclipse.persistence.queries.ReadAllQuery.executeObjectLevelReadQuery(ReadAllQuery.java:573)
org.eclipse.persistence.queries.ObjectLevelReadQuery.executeDatabaseQuery(ObjectLevelReadQuery.java:1175)
org.eclipse.persistence.queries.DatabaseQuery.execute(DatabaseQuery.java:904)
org.eclipse.persistence.queries.ObjectLevelReadQuery.execute(ObjectLevelReadQuery.java:1134)
org.eclipse.persistence.queries.ReadAllQuery.execute(ReadAllQuery.java:460)
org.eclipse.persistence.queries.ObjectLevelReadQuery.executeInUnitOfWork(ObjectLevelReadQuery.java:1222)
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.internalExecuteQuery(UnitOfWorkImpl.java:2896)
org.eclipse.persistence.internal.sessions.AbstractSession.executeQuery(AbstractSession.java:1857)
org.eclipse.persistence.internal.sessions.AbstractSession.executeQuery(AbstractSession.java:1839)
org.eclipse.persistence.internal.sessions.AbstractSession.executeQuery(AbstractSession.java:1804)
org.eclipse.persistence.internal.jpa.QueryImpl.executeReadQuery(QueryImpl.java:258)
org.eclipse.persistence.internal.jpa.QueryImpl.getSingleResult(QueryImpl.java:521)
org.eclipse.persistence.internal.jpa.EJBQueryImpl.getSingleResult(EJBQueryImpl.java:400)
com.bst.orasis.dal.UserCompanySelectionDAO.findByUserName(UserCompanySelectionDAO.java:31)
com.bst.orasis.service.UserCompanySelectionService.findByUserName(UserCompanySelectionService.java:72)
com.bst.orasis.dal.BaseService.getLoggedUserCompanySelectioEntity(BaseService.java:266)
com.bst.orasis.dal.BaseService.setLoggedInUserEntity(BaseService.java:188)
com.bst.orasis.dal.BaseService.<init>(BaseService.java:167)
com.bst.orasis.ui.portal.ServiceBean.<init>(ServiceBean.java:15)
sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
java.lang.reflect.Constructor.newInstance(Constructor.java:423)
java.lang.Class.newInstance(Class.java:442)
com.sun.faces.mgbean.BeanBuilder.newBeanInstance(BeanBuilder.java:186)
com.sun.faces.mgbean.BeanBuilder.build(BeanBuilder.java:100)
com.sun.faces.mgbean.BeanManager.createAndPush(BeanManager.java:409)
com.sun.faces.mgbean.BeanManager.create(BeanManager.java:269)
com.sun.faces.el.ManagedBeanELResolver.resolveBean(ManagedBeanELResolver.java:257)
com.sun.faces.el.ManagedBeanELResolver.getValue(ManagedBeanELResolver.java:117)
com.sun.faces.el.DemuxCompositeELResolver._getValue(DemuxCompositeELResolver.java:176)
com.sun.faces.el.DemuxCompositeELResolver.getValue(DemuxCompositeELResolver.java:203)
org.apache.el.parser.AstIdentifier.getValue(AstIdentifier.java:94)
org.apache.el.ValueExpressionImpl.getValue(ValueExpressionImpl.java:184)
com.sun.faces.mgbean.BeanBuilder$Expression.evaluate(BeanBuilder.java:561)
com.sun.faces.mgbean.ManagedBeanBuilder$BakedBeanProperty.set(ManagedBeanBuilder.java:606)
com.sun.faces.mgbean.ManagedBeanBuilder.buildBean(ManagedBeanBuilder.java:133)
com.sun.faces.mgbean.BeanBuilder.build(BeanBuilder.java:102)
com.sun.faces.mgbean.BeanManager.createAndPush(BeanManager.java:409)
com.sun.faces.mgbean.BeanManager.create(BeanManager.java:269)
com.sun.faces.el.ManagedBeanELResolver.resolveBean(ManagedBeanELResolver.java:257)
com.sun.faces.el.ManagedBeanELResolver.getValue(ManagedBeanELResolver.java:117)
com.sun.faces.el.DemuxCompositeELResolver._getValue(DemuxCompositeELResolver.java:176)
com.sun.faces.el.DemuxCompositeELResolver.getValue(DemuxCompositeELResolver.java:203)
org.apache.el.parser.AstIdentifier.getValue(AstIdentifier.java:94)
org.apache.el.parser.AstValue.getValue(AstValue.java:137)
org.apache.el.ValueExpressionImpl.getValue(ValueExpressionImpl.java:184)
com.sun.faces.facelets.el.ELText$ELTextVariable.writeText(ELText.java:238)
com.sun.faces.facelets.el.ELText$ELTextComposite.writeText(ELText.java:154)
com.sun.faces.facelets.compiler.TextInstruction.write(TextInstruction.java:85)
com.sun.faces.facelets.compiler.UIInstructions.encodeBegin(UIInstructions.java:82)
com.sun.faces.facelets.compiler.UILeaf.encodeAll(UILeaf.java:183)
javax.faces.component.UIComponent.encodeAll(UIComponent.java:1859)
javax.faces.component.UIComponent.encodeAll(UIComponent.java:1859)
com.sun.faces.application.view.FaceletViewHandlingStrategy.renderView(FaceletViewHandlingStrategy.java:458)
com.sun.faces.application.view.MultiViewHandler.renderView(MultiViewHandler.java:134)
com.sun.faces.lifecycle.RenderResponsePhase.execute(RenderResponsePhase.java:120)
com.sun.faces.lifecycle.Phase.doPhase(Phase.java:101)
com.sun.faces.lifecycle.LifecycleImpl.render(LifecycleImpl.java:219)
javax.faces.webapp.FacesServlet.service(FacesServlet.java:659)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:292)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
com.bst.orasis.entities.userauth.UserAuthFilter.doFilter(UserAuthFilter.java:57)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
org.jasig.cas.client.util.HttpServletRequestWrapperFilter.doFilter(HttpServletRequestWrapperFilter.java:62)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
org.jasig.cas.client.validation.AbstractTicketValidationFilter.doFilter(AbstractTicketValidationFilter.java:196)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
org.jasig.cas.client.authentication.AuthenticationFilter.doFilter(AuthenticationFilter.java:93)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
org.jasig.cas.client.session.SingleSignOutFilter.doFilter(SingleSignOutFilter.java:65)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:212)
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:106)
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:502)
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:141)
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)
org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:676)
org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:616)
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:88)
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:528)
org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1099)
org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:672)
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1520)
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1476)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
java.lang.Thread.run(Thread.java:745)