| [jetty-users] IllegalStateException followed by SocketTimeoutException until system restarted |
|
Hello there, We are running our own Jetty base reverse proxy using AsyncMiddleManServlet. For most part it runs flawlessly, however recently we have come across situations where our application gets into a bad state where
its unable to connect to external systems due to “java.net.SocketTimeoutException: Connect Timeout”. From the logs we see there is an IllegalStateException that happens “java.lang.IllegalStateException:
org.eclipse.jetty.server.HttpConnection$SendCallback@72dcb5d8[CLOSED][i=HTTP/1.1{s=500,h=1,cl=0},cb=org.eclipse.jetty.server.HttpChannel$CommitCallback$1@5c33a349]” and imminently after that all connections get “java.net.SocketTimeoutException: Connect Timeout” and the only way to recovery is to restart the system. Any hint you can provide is greatly appreciated.
Thank you, -Lu Here are more details: JRE: openjdk 10.0.2 OS: ubuntu 18.04.1 LTS (Bionic Beaver) Jetty Version: 9.4.12.v20180830 Full stack traces: java.lang.IllegalStateException: org.eclipse.jetty.server.HttpConnection$SendCallback@72dcb5d8[CLOSED][i=HTTP/1.1{s=500,h=1,cl=0},cb=org.eclipse.jetty.server.HttpChannel$CommitCallback$1@5c33a349] at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:219) at org.eclipse.jetty.server.HttpConnection.send(HttpConnection.java:538) at org.eclipse.jetty.server.HttpChannel$CommitCallback.failed(HttpChannel.java:1252) at org.eclipse.jetty.io.AbstractConnection.lambda$failedCallback$0(AbstractConnection.java:92) at org.eclipse.jetty.io.AbstractConnection.failedCallback(AbstractConnection.java:115) at org.eclipse.jetty.server.HttpConnection.access$1300(HttpConnection.java:53) at org.eclipse.jetty.server.HttpConnection$SendCallback.onCompleteFailure(HttpConnection.java:868) at org.eclipse.jetty.util.IteratingCallback.failed(IteratingCallback.java:401) at org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:245) at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:224) at org.eclipse.jetty.server.HttpConnection.send(HttpConnection.java:538) at org.eclipse.jetty.server.HttpChannel.sendResponse(HttpChannel.java:831) at org.eclipse.jetty.server.HttpChannel.write(HttpChannel.java:887) at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:239) at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:215) at org.eclipse.jetty.server.HttpOutput.close(HttpOutput.java:297) at org.eclipse.jetty.server.Response.closeOutput(Response.java:1048) at org.eclipse.jetty.server.Response.sendError(Response.java:658) at org.eclipse.jetty.server.Response.sendError(Response.java:590) at org.eclipse.jetty.proxy.AbstractProxyServlet.sendProxyResponseError(AbstractProxyServlet.java:666) at com.opentable.frontdoor.proxy.RealFrontdoorProxyServlet.sendProxyResponseError(RealFrontdoorProxyServlet.java:531) at org.eclipse.jetty.proxy.AbstractProxyServlet.onProxyResponseFailure(AbstractProxyServlet.java:648) at com.opentable.frontdoor.proxy.RealFrontdoorProxyServlet.onProxyResponseFailure(RealFrontdoorProxyServlet.java:430) at org.eclipse.jetty.proxy.AsyncMiddleManServlet$ProxyResponseListener.failed(AsyncMiddleManServlet.java:593) at org.eclipse.jetty.util.Callback$Nested.failed(Callback.java:141) at org.eclipse.jetty.util.CountingCallback.failed(CountingCallback.java:88) at org.eclipse.jetty.proxy.AsyncMiddleManServlet$ProxyResponseListener.onComplete(AsyncMiddleManServlet.java:579) at org.eclipse.jetty.client.ResponseNotifier.notifyComplete(ResponseNotifier.java:202) at org.eclipse.jetty.client.ResponseNotifier.notifyComplete(ResponseNotifier.java:194) at org.eclipse.jetty.client.HttpExchange.notifyFailureComplete(HttpExchange.java:269) at org.eclipse.jetty.client.HttpExchange.abort(HttpExchange.java:240) at org.eclipse.jetty.client.HttpConversation.abort(HttpConversation.java:141) at org.eclipse.jetty.client.HttpRequest.abort(HttpRequest.java:767) at org.eclipse.jetty.client.HttpDestination.abort(HttpDestination.java:453) at org.eclipse.jetty.client.HttpDestination.failed(HttpDestination.java:235) at org.eclipse.jetty.client.AbstractConnectionPool$1.failed(AbstractConnectionPool.java:141) at org.eclipse.jetty.util.Promise$Wrapper.failed(Promise.java:136) at org.eclipse.jetty.client.HttpClient$1$1.failed(HttpClient.java:610) at org.eclipse.jetty.client.AbstractConnectorHttpClientTransport.connectFailed(AbstractConnectorHttpClientTransport.java:138) at org.eclipse.jetty.client.AbstractConnectorHttpClientTransport$ClientSelectorManager.connectionFailed(AbstractConnectorHttpClientTransport.java:183) at org.eclipse.jetty.io.ManagedSelector$Connect.failed(ManagedSelector.java:775) at org.eclipse.jetty.io.ManagedSelector.processConnect(ManagedSelector.java:229) at org.eclipse.jetty.io.ManagedSelector.access$1400(ManagedSelector.java:61) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.processSelected(ManagedSelector.java:485) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:351) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:357) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:181) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at com.opentable.metrics.JettyServerMetricsConfiguration$OTQueuedThreadPool.runJob(JettyServerMetricsConfiguration.java:84) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.base/java.lang.Thread.run(Thread.java:844) java.net.SocketTimeoutException: Connect Timeout at org.eclipse.jetty.io.ManagedSelector$Connect.run(ManagedSelector.java:765) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:514) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:844) |