Re: [jetty-users] Websocket Cleanup

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [List Home]

Re: [jetty-users] Websocket Cleanup

From: Mack Gerhardt <mack@xxxxxxxxxxxxxx>
Date: Thu, 13 Aug 2015 17:10:56 -0400
Delivered-to: jetty-users@xxxxxxxxxxx
List-archive: <https://dev.eclipse.org/mailman/private/jetty-users>
List-help: <mailto:jetty-users-request@eclipse.org?subject=help>
List-subscribe: <https://dev.eclipse.org/mailman/listinfo/jetty-users>, <mailto:jetty-users-request@eclipse.org?subject=subscribe>
List-unsubscribe: <https://dev.eclipse.org/mailman/options/jetty-users>, <mailto:jetty-users-request@eclipse.org?subject=unsubscribe>

Here are a few sessions where isOpen() is false.

[44]

wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection

aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection

endPoint = SelectChannelEndPoint@3621d32d{/172.22.4.102:15537<->8080,CLOSED,ISHUT,OSHUT,-,-,294685/30000,WebSocketServerConnection}{io=1,kio=-1,kro=-1}

endPoint.isOpen() = false

endPoint.isOutputShutdown() = true

endPoint.isInputShutdown() = true

wsSession.isOpen() = false

wsSession.isFailed() = false

wsSession.isRunning() = true

wsSession.isStarted() = true

wsSession.isStarting() = false

wsSession.isStopped() = false

wsSession.isStopping() = false

wsSession.getConnection().getIOState() = IOState@183f1b89[OPEN,in,out]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@32fbaa18],behavior=SERVER,connection=WebSocketServerConnection@14695a24{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@7fb3b883[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@1f3007af[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@7074c53b[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@32fbaa18],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[44]

wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection

aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection

endPoint = SelectChannelEndPoint@3621d32d{/172.22.4.102:15537<->8080,CLOSED,ISHUT,OSHUT,-,-,294696/30000,WebSocketServerConnection}{io=1,kio=-1,kro=-1}

endPoint.isOpen() = false

endPoint.isOutputShutdown() = true

endPoint.isInputShutdown() = true

wsSession.isOpen() = false

wsSession.isFailed() = false

wsSession.isRunning() = true

wsSession.isStarted() = true

wsSession.isStarting() = false

wsSession.isStopped() = false

wsSession.isStopping() = false

wsSession.getConnection().getIOState() = IOState@183f1b89[OPEN,in,out]]

[44]

wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection

aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection

endPoint = SelectChannelEndPoint@3621d32d{/172.22.4.102:15537<->8080,CLOSED,ISHUT,OSHUT,-,-,294697/30000,WebSocketServerConnection}{io=1,kio=-1,kro=-1}

endPoint.isOpen() = false

endPoint.isOutputShutdown() = true

endPoint.isInputShutdown() = true

wsSession.isOpen() = false

wsSession.isFailed() = false

wsSession.isRunning() = true

wsSession.isStarted() = true

wsSession.isStarting() = false

wsSession.isStopped() = false

wsSession.isStopping() = false

wsSession.getConnection().getIOState() = IOState@183f1b89[OPEN,in,out]]

[51]

wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection

aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection

endPoint = SelectChannelEndPoint@736dbbfb{/172.22.4.102:3135<->8080,CLOSED,ISHUT,OSHUT,-,-,154515/30000,WebSocketServerConnection}{io=1,kio=-1,kro=-1}

endPoint.isOpen() = false

endPoint.isOutputShutdown() = true

endPoint.isInputShutdown() = true

wsSession.isOpen() = false

wsSession.isFailed() = false

wsSession.isRunning() = true

wsSession.isStarted() = true

wsSession.isStarting() = false

wsSession.isStopped() = false

wsSession.isStopping() = false

wsSession.getConnection().getIOState() = IOState@4c87cb8a[OPEN,in,out]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@5315a0ca],behavior=SERVER,connection=WebSocketServerConnection@51eea8e{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@3b797207[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@49502579[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@37bdcf69[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@5315a0ca],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[53]

wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection

aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection

endPoint = SelectChannelEndPoint@5ed1e6c4{/172.22.4.102:17994<->8080,CLOSED,ISHUT,OSHUT,-,-,499106/30000,WebSocketServerConnection}{io=1,kio=-1,kro=-1}

endPoint.isOpen() = false

endPoint.isOutputShutdown() = true

endPoint.isInputShutdown() = true

wsSession.isOpen() = false

wsSession.isFailed() = false

wsSession.isRunning() = true

wsSession.isStarted() = true

wsSession.isStarting() = false

wsSession.isStopped() = false

wsSession.isStopping() = false

wsSession.getConnection().getIOState() = IOState@4505098a[OPEN,in,out]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@359ca267],behavior=SERVER,connection=WebSocketServerConnection@7d2f3240{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@24b06c0b[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@4994684e[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@19cb6481[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@359ca267],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[70]

wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection

aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection

endPoint = SelectChannelEndPoint@2f8ca5d6{/172.22.4.102:50384<->8080,CLOSED,ISHUT,OSHUT,-,-,174541/30000,WebSocketServerConnection}{io=1,kio=-1,kro=-1}

endPoint.isOpen() = false

endPoint.isOutputShutdown() = true

endPoint.isInputShutdown() = true

wsSession.isOpen() = false

wsSession.isFailed() = false

wsSession.isRunning() = true

wsSession.isStarted() = true

wsSession.isStarting() = false

wsSession.isStopped() = false

wsSession.isStopping() = false

wsSession.getConnection().getIOState() = IOState@1df2e8ca[OPEN,in,out]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@3a051f4b],behavior=SERVER,connection=WebSocketServerConnection@50b9c65d{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@12145fa7[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@18aa4197[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@ac7ea8e[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@3a051f4b],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[87]

wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection

aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection

endPoint = SelectChannelEndPoint@4d5a4017{/172.22.4.102:45964<->8080,CLOSED,ISHUT,OSHUT,-,-,24351/30000,WebSocketServerConnection}{io=1,kio=-1,kro=-1}

endPoint.isOpen() = false

endPoint.isOutputShutdown() = true

endPoint.isInputShutdown() = true

wsSession.isOpen() = false

wsSession.isFailed() = false

wsSession.isRunning() = true

wsSession.isStarted() = true

wsSession.isStarting() = false

wsSession.isStopped() = false

wsSession.isStopping() = false

wsSession.getConnection().getIOState() = IOState@14524ea5[OPEN,in,out]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@248bc2ef],behavior=SERVER,connection=WebSocketServerConnection@6ee79910{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@5b418dd8[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@5de0d773[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@2706fa9a[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@248bc2ef],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[90]

wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection

aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection

endPoint = SelectChannelEndPoint@60ee84f2{/172.22.4.102:23438<->8080,CLOSED,ISHUT,OSHUT,-,-,344754/30000,WebSocketServerConnection}{io=1,kio=-1,kro=-1}

endPoint.isOpen() = false

endPoint.isOutputShutdown() = true

endPoint.isInputShutdown() = true

wsSession.isOpen() = false

wsSession.isFailed() = false

wsSession.isRunning() = true

wsSession.isStarted() = true

wsSession.isStarting() = false

wsSession.isStopped() = false

wsSession.isStopping() = false

wsSession.getConnection().getIOState() = IOState@75fbc873[OPEN,in,out]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@2203b459],behavior=SERVER,connection=WebSocketServerConnection@7903a425{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@69c153b5[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@4252b6ae[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@7dcc63c1[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@2203b459],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

On Thu, Aug 13, 2015 at 12:40 PM, Joakim Erdfelt <joakim@xxxxxxxxxxx> wrote:

The last 3 that you pasted show OPEN connections.
In other words, the normal state.
I'm more interested in the ones that stay in the openSessions + has wsSession.isOpen=false
The endpoint.toString() will be most useful to know for those.

Joakim Erdfelt / joakim@xxxxxxxxxxx

On Thu, Aug 13, 2015 at 9:36 AM, Mack Gerhardt <mack@xxxxxxxxxxxxxx> wrote:
Where you able to find anything useful in the data dump? I can provide more info if needed.

On Tue, Aug 11, 2015 at 10:33 PM, Mack Gerhardt <mack@xxxxxxxxxxxxxx> wrote:
Here is some detailed info we were able to extract. The endpoint 172.22.4.102 is a haproxy server

[1]
wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection
aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection
endPoint = SelectChannelEndPoint@41ce2b7a{/172.22.4.102:60455<->8080,Open,in,out,R,-,1656/30000,WebSocketServerConnection}{io=1,kio=1,kro=1}
endPoint.isOpen() = true
endPoint.isOutputShutdown() = false
endPoint.isInputShutdown() = false
wsSession.isOpen() = true
wsSession.isFailed() = false
wsSession.isRunning() = true
wsSession.isStarted() = true
wsSession.isStarting() = false
wsSession.isStopped() = false
wsSession.isStopping() = false
wsSession.getConnection().getIOState() = IOState@74964a1f[OPEN,in,out]]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@276cd7e7],behavior=SERVER,connection=WebSocketServerConnection@599eae20{FILL_INTERESTED}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@5e6de002[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@18ed8f62[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@597fa077[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@276cd7e7],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]
[1]
wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection
aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection
endPoint = SelectChannelEndPoint@41ce2b7a{/172.22.4.102:60455<->8080,Open,in,out,R,-,1662/30000,WebSocketServerConnection}{io=1,kio=1,kro=1}
endPoint.isOpen() = true
endPoint.isOutputShutdown() = false
endPoint.isInputShutdown() = false
wsSession.isOpen() = true
wsSession.isFailed() = false
wsSession.isRunning() = true
wsSession.isStarted() = true
wsSession.isStarting() = false
wsSession.isStopped() = false
wsSession.isStopping() = false
wsSession.getConnection().getIOState() = IOState@74964a1f[OPEN,in,out]]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@276cd7e7],behavior=SERVER,connection=WebSocketServerConnection@599eae20{FILL_INTERESTED}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@5e6de002[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@18ed8f62[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@597fa077[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@276cd7e7],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]
[1]
wsSession.connection = org.eclipse.jetty.websocket.server.WebSocketServerConnection
aws = org.eclipse.jetty.websocket.server.WebSocketServerConnection
endPoint = SelectChannelEndPoint@41ce2b7a{/172.22.4.102:60455<->8080,Open,in,out,R,-,1663/30000,WebSocketServerConnection}{io=1,kio=1,kro=1}
endPoint.isOpen() = true
endPoint.isOutputShutdown() = false
endPoint.isInputShutdown() = false
wsSession.isOpen() = true
wsSession.isFailed() = false
wsSession.isRunning() = true
wsSession.isStarted() = true
wsSession.isStarting() = false
wsSession.isStopped() = false
wsSession.isStopping() = false
wsSession.getConnection().getIOState() = IOState@74964a1f[OPEN,in,out]]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@276cd7e7],behavior=SERVER,connection=WebSocketServerConnection@599eae20{FILL_INTERESTED}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@5e6de002[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@18ed8f62[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@597fa077[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@276cd7e7],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

On Tue, Aug 11, 2015 at 6:37 PM, Joakim Erdfelt <joakim@xxxxxxxxxxx> wrote:
I'm trying to understand how IOState thinks its OPEN, but the wsSession.isOpen() doesn't comply.
I'm needing to know some state from the wsSession.connection.endpoint to get a better idea on the state your scenario is in.

Joakim Erdfelt / joakim@xxxxxxxxxxx

On Tue, Aug 11, 2015 at 3:34 PM, Alex Jeannopoulos <alexj@xxxxxxxxxxxxxx> wrote:
closed is - !wsSession.isOpen()

For now, I was able to install code into production dynamically to call

if ( !wsSession.isOpen() ){
factory.onSessionClosed(wsSession);
closed++;
}

I am letting it run in production under load, to see how memory usage is, so far in about 10 mins the memory freed up and is running at the levels of a traditional embedded jetty server doing http polling only no websockets.

On Tue, Aug 11, 2015 at 6:16 PM, Joakim Erdfelt <joakim@xxxxxxxxxxx> wrote:
I take it "open" is WebSocketSession.isOpen()
What is "closed" in your labels?

Joakim Erdfelt / joakim@xxxxxxxxxxx

On Tue, Aug 11, 2015 at 2:33 PM, Mack Gerhardt <mack@xxxxxxxxxxxxxx> wrote:
I will take care of that now, some other details from my dump, I iterated teh set of openSessions (35359), and I tallied up the following from methods on WebSocketSession

cnt = 35359
open = 374
closed = 34985
failed = 0
running = 35359
started = 35359
starting = 0
stopped = 0
stopping = 0

On Tue, Aug 11, 2015 at 5:15 PM, Joakim Erdfelt <joakim@xxxxxxxxxxx> wrote:
man, i've got to clean that output up. too much noise/duplicates.

The raw connection details are missing *groan*, and the ioState is also not present *double-groan*.
Would you mind running a snapshot with some updates to the WebSocketSession.toString() to help use figure this out.
I'm really curious about the raw connection state (will let me know what java/jetty thinks the physical connection is doing), as well as the higher level websocket ioState (let me know what the websocket level thinks is reality).

Joakim Erdfelt / joakim@xxxxxxxxxxx

On Tue, Aug 11, 2015 at 1:54 PM, Mack Gerhardt <mack@xxxxxxxxxxxxxx> wrote:
I know this looks messy, its the first 10 sessions that I dumped.

[0]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@3f4c0042],behavior=SERVER,connection=WebSocketServerConnection@616c1084{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@1e934e02[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@775db967[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@28cce9de[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@3f4c0042],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]] - STARTED
+- incomingHandler : JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@3f4c0042]
+- outgoingHandler : ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection] - STARTED
+- Stack
+- Network : WebSocketServerConnection@616c1084{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@1e934e02[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@775db967[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]}
+- Websocket: WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@3f4c0042],behavior=SERVER,connection=WebSocketServerConnection@616c1084{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@1e934e02[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@775db967[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@28cce9de[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@3f4c0042],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@3f4c0042],behavior=SERVER,connection=WebSocketServerConnection@616c1084{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@1e934e02[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@775db967[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@28cce9de[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@3f4c0042],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[1]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@25b3fc31],behavior=SERVER,connection=WebSocketServerConnection@6b96034a{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@74ce2e72[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@1f38eda1[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@25d0d803[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@25b3fc31],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]] - STARTED
+- incomingHandler : JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@25b3fc31]
+- outgoingHandler : ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection] - STARTED
+- Stack
+- Network : WebSocketServerConnection@6b96034a{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@74ce2e72[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@1f38eda1[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]}
+- Websocket: WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@25b3fc31],behavior=SERVER,connection=WebSocketServerConnection@6b96034a{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@74ce2e72[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@1f38eda1[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@25d0d803[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@25b3fc31],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@25b3fc31],behavior=SERVER,connection=WebSocketServerConnection@6b96034a{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@74ce2e72[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@1f38eda1[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@25d0d803[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@25b3fc31],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[2]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@7e0703a0],behavior=SERVER,connection=WebSocketServerConnection@7bbf0fac{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@23b2b09f[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@1642a[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@465d6d89[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@7e0703a0],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]] - STARTED
+- incomingHandler : JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@7e0703a0]
+- outgoingHandler : ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection] - STARTED
+- Stack
+- Network : WebSocketServerConnection@7bbf0fac{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@23b2b09f[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@1642a[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]}
+- Websocket: WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@7e0703a0],behavior=SERVER,connection=WebSocketServerConnection@7bbf0fac{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@23b2b09f[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@1642a[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@465d6d89[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@7e0703a0],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@7e0703a0],behavior=SERVER,connection=WebSocketServerConnection@7bbf0fac{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@23b2b09f[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@1642a[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@465d6d89[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@7e0703a0],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[3]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@451cd881],behavior=SERVER,connection=WebSocketServerConnection@739caef5{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@7837a498[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@552bbf32[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@7650bdc6[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@451cd881],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]] - STARTED
+- incomingHandler : JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@451cd881]
+- outgoingHandler : ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection] - STARTED
+- Stack
+- Network : WebSocketServerConnection@739caef5{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@7837a498[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@552bbf32[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]}
+- Websocket: WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@451cd881],behavior=SERVER,connection=WebSocketServerConnection@739caef5{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@7837a498[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@552bbf32[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@7650bdc6[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@451cd881],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@451cd881],behavior=SERVER,connection=WebSocketServerConnection@739caef5{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@7837a498[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@552bbf32[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@7650bdc6[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@451cd881],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[4]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@60ad4eff],behavior=SERVER,connection=WebSocketServerConnection@257b2516{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@2da1dca4[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@5e6cea5b[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@543c8ddf[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@60ad4eff],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]] - STARTED
+- incomingHandler : JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@60ad4eff]
+- outgoingHandler : ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection] - STARTED
+- Stack
+- Network : WebSocketServerConnection@257b2516{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@2da1dca4[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@5e6cea5b[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]}
+- Websocket: WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@60ad4eff],behavior=SERVER,connection=WebSocketServerConnection@257b2516{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@2da1dca4[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@5e6cea5b[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@543c8ddf[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@60ad4eff],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@60ad4eff],behavior=SERVER,connection=WebSocketServerConnection@257b2516{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@2da1dca4[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@5e6cea5b[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@543c8ddf[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@60ad4eff],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[5]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@289351db],behavior=SERVER,connection=WebSocketServerConnection@bf4ec91{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@32f3e186[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@291c072[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@303e5472[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@289351db],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]] - STARTED
+- incomingHandler : JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@289351db]
+- outgoingHandler : ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection] - STARTED
+- Stack
+- Network : WebSocketServerConnection@bf4ec91{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@32f3e186[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@291c072[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]}
+- Websocket: WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@289351db],behavior=SERVER,connection=WebSocketServerConnection@bf4ec91{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@32f3e186[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@291c072[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@303e5472[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@289351db],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@289351db],behavior=SERVER,connection=WebSocketServerConnection@bf4ec91{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@32f3e186[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@291c072[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@303e5472[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@289351db],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[6]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@1c02b680],behavior=SERVER,connection=WebSocketServerConnection@31509b57{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@3d4e4f5f[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@7f911140[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@762e1e84[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@1c02b680],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]] - STARTED
+- incomingHandler : JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@1c02b680]
+- outgoingHandler : ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection] - STARTED
+- Stack
+- Network : WebSocketServerConnection@31509b57{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@3d4e4f5f[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@7f911140[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]}
+- Websocket: WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@1c02b680],behavior=SERVER,connection=WebSocketServerConnection@31509b57{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@3d4e4f5f[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@7f911140[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@762e1e84[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@1c02b680],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@1c02b680],behavior=SERVER,connection=WebSocketServerConnection@31509b57{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@3d4e4f5f[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@7f911140[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@762e1e84[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@1c02b680],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[7]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@4391b935],behavior=SERVER,connection=WebSocketServerConnection@cea32af{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@7b063963[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@7ddac2b9[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@334c9426[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@4391b935],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]] - STARTED
+- incomingHandler : JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@4391b935]
+- outgoingHandler : ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection] - STARTED
+- Stack
+- Network : WebSocketServerConnection@cea32af{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@7b063963[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@7ddac2b9[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]}
+- Websocket: WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@4391b935],behavior=SERVER,connection=WebSocketServerConnection@cea32af{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@7b063963[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@7ddac2b9[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@334c9426[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@4391b935],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@4391b935],behavior=SERVER,connection=WebSocketServerConnection@cea32af{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@7b063963[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@7ddac2b9[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@334c9426[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@4391b935],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[8]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@6e6ea7aa],behavior=SERVER,connection=WebSocketServerConnection@24694625{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@f8d971b[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@5837cf68[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@437f7033[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@6e6ea7aa],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]] - STARTED
+- incomingHandler : JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@6e6ea7aa]
+- outgoingHandler : ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection] - STARTED
+- Stack
+- Network : WebSocketServerConnection@24694625{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@f8d971b[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@5837cf68[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]}
+- Websocket: WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@6e6ea7aa],behavior=SERVER,connection=WebSocketServerConnection@24694625{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@f8d971b[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@5837cf68[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@437f7033[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@6e6ea7aa],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@6e6ea7aa],behavior=SERVER,connection=WebSocketServerConnection@24694625{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@f8d971b[ExtensionStack,s=START,c=0,len=0,f=null,p=WebSocketPolicy@5837cf68[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@437f7033[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@6e6ea7aa],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

[9]
WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@2dc95e17],behavior=SERVER,connection=WebSocketServerConnection@4e1f21fa{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@2835fbde[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@10f3f803[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@77ad35ec[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@2dc95e17],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]] - STARTED
+- incomingHandler : JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@2dc95e17]
+- outgoingHandler : ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection] - STARTED
+- Stack
+- Network : WebSocketServerConnection@4e1f21fa{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@2835fbde[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@10f3f803[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]}
+- Websocket: WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@2dc95e17],behavior=SERVER,connection=WebSocketServerConnection@4e1f21fa{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@2835fbde[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@10f3f803[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@77ad35ec[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@2dc95e17],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

WebSocketSession[websocket=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@2dc95e17],behavior=SERVER,connection=WebSocketServerConnection@4e1f21fa{IDLE}{f=Flusher[queueSize=0,aggregateSize=0,failure=null],g=Generator[SERVER,validating],p=Parser@2835fbde[ExtensionStack,s=START,c=0,len=5,f=null,p=WebSocketPolicy@10f3f803[behavior=SERVER,maxTextMessageSize=65535,maxTextMessageBufferSize=32768,maxBinaryMessageSize=65536,maxBinaryMessageBufferSize=32768,asyncWriteTimeout=10000,idleTimeout=30000,inputBufferSize=4096]]},remote=WebSocketRemoteEndpoint@77ad35ec[batching=true],incoming=JettyAnnotatedEventDriver[com.backpocket.servlet.jetty9.ChatWebSocket@2dc95e17],outgoing=ExtensionStack[queueSize=0,extensions=[],incoming=org.eclipse.jetty.websocket.common.WebSocketSession,outgoing=org.eclipse.jetty.websocket.server.WebSocketServerConnection]]

On Tue, Aug 11, 2015 at 3:57 PM, Joakim Erdfelt <joakim@xxxxxxxxxxx> wrote:
If you can access the session in that openSession list, can you iterate through them for state information?

A simple toString() on the individual objects in that list should produce reams of useful information.

I'm curious about the various connection and ioState values atm.

Joakim Erdfelt / joakim@xxxxxxxxxxx

On Tue, Aug 11, 2015 at 12:42 PM, Mack Gerhardt <mack@xxxxxxxxxxxxxx> wrote:
I am seeing this in 9.2.11 but I also saw this in 9.3.0. I do see sessions closing and removed but I am seeing a leak where my embedded jetty engine will die on a OOM after about 12 hours of heave use in production. If there is anything I could do or provide please let me know. I was even thinking of recompiling the websocket jar exposing the openSessions so I could attempt to introspect the sessions.

Alex

On Tue, Aug 11, 2015 at 3:26 PM, Joakim Erdfelt <joakim@xxxxxxxxxxx> wrote:
Sounds like a bug.
What version of Jetty are you using?

Joakim Erdfelt / joakim@xxxxxxxxxxx

On Tue, Aug 11, 2015 at 12:02 PM, Mack Gerhardt <mack@xxxxxxxxxxxxxx> wrote:
I am experiencing an issue with a leak of WebSocketSession objects. I believe I might not be closing down sessions properly. I was using the plumbr.eu tool and it detected the following. Any ideas of what should I look for.

At the time the leak was detected, 30724 objects existed of class
org.eclipse.jetty.websocket.common.WebSocketSession
These objects had been created at
org.eclipse.jetty.websocket.common.WebSocketSessionFactory.createSession(java.net.URI, org.eclipse.jetty.websocket.common.events.EventDriver, org.eclipse.jetty.websocket.common.LogicalConnection):48
and were being held
in array of java.util.concurrent.CopyOnWriteArrayList
in al of java.util.concurrent.CopyOnWriteArraySet
in openSessions of org.eclipse.jetty.websocket.server.WebSocketServerFactory

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

_______________________________________________
jetty-users mailing list
jetty-users@xxxxxxxxxxx
To change your delivery options, retrieve your password, or unsubscribe from this list, visit
https://dev.eclipse.org/mailman/listinfo/jetty-users

Follow-Ups:
- Re: [jetty-users] Websocket Cleanup
  - From: Joakim Erdfelt

References:
- [jetty-users] Websocket Cleanup
  - From: Mack Gerhardt
- Re: [jetty-users] Websocket Cleanup
  - From: Joakim Erdfelt
- Re: [jetty-users] Websocket Cleanup
  - From: Mack Gerhardt
- Re: [jetty-users] Websocket Cleanup
  - From: Joakim Erdfelt
- Re: [jetty-users] Websocket Cleanup
  - From: Mack Gerhardt
- Re: [jetty-users] Websocket Cleanup
  - From: Joakim Erdfelt
- Re: [jetty-users] Websocket Cleanup
  - From: Mack Gerhardt
- Re: [jetty-users] Websocket Cleanup
  - From: Joakim Erdfelt
- Re: [jetty-users] Websocket Cleanup
  - From: Alex Jeannopoulos
- Re: [jetty-users] Websocket Cleanup
  - From: Joakim Erdfelt
- Re: [jetty-users] Websocket Cleanup
  - From: Mack Gerhardt
- Re: [jetty-users] Websocket Cleanup
  - From: Mack Gerhardt
- Re: [jetty-users] Websocket Cleanup
  - From: Joakim Erdfelt

Prev by Date: [jetty-users] Migrating from jetty-9.2.11.v20150529-all.jar to jetty-9.3.2.v20150730-uber.jar results in a HTTP Error 404
Next by Date: [jetty-users] unsubscribe
Previous by thread: Re: [jetty-users] Websocket Cleanup
Next by thread: Re: [jetty-users] Websocket Cleanup
Index(es):
- Date
- Thread

Breadcrumbs