Last active
August 29, 2015 14:01
-
-
Save pcarrier/ccd873fee1105d7c7714 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| From git@github.com:airbnb/plog branch master | |
| - Create a config file with: | |
| $ echo 'plog.server.udp.listeners=[{handlers=[{provider="com.airbnb.plog.stress.EaterProvider"}]}]' > plog-server/src/main/resources/application.conf | |
| - Fire up the server with './gradlew run' | |
| - Package with './gradlew shadowJar' | |
| - Run the stresser with: | |
| $ java -server -cp .:plog-distro/build/distributions/plog-distro-*.jar -Dplog.stress.udp.loss=0.1 -Dplog.stress.rate=100 -Dplog.stress.min_size=100000 -Dplog.stress.max_size=200000 plog.Stress | |
| After a while it'll start logging a lot of: | |
| 5:47:40.274 [nioEventLoopGroup-2-1] ERROR com.airbnb.plog.server.pipeline.EndOfPipeline - Exception down the pipeline | |
| java.lang.OutOfMemoryError: Direct buffer memory | |
| at java.nio.Bits.reserveMemory(Bits.java:658) ~[na:1.8.0_05] | |
| at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123) ~[na:1.8.0_05] | |
| at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311) ~[na:1.8.0_05] | |
| at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:434) ~[netty-buffer-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:179) ~[netty-buffer-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.buffer.PoolArena.allocate(PoolArena.java:168) ~[netty-buffer-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.buffer.PoolArena.allocate(PoolArena.java:98) ~[netty-buffer-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:261) ~[netty-buffer-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:155) ~[netty-buffer-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:91) ~[netty-buffer-4.0.19.Final.jar:4.0.19.Final] | |
| at com.airbnb.plog.server.fragmentation.FragmentedMessage.<init>(FragmentedMessage.java:36) ~[plog-server-v4.0.0-ALPHA-44-g8b81245.jar:na] | |
| at com.airbnb.plog.server.fragmentation.FragmentedMessage.fromFragment(FragmentedMessage.java:49) ~[plog-server-v4.0.0-ALPHA-44-g8b81245.jar:na] | |
| at com.airbnb.plog.server.fragmentation.Defragmenter.ingestIntoIncompleteMessage(Defragmenter.java:84) ~[plog-server-v4.0.0-ALPHA-44-g8b81245.jar:na] | |
| at com.airbnb.plog.server.fragmentation.Defragmenter.decode(Defragmenter.java:102) ~[plog-server-v4.0.0-ALPHA-44-g8b81245.jar:na] | |
| at com.airbnb.plog.server.fragmentation.Defragmenter.decode(Defragmenter.java:19) ~[plog-server-v4.0.0-ALPHA-44-g8b81245.jar:na] | |
| at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:89) [netty-codec-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:341) [netty-transport-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.channel.DefaultChannelHandlerContext.fireChannelRead(DefaultChannelHandlerContext.java:327) [netty-transport-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:341) [netty-transport-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.channel.DefaultChannelHandlerContext.access$700(DefaultChannelHandlerContext.java:30) [netty-transport-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.channel.DefaultChannelHandlerContext$8.run(DefaultChannelHandlerContext.java:332) [netty-transport-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:370) [netty-common-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:353) [netty-transport-4.0.19.Final.jar:4.0.19.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:116) [netty-common-4.0.19.Final.jar:4.0.19.Final] | |
| at java.lang.Thread.null(Unknown Source) [na:1.8.0_05] | |
| Sticking to smaller messages makes the problem disappear. | |
| NEVERMIND, got it: | |
| 15:26:15.119 [nioEventLoopGroup-2-1] ERROR io.netty.util.ResourceLeakDetector - LEAK: ByteBuf.release() was not called before it's garbage-collected. | |
| Recent access records: 3 | |
| #3: | |
| io.netty.buffer.AdvancedLeakAwareByteBuf.release(AdvancedLeakAwareByteBuf.java:45) | |
| io.netty.buffer.DefaultByteBufHolder.release(DefaultByteBufHolder.java:73) | |
| com.airbnb.plog.server.fragmentation.Defragmenter$1.onRemoval(Defragmenter.java:54) | |
| com.google.common.cache.LocalCache.processPendingNotifications(LocalCache.java:1954) | |
| com.google.common.cache.LocalCache$Segment.runUnlockedCleanup(LocalCache.java:3457) | |
| com.google.common.cache.LocalCache$Segment.postWriteCleanup(LocalCache.java:3433) | |
| com.google.common.cache.LocalCache$Segment.put(LocalCache.java:2888) | |
| com.google.common.cache.LocalCache.put(LocalCache.java:4146) | |
| com.google.common.cache.LocalCache$LocalManualCache.put(LocalCache.java:4751) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.ingestIntoIncompleteMessage(Defragmenter.java:87) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.decode(Defragmenter.java:102) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.decode(Defragmenter.java:19) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:89) | |
| io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:341) | |
| io.netty.channel.DefaultChannelHandlerContext.fireChannelRead(DefaultChannelHandlerContext.java:327) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:341) | |
| io.netty.channel.DefaultChannelHandlerContext.access$700(DefaultChannelHandlerContext.java:30) | |
| io.netty.channel.DefaultChannelHandlerContext$8.run(DefaultChannelHandlerContext.java:332) | |
| io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:370) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:353) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:116) | |
| java.lang.Thread.run(Thread.java:745) | |
| #2: | |
| io.netty.buffer.AdvancedLeakAwareByteBuf.retain(AdvancedLeakAwareByteBuf.java:709) | |
| io.netty.buffer.DefaultByteBufHolder.retain(DefaultByteBufHolder.java:61) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.ingestIntoIncompleteMessage(Defragmenter.java:86) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.decode(Defragmenter.java:102) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.decode(Defragmenter.java:19) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:89) | |
| io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:341) | |
| io.netty.channel.DefaultChannelHandlerContext.fireChannelRead(DefaultChannelHandlerContext.java:327) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:341) | |
| io.netty.channel.DefaultChannelHandlerContext.access$700(DefaultChannelHandlerContext.java:30) | |
| io.netty.channel.DefaultChannelHandlerContext$8.run(DefaultChannelHandlerContext.java:332) | |
| io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:370) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:353) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:116) | |
| java.lang.Thread.run(Thread.java:745) | |
| #1: | |
| io.netty.buffer.AdvancedLeakAwareByteBuf.setBytes(AdvancedLeakAwareByteBuf.java:313) | |
| com.airbnb.plog.server.fragmentation.FragmentedMessage.ingestFragment(FragmentedMessage.java:92) | |
| com.airbnb.plog.server.fragmentation.FragmentedMessage.fromFragment(FragmentedMessage.java:50) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.ingestIntoIncompleteMessage(Defragmenter.java:84) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.decode(Defragmenter.java:102) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.decode(Defragmenter.java:19) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:89) | |
| io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:341) | |
| io.netty.channel.DefaultChannelHandlerContext.fireChannelRead(DefaultChannelHandlerContext.java:327) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:341) | |
| io.netty.channel.DefaultChannelHandlerContext.access$700(DefaultChannelHandlerContext.java:30) | |
| io.netty.channel.DefaultChannelHandlerContext$8.run(DefaultChannelHandlerContext.java:332) | |
| io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:370) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:353) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:116) | |
| java.lang.Thread.run(Thread.java:745) | |
| Created at: | |
| io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:270) | |
| io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:155) | |
| io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:91) | |
| com.airbnb.plog.server.fragmentation.FragmentedMessage.<init>(FragmentedMessage.java:36) | |
| com.airbnb.plog.server.fragmentation.FragmentedMessage.fromFragment(FragmentedMessage.java:49) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.ingestIntoIncompleteMessage(Defragmenter.java:84) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.decode(Defragmenter.java:102) | |
| com.airbnb.plog.server.fragmentation.Defragmenter.decode(Defragmenter.java:19) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:89) | |
| io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:341) | |
| io.netty.channel.DefaultChannelHandlerContext.fireChannelRead(DefaultChannelHandlerContext.java:327) | |
| io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) | |
| io.netty.channel.DefaultChannelHandlerContext.invokeChannelRead(DefaultChannelHandlerContext.java:341) | |
| io.netty.channel.DefaultChannelHandlerContext.access$700(DefaultChannelHandlerContext.java:30) | |
| io.netty.channel.DefaultChannelHandlerContext$8.run(DefaultChannelHandlerContext.java:332) | |
| io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:370) | |
| io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:353) | |
| io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:116) | |
| java.lang.Thread.run(Thread.java:745) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment