Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Cassandra 1.2.11 OOMs during relatively high read load, possibly around/in compaction?
ERROR [ReadStage:51] 2013-12-06 00:02:54,065 CassandraDaemon.java (line 191) Exception in thread Thread[ReadStage:51,5,main]
ERROR [ReadStage:51] 2013-12-06 00:02:54,065 CassandraDaemon.java (line 191) Exception in thread Thread[ReadStage:51,5,main]
java.lang.OutOfMemoryError: Java heap space
at org.apache.cassandra.io.util.RandomAccessReader.readBytes(RandomAccessReader.java:376)
at org.apache.cassandra.utils.ByteBufferUtil.read(ByteBufferUtil.java:392)
at org.apache.cassandra.utils.ByteBufferUtil.readWithLength(ByteBufferUtil.java:355)
at org.apache.cassandra.db.ColumnSerializer.deserializeColumnBody(ColumnSerializer.java:108)
at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(OnDiskAtom.java:92)
at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(OnDiskAtom.java:73)
at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext(SimpleSliceReader.java:106)
at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext(SimpleSliceReader.java:40)
at com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143)
at com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138)
at org.apache.cassandra.db.columniterator.SSTableSliceIterator.hasNext(SSTableSliceIterator.java:90)
at org.apache.cassandra.db.filter.QueryFilter$2.getNext(QueryFilter.java:171)
at org.apache.cassandra.db.filter.QueryFilter$2.hasNext(QueryFilter.java:154)
at org.apache.cassandra.utils.MergeIterator$Candidate.advance(MergeIterator.java:143)
at org.apache.cassandra.utils.MergeIterator$ManyToOne.<init>(MergeIterator.java:86)
at org.apache.cassandra.utils.MergeIterator.get(MergeIterator.java:45)
at org.apache.cassandra.db.filter.QueryFilter.collateColumns(QueryFilter.java:134)
at org.apache.cassandra.db.filter.QueryFilter.collateOnDiskAtom(QueryFilter.java:84)
at org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:291)
at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:65)
at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1391)
at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1207)
at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1123)
at org.apache.cassandra.db.Table.getRow(Table.java:347)
at org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:70)
at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1062)
at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1614)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext(SimpleSliceReader.java:40)
at com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143)
at com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138)
at org.apache.cassandra.db.columniterator.SSTableSliceIterator.hasNext(SSTableSliceIterator.java:90)
at org.apache.cassandra.db.filter.QueryFilter$2.getNext(QueryFilter.java:171)
at org.apache.cassandra.db.filter.QueryFilter$2.hasNext(QueryFilter.java:154)
at org.apache.cassandra.utils.MergeIterator$Candidate.advance(MergeIterator.java:143)
at org.apache.cassandra.utils.MergeIterator$ManyToOne.<init>(MergeIterator.java:86)
at org.apache.cassandra.utils.MergeIterator.get(MergeIterator.java:45)
at org.apache.cassandra.db.filter.QueryFilter.collateColumns(QueryFilter.java:134)
at org.apache.cassandra.db.filter.QueryFilter.collateOnDiskAtom(QueryFilter.java:84)
at org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:291)
at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:65)
at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1391)
at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1207)
at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1123)
at org.apache.cassandra.db.Table.getRow(Table.java:347)
at org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:70)
at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1062)
at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1614)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:724)
ReadStage:323
at sun.nio.ch.NativeThread.current()J (Native Method)
at sun.nio.ch.NativeThreadSet.add()I (NativeThreadSet.java:46)
at sun.nio.ch.FileChannelImpl.position()J (FileChannelImpl.java:243)
at org.apache.cassandra.io.compress.CompressedRandomAccessReader.decompressChunk(Lorg/apache/cassandra/io/compress/CompressionMetadata$Chunk;)V (CompressedRandomAccessReader.java:99)
at org.apache.cassandra.io.compress.CompressedRandomAccessReader.reBuffer()V (CompressedRandomAccessReader.java:85)
at org.apache.cassandra.io.util.RandomAccessReader.read([BII)I (RandomAccessReader.java:355)
at java.io.RandomAccessFile.readFully([BII)V (RandomAccessFile.java:414)
at java.io.RandomAccessFile.readFully([B)V (RandomAccessFile.java:394)
at org.apache.cassandra.io.util.RandomAccessReader.readBytes(I)Ljava/nio/ByteBuffer; (RandomAccessReader.java:380)
at org.apache.cassandra.utils.ByteBufferUtil.read(Ljava/io/DataInput;I)Ljava/nio/ByteBuffer; (ByteBufferUtil.java:392)
at org.apache.cassandra.utils.ByteBufferUtil.readWithLength(Ljava/io/DataInput;)Ljava/nio/ByteBuffer; (ByteBufferUtil.java:355)
at org.apache.cassandra.db.ColumnSerializer.deserializeColumnBody(Ljava/io/DataInput;Ljava/nio/ByteBuffer;ILorg/apache/cassandra/io/IColumnSerializer$Flag;I)Lorg/apache/cassandra/db/Column; (ColumnSerializer.java:94)
at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(Ljava/io/DataInput;Lorg/apache/cassandra/io/IColumnSerializer$Flag;ILorg/apache/cassandra/io/sstable/Descriptor$Version;)Lorg/apache/cassandra/db/OnDiskAtom; (OnDiskAtom.java:92)
at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(Ljava/io/DataInput;Lorg/apache/cassandra/io/sstable/Descriptor$Version;)Lorg/apache/cassandra/db/OnDiskAtom; (OnDiskAtom.java:73)
at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext()Lorg/apache/cassandra/db/OnDiskAtom; (SimpleSliceReader.java:106)
at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext()Ljava/lang/Object; (SimpleSliceReader.java:40)
at com.google.common.collect.AbstractIterator.tryToComputeNext()Z (AbstractIterator.java:143)
at com.google.common.collect.AbstractIterator.hasNext()Z (AbstractIterator.java:138)
at org.apache.cassandra.db.columniterator.SSTableSliceIterator.hasNext()Z (SSTableSliceIterator.java:90)
at org.apache.cassandra.db.filter.QueryFilter$2.getNext()V (QueryFilter.java:171)
at org.apache.cassandra.db.filter.QueryFilter$2.hasNext()Z (QueryFilter.java:154)
at org.apache.cassandra.utils.MergeIterator$OneToOne.computeNext()Ljava/lang/Object; (MergeIterator.java:199)
at com.google.common.collect.AbstractIterator.tryToComputeNext()Z (AbstractIterator.java:143)
at com.google.common.collect.AbstractIterator.hasNext()Z (AbstractIterator.java:138)
at org.apache.cassandra.db.filter.SliceQueryFilter.collectReducedColumns(Lorg/apache/cassandra/db/IColumnContainer;Ljava/util/Iterator;I)V (SliceQueryFilter.java:160)
at org.apache.cassandra.db.filter.QueryFilter.collateColumns(Lorg/apache/cassandra/db/ColumnFamily;Ljava/util/List;I)V (QueryFilter.java:136)
at org.apache.cassandra.db.filter.QueryFilter.collateOnDiskAtom(Lorg/apache/cassandra/db/ColumnFamily;Ljava/util/List;I)V (QueryFilter.java:84)
at org.apache.cassandra.db.CollationController.collectAllData()Lorg/apache/cassandra/db/ColumnFamily; (CollationController.java:291)
at org.apache.cassandra.db.CollationController.getTopLevelColumns()Lorg/apache/cassandra/db/ColumnFamily; (CollationController.java:65)
at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(Lorg/apache/cassandra/db/filter/QueryFilter;IZ)Lorg/apache/cassandra/db/ColumnFamily; (ColumnFamilyStore.java:1391)
at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(Lorg/apache/cassandra/db/filter/QueryFilter;I)Lorg/apache/cassandra/db/ColumnFamily; (ColumnFamilyStore.java:1207)
at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(Lorg/apache/cassandra/db/filter/QueryFilter;)Lorg/apache/cassandra/db/ColumnFamily; (ColumnFamilyStore.java:1123)
at org.apache.cassandra.db.Table.getRow(Lorg/apache/cassandra/db/filter/QueryFilter;)Lorg/apache/cassandra/db/Row; (Table.java:347)
at org.apache.cassandra.db.SliceFromReadCommand.getRow(Lorg/apache/cassandra/db/Table;)Lorg/apache/cassandra/db/Row; (SliceFromReadCommand.java:70)
at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow()V (StorageProxy.java:1062)
at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run()V (StorageProxy.java:1614)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V (ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run()V (ThreadPoolExecutor.java:615)
at java.lang.Thread.run()V (Thread.java:724)
ERROR [ReadStage:23] 2013-12-06 09:47:22,434 CassandraDaemon.java (line 191) Exception in thread Thread[ReadStage:23,5,main]
java.lang.OutOfMemoryError: Java heap space
at org.apache.cassandra.io.util.RandomAccessReader.readBytes(RandomAccessReader.java:376)
at org.apache.cassandra.utils.ByteBufferUtil.read(ByteBufferUtil.java:392)
at org.apache.cassandra.utils.ByteBufferUtil.readWithLength(ByteBufferUtil.java:355)
at org.apache.cassandra.db.ColumnSerializer.deserializeColumnBody(ColumnSerializer.java:108)
at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(OnDiskAtom.java:92)
at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(OnDiskAtom.java:73)
at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext(SimpleSliceReader.java:106)
at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext(SimpleSliceReader.java:40)
at com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143)
at com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138)
at org.apache.cassandra.db.columniterator.SSTableSliceIterator.hasNext(SSTableSliceIterator.java:90)
at org.apache.cassandra.db.filter.QueryFilter$2.getNext(QueryFilter.java:171)
at org.apache.cassandra.db.filter.QueryFilter$2.hasNext(QueryFilter.java:154)
at org.apache.cassandra.utils.MergeIterator$Candidate.advance(MergeIterator.java:143)
at org.apache.cassandra.utils.MergeIterator$ManyToOne.<init>(MergeIterator.java:86)
at org.apache.cassandra.utils.MergeIterator.get(MergeIterator.java:45)
at org.apache.cassandra.db.filter.QueryFilter.collateColumns(QueryFilter.java:134)
at org.apache.cassandra.db.filter.QueryFilter.collateOnDiskAtom(QueryFilter.java:84)
at org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:291)
at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:65)
at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1391)
at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1207)
at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1123)
at org.apache.cassandra.db.Table.getRow(Table.java:347)
at org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:70)
at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1062)
at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1614)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:724)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment