Last active
December 30, 2015 11:09
-
-
Save klausbrunner/7820902 to your computer and use it in GitHub Desktop.
Cassandra 1.2.11 OOMs during relatively high read load, possibly around/in compaction?
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ERROR [ReadStage:51] 2013-12-06 00:02:54,065 CassandraDaemon.java (line 191) Exception in thread Thread[ReadStage:51,5,main] | |
| ERROR [ReadStage:51] 2013-12-06 00:02:54,065 CassandraDaemon.java (line 191) Exception in thread Thread[ReadStage:51,5,main] | |
| java.lang.OutOfMemoryError: Java heap space | |
| at org.apache.cassandra.io.util.RandomAccessReader.readBytes(RandomAccessReader.java:376) | |
| at org.apache.cassandra.utils.ByteBufferUtil.read(ByteBufferUtil.java:392) | |
| at org.apache.cassandra.utils.ByteBufferUtil.readWithLength(ByteBufferUtil.java:355) | |
| at org.apache.cassandra.db.ColumnSerializer.deserializeColumnBody(ColumnSerializer.java:108) | |
| at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(OnDiskAtom.java:92) | |
| at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(OnDiskAtom.java:73) | |
| at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext(SimpleSliceReader.java:106) | |
| at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext(SimpleSliceReader.java:40) | |
| at com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143) | |
| at com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138) | |
| at org.apache.cassandra.db.columniterator.SSTableSliceIterator.hasNext(SSTableSliceIterator.java:90) | |
| at org.apache.cassandra.db.filter.QueryFilter$2.getNext(QueryFilter.java:171) | |
| at org.apache.cassandra.db.filter.QueryFilter$2.hasNext(QueryFilter.java:154) | |
| at org.apache.cassandra.utils.MergeIterator$Candidate.advance(MergeIterator.java:143) | |
| at org.apache.cassandra.utils.MergeIterator$ManyToOne.<init>(MergeIterator.java:86) | |
| at org.apache.cassandra.utils.MergeIterator.get(MergeIterator.java:45) | |
| at org.apache.cassandra.db.filter.QueryFilter.collateColumns(QueryFilter.java:134) | |
| at org.apache.cassandra.db.filter.QueryFilter.collateOnDiskAtom(QueryFilter.java:84) | |
| at org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:291) | |
| at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:65) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1391) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1207) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1123) | |
| at org.apache.cassandra.db.Table.getRow(Table.java:347) | |
| at org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:70) | |
| at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1062) | |
| at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1614) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext(SimpleSliceReader.java:40) | |
| at com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143) | |
| at com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138) | |
| at org.apache.cassandra.db.columniterator.SSTableSliceIterator.hasNext(SSTableSliceIterator.java:90) | |
| at org.apache.cassandra.db.filter.QueryFilter$2.getNext(QueryFilter.java:171) | |
| at org.apache.cassandra.db.filter.QueryFilter$2.hasNext(QueryFilter.java:154) | |
| at org.apache.cassandra.utils.MergeIterator$Candidate.advance(MergeIterator.java:143) | |
| at org.apache.cassandra.utils.MergeIterator$ManyToOne.<init>(MergeIterator.java:86) | |
| at org.apache.cassandra.utils.MergeIterator.get(MergeIterator.java:45) | |
| at org.apache.cassandra.db.filter.QueryFilter.collateColumns(QueryFilter.java:134) | |
| at org.apache.cassandra.db.filter.QueryFilter.collateOnDiskAtom(QueryFilter.java:84) | |
| at org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:291) | |
| at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:65) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1391) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1207) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1123) | |
| at org.apache.cassandra.db.Table.getRow(Table.java:347) | |
| at org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:70) | |
| at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1062) | |
| at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1614) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:724) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ReadStage:323 | |
| at sun.nio.ch.NativeThread.current()J (Native Method) | |
| at sun.nio.ch.NativeThreadSet.add()I (NativeThreadSet.java:46) | |
| at sun.nio.ch.FileChannelImpl.position()J (FileChannelImpl.java:243) | |
| at org.apache.cassandra.io.compress.CompressedRandomAccessReader.decompressChunk(Lorg/apache/cassandra/io/compress/CompressionMetadata$Chunk;)V (CompressedRandomAccessReader.java:99) | |
| at org.apache.cassandra.io.compress.CompressedRandomAccessReader.reBuffer()V (CompressedRandomAccessReader.java:85) | |
| at org.apache.cassandra.io.util.RandomAccessReader.read([BII)I (RandomAccessReader.java:355) | |
| at java.io.RandomAccessFile.readFully([BII)V (RandomAccessFile.java:414) | |
| at java.io.RandomAccessFile.readFully([B)V (RandomAccessFile.java:394) | |
| at org.apache.cassandra.io.util.RandomAccessReader.readBytes(I)Ljava/nio/ByteBuffer; (RandomAccessReader.java:380) | |
| at org.apache.cassandra.utils.ByteBufferUtil.read(Ljava/io/DataInput;I)Ljava/nio/ByteBuffer; (ByteBufferUtil.java:392) | |
| at org.apache.cassandra.utils.ByteBufferUtil.readWithLength(Ljava/io/DataInput;)Ljava/nio/ByteBuffer; (ByteBufferUtil.java:355) | |
| at org.apache.cassandra.db.ColumnSerializer.deserializeColumnBody(Ljava/io/DataInput;Ljava/nio/ByteBuffer;ILorg/apache/cassandra/io/IColumnSerializer$Flag;I)Lorg/apache/cassandra/db/Column; (ColumnSerializer.java:94) | |
| at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(Ljava/io/DataInput;Lorg/apache/cassandra/io/IColumnSerializer$Flag;ILorg/apache/cassandra/io/sstable/Descriptor$Version;)Lorg/apache/cassandra/db/OnDiskAtom; (OnDiskAtom.java:92) | |
| at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(Ljava/io/DataInput;Lorg/apache/cassandra/io/sstable/Descriptor$Version;)Lorg/apache/cassandra/db/OnDiskAtom; (OnDiskAtom.java:73) | |
| at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext()Lorg/apache/cassandra/db/OnDiskAtom; (SimpleSliceReader.java:106) | |
| at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext()Ljava/lang/Object; (SimpleSliceReader.java:40) | |
| at com.google.common.collect.AbstractIterator.tryToComputeNext()Z (AbstractIterator.java:143) | |
| at com.google.common.collect.AbstractIterator.hasNext()Z (AbstractIterator.java:138) | |
| at org.apache.cassandra.db.columniterator.SSTableSliceIterator.hasNext()Z (SSTableSliceIterator.java:90) | |
| at org.apache.cassandra.db.filter.QueryFilter$2.getNext()V (QueryFilter.java:171) | |
| at org.apache.cassandra.db.filter.QueryFilter$2.hasNext()Z (QueryFilter.java:154) | |
| at org.apache.cassandra.utils.MergeIterator$OneToOne.computeNext()Ljava/lang/Object; (MergeIterator.java:199) | |
| at com.google.common.collect.AbstractIterator.tryToComputeNext()Z (AbstractIterator.java:143) | |
| at com.google.common.collect.AbstractIterator.hasNext()Z (AbstractIterator.java:138) | |
| at org.apache.cassandra.db.filter.SliceQueryFilter.collectReducedColumns(Lorg/apache/cassandra/db/IColumnContainer;Ljava/util/Iterator;I)V (SliceQueryFilter.java:160) | |
| at org.apache.cassandra.db.filter.QueryFilter.collateColumns(Lorg/apache/cassandra/db/ColumnFamily;Ljava/util/List;I)V (QueryFilter.java:136) | |
| at org.apache.cassandra.db.filter.QueryFilter.collateOnDiskAtom(Lorg/apache/cassandra/db/ColumnFamily;Ljava/util/List;I)V (QueryFilter.java:84) | |
| at org.apache.cassandra.db.CollationController.collectAllData()Lorg/apache/cassandra/db/ColumnFamily; (CollationController.java:291) | |
| at org.apache.cassandra.db.CollationController.getTopLevelColumns()Lorg/apache/cassandra/db/ColumnFamily; (CollationController.java:65) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(Lorg/apache/cassandra/db/filter/QueryFilter;IZ)Lorg/apache/cassandra/db/ColumnFamily; (ColumnFamilyStore.java:1391) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(Lorg/apache/cassandra/db/filter/QueryFilter;I)Lorg/apache/cassandra/db/ColumnFamily; (ColumnFamilyStore.java:1207) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(Lorg/apache/cassandra/db/filter/QueryFilter;)Lorg/apache/cassandra/db/ColumnFamily; (ColumnFamilyStore.java:1123) | |
| at org.apache.cassandra.db.Table.getRow(Lorg/apache/cassandra/db/filter/QueryFilter;)Lorg/apache/cassandra/db/Row; (Table.java:347) | |
| at org.apache.cassandra.db.SliceFromReadCommand.getRow(Lorg/apache/cassandra/db/Table;)Lorg/apache/cassandra/db/Row; (SliceFromReadCommand.java:70) | |
| at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow()V (StorageProxy.java:1062) | |
| at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run()V (StorageProxy.java:1614) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V (ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run()V (ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run()V (Thread.java:724) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ERROR [ReadStage:23] 2013-12-06 09:47:22,434 CassandraDaemon.java (line 191) Exception in thread Thread[ReadStage:23,5,main] | |
| java.lang.OutOfMemoryError: Java heap space | |
| at org.apache.cassandra.io.util.RandomAccessReader.readBytes(RandomAccessReader.java:376) | |
| at org.apache.cassandra.utils.ByteBufferUtil.read(ByteBufferUtil.java:392) | |
| at org.apache.cassandra.utils.ByteBufferUtil.readWithLength(ByteBufferUtil.java:355) | |
| at org.apache.cassandra.db.ColumnSerializer.deserializeColumnBody(ColumnSerializer.java:108) | |
| at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(OnDiskAtom.java:92) | |
| at org.apache.cassandra.db.OnDiskAtom$Serializer.deserializeFromSSTable(OnDiskAtom.java:73) | |
| at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext(SimpleSliceReader.java:106) | |
| at org.apache.cassandra.db.columniterator.SimpleSliceReader.computeNext(SimpleSliceReader.java:40) | |
| at com.google.common.collect.AbstractIterator.tryToComputeNext(AbstractIterator.java:143) | |
| at com.google.common.collect.AbstractIterator.hasNext(AbstractIterator.java:138) | |
| at org.apache.cassandra.db.columniterator.SSTableSliceIterator.hasNext(SSTableSliceIterator.java:90) | |
| at org.apache.cassandra.db.filter.QueryFilter$2.getNext(QueryFilter.java:171) | |
| at org.apache.cassandra.db.filter.QueryFilter$2.hasNext(QueryFilter.java:154) | |
| at org.apache.cassandra.utils.MergeIterator$Candidate.advance(MergeIterator.java:143) | |
| at org.apache.cassandra.utils.MergeIterator$ManyToOne.<init>(MergeIterator.java:86) | |
| at org.apache.cassandra.utils.MergeIterator.get(MergeIterator.java:45) | |
| at org.apache.cassandra.db.filter.QueryFilter.collateColumns(QueryFilter.java:134) | |
| at org.apache.cassandra.db.filter.QueryFilter.collateOnDiskAtom(QueryFilter.java:84) | |
| at org.apache.cassandra.db.CollationController.collectAllData(CollationController.java:291) | |
| at org.apache.cassandra.db.CollationController.getTopLevelColumns(CollationController.java:65) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getTopLevelColumns(ColumnFamilyStore.java:1391) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1207) | |
| at org.apache.cassandra.db.ColumnFamilyStore.getColumnFamily(ColumnFamilyStore.java:1123) | |
| at org.apache.cassandra.db.Table.getRow(Table.java:347) | |
| at org.apache.cassandra.db.SliceFromReadCommand.getRow(SliceFromReadCommand.java:70) | |
| at org.apache.cassandra.service.StorageProxy$LocalReadRunnable.runMayThrow(StorageProxy.java:1062) | |
| at org.apache.cassandra.service.StorageProxy$DroppableRunnable.run(StorageProxy.java:1614) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:724) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment