Skip to content

Instantly share code, notes, and snippets.

@brettrann brettrann/broker.log Secret
Created Nov 8, 2017

Embed
What would you like to do?
kafka 1.0.0 translating from 0.11.0.1 to 0.9, 0.10, 0.11 clients OutOfMemoryError
{"timestamp":"2017-11-07 01:08:51,836","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-5","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.accounts,partitions=[{partition=0,fetch_offset=541011,max_bytes=11000000},{partition=1,fetch_offset=547489,max_bytes=11000000},{partition=5,fetch_offset=547902,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57)
at java.nio.ByteBuffer.allocate(ByteBuffer.java:335)
at org.apache.kafka.common.record.AbstractRecords.downConvert(AbstractRecords.java:101)
at org.apache.kafka.common.record.FileRecords.downConvert(FileRecords.java:253)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1$$anonfun$apply$4.apply(KafkaApis.scala:520)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1$$anonfun$apply$4.apply(KafkaApis.scala:518)
at scala.Option.map(Option.scala:146)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1.apply(KafkaApis.scala:518)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1.apply(KafkaApis.scala:508)
at scala.Option.flatMap(Option.scala:171)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$convertedPartitionData$1(KafkaApis.scala:508)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$createResponse$2$1.apply(KafkaApis.scala:556)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$createResponse$2$1.apply(KafkaApis.scala:555)
at scala.collection.Iterator$class.foreach(Iterator.scala:891)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1334)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$createResponse$2(KafkaApis.scala:555)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$fetchResponseCallback$1$1.apply(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$fetchResponseCallback$1$1.apply(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$sendResponseMaybeThrottle$1.apply$mcVI$sp(KafkaApis.scala:2034)
at kafka.server.ClientRequestQuotaManager.maybeRecordAndThrottle(ClientRequestQuotaManager.scala:52)
at kafka.server.KafkaApis.sendResponseMaybeThrottle(KafkaApis.scala:2033)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$fetchResponseCallback$1(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$processResponseCallback$1$1.apply$mcVI$sp(KafkaApis.scala:588)
at kafka.server.ClientQuotaManager.maybeRecordAndThrottle(ClientQuotaManager.scala:175)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:587)
at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:604)
at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:604)
at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:820)
at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:596)
at kafka.server.KafkaApis.handle(KafkaApis.scala:100)
{"timestamp":"2017-11-07 01:10:06,184","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Member <redacted> in group <redacted> has failed, removing it from the group"}
{"timestamp":"2017-11-07 01:10:06,184","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 3189 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 01:10:08,464","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Member <redacted> in group <redacted> has failed, removing it from the group"}
{"timestamp":"2017-11-07 01:10:08,464","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: group <redacted> with generation 3190 is now empty (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 01:13:22,190","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 17 expired offsets in 6 milliseconds."}
{"timestamp":"2017-11-07 01:18:53,830","level":"INFO","logger":"kafka.log.Log","thread":"ReplicaFetcherThread-0-4","message":"Incrementing log start offset of partition maxwell.transactions-15 to 11485016 in dir /data/kafka/logs"}
{"timestamp":"2017-11-07 01:18:53,833","level":"INFO","logger":"kafka.server.epoch.LeaderEpochFileCache","thread":"ReplicaFetcherThread-0-4","message":"Cleared earliest 0 entries from epoch cache based on passed offset 11485016 leaving 7 in EpochFile for partition maxwell.transactions-15"}
{"timestamp":"2017-11-07 01:23:22,509","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 216 milliseconds."}
{"timestamp":"2017-11-07 01:23:51,953","level":"INFO","logger":"kafka.log.Log","thread":"kafka-scheduler-2","message":"Found deletable segments with base offsets [10947606] due to log start offset 11485016 breach"}
{"timestamp":"2017-11-07 01:23:51,953","level":"INFO","logger":"kafka.log.Log","thread":"kafka-scheduler-2","message":"Scheduling log segment 10947606 for log maxwell.transactions-15 for deletion."}
{"timestamp":"2017-11-07 01:24:51,954","level":"INFO","logger":"kafka.log.Log","thread":"kafka-scheduler-3","message":"Deleting segment 10947606 from log maxwell.transactions-15."}
{"timestamp":"2017-11-07 01:24:52,145","level":"INFO","logger":"kafka.log.OffsetIndex","thread":"kafka-scheduler-3","message":"Deleting index /data/kafka/logs/maxwell.transactions-15/00000000000010947606.index.deleted"}
{"timestamp":"2017-11-07 01:24:52,146","level":"INFO","logger":"kafka.log.TimeIndex","thread":"kafka-scheduler-3","message":"Deleting index /data/kafka/logs/maxwell.transactions-15/00000000000010947606.timeindex.deleted"}
{"timestamp":"2017-11-07 01:32:43,702","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-1","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 3190 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 01:32:46,703","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Rebalance","message":"[GroupCoordinator 2]: Stabilized group pravda.change-notification-acme-app generation 3191 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 01:32:46,704","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-3","message":"[GroupCoordinator 2]: Assignment received from leader for <redacted> for generation 3191"}
{"timestamp":"2017-11-07 01:32:47,275","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-0","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.accounts,partitions=[{partition=0,fetch_offset=541011,max_bytes=11000000},{partition=1,fetch_offset=547489,max_bytes=11000000},{partition=5,fetch_offset=547902,max_bytes=11000000},{partition=9,fetch_offset=548856,max_bytes=11000000},{partition=10,fetch_offset=549934,max_bytes=11000000},{partition=13,fetch_offset=542394,max_bytes=11000000},{partition=15,fetch_offset=552518,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57)
at java.nio.ByteBuffer.allocate(ByteBuffer.java:335)
at org.apache.kafka.common.record.AbstractRecords.downConvert(AbstractRecords.java:101)
at org.apache.kafka.common.record.FileRecords.downConvert(FileRecords.java:253)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1$$anonfun$apply$4.apply(KafkaApis.scala:520)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1$$anonfun$apply$4.apply(KafkaApis.scala:518)
at scala.Option.map(Option.scala:146)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1.apply(KafkaApis.scala:518)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1.apply(KafkaApis.scala:508)
at scala.Option.flatMap(Option.scala:171)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$convertedPartitionData$1(KafkaApis.scala:508)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$createResponse$2$1.apply(KafkaApis.scala:556)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$createResponse$2$1.apply(KafkaApis.scala:555)
at scala.collection.Iterator$class.foreach(Iterator.scala:891)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1334)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$createResponse$2(KafkaApis.scala:555)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$fetchResponseCallback$1$1.apply(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$fetchResponseCallback$1$1.apply(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$sendResponseMaybeThrottle$1.apply$mcVI$sp(KafkaApis.scala:2034)
at kafka.server.ClientRequestQuotaManager.maybeRecordAndThrottle(ClientRequestQuotaManager.scala:52)
at kafka.server.KafkaApis.sendResponseMaybeThrottle(KafkaApis.scala:2033)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$fetchResponseCallback$1(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$processResponseCallback$1$1.apply$mcVI$sp(KafkaApis.scala:588)
at kafka.server.ClientQuotaManager.maybeRecordAndThrottle(ClientQuotaManager.scala:175)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:587)
at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:604)
at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:604)
at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:820)
at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:596)
at kafka.server.KafkaApis.handle(KafkaApis.scala:100)
{"timestamp":"2017-11-07 01:33:21,883","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Member <redacted> in group <redacted> has failed, removing it from the group"}
{"timestamp":"2017-11-07 01:33:21,883","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 3191 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 01:33:21,883","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: group <redacted> with generation 3192 is now empty (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 01:33:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 01:41:50,984","level":"INFO","logger":"kafka.log.Log","thread":"kafka-request-handler-3","message":"Rolled new log segment for '__consumer_offsets-1' in 1 ms."}
{"timestamp":"2017-11-07 01:42:57,907","level":"INFO","logger":"kafka.log.Log","thread":"kafka-scheduler-6","message":"Deleting segment 0 from log __consumer_offsets-1."}
{"timestamp":"2017-11-07 01:42:57,907","level":"INFO","logger":"kafka.log.Log","thread":"kafka-scheduler-7","message":"Deleting segment 119599309 from log __consumer_offsets-1."}
{"timestamp":"2017-11-07 01:42:57,907","level":"INFO","logger":"kafka.log.OffsetIndex","thread":"kafka-scheduler-7","message":"Deleting index /data/kafka/logs/__consumer_offsets-1/00000000000119599309.index.deleted"}
{"timestamp":"2017-11-07 01:42:57,907","level":"INFO","logger":"kafka.log.OffsetIndex","thread":"kafka-scheduler-6","message":"Deleting index /data/kafka/logs/__consumer_offsets-1/00000000000000000000.index.deleted"}
{"timestamp":"2017-11-07 01:42:57,907","level":"INFO","logger":"kafka.log.TimeIndex","thread":"kafka-scheduler-7","message":"Deleting index /data/kafka/logs/__consumer_offsets-1/00000000000119599309.timeindex.deleted"}
{"timestamp":"2017-11-07 01:42:57,907","level":"INFO","logger":"kafka.log.TimeIndex","thread":"kafka-scheduler-6","message":"Deleting index /data/kafka/logs/__consumer_offsets-1/00000000000000000000.timeindex.deleted"}
{"timestamp":"2017-11-07 01:42:58,916","level":"INFO","logger":"kafka.log.Log","thread":"kafka-scheduler-5","message":"Deleting segment 120590073 from log __consumer_offsets-1."}
{"timestamp":"2017-11-07 01:42:58,949","level":"INFO","logger":"kafka.log.OffsetIndex","thread":"kafka-scheduler-5","message":"Deleting index /data/kafka/logs/__consumer_offsets-1/00000000000120590073.index.deleted"}
{"timestamp":"2017-11-07 01:42:58,949","level":"INFO","logger":"kafka.log.TimeIndex","thread":"kafka-scheduler-5","message":"Deleting index /data/kafka/logs/__consumer_offsets-1/00000000000120590073.timeindex.deleted"}
{"timestamp":"2017-11-07 01:43:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 18 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 01:53:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] group <redacted> transitioned to Dead in generation 2"}
{"timestamp":"2017-11-07 01:53:22,189","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 9 expired offsets in 5 milliseconds."}
{"timestamp":"2017-11-07 01:54:34,654","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-0","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 3192 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 01:54:37,654","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Rebalance","message":"[GroupCoordinator 2]: Stabilized group pravda.change-notification-acme-app generation 3193 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 01:54:37,655","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-6","message":"[GroupCoordinator 2]: Assignment received from leader for <redacted> for generation 3193"}
{"timestamp":"2017-11-07 01:54:38,188","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-7","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.accounts,partitions=[{partition=9,fetch_offset=548856,max_bytes=11000000},{partition=10,fetch_offset=549934,max_bytes=11000000},{partition=13,fetch_offset=542394,max_bytes=11000000},{partition=15,fetch_offset=552518,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57)
at java.nio.ByteBuffer.allocate(ByteBuffer.java:335)
at org.apache.kafka.common.record.AbstractRecords.downConvert(AbstractRecords.java:101)
at org.apache.kafka.common.record.FileRecords.downConvert(FileRecords.java:253)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1$$anonfun$apply$4.apply(KafkaApis.scala:520)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1$$anonfun$apply$4.apply(KafkaApis.scala:518)
at scala.Option.map(Option.scala:146)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1.apply(KafkaApis.scala:518)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1.apply(KafkaApis.scala:508)
at scala.Option.flatMap(Option.scala:171)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$convertedPartitionData$1(KafkaApis.scala:508)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$createResponse$2$1.apply(KafkaApis.scala:556)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$createResponse$2$1.apply(KafkaApis.scala:555)
at scala.collection.Iterator$class.foreach(Iterator.scala:891)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1334)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$createResponse$2(KafkaApis.scala:555)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$fetchResponseCallback$1$1.apply(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$fetchResponseCallback$1$1.apply(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$sendResponseMaybeThrottle$1.apply$mcVI$sp(KafkaApis.scala:2034)
at kafka.server.ClientRequestQuotaManager.maybeRecordAndThrottle(ClientRequestQuotaManager.scala:52)
at kafka.server.KafkaApis.sendResponseMaybeThrottle(KafkaApis.scala:2033)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$fetchResponseCallback$1(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$processResponseCallback$1$1.apply$mcVI$sp(KafkaApis.scala:588)
at kafka.server.ClientQuotaManager.maybeRecordAndThrottle(ClientQuotaManager.scala:175)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:587)
at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:604)
at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:604)
at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:820)
at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:596)
at kafka.server.KafkaApis.handle(KafkaApis.scala:100)
{"timestamp":"2017-11-07 01:54:40,133","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-1","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.accounts,partitions=[{partition=0,fetch_offset=541011,max_bytes=11000000},{partition=1,fetch_offset=547489,max_bytes=11000000},{partition=5,fetch_offset=547902,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57)
at java.nio.ByteBuffer.allocate(ByteBuffer.java:335)
at org.apache.kafka.common.record.AbstractRecords.downConvert(AbstractRecords.java:101)
at org.apache.kafka.common.record.FileRecords.downConvert(FileRecords.java:253)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1$$anonfun$apply$4.apply(KafkaApis.scala:520)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1$$anonfun$apply$4.apply(KafkaApis.scala:518)
at scala.Option.map(Option.scala:146)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1.apply(KafkaApis.scala:518)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$convertedPartitionData$1$1.apply(KafkaApis.scala:508)
at scala.Option.flatMap(Option.scala:171)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$convertedPartitionData$1(KafkaApis.scala:508)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$createResponse$2$1.apply(KafkaApis.scala:556)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$createResponse$2$1.apply(KafkaApis.scala:555)
at scala.collection.Iterator$class.foreach(Iterator.scala:891)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1334)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$createResponse$2(KafkaApis.scala:555)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$fetchResponseCallback$1$1.apply(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$fetchResponseCallback$1$1.apply(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$sendResponseMaybeThrottle$1.apply$mcVI$sp(KafkaApis.scala:2034)
at kafka.server.ClientRequestQuotaManager.maybeRecordAndThrottle(ClientRequestQuotaManager.scala:52)
at kafka.server.KafkaApis.sendResponseMaybeThrottle(KafkaApis.scala:2033)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$fetchResponseCallback$1(KafkaApis.scala:569)
at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$processResponseCallback$1$1.apply$mcVI$sp(KafkaApis.scala:588)
at kafka.server.ClientQuotaManager.maybeRecordAndThrottle(ClientQuotaManager.scala:175)
at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:587)
at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:604)
at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:604)
at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:820)
at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:596)
at kafka.server.KafkaApis.handle(KafkaApis.scala:100)
{"timestamp":"2017-11-07 01:55:30,614","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Member <redacted> in group <redacted> has failed, removing it from the group"}
{"timestamp":"2017-11-07 01:55:30,614","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 3193 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 01:55:30,757","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Member <redacted> in group <redacted> has failed, removing it from the group"}
{"timestamp":"2017-11-07 01:55:30,757","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: group <redacted> with generation 3194 is now empty (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 02:03:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 02:13:22,443","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 2 expired offsets in 231 milliseconds."}
{"timestamp":"2017-11-07 02:23:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 02:33:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] group <redacted> transitioned to Dead in generation 2"}
{"timestamp":"2017-11-07 02:33:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] group <redacted> transitioned to Dead in generation 2"}
{"timestamp":"2017-11-07 02:33:22,188","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 2 expired offsets in 4 milliseconds."}
{"timestamp":"2017-11-07 02:43:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 02:50:18,838","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-2","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 2133 (__consumer_offsets-7)"}
{"timestamp":"2017-11-07 02:50:21,838","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Rebalance","message":"[GroupCoordinator 2]: Stabilized group metropolis.change-notification-acme-app generation 2134 (__consumer_offsets-7)"}
{"timestamp":"2017-11-07 02:50:21,840","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-4","message":"[GroupCoordinator 2]: Assignment received from leader for <redacted> for generation 2134"}
{"timestamp":"2017-11-07 02:50:22,437","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-3","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.users,partitions=[{partition=1,fetch_offset=401121,max_bytes=11000000},{partition=6,fetch_offset=394540,max_bytes=11000000},{partition=9,fetch_offset=358797,max_bytes=11000000},{partition=11,fetch_offset=390987,max_bytes=11000000},{partition=13,fetch_offset=255611,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 02:50:22,897","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-0","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.users,partitions=[{partition=1,fetch_offset=353423,max_bytes=11000000},{partition=6,fetch_offset=399478,max_bytes=11000000},{partition=9,fetch_offset=372926,max_bytes=11000000},{partition=11,fetch_offset=330724,max_bytes=11000000},{partition=13,fetch_offset=252317,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 02:50:23,338","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-1","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.users,partitions=[{partition=1,fetch_offset=401121,max_bytes=11000000},{partition=6,fetch_offset=394540,max_bytes=11000000},{partition=9,fetch_offset=358797,max_bytes=11000000},{partition=11,fetch_offset=390987,max_bytes=11000000},{partition=13,fetch_offset=255611,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 02:50:23,831","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-4","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.users,partitions=[{partition=1,fetch_offset=401121,max_bytes=11000000},{partition=6,fetch_offset=394540,max_bytes=11000000},{partition=9,fetch_offset=358797,max_bytes=11000000},{partition=11,fetch_offset=390987,max_bytes=11000000},{partition=13,fetch_offset=255611,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 02:50:24,275","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-2","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.users,partitions=[{partition=1,fetch_offset=353423,max_bytes=11000000},{partition=6,fetch_offset=399478,max_bytes=11000000},{partition=9,fetch_offset=372926,max_bytes=11000000},{partition=11,fetch_offset=330724,max_bytes=11000000},{partition=13,fetch_offset=252317,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 02:50:51,853","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Member <redacted> in group <redacted> has failed, removing it from the group"}
{"timestamp":"2017-11-07 02:50:51,853","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 2134 (__consumer_offsets-7)"}
{"timestamp":"2017-11-07 02:50:51,853","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: group <redacted> with generation 2135 is now empty (__consumer_offsets-7)"}
{"timestamp":"2017-11-07 02:53:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 03:03:12,741","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-2","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 2135 (__consumer_offsets-7)"}
{"timestamp":"2017-11-07 03:03:15,742","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Rebalance","message":"[GroupCoordinator 2]: Stabilized group metropolis.change-notification-acme-app generation 2136 (__consumer_offsets-7)"}
{"timestamp":"2017-11-07 03:03:15,743","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-7","message":"[GroupCoordinator 2]: Assignment received from leader for <redacted> for generation 2136"}
{"timestamp":"2017-11-07 03:03:16,261","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-1","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.users,partitions=[{partition=1,fetch_offset=401121,max_bytes=11000000},{partition=6,fetch_offset=394540,max_bytes=11000000},{partition=9,fetch_offset=358797,max_bytes=11000000},{partition=11,fetch_offset=390987,max_bytes=11000000},{partition=13,fetch_offset=255611,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 03:03:16,838","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-0","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.users,partitions=[{partition=1,fetch_offset=401121,max_bytes=11000000},{partition=6,fetch_offset=394540,max_bytes=11000000},{partition=9,fetch_offset=358797,max_bytes=11000000},{partition=11,fetch_offset=390987,max_bytes=11000000},{partition=13,fetch_offset=255611,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 03:03:17,280","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-2","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.users,partitions=[{partition=1,fetch_offset=353423,max_bytes=11000000},{partition=6,fetch_offset=399478,max_bytes=11000000},{partition=9,fetch_offset=372926,max_bytes=11000000},{partition=11,fetch_offset=330724,max_bytes=11000000},{partition=13,fetch_offset=252317,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 03:03:17,750","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-5","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.users,partitions=[{partition=1,fetch_offset=401121,max_bytes=11000000},{partition=6,fetch_offset=394540,max_bytes=11000000},{partition=9,fetch_offset=358797,max_bytes=11000000},{partition=11,fetch_offset=390987,max_bytes=11000000},{partition=13,fetch_offset=255611,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 03:03:18,149","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-6","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.users,partitions=[{partition=1,fetch_offset=353423,max_bytes=11000000},{partition=6,fetch_offset=399478,max_bytes=11000000},{partition=9,fetch_offset=372926,max_bytes=11000000},{partition=11,fetch_offset=330724,max_bytes=11000000},{partition=13,fetch_offset=252317,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 03:03:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 03:03:45,756","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Member <redacted> in group <redacted> has failed, removing it from the group"}
{"timestamp":"2017-11-07 03:03:45,757","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 2136 (__consumer_offsets-7)"}
{"timestamp":"2017-11-07 03:03:45,757","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: group <redacted> with generation 2137 is now empty (__consumer_offsets-7)"}
{"timestamp":"2017-11-07 03:13:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 03:15:44,549","level":"INFO","logger":"kafka.log.Log","thread":"ReplicaFetcherThread-0-1","message":"Rolled new log segment for 'fluent.default-2' in 1 ms."}
{"timestamp":"2017-11-07 03:23:12,406","level":"INFO","logger":"kafka.log.Log","thread":"kafka-request-handler-6","message":"Rolled new log segment for '__consumer_offsets-11' in 1 ms."}
{"timestamp":"2017-11-07 03:23:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 03:24:24,052","level":"INFO","logger":"kafka.log.Log","thread":"kafka-scheduler-9","message":"Deleting segment 0 from log __consumer_offsets-11."}
{"timestamp":"2017-11-07 03:24:24,052","level":"INFO","logger":"kafka.log.Log","thread":"kafka-scheduler-6","message":"Deleting segment 1925914 from log __consumer_offsets-11."}
{"timestamp":"2017-11-07 03:24:24,052","level":"INFO","logger":"kafka.log.OffsetIndex","thread":"kafka-scheduler-9","message":"Deleting index /data/kafka/logs/__consumer_offsets-11/00000000000000000000.index.deleted"}
{"timestamp":"2017-11-07 03:24:24,052","level":"INFO","logger":"kafka.log.TimeIndex","thread":"kafka-scheduler-9","message":"Deleting index /data/kafka/logs/__consumer_offsets-11/00000000000000000000.timeindex.deleted"}
{"timestamp":"2017-11-07 03:24:24,053","level":"INFO","logger":"kafka.log.OffsetIndex","thread":"kafka-scheduler-6","message":"Deleting index /data/kafka/logs/__consumer_offsets-11/00000000000001925914.index.deleted"}
{"timestamp":"2017-11-07 03:24:24,053","level":"INFO","logger":"kafka.log.TimeIndex","thread":"kafka-scheduler-6","message":"Deleting index /data/kafka/logs/__consumer_offsets-11/00000000000001925914.timeindex.deleted"}
{"timestamp":"2017-11-07 03:33:22,247","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 0 milliseconds."}
{"timestamp":"2017-11-07 03:39:43,348","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-4","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 3194 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 03:39:46,349","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Rebalance","message":"[GroupCoordinator 2]: Stabilized group pravda.change-notification-acme-app generation 3195 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 03:39:46,350","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-0","message":"[GroupCoordinator 2]: Assignment received from leader for <redacted> for generation 3195"}
{"timestamp":"2017-11-07 03:39:46,900","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-7","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.accounts,partitions=[{partition=0,fetch_offset=541011,max_bytes=11000000},{partition=1,fetch_offset=547489,max_bytes=11000000},{partition=5,fetch_offset=547902,max_bytes=11000000},{partition=9,fetch_offset=548856,max_bytes=11000000},{partition=10,fetch_offset=549934,max_bytes=11000000},{partition=13,fetch_offset=542394,max_bytes=11000000},{partition=15,fetch_offset=553927,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
{"timestamp":"2017-11-07 03:40:58,641","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Member <redacted> in group <redacted> has failed, removing it from the group"}
{"timestamp":"2017-11-07 03:40:58,641","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 3195 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 03:40:58,641","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Heartbeat","message":"[GroupCoordinator 2]: group <redacted> with generation 3196 is now empty (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 03:43:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 03:47:39,741","level":"INFO","logger":"kafka.log.Log","thread":"ReplicaFetcherThread-0-1","message":"Rolled new log segment for 'fluent.default-1' in 1 ms."}
{"timestamp":"2017-11-07 03:53:22,185","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 04:03:22,223","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 04:13:22,196","level":"INFO","logger":"kafka.coordinator.group.GroupMetadataManager","thread":"group-metadata-manager-0","message":"[GroupMetadataManager brokerId=2] Removed 0 expired offsets in 1 milliseconds."}
{"timestamp":"2017-11-07 04:18:05,317","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-6","message":"[GroupCoordinator 2]: Preparing to rebalance group <redacted> with old generation 3196 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 04:18:08,318","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"executor-Rebalance","message":"[GroupCoordinator 2]: Stabilized group pravda.change-notification-acme-app generation 3197 (__consumer_offsets-5)"}
{"timestamp":"2017-11-07 04:18:08,319","level":"INFO","logger":"kafka.coordinator.group.GroupCoordinator","thread":"kafka-request-handler-5","message":"[GroupCoordinator 2]: Assignment received from leader for <redacted> for generation 3197"}
{"timestamp":"2017-11-07 04:18:08,841","level":"ERROR","logger":"kafka.server.KafkaApis","thread":"kafka-request-handler-3","message":"[KafkaApi-2] Error when handling request {replica_id=-1,max_wait_time=500,min_bytes=1,topics=[{topic=maxwell.accounts,partitions=[{partition=0,fetch_offset=541011,max_bytes=11000000},{partition=1,fetch_offset=547489,max_bytes=11000000},{partition=5,fetch_offset=547902,max_bytes=11000000},{partition=9,fetch_offset=548856,max_bytes=11000000},{partition=10,fetch_offset=549934,max_bytes=11000000},{partition=13,fetch_offset=542394,max_bytes=11000000},{partition=15,fetch_offset=553927,max_bytes=11000000}]}]}"}
java.lang.OutOfMemoryError: Java heap space
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.