Navigation Menu

Skip to content

Instantly share code, notes, and snippets.

@Srinathc

Srinathc/oom.log

Created August 1, 2015 03:27
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Srinathc/e22237b578fcfd73d57b to your computer and use it in GitHub Desktop.
Save Srinathc/e22237b578fcfd73d57b to your computer and use it in GitHub Desktop.
Download ZIP
Elasticsearch_OOM
Raw
oom.log
[2015-08-01 01:17:12,416][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][78377][4873] duration [4.1s], collections [1]/[5s], total [4.1s]/[1h], memory [2.5gb]->[3.2gb]/[7.1gb], all_pools {[young] [1.4gb]->[20mb]/[3gb]}{[survivor] [371mb]->[384mb]/[384mb]}{[old] [785.2mb]->[2.8gb]/[3.7gb]}
[2015-08-01 01:18:20,587][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][78434][4878] duration [3.3s], collections [1]/[3.3s], total [3.3s]/[1h], memory [3.4gb]->[2.1gb]/[7.1gb], all_pools {[young] [2.9gb]->[76.7mb]/[3gb]}{[survivor] [77.5mb]->[384mb]/[384mb]}{[old] [435.4mb]->[1.7gb]/[3.7gb]}
[2015-08-01 01:18:29,929][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78435][764] duration [8.8s], collections [1]/[9.3s], total [8.8s]/[41.2m], memory [2.1gb]->[4.4gb]/[7.1gb], all_pools {[young] [76.7mb]->[730.8mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [1.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:19:29,555][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][78466][4882] duration [17s], collections [1]/[25.1s], total [17s]/[1h], memory [1.5gb]->[3.6gb]/[7.1gb], all_pools {[young] [189.8mb]->[46.3mb]/[3gb]}{[survivor] [267.3mb]->[0b]/[384mb]}{[old] [1gb]->[3.6gb]/[3.7gb]}
[2015-08-01 01:19:29,555][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78466][766] duration [7.7s], collections [1]/[25.1s], total [7.7s]/[41.4m], memory [1.5gb]->[3.6gb]/[7.1gb], all_pools {[young] [189.8mb]->[46.3mb]/[3gb]}{[survivor] [267.3mb]->[0b]/[384mb]}{[old] [1gb]->[3.6gb]/[3.7gb]}
[2015-08-01 01:19:46,294][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78468][767] duration [15.7s], collections [1]/[15.7s], total [15.7s]/[41.6m], memory [6.4gb]->[6.3gb]/[7.1gb], all_pools {[young] [2.8gb]->[2.5gb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [3.6gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:19:48,513][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@72cf1905]
[2015-08-01 01:20:07,078][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78472][768] duration [15.8s], collections [1]/[16.8s], total [15.8s]/[41.9m], memory [7gb]->[6.7gb]/[7.1gb], all_pools {[young] [3gb]->[3gb]/[3gb]}{[survivor] [375.1mb]->[66.1mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:20:07,209][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@2d92215a]
[2015-08-01 01:20:23,456][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78474][769] duration [15s], collections [1]/[15.3s], total [15s]/[42.2m], memory [7gb]->[6.8gb]/[7.1gb], all_pools {[young] [3gb]->[2.9gb]/[3gb]}{[survivor] [362.5mb]->[151.4mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:20:42,362][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78477][770] duration [16.5s], collections [1]/[16.8s], total [16.5s]/[42.5m], memory [7.1gb]->[6.9gb]/[7.1gb], all_pools {[young] [2.9gb]->[3gb]/[3gb]}{[survivor] [383.9mb]->[216.1mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:20:44,651][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@21c53c07]
[2015-08-01 01:21:03,239][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78480][771] duration [17s], collections [1]/[17.5s], total [17s]/[42.8m], memory [7.1gb]->[6.9gb]/[7.1gb], all_pools {[young] [3gb]->[3gb]/[3gb]}{[survivor] [383.9mb]->[273.5mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:21:05,636][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@51c7c3fa]
[2015-08-01 01:21:25,515][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78483][772] duration [18.7s], collections [1]/[16.7s], total [18.7s]/[43.1m], memory [7.1gb]->[7gb]/[7.1gb], all_pools {[young] [3gb]->[3gb]/[3gb]}{[survivor] [383.9mb]->[307.1mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:21:42,930][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78484][773] duration [16.3s], collections [1]/[3.1s], total [16.3s]/[43.4m], memory [7gb]->[7gb]/[7.1gb], all_pools {[young] [3gb]->[3gb]/[3gb]}{[survivor] [307.1mb]->[338.6mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:21:45,138][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@6b89f28]
[2015-08-01 01:22:02,828][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78486][774] duration [17.1s], collections [1]/[17.6s], total [17.1s]/[43.7m], memory [7.1gb]->[7gb]/[7.1gb], all_pools {[young] [3gb]->[2.9gb]/[3gb]}{[survivor] [379.3mb]->[354.8mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:22:34,336][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78487][776] duration [30.9s], collections [2]/[15.6s], total [30.9s]/[44.2m], memory [7gb]->[7.1gb]/[7.1gb], all_pools {[young] [2.9gb]->[3gb]/[3gb]}{[survivor] [354.8mb]->[383.9mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:22:46,126][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78488][777] duration [11.6s], collections [1]/[27.6s], total [11.6s]/[44.4m], memory [7.1gb]->[7.1gb]/[7.1gb], all_pools {[young] [3gb]->[3gb]/[3gb]}{[survivor] [383.9mb]->[380.5mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:23:04,159][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78490][778] duration [15.7s], collections [1]/[15.9s], total [15.7s]/[44.7m], memory [7.1gb]->[7.1gb]/[7.1gb], all_pools {[young] [3gb]->[2.9gb]/[3gb]}{[survivor] [381mb]->[375.1mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:23:22,279][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78491][779] duration [17.8s], collections [1]/[18.1s], total [17.8s]/[45m], memory [7.1gb]->[7.1gb]/[7.1gb], all_pools {[young] [2.9gb]->[3gb]/[3gb]}{[survivor] [375.1mb]->[378.6mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:27:36,507][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78492][790] duration [2.7m], collections [11]/[2.7m], total [2.7m]/[47.7m], memory [7.1gb]->[7.1gb]/[7.1gb], all_pools {[young] [3gb]->[3gb]/[3gb]}{[survivor] [378.6mb]->[384mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:28:39,731][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78493][797] duration [1.7m], collections [7]/[1.7m], total [1.7m]/[49.4m], memory [7.1gb]->[7.1gb]/[7.1gb], all_pools {[young] [3gb]->[2.9gb]/[3gb]}{[survivor] [384mb]->[383.8mb]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:44:52,940][ERROR][marvel.agent ] [metrics-datastore-5-es-tune-2] exporter [es_exporter] has thrown an exception:
java.lang.NullPointerException
at org.elasticsearch.marvel.agent.Utils.santizeUrlPwds(Utils.java:163)
at org.elasticsearch.marvel.agent.exporter.ESExporter.logErrorBasedOnLevel(ESExporter.java:457)
at org.elasticsearch.marvel.agent.exporter.ESExporter.openConnection(ESExporter.java:451)
at org.elasticsearch.marvel.agent.exporter.ESExporter.openAndValidateConnection(ESExporter.java:392)
at org.elasticsearch.marvel.agent.exporter.ESExporter.openExportingConnection(ESExporter.java:233)
at org.elasticsearch.marvel.agent.exporter.ESExporter.exportXContent(ESExporter.java:296)
at org.elasticsearch.marvel.agent.exporter.ESExporter.exportNodeStats(ESExporter.java:190)
at org.elasticsearch.marvel.agent.AgentService$ExportingWorker.exportNodeStats(AgentService.java:347)
at org.elasticsearch.marvel.agent.AgentService$ExportingWorker.run(AgentService.java:254)
at java.lang.Thread.run(Thread.java:745)
[2015-08-01 01:44:52,947][WARN ][index.merge.scheduler ] [metrics-datastore-5-es-tune-2] [perf3106556010][0] failed to merge
org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:354)
at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:719)
at org.apache.lucene.index.IndexFileDeleter.refresh(IndexFileDeleter.java:451)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3826)
at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:409)
at org.apache.lucene.index.TrackingConcurrentMergeScheduler.doMerge(TrackingConcurrentMergeScheduler.java:107)
at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:486)
Caused by: java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:89)
at org.apache.lucene.util.fst.FST.<init>(FST.java:302)
at org.apache.lucene.util.fst.Builder.<init>(Builder.java:155)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:476)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:656)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:943)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finishTerm(BlockTreeTermsWriter.java:912)
at org.apache.lucene.index.FreqProxTermsWriterPerField.flush(FreqProxTermsWriterPerField.java:500)
at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:80)
at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:114)
at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:441)
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:521)
at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:364)
at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:409)
at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1353)
at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1311)
at org.elasticsearch.index.engine.InternalEngine.innerCreateNoLock(InternalEngine.java:343)
at org.elasticsearch.index.engine.InternalEngine.innerCreate(InternalEngine.java:287)
at org.elasticsearch.index.engine.InternalEngine.create(InternalEngine.java:259)
at org.elasticsearch.index.shard.IndexShard.create(IndexShard.java:483)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:423)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:148)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase.performOnPrimary(TransportShardReplicationOperationAction.java:574)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1.doRun(TransportShardReplicationOperationAction.java:440)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
[2015-08-01 01:44:52,948][WARN ][index.engine ] [metrics-datastore-5-es-tune-2] [perf1106556008][0] failed engine [lucene commit failed]
java.lang.OutOfMemoryError: Java heap space
at java.util.HashMap.resize(HashMap.java:580)
at java.util.HashMap.addEntry(HashMap.java:879)
at java.util.HashMap.put(HashMap.java:505)
at java.util.HashSet.add(HashSet.java:217)
at java.util.AbstractCollection.addAll(AbstractCollection.java:342)
at org.apache.lucene.index.SegmentInfos.files(SegmentInfos.java:1056)
at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3007)
at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3134)
at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3101)
at org.elasticsearch.index.engine.InternalEngine.commitIndexWriter(InternalEngine.java:1293)
at org.elasticsearch.index.engine.InternalEngine.flush(InternalEngine.java:669)
at org.elasticsearch.index.engine.InternalEngine.flush(InternalEngine.java:631)
at org.elasticsearch.index.shard.IndexShard.flush(IndexShard.java:723)
at org.elasticsearch.indices.flush.SyncedFlushService.performPreSyncedFlush(SyncedFlushService.java:467)
at org.elasticsearch.indices.flush.SyncedFlushService.access$1100(SyncedFlushService.java:68)
at org.elasticsearch.indices.flush.SyncedFlushService$9.call(SyncedFlushService.java:435)
at org.elasticsearch.indices.flush.SyncedFlushService$9.call(SyncedFlushService.java:432)
at org.elasticsearch.indices.flush.SyncedFlushService$11.run(SyncedFlushService.java:821)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
[2015-08-01 01:44:52,954][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78494][884] duration [17m], collections [87]/[17m], total [17m]/[1.1h], memory [7.1gb]->[6.6gb]/[7.1gb], all_pools {[young] [2.9gb]->[2.8gb]/[3gb]}{[survivor] [383.8mb]->[0b]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:44:52,956][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] [perf3106556010][0] failed to execute bulk item (index) index {[perf3106556010][accesslog][AU7m12_rWLUv9z2-Ghr2], source[{"version":1.0,"timestamp":1438391953129,"accountId":"perf3","applicationId":"perf2","hostId":"default-host","domainId":"perf3.com","serviceId":"default-service","flowId":"default-smartflow","protocol":"HTTP/1.0","method":"GET","url":"/","clientIP":"52.26.91.26","clientPort":59413,"responseCode":200,"requestSize":77,"responseSize":27121,"compressed":false,"uncompressedResponseSize":0,"compressedResponseSize":0,"cached":true,"referrer":"","xForwardedFor":"","virtualHost":"perf3.com","modSecPhase1":null,"modSecSeverity1":null,"modSecMessage1":"","modSecPhase2":null,"modSecSeverity2":null,"modSecMessage2":"","modSecPhase3":null,"modSecSeverity3":null,"modSecMessage3":"","modSecPhase4":null,"modSecSeverity4":null,"modSecMessage4":"","totalWafEvents":0,"clientSRTT":14,"internal":false,"userAgent":"ApacheBench/2.3","bot":false,"sslHandshakeTime":0,"clientSSL":false,"appServerIP":"","appServerPort":0,"servedByAppServer":false,"appServerSSL":false,"ddosEventId":0,"requestStartTime":1,"requestEndTime":1,"appServerConnStartTime":0,"appServerConnEstablishedTime":0,"appServerRequestStartTime":0,"appServerRequestEndTime":0,"appServerResponseStartTime":0,"appServerResponseEndTime":0,"responseStartTime":1,"responseEndTime":1,"appServerSRTT":0,"appServerQueueSize":0,"clientCountryCode":"US","clientFamily":"AB (Apache Bench)","clientFamilyVersion":"2.3","clientType":"Other","clientOS":"unknown","clientDevice":"Other","responseTime":0,"denialReason":"","times":{"clientAccess":14,"clientRequestNetwork":0,"clientResponseNetwork":0,"totalClientNetwork":0,"requestWait":0,"requestProcessing":-1,"responseProcessing":1,"totalProcessing":0,"appServerAccess":0,"appServerExecution":0,"appServerRequestNetwork":0,"appServerResponseNetwork":0,"totalAppServerNetwork":0}}]}
org.elasticsearch.index.engine.CreateFailedEngineException: [perf3106556010][0] Create failed for [accesslog#AU7m12_rWLUv9z2-Ghr2]
at org.elasticsearch.index.engine.InternalEngine.create(InternalEngine.java:264)
at org.elasticsearch.index.shard.IndexShard.create(IndexShard.java:483)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:423)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:148)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase.performOnPrimary(TransportShardReplicationOperationAction.java:574)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1.doRun(TransportShardReplicationOperationAction.java:440)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:89)
at org.apache.lucene.util.fst.FST.<init>(FST.java:302)
at org.apache.lucene.util.fst.Builder.<init>(Builder.java:155)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:476)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:656)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:943)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finishTerm(BlockTreeTermsWriter.java:912)
at org.apache.lucene.index.FreqProxTermsWriterPerField.flush(FreqProxTermsWriterPerField.java:500)
at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:80)
at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:114)
at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:441)
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:521)
at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:364)
at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:409)
at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1353)
at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1311)
at org.elasticsearch.index.engine.InternalEngine.innerCreateNoLock(InternalEngine.java:343)
at org.elasticsearch.index.engine.InternalEngine.innerCreate(InternalEngine.java:287)
at org.elasticsearch.index.engine.InternalEngine.create(InternalEngine.java:259)
... 9 more
[2015-08-01 01:44:52,957][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.3m]
[2015-08-01 01:44:52,958][WARN ][index.engine ] [metrics-datastore-5-es-tune-2] [perf3106556010][0] failed engine [already closed by tragic event]
java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:89)
at org.apache.lucene.util.fst.FST.<init>(FST.java:302)
at org.apache.lucene.util.fst.Builder.<init>(Builder.java:155)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:476)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:656)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:943)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finishTerm(BlockTreeTermsWriter.java:912)
at org.apache.lucene.index.FreqProxTermsWriterPerField.flush(FreqProxTermsWriterPerField.java:500)
at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:80)
at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:114)
at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:441)
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:521)
at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:364)
at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:409)
at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1353)
at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1311)
at org.elasticsearch.index.engine.InternalEngine.innerCreateNoLock(InternalEngine.java:343)
at org.elasticsearch.index.engine.InternalEngine.innerCreate(InternalEngine.java:287)
at org.elasticsearch.index.engine.InternalEngine.create(InternalEngine.java:259)
at org.elasticsearch.index.shard.IndexShard.create(IndexShard.java:483)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:423)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:148)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase.performOnPrimary(TransportShardReplicationOperationAction.java:574)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1.doRun(TransportShardReplicationOperationAction.java:440)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
[2015-08-01 01:44:53,033][WARN ][transport.netty ] [metrics-datastore-5-es-tune-2] Message not fully read (request) for requestId [274406], action [indices:data/write/bulk[s][r]], readerIndex [265634] vs expected [268642]; resetting
[2015-08-01 01:44:53,047][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] [perf3106556010][0] failed to execute bulk item (index) index {[perf3106556010][accesslog][AU7m13vMWLUv9z2-Giif], source[{"version":1.0,"timestamp":1438391957455,"accountId":"perf3","applicationId":"perf2","hostId":"default-host","domainId":"perf3.com","serviceId":"default-service","flowId":"default-smartflow","protocol":"HTTP/1.0","method":"GET","url":"/","clientIP":"52.26.91.26","clientPort":56200,"responseCode":200,"requestSize":77,"responseSize":27121,"compressed":false,"uncompressedResponseSize":0,"compressedResponseSize":0,"cached":true,"referrer":"","xForwardedFor":"","virtualHost":"perf3.com","modSecPhase1":null,"modSecSeverity1":null,"modSecMessage1":"","modSecPhase2":null,"modSecSeverity2":null,"modSecMessage2":"","modSecPhase3":null,"modSecSeverity3":null,"modSecMessage3":"","modSecPhase4":null,"modSecSeverity4":null,"modSecMessage4":"","totalWafEvents":0,"clientSRTT":308,"internal":false,"userAgent":"ApacheBench/2.3","bot":false,"sslHandshakeTime":0,"clientSSL":false,"appServerIP":"","appServerPort":0,"servedByAppServer":false,"appServerSSL":false,"ddosEventId":0,"requestStartTime":1,"requestEndTime":1,"appServerConnStartTime":0,"appServerConnEstablishedTime":0,"appServerRequestStartTime":0,"appServerRequestEndTime":0,"appServerResponseStartTime":0,"appServerResponseEndTime":0,"responseStartTime":1,"responseEndTime":1,"appServerSRTT":0,"appServerQueueSize":0,"clientCountryCode":"US","clientFamily":"AB (Apache Bench)","clientFamilyVersion":"2.3","clientType":"Other","clientOS":"unknown","clientDevice":"Other","responseTime":0,"denialReason":"","times":{"clientAccess":308,"clientRequestNetwork":0,"clientResponseNetwork":0,"totalClientNetwork":0,"requestWait":0,"requestProcessing":-1,"responseProcessing":1,"totalProcessing":0,"appServerAccess":0,"appServerExecution":0,"appServerRequestNetwork":0,"appServerResponseNetwork":0,"totalAppServerNetwork":0}}]}
org.elasticsearch.index.engine.CreateFailedEngineException: [perf3106556010][0] Create failed for [accesslog#AU7m13vMWLUv9z2-Giif]
at org.elasticsearch.index.engine.InternalEngine.create(InternalEngine.java:264)
at org.elasticsearch.index.shard.IndexShard.create(IndexShard.java:483)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:423)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:148)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase.performOnPrimary(TransportShardReplicationOperationAction.java:574)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1.doRun(TransportShardReplicationOperationAction.java:440)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:700)
at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:714)
at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1349)
at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1311)
at org.elasticsearch.index.engine.InternalEngine.innerCreateNoLock(InternalEngine.java:343)
at org.elasticsearch.index.engine.InternalEngine.innerCreate(InternalEngine.java:287)
at org.elasticsearch.index.engine.InternalEngine.create(InternalEngine.java:259)
... 9 more
Caused by: java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:89)
at org.apache.lucene.util.fst.FST.<init>(FST.java:302)
at org.apache.lucene.util.fst.Builder.<init>(Builder.java:155)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:476)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:656)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:943)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finishTerm(BlockTreeTermsWriter.java:912)
at org.apache.lucene.index.FreqProxTermsWriterPerField.flush(FreqProxTermsWriterPerField.java:500)
at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:80)
at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:114)
at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:441)
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:521)
at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:364)
at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:409)
at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1353)
... 13 more
[2015-08-01 01:44:53,048][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.3m]
[2015-08-01 01:44:53,050][WARN ][index.shard ] [metrics-datastore-5-es-tune-2] [perf3106556010][0] Failed to perform scheduled engine refresh
org.elasticsearch.index.engine.RefreshFailedEngineException: [perf3106556010][0] Refresh failed
at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:575)
at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:595)
at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1186)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:89)
at org.apache.lucene.util.fst.FST.<init>(FST.java:302)
at org.apache.lucene.util.fst.Builder.<init>(Builder.java:155)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:476)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:656)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finish(BlockTreeTermsWriter.java:971)
at org.apache.lucene.index.FreqProxTermsWriterPerField.flush(FreqProxTermsWriterPerField.java:505)
at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:80)
at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:114)
at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:441)
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:521)
at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:632)
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:410)
at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:171)
at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:118)
at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:567)
... 5 more
[2015-08-01 01:44:53,033][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.3m]
[2015-08-01 01:44:53,021][INFO ][discovery.zen ] [metrics-datastore-5-es-tune-2] master_left [[metrics-datastore-1-es-tune-2][7PdAs7byRHONpe-0LfUcJg][ip-172-31-37-67.us-west-2.compute.internal][inet[/172.31.37.67:9300]]{data=false, max_local_storage_nodes=1, master=true}], reason [do not exists on master, act as master failure]
[2015-08-01 01:44:52,968][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.3m]
[2015-08-01 01:44:53,402][WARN ][transport.netty ] [metrics-datastore-5-es-tune-2] Message read past expected size (request) for requestId=[380181], action [indices:data/write/bulk[s]], readerIndex [585485] vs expected [176369]; resetting
[2015-08-01 01:44:53,411][WARN ][transport.netty ] [metrics-datastore-5-es-tune-2] Message read past expected size (request) for requestId=[442305], action [indices:data/write/bulk[s]], readerIndex [547554] vs expected [138349]; resetting
[2015-08-01 01:44:53,585][WARN ][transport.netty ] [metrics-datastore-5-es-tune-2] Message read past expected size (request) for requestId=[440983], action [indices:data/write/bulk[s]], readerIndex [610579] vs expected [180950]; resetting
[2015-08-01 01:44:53,701][WARN ][indices.cluster ] [metrics-datastore-5-es-tune-2] [[perf3106556010][0]] marking and sending shard failed due to [engine failure, reason [already closed by tragic event]]
java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.util.fst.BytesStore.writeByte(BytesStore.java:89)
at org.apache.lucene.util.fst.FST.<init>(FST.java:302)
at org.apache.lucene.util.fst.Builder.<init>(Builder.java:155)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:476)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:656)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:943)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finishTerm(BlockTreeTermsWriter.java:912)
at org.apache.lucene.index.FreqProxTermsWriterPerField.flush(FreqProxTermsWriterPerField.java:500)
at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:80)
at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:114)
at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:441)
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:521)
at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:364)
at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:409)
at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1353)
at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1311)
at org.elasticsearch.index.engine.InternalEngine.innerCreateNoLock(InternalEngine.java:343)
at org.elasticsearch.index.engine.InternalEngine.innerCreate(InternalEngine.java:287)
at org.elasticsearch.index.engine.InternalEngine.create(InternalEngine.java:259)
at org.elasticsearch.index.shard.IndexShard.create(IndexShard.java:483)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:423)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:148)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase.performOnPrimary(TransportShardReplicationOperationAction.java:574)
at org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1.doRun(TransportShardReplicationOperationAction.java:440)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:36)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
[2015-08-01 01:44:53,707][WARN ][indices.cluster ] [metrics-datastore-5-es-tune-2] [[perf1106556008][0]] marking and sending shard failed due to [engine failure, reason [lucene commit failed]]
java.lang.OutOfMemoryError: Java heap space
at java.util.HashMap.resize(HashMap.java:580)
at java.util.HashMap.addEntry(HashMap.java:879)
at java.util.HashMap.put(HashMap.java:505)
at java.util.HashSet.add(HashSet.java:217)
at java.util.AbstractCollection.addAll(AbstractCollection.java:342)
at org.apache.lucene.index.SegmentInfos.files(SegmentInfos.java:1056)
at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3007)
at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3134)
at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3101)
at org.elasticsearch.index.engine.InternalEngine.commitIndexWriter(InternalEngine.java:1293)
at org.elasticsearch.index.engine.InternalEngine.flush(InternalEngine.java:669)
at org.elasticsearch.index.engine.InternalEngine.flush(InternalEngine.java:631)
at org.elasticsearch.index.shard.IndexShard.flush(IndexShard.java:723)
at org.elasticsearch.indices.flush.SyncedFlushService.performPreSyncedFlush(SyncedFlushService.java:467)
at org.elasticsearch.indices.flush.SyncedFlushService.access$1100(SyncedFlushService.java:68)
at org.elasticsearch.indices.flush.SyncedFlushService$9.call(SyncedFlushService.java:435)
at org.elasticsearch.indices.flush.SyncedFlushService$9.call(SyncedFlushService.java:432)
at org.elasticsearch.indices.flush.SyncedFlushService$11.run(SyncedFlushService.java:821)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
[2015-08-01 01:44:53,708][WARN ][transport.netty ] [metrics-datastore-5-es-tune-2] Message read past expected size (request) for requestId=[380164], action [indices:data/write/bulk[s]], readerIndex [613111] vs expected [121751]; resetting
[2015-08-01 01:44:53,802][WARN ][cluster.service ] [metrics-datastore-5-es-tune-2] cluster state update task [zen-disco-receive(from master [[metrics-datastore-1-es-tune-2][7PdAs7byRHONpe-0LfUcJg][ip-172-31-37-67.us-west-2.compute.internal][inet[/172.31.37.67:9300]]{data=false, max_local_storage_nodes=1, master=true}])] took 22.1m above the warn threshold of 30s
[2015-08-01 01:44:53,803][WARN ][discovery.zen ] [metrics-datastore-5-es-tune-2] master left (reason = do not exists on master, act as master failure), current nodes: {[metrics-datastore-3-es-tune-2][ekRTpruPQQSx8Hp25CN5sA][ip-172-31-36-68.us-west-2.compute.internal][inet[/172.31.36.68:9300]]{max_local_storage_nodes=1, master=false},[Ereshkigal][7Lme8KGgQTCzUGgZuIlfMQ][ip-172-31-43-121.us-west-2.compute.internal][inet[/172.31.43.121:9300]]{client=true, data=false},[metrics-datastore-2-es-tune-2][8WWgb17ATA24ftDV34_IyQ][ip-172-31-43-228.us-west-2.compute.internal][inet[/172.31.43.228:9300]]{max_local_storage_nodes=1, master=false},[Shrunken Bones][2R1jD6I-QYGsXBWRI9PTCw][ip-172-31-34-71.us-west-2.compute.internal][inet[/172.31.34.71:9300]]{client=true, data=false},[metrics-datastore-5-es-tune-2][xrbldxSoRBu1doZDTG610A][ip-172-31-44-114.us-west-2.compute.internal][inet[ip-172-31-44-114.us-west-2.compute.internal/172.31.44.114:9300]]{max_local_storage_nodes=1, master=false},[metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false},[Blowhard][xhPTiIAqRWiwbestdP_zdA][ip-172-31-35-196.us-west-2.compute.internal][inet[/172.31.35.196:9300]]{client=true, data=false},[metrics-datastore-6-es-tune-2][DpDY2ciBTzeaERknhAIbSw][ip-172-31-45-34.us-west-2.compute.internal][inet[/172.31.45.34:9300]]{max_local_storage_nodes=1, master=false},}
[2015-08-01 01:44:53,803][INFO ][cluster.service ] [metrics-datastore-5-es-tune-2] removed {[metrics-datastore-1-es-tune-2][7PdAs7byRHONpe-0LfUcJg][ip-172-31-37-67.us-west-2.compute.internal][inet[/172.31.37.67:9300]]{data=false, max_local_storage_nodes=1, master=true},}, reason: zen-disco-master_failed ([metrics-datastore-1-es-tune-2][7PdAs7byRHONpe-0LfUcJg][ip-172-31-37-67.us-west-2.compute.internal][inet[/172.31.37.67:9300]]{data=false, max_local_storage_nodes=1, master=true})
[2015-08-01 01:44:53,847][WARN ][indices.cluster ] [metrics-datastore-5-es-tune-2] [[perf3106556010][0]] marking and sending shard failed due to [master [null] marked shard as started, but shard has not been created, mark shard as failed]
[2015-08-01 01:44:53,847][WARN ][cluster.action.shard ] [metrics-datastore-5-es-tune-2] can't send shard failed for [perf3106556010][0], node[xrbldxSoRBu1doZDTG610A], [P], s[STARTED], no master known.
[2015-08-01 01:44:53,847][WARN ][indices.cluster ] [metrics-datastore-5-es-tune-2] [[perf1106556008][0]] marking and sending shard failed due to [master [null] marked shard as started, but shard has not been created, mark shard as failed]
[2015-08-01 01:44:53,848][WARN ][cluster.action.shard ] [metrics-datastore-5-es-tune-2] can't send shard failed for [perf1106556008][0], node[xrbldxSoRBu1doZDTG610A], [P], s[STARTED], no master known.
[2015-08-01 01:44:53,941][WARN ][transport.netty ] [metrics-datastore-5-es-tune-2] Message read past expected size (request) for requestId=[380239], action [indices:data/write/bulk[s]], readerIndex [577379] vs expected [134765]; resetting
[2015-08-01 01:44:53,955][INFO ][discovery.zen ] [metrics-datastore-5-es-tune-2] master_left [null], reason [failed to perform initial connect [ can't connect to a null node]]
[2015-08-01 01:44:53,949][INFO ][discovery.zen ] [metrics-datastore-5-es-tune-2] master_left [null], reason [failed to perform initial connect [ can't connect to a null node]]
[2015-08-01 01:44:54,003][ERROR][discovery.zen ] [metrics-datastore-5-es-tune-2] unexpected failure during [zen-disco-master_failed (null)]
java.lang.NullPointerException
at org.elasticsearch.discovery.zen.ZenDiscovery$8.execute(ZenDiscovery.java:639)
at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:374)
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:188)
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:158)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
[2015-08-01 01:44:54,004][ERROR][discovery.zen ] [metrics-datastore-5-es-tune-2] unexpected failure during [zen-disco-master_failed (null)]
java.lang.NullPointerException
at org.elasticsearch.discovery.zen.ZenDiscovery$8.execute(ZenDiscovery.java:639)
at org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:374)
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:188)
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:158)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
[2015-08-01 01:44:55,145][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.4m]
[2015-08-01 01:45:09,349][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78497][885] duration [14.1s], collections [1]/[14.3s], total [14.1s]/[1.1h], memory [7.1gb]->[5.7gb]/[7.1gb], all_pools {[young] [3gb]->[2gb]/[3gb]}{[survivor] [382.6mb]->[0b]/[384mb]}{[old] [3.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 01:45:10,622][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.6m]
[2015-08-01 01:45:10,690][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] remote target didn't respond with 200 OK response code [503 Service Unavailable]. content: [:)
?error?ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master];]?status$??]
[2015-08-01 01:45:10,760][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.6m]
[2015-08-01 01:45:10,808][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.6m]
[2015-08-01 01:45:11,065][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.6m]
[2015-08-01 01:45:11,427][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.6m]
[2015-08-01 01:45:11,434][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.6m]
[2015-08-01 01:45:11,554][INFO ][cluster.service ] [metrics-datastore-5-es-tune-2] detected_master [metrics-datastore-1-es-tune-2][7PdAs7byRHONpe-0LfUcJg][ip-172-31-37-67.us-west-2.compute.internal][inet[/172.31.37.67:9300]]{data=false, max_local_storage_nodes=1, master=true}, added {[metrics-datastore-1-es-tune-2][7PdAs7byRHONpe-0LfUcJg][ip-172-31-37-67.us-west-2.compute.internal][inet[/172.31.37.67:9300]]{data=false, max_local_storage_nodes=1, master=true},}, reason: zen-disco-receive(from master [[metrics-datastore-1-es-tune-2][7PdAs7byRHONpe-0LfUcJg][ip-172-31-37-67.us-west-2.compute.internal][inet[/172.31.37.67:9300]]{data=false, max_local_storage_nodes=1, master=true}])
[2015-08-01 01:45:11,561][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.6m]
[2015-08-01 01:45:12,358][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,367][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,367][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,380][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,380][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,381][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,389][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,392][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,406][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,422][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,463][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,478][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,484][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,487][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,494][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,494][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,498][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,498][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,500][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,523][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,531][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,532][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] observer timed out. notifying listener. timeout setting [1m], time since start [25.7m]
[2015-08-01 01:45:12,570][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] ignoring failed replica [perf2106556009][2] because index was already removed.
[2015-08-01 01:45:12,591][DEBUG][action.bulk ] [metrics-datastore-5-es-tune-2] ignoring failed replica [perf2106556009][2] because index was already removed.
[2015-08-01 01:45:21,854][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][78503][886] duration [6.2s], collections [1]/[7.2s], total [6.2s]/[1.1h], memory [7gb]->[2.4gb]/[7.1gb], all_pools {[young] [3gb]->[31.9mb]/[3gb]}{[survivor] [285.6mb]->[0b]/[384mb]}{[old] [3.7gb]->[2.4gb]/[3.7gb]}
[2015-08-01 01:47:16,050][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] error sending data to [http://0.0.0.0:9200/.marvel-2015.08.01/_bulk]: SocketTimeoutException[Read timed out]
[2015-08-01 01:47:26,953][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-3-es-tune-2][inet[/172.31.36.68:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@6ffd4627];
[2015-08-01 01:49:19,129][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-3-es-tune-2][inet[/172.31.36.68:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@7d4c8bb3];
[2015-08-01 01:49:41,472][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-3-es-tune-2][inet[/172.31.36.68:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@2989da5b];
[2015-08-01 01:52:54,713][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@1379b659];
[2015-08-01 01:54:08,632][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@1a16faf2];
[2015-08-01 01:54:19,345][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@65ca7a24];
[2015-08-01 01:54:34,239][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@3c21f82];
[2015-08-01 01:54:49,900][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@5f1f1dde];
[2015-08-01 01:55:13,074][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79092][4894] duration [2.3s], collections [1]/[2.5s], total [2.3s]/[1h], memory [3.8gb]->[1.7gb]/[7.1gb], all_pools {[young] [2.9gb]->[4.4mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [567.2mb]->[1.3gb]/[3.7gb]}
[2015-08-01 01:55:20,586][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@241b972];
[2015-08-01 01:55:35,088][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@430faf28];
[2015-08-01 01:55:45,113][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@21d5ea98];
[2015-08-01 01:55:48,649][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79127][4895] duration [1.1s], collections [1]/[1.5s], total [1.1s]/[1h], memory [4.7gb]->[2gb]/[7.1gb], all_pools {[young] [2.9gb]->[755.2kb]/[3gb]}{[survivor] [384mb]->[333.7mb]/[384mb]}{[old] [1.3gb]->[1.7gb]/[3.7gb]}
[2015-08-01 01:56:16,512][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@6d3b0673];
[2015-08-01 01:56:36,571][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@6b860785];
[2015-08-01 01:56:51,903][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@36f2b0c];
[2015-08-01 01:57:19,157][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@6018fcf9];
[2015-08-01 01:57:27,074][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79222][4899] duration [3.1s], collections [1]/[4.3s], total [3.1s]/[1h], memory [3.8gb]->[3.2gb]/[7.1gb], all_pools {[young] [1.7gb]->[74.7mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.7gb]->[2.8gb]/[3.7gb]}
[2015-08-01 01:57:46,859][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79227][4900] duration [7.1s], collections [1]/[15.7s], total [7.1s]/[1h], memory [4.2gb]->[2.2gb]/[7.1gb], all_pools {[young] [1.5gb]->[48.2mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [2.3gb]->[2.1gb]/[3.7gb]}
[2015-08-01 01:58:07,034][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@38ff6e3d];
[2015-08-01 01:59:06,534][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79303][4904] duration [3.3s], collections [1]/[3.4s], total [3.3s]/[1h], memory [3.3gb]->[2.1gb]/[7.1gb], all_pools {[young] [2.9gb]->[10.3mb]/[3gb]}{[survivor] [229.7mb]->[384mb]/[384mb]}{[old] [169.2mb]->[1.7gb]/[3.7gb]}
[2015-08-01 01:59:40,088][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79333][4906] duration [1.7s], collections [1]/[2.6s], total [1.7s]/[1h], memory [3gb]->[1.8gb]/[7.1gb], all_pools {[young] [2gb]->[17.8mb]/[3gb]}{[survivor] [175mb]->[384mb]/[384mb]}{[old] [906mb]->[1.4gb]/[3.7gb]}
[2015-08-01 01:59:48,995][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79340][4907] duration [2.4s], collections [1]/[2.9s], total [2.4s]/[1h], memory [3.7gb]->[2.5gb]/[7.1gb], all_pools {[young] [1.8gb]->[9.8mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.4gb]->[2.1gb]/[3.7gb]}
[2015-08-01 02:00:14,001][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79365][4910] duration [785ms], collections [1]/[1s], total [785ms]/[1h], memory [4.1gb]->[2.1gb]/[7.1gb], all_pools {[young] [2.9gb]->[428.4mb]/[3gb]}{[survivor] [39.2mb]->[384mb]/[384mb]}{[old] [1.2gb]->[1.3gb]/[3.7gb]}
[2015-08-01 02:00:17,658][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79367][4911] duration [1.5s], collections [1]/[2.6s], total [1.5s]/[1h], memory [4gb]->[2.3gb]/[7.1gb], all_pools {[young] [2.3gb]->[24.3mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.3gb]->[1.9gb]/[3.7gb]}
[2015-08-01 02:00:29,582][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79376][4912] duration [1.4s], collections [1]/[2.2s], total [1.4s]/[1h], memory [4.3gb]->[1.8gb]/[7.1gb], all_pools {[young] [2.3gb]->[37.2mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.6gb]->[1.4gb]/[3.7gb]}
[2015-08-01 02:00:57,948][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@5ecc07af];
[2015-08-01 02:01:31,626][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79434][4917] duration [4.4s], collections [1]/[5s], total [4.4s]/[1h], memory [3.2gb]->[3.6gb]/[7.1gb], all_pools {[young] [1.8gb]->[38.5mb]/[3gb]}{[survivor] [88.5mb]->[384mb]/[384mb]}{[old] [1.3gb]->[3.2gb]/[3.7gb]}
[2015-08-01 02:01:41,681][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79437][896] duration [7.4s], collections [1]/[8s], total [7.4s]/[1.1h], memory [5.7gb]->[2.3gb]/[7.1gb], all_pools {[young] [2gb]->[563.6kb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [3.2gb]->[2.2gb]/[3.7gb]}
[2015-08-01 02:02:04,457][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79458][4919] duration [707ms], collections [1]/[1.3s], total [707ms]/[1h], memory [3.7gb]->[1.8gb]/[7.1gb], all_pools {[young] [2.4gb]->[7.8mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [1.3gb]->[1.4gb]/[3.7gb]}
[2015-08-01 02:03:32,000][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] error sending data to [http://0.0.0.0:9200/.marvel-2015.08.01/_bulk]: SocketTimeoutException[Read timed out]
[2015-08-01 02:03:46,038][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-2-es-tune-2][inet[/172.31.43.228:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@1cf1820f];
[2015-08-01 02:04:22,543][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79595][4924] duration [1.6s], collections [1]/[2s], total [1.6s]/[1h], memory [3.8gb]->[2.3gb]/[7.1gb], all_pools {[young] [2.2gb]->[10.6mb]/[3gb]}{[survivor] [135mb]->[384mb]/[384mb]}{[old] [1.4gb]->[2gb]/[3.7gb]}
[2015-08-01 02:04:33,578][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79605][4925] duration [1.9s], collections [1]/[2s], total [1.9s]/[1h], memory [5.2gb]->[2.8gb]/[7.1gb], all_pools {[young] [2.8gb]->[563.5kb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [2gb]->[2.4gb]/[3.7gb]}
[2015-08-01 02:04:49,739][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79619][4928] duration [2.1s], collections [1]/[2.5s], total [2.1s]/[1h], memory [3.1gb]->[2.2gb]/[7.1gb], all_pools {[young] [1.8gb]->[13.1mb]/[3gb]}{[survivor] [46.3mb]->[384mb]/[384mb]}{[old] [1.2gb]->[1.8gb]/[3.7gb]}
[2015-08-01 02:04:54,630][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79621][4929] duration [3.6s], collections [1]/[3.8s], total [3.6s]/[1h], memory [4.3gb]->[3.6gb]/[7.1gb], all_pools {[young] [2.1gb]->[54.8mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.8gb]->[3.1gb]/[3.7gb]}
[2015-08-01 02:05:54,221][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79677][4935] duration [733ms], collections [1]/[1.1s], total [733ms]/[1h], memory [4.1gb]->[1.9gb]/[7.1gb], all_pools {[young] [2.5gb]->[4.1kb]/[3gb]}{[survivor] [116.3mb]->[384mb]/[384mb]}{[old] [1.4gb]->[1.5gb]/[3.7gb]}
[2015-08-01 02:06:00,423][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79679][4936] duration [3.9s], collections [1]/[5.2s], total [3.9s]/[1h], memory [2.9gb]->[4gb]/[7.1gb], all_pools {[young] [1gb]->[222.2mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.5gb]->[3.4gb]/[3.7gb]}
[2015-08-01 02:06:35,028][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79704][4938] duration [4.3s], collections [1]/[4.5s], total [4.3s]/[1h], memory [4.4gb]->[3.5gb]/[7.1gb], all_pools {[young] [2.9gb]->[994.4kb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [1.4gb]->[3.1gb]/[3.7gb]}
[2015-08-01 02:06:38,885][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-3-es-tune-2][inet[/172.31.36.68:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@2e29c583];
[2015-08-01 02:08:03,166][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79787][4942] duration [5.1s], collections [1]/[6.1s], total [5.1s]/[1h], memory [1.6gb]->[3.3gb]/[7.1gb], all_pools {[young] [902.7mb]->[14.1mb]/[3gb]}{[survivor] [45.6mb]->[384mb]/[384mb]}{[old] [757mb]->[3gb]/[3.7gb]}
[2015-08-01 02:08:12,384][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79791][903] duration [5.3s], collections [1]/[6.2s], total [5.3s]/[1.1h], memory [5.2gb]->[1.4gb]/[7.1gb], all_pools {[young] [1.8gb]->[65mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [3gb]->[1.3gb]/[3.7gb]}
[2015-08-01 02:08:44,256][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79814][905] duration [6.4s], collections [1]/[7.1s], total [6.4s]/[1.1h], memory [1.6gb]->[3gb]/[7.1gb], all_pools {[young] [5.3kb]->[12.8mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [1.6gb]->[3gb]/[3.7gb]}
[2015-08-01 02:08:56,003][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79816][906] duration [10.1s], collections [1]/[10.7s], total [10.1s]/[1.1h], memory [4.7gb]->[3gb]/[7.1gb], all_pools {[young] [1.6gb]->[82.9mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [3gb]->[2.9gb]/[3.7gb]}
[2015-08-01 02:09:07,289][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79820][907] duration [7.7s], collections [1]/[8.2s], total [7.7s]/[1.1h], memory [5.8gb]->[1.9gb]/[7.1gb], all_pools {[young] [2.8gb]->[1.8mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2.9gb]->[1.9gb]/[3.7gb]}
[2015-08-01 02:09:17,066][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79823][908] duration [6.1s], collections [1]/[6.9s], total [6.1s]/[1.1h], memory [3.7gb]->[2.8gb]/[7.1gb], all_pools {[young] [1.7gb]->[43.8mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [1.9gb]->[2.7gb]/[3.7gb]}
[2015-08-01 02:09:31,633][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79828][910] duration [6.2s], collections [1]/[7s], total [6.2s]/[1.1h], memory [1.7gb]->[2.9gb]/[7.1gb], all_pools {[young] [217.9mb]->[11mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [1.5gb]->[2.8gb]/[3.7gb]}
[2015-08-01 02:10:34,024][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79880][4954] duration [988ms], collections [1]/[1.7s], total [988ms]/[1h], memory [3.2gb]->[1.1gb]/[7.1gb], all_pools {[young] [2.3gb]->[2.6mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [484.3mb]->[752.2mb]/[3.7gb]}
[2015-08-01 02:11:20,387][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79882][4955] duration [38.1s], collections [1]/[45.3s], total [38.1s]/[1h], memory [2gb]->[2.4gb]/[7.1gb], all_pools {[young] [924.7mb]->[83.6mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [752.2mb]->[2.4gb]/[3.7gb]}
[2015-08-01 02:11:20,387][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79882][915] duration [6.3s], collections [1]/[45.3s], total [6.3s]/[1.1h], memory [2gb]->[2.4gb]/[7.1gb], all_pools {[young] [924.7mb]->[83.6mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [752.2mb]->[2.4gb]/[3.7gb]}
[2015-08-01 02:11:31,034][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79885][916] duration [8.5s], collections [1]/[8.6s], total [8.5s]/[1.1h], memory [5.4gb]->[2.9gb]/[7.1gb], all_pools {[young] [2.9gb]->[2.5mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2.4gb]->[2.9gb]/[3.7gb]}
[2015-08-01 02:11:53,907][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79893][919] duration [7s], collections [1]/[7.4s], total [7s]/[1.1h], memory [4gb]->[3.4gb]/[7.1gb], all_pools {[young] [2.4gb]->[1.5mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [1.6gb]->[3.4gb]/[3.7gb]}
[2015-08-01 02:12:03,852][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79898][920] duration [5.7s], collections [1]/[5.9s], total [5.7s]/[1.1h], memory [5.8gb]->[2.3gb]/[7.1gb], all_pools {[young] [2.4gb]->[11.4mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [3.4gb]->[2.3gb]/[3.7gb]}
[2015-08-01 02:12:25,110][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79906][923] duration [6.1s], collections [1]/[7s], total [6.1s]/[1.1h], memory [4.8gb]->[2.6gb]/[7.1gb], all_pools {[young] [2.8gb]->[20.6mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2gb]->[2.6gb]/[3.7gb]}
[2015-08-01 02:13:13,220][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79912][4964] duration [29.2s], collections [1]/[36.8s], total [29.2s]/[1h], memory [1gb]->[2.9gb]/[7.1gb], all_pools {[young] [6.9mb]->[261.1mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [1019.4mb]->[2.6gb]/[3.7gb]}
[2015-08-01 02:13:13,220][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79912][926] duration [6.7s], collections [1]/[36.8s], total [6.7s]/[1.1h], memory [1gb]->[2.9gb]/[7.1gb], all_pools {[young] [6.9mb]->[261.1mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [1019.4mb]->[2.6gb]/[3.7gb]}
[2015-08-01 02:13:43,486][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79937][4966] duration [2.9s], collections [1]/[3.5s], total [2.9s]/[1h], memory [2gb]->[1.9gb]/[7.1gb], all_pools {[young] [1.7gb]->[13.1mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [389.6mb]->[1.5gb]/[3.7gb]}
[2015-08-01 02:14:20,674][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79952][4968] duration [11.4s], collections [1]/[19.4s], total [11.4s]/[1h], memory [3.5gb]->[3.2gb]/[7.1gb], all_pools {[young] [2.7gb]->[295.7kb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [808.7mb]->[3.2gb]/[3.7gb]}
[2015-08-01 02:14:20,674][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][79952][929] duration [7.9s], collections [1]/[19.4s], total [7.9s]/[1.1h], memory [3.5gb]->[3.2gb]/[7.1gb], all_pools {[young] [2.7gb]->[295.7kb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [808.7mb]->[3.2gb]/[3.7gb]}
[2015-08-01 02:14:58,338][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79985][4970] duration [1.9s], collections [1]/[2.3s], total [1.9s]/[1h], memory [2.4gb]->[1.4gb]/[7.1gb], all_pools {[young] [2gb]->[8.9mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [379.6mb]->[1gb]/[3.7gb]}
[2015-08-01 02:15:13,825][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][79999][4971] duration [2.3s], collections [1]/[2.4s], total [2.3s]/[1h], memory [4.1gb]->[2.3gb]/[7.1gb], all_pools {[young] [2.6gb]->[47.1kb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1gb]->[1.9gb]/[3.7gb]}
[2015-08-01 02:16:27,827][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] error sending data to [http://0.0.0.0:9200/.marvel-2015.08.01/_bulk]: SocketTimeoutException[Read timed out]
[2015-08-01 02:16:27,857][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80000][4972] duration [1m], collections [1]/[1.2m], total [1m]/[1.1h], memory [2.3gb]->[2.7gb]/[7.1gb], all_pools {[young] [47.1kb]->[325.3mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [1.9gb]->[2.4gb]/[3.7gb]}
[2015-08-01 02:16:27,858][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80000][931] duration [8.4s], collections [1]/[1.2m], total [8.4s]/[1.1h], memory [2.3gb]->[2.7gb]/[7.1gb], all_pools {[young] [47.1kb]->[325.3mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [1.9gb]->[2.4gb]/[3.7gb]}
[2015-08-01 02:16:40,753][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80002][932] duration [11.4s], collections [1]/[11.8s], total [11.4s]/[1.1h], memory [4.6gb]->[4.7gb]/[7.1gb], all_pools {[young] [2.1gb]->[1009.5mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2.4gb]->[3.7gb]/[3.7gb]}
[2015-08-01 02:16:41,559][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@7a7bc010]
[2015-08-01 02:17:17,304][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80032][4976] duration [4.9s], collections [1]/[5.6s], total [4.9s]/[1.1h], memory [2gb]->[3.3gb]/[7.1gb], all_pools {[young] [1.5gb]->[101.5mb]/[3gb]}{[survivor] [70.5mb]->[384mb]/[384mb]}{[old] [453.9mb]->[2.8gb]/[3.7gb]}
[2015-08-01 02:17:59,210][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80059][4979] duration [5.3s], collections [1]/[6.3s], total [5.3s]/[1.1h], memory [2.3gb]->[3.3gb]/[7.1gb], all_pools {[young] [1.9gb]->[45.1mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [344.2mb]->[2.9gb]/[3.7gb]}
[2015-08-01 02:18:14,539][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80066][940] duration [5.4s], collections [1]/[6s], total [5.4s]/[1.1h], memory [1019.4mb]->[2.7gb]/[7.1gb], all_pools {[young] [20.6kb]->[21.9mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [1019.3mb]->[2.7gb]/[3.7gb]}
[2015-08-01 02:19:23,247][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80130][4985] duration [1.6s], collections [1]/[2.5s], total [1.6s]/[1.1h], memory [2.7gb]->[1.3gb]/[7.1gb], all_pools {[young] [2.2gb]->[70.8mb]/[3gb]}{[survivor] [242.7mb]->[384mb]/[384mb]}{[old] [355.1mb]->[907.3mb]/[3.7gb]}
[2015-08-01 02:19:29,920][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80132][4986] duration [4.9s], collections [1]/[5.6s], total [4.9s]/[1.1h], memory [1.8gb]->[3.5gb]/[7.1gb], all_pools {[young] [611.3mb]->[9mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [907.3mb]->[3.1gb]/[3.7gb]}
[2015-08-01 02:20:58,920][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80208][4989] duration [3.2s], collections [1]/[3.5s], total [3.2s]/[1.1h], memory [3.6gb]->[2.3gb]/[7.1gb], all_pools {[young] [2.9gb]->[197.5mb]/[3gb]}{[survivor] [205.6mb]->[384mb]/[384mb]}{[old] [569.1mb]->[1.8gb]/[3.7gb]}
[2015-08-01 02:21:13,034][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80212][946] duration [5.9s], collections [1]/[6.6s], total [5.9s]/[1.1h], memory [4.3gb]->[1.7gb]/[7.1gb], all_pools {[young] [1.9gb]->[3.1mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2.3gb]->[1.7gb]/[3.7gb]}
[2015-08-01 02:21:19,549][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80216][4992] duration [2.3s], collections [1]/[3.5s], total [2.3s]/[1.1h], memory [4.6gb]->[2.9gb]/[7.1gb], all_pools {[young] [2.8gb]->[10.5mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [1.7gb]->[2.5gb]/[3.7gb]}
[2015-08-01 02:21:51,316][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80227][4993] duration [13.9s], collections [1]/[21.7s], total [13.9s]/[1.1h], memory [4.8gb]->[2.3gb]/[7.1gb], all_pools {[young] [2.5gb]->[293.5mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [1.9gb]->[2gb]/[3.7gb]}
[2015-08-01 02:22:00,341][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80229][949] duration [7s], collections [1]/[8s], total [7s]/[1.1h], memory [3.4gb]->[2.6gb]/[7.1gb], all_pools {[young] [1.3gb]->[65.3mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2gb]->[2.5gb]/[3.7gb]}
[2015-08-01 02:22:00,897][ERROR][indices.ttl ] [metrics-datastore-5-es-tune-2] bulk deletion failures for [10000]/[10000] items
[2015-08-01 02:22:01,030][ERROR][indices.ttl ] [metrics-datastore-5-es-tune-2] bulk deletion failures for [10000]/[10000] items
[2015-08-01 02:22:01,115][ERROR][indices.ttl ] [metrics-datastore-5-es-tune-2] bulk deletion failures for [2611]/[2611] items
[2015-08-01 02:22:10,032][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80232][950] duration [6.1s], collections [1]/[6.7s], total [6.1s]/[1.1h], memory [4.9gb]->[2.5gb]/[7.1gb], all_pools {[young] [2.3gb]->[11.5mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2.5gb]->[2.5gb]/[3.7gb]}
[2015-08-01 02:22:31,481][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80243][4997] duration [4.3s], collections [1]/[4.9s], total [4.3s]/[1.1h], memory [2.9gb]->[3.9gb]/[7.1gb], all_pools {[young] [1.5gb]->[33.1mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [1.4gb]->[3.5gb]/[3.7gb]}
[2015-08-01 02:22:56,858][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80254][956] duration [6.3s], collections [1]/[7s], total [6.3s]/[1.2h], memory [1.4gb]->[3.2gb]/[7.1gb], all_pools {[young] [1.5mb]->[32.1mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [1.4gb]->[3.1gb]/[3.7gb]}
[2015-08-01 02:23:30,234][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80281][5003] duration [4.6s], collections [1]/[4.8s], total [4.6s]/[1.1h], memory [3.1gb]->[2.8gb]/[7.1gb], all_pools {[young] [2.2gb]->[17.3mb]/[3gb]}{[survivor] [382.7mb]->[384mb]/[384mb]}{[old] [560.2mb]->[2.4gb]/[3.7gb]}
[2015-08-01 02:23:38,604][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80284][958] duration [5.6s], collections [1]/[6.3s], total [5.6s]/[1.2h], memory [5.1gb]->[2.6gb]/[7.1gb], all_pools {[young] [2.3gb]->[104.1mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [2.4gb]->[2.5gb]/[3.7gb]}
[2015-08-01 02:24:40,481][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80341][5008] duration [1.2s], collections [1]/[1.8s], total [1.2s]/[1.1h], memory [2.8gb]->[1gb]/[7.1gb], all_pools {[young] [2.3gb]->[4.8mb]/[3gb]}{[survivor] [65.5mb]->[384mb]/[384mb]}{[old] [396.9mb]->[729.2mb]/[3.7gb]}
[2015-08-01 02:24:48,594][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80346][5009] duration [3.5s], collections [1]/[4.1s], total [3.5s]/[1.1h], memory [3.6gb]->[2.4gb]/[7.1gb], all_pools {[young] [2.5gb]->[33.2mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [729.2mb]->[2gb]/[3.7gb]}
[2015-08-01 02:25:36,387][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-3-es-tune-2][inet[/172.31.36.68:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@1454b67e];
[2015-08-01 02:25:42,240][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80390][5011] duration [897ms], collections [1]/[1.1s], total [897ms]/[1.1h], memory [3.7gb]->[1.4gb]/[7.1gb], all_pools {[young] [2.8gb]->[505.7kb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [955.8mb]->[1gb]/[3.7gb]}
[2015-08-01 02:26:18,580][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80425][5013] duration [1.9s], collections [1]/[2.2s], total [1.9s]/[1.1h], memory [3.7gb]->[2.2gb]/[7.1gb], all_pools {[young] [2.2gb]->[219.3kb]/[3gb]}{[survivor] [327.9mb]->[384mb]/[384mb]}{[old] [1.2gb]->[1.8gb]/[3.7gb]}
[2015-08-01 02:27:14,120][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80477][5015] duration [3.8s], collections [1]/[4.2s], total [3.8s]/[1.1h], memory [3.4gb]->[3gb]/[7.1gb], all_pools {[young] [2.2gb]->[10.4mb]/[3gb]}{[survivor] [165.4mb]->[384mb]/[384mb]}{[old] [1gb]->[2.6gb]/[3.7gb]}
[2015-08-01 02:27:37,009][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] error sending data to [http://0.0.0.0:9200/.marvel-2015.08.01/_bulk]: SocketTimeoutException[Read timed out]
[2015-08-01 02:27:59,536][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80510][5018] duration [2.4s], collections [1]/[3.2s], total [2.4s]/[1.1h], memory [3gb]->[1.9gb]/[7.1gb], all_pools {[young] [2.1gb]->[15mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [469.8mb]->[1.5gb]/[3.7gb]}
[2015-08-01 02:28:31,862][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-3-es-tune-2][inet[/172.31.36.68:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@7fea2562];
[2015-08-01 02:29:41,938][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] error sending data to [http://0.0.0.0:9200/.marvel-2015.08.01/_bulk]: SocketTimeoutException[Read timed out]
[2015-08-01 02:31:18,322][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80695][5022] duration [7.2s], collections [1]/[14.6s], total [7.2s]/[1.1h], memory [2.6gb]->[2.8gb]/[7.1gb], all_pools {[young] [851.9mb]->[278.1mb]/[3gb]}{[survivor] [263.3mb]->[0b]/[384mb]}{[old] [1.5gb]->[2.5gb]/[3.7gb]}
[2015-08-01 02:31:18,322][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80695][969] duration [6.4s], collections [1]/[14.6s], total [6.4s]/[1.2h], memory [2.6gb]->[2.8gb]/[7.1gb], all_pools {[young] [851.9mb]->[278.1mb]/[3gb]}{[survivor] [263.3mb]->[0b]/[384mb]}{[old] [1.5gb]->[2.5gb]/[3.7gb]}
[2015-08-01 02:31:28,486][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80700][970] duration [6.1s], collections [1]/[6.1s], total [6.1s]/[1.2h], memory [5.5gb]->[2.2gb]/[7.1gb], all_pools {[young] [2.9gb]->[37.3mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2.5gb]->[2.1gb]/[3.7gb]}
[2015-08-01 02:32:12,191][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80738][5025] duration [1.7s], collections [1]/[1.8s], total [1.7s]/[1.1h], memory [2.9gb]->[1.3gb]/[7.1gb], all_pools {[young] [2.4gb]->[13.5mb]/[3gb]}{[survivor] [168.6mb]->[384mb]/[384mb]}{[old] [365.1mb]->[934.7mb]/[3.7gb]}
[2015-08-01 02:32:33,458][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80756][5027] duration [3.3s], collections [1]/[4.2s], total [3.3s]/[1.1h], memory [3gb]->[2.5gb]/[7.1gb], all_pools {[young] [1.6gb]->[59.5mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1009mb]->[2.1gb]/[3.7gb]}
[2015-08-01 02:33:08,899][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80789][5028] duration [3s], collections [1]/[3.4s], total [3s]/[1.1h], memory [3.7gb]->[2.9gb]/[7.1gb], all_pools {[young] [1.7gb]->[167.7kb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.6gb]->[2.5gb]/[3.7gb]}
[2015-08-01 02:33:16,914][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][80792][974] duration [5.1s], collections [1]/[6s], total [5.1s]/[1.2h], memory [4.8gb]->[2.1gb]/[7.1gb], all_pools {[young] [1.8gb]->[43.2mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [2.5gb]->[2gb]/[3.7gb]}
[2015-08-01 02:34:43,826][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80874][5033] duration [3s], collections [1]/[3s], total [3s]/[1.1h], memory [4.9gb]->[3.5gb]/[7.1gb], all_pools {[young] [2.9gb]->[248.4mb]/[3gb]}{[survivor] [304mb]->[384mb]/[384mb]}{[old] [1.7gb]->[2.9gb]/[3.7gb]}
[2015-08-01 02:34:59,236][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80887][5034] duration [1.6s], collections [1]/[1.8s], total [1.6s]/[1.1h], memory [4.1gb]->[2.2gb]/[7.1gb], all_pools {[young] [2.2gb]->[5.1mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.5gb]->[1.9gb]/[3.7gb]}
[2015-08-01 02:35:57,976][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80941][5038] duration [4.6s], collections [1]/[5.6s], total [4.6s]/[1.1h], memory [2.2gb]->[3.7gb]/[7.1gb], all_pools {[young] [946.6mb]->[21.6mb]/[3gb]}{[survivor] [103.4mb]->[384mb]/[384mb]}{[old] [1.2gb]->[3.3gb]/[3.7gb]}
[2015-08-01 02:36:46,735][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][80982][5041] duration [949ms], collections [1]/[1.2s], total [949ms]/[1.1h], memory [4.4gb]->[2.1gb]/[7.1gb], all_pools {[young] [2.8gb]->[178.3mb]/[3gb]}{[survivor] [384mb]->[301.4mb]/[384mb]}{[old] [1.2gb]->[1.6gb]/[3.7gb]}
[2015-08-01 02:37:41,034][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81034][5047] duration [2.1s], collections [1]/[3s], total [2.1s]/[1.1h], memory [3.9gb]->[2.7gb]/[7.1gb], all_pools {[young] [2gb]->[43.3mb]/[3gb]}{[survivor] [68.9mb]->[384mb]/[384mb]}{[old] [1.7gb]->[2.3gb]/[3.7gb]}
[2015-08-01 02:38:03,157][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81053][5049] duration [3.3s], collections [1]/[3.4s], total [3.3s]/[1.1h], memory [3.9gb]->[2.6gb]/[7.1gb], all_pools {[young] [2.6gb]->[72.4kb]/[3gb]}{[survivor] [144.6mb]->[384mb]/[384mb]}{[old] [1.1gb]->[2.2gb]/[3.7gb]}
[2015-08-01 02:38:12,888][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81055][981] duration [8.1s], collections [1]/[8.7s], total [8.1s]/[1.2h], memory [3.5gb]->[3gb]/[7.1gb], all_pools {[young] [916.8mb]->[61.1kb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [2.2gb]->[3gb]/[3.7gb]}
[2015-08-01 02:38:48,326][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81087][5052] duration [2s], collections [1]/[2.2s], total [2s]/[1.1h], memory [2.8gb]->[1.5gb]/[7.1gb], all_pools {[young] [2.3gb]->[47.6mb]/[3gb]}{[survivor] [150.5mb]->[384mb]/[384mb]}{[old] [389.3mb]->[1gb]/[3.7gb]}
[2015-08-01 02:39:32,811][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81130][5057] duration [1.6s], collections [1]/[2.3s], total [1.6s]/[1.1h], memory [3.6gb]->[1.9gb]/[7.1gb], all_pools {[young] [2.2gb]->[32mb]/[3gb]}{[survivor] [248.2mb]->[384mb]/[384mb]}{[old] [1.1gb]->[1.5gb]/[3.7gb]}
[2015-08-01 02:39:45,754][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81142][5058] duration [963ms], collections [1]/[1.9s], total [963ms]/[1.1h], memory [4.3gb]->[2.2gb]/[7.1gb], all_pools {[young] [2.3gb]->[41.5mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.5gb]->[1.8gb]/[3.7gb]}
[2015-08-01 02:40:18,410][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81174][5060] duration [1.3s], collections [1]/[1.6s], total [1.3s]/[1.1h], memory [4.4gb]->[2.4gb]/[7.1gb], all_pools {[young] [2.3gb]->[6.3mb]/[3gb]}{[survivor] [327.3mb]->[384mb]/[384mb]}{[old] [1.8gb]->[2.1gb]/[3.7gb]}
[2015-08-01 02:41:11,588][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81223][5064] duration [2s], collections [1]/[2.5s], total [2s]/[1.1h], memory [2.9gb]->[1.9gb]/[7.1gb], all_pools {[young] [2gb]->[194.3mb]/[3gb]}{[survivor] [65mb]->[384mb]/[384mb]}{[old] [859.1mb]->[1.4gb]/[3.7gb]}
[2015-08-01 02:41:16,038][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81225][5065] duration [2.5s], collections [1]/[3.4s], total [2.5s]/[1.1h], memory [3.3gb]->[2.8gb]/[7.1gb], all_pools {[young] [1.5gb]->[25.5mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.4gb]->[2.4gb]/[3.7gb]}
[2015-08-01 02:41:42,599][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81248][5066] duration [2.2s], collections [1]/[3.5s], total [2.2s]/[1.1h], memory [4.8gb]->[2.7gb]/[7.1gb], all_pools {[young] [2.9gb]->[190.2mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.5gb]->[2.2gb]/[3.7gb]}
[2015-08-01 02:41:47,008][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81251][5067] duration [1.5s], collections [1]/[2.4s], total [1.5s]/[1.1h], memory [5.2gb]->[3.3gb]/[7.1gb], all_pools {[young] [2.6gb]->[99.2mb]/[3gb]}{[survivor] [384mb]->[339.7mb]/[384mb]}{[old] [2.2gb]->[2.9gb]/[3.7gb]}
[2015-08-01 02:42:05,738][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81265][5068] duration [5.3s], collections [1]/[5.7s], total [5.3s]/[1.1h], memory [3.5gb]->[3.7gb]/[7.1gb], all_pools {[young] [1.8gb]->[254.4kb]/[3gb]}{[survivor] [339.7mb]->[384mb]/[384mb]}{[old] [1.3gb]->[3.3gb]/[3.7gb]}
[2015-08-01 02:42:21,847][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81273][5070] duration [4.4s], collections [1]/[5s], total [4.4s]/[1.1h], memory [1.2gb]->[3.4gb]/[7.1gb], all_pools {[young] [22.7kb]->[786.6kb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [1.2gb]->[3gb]/[3.7gb]}
[2015-08-01 02:43:00,874][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81287][5072] duration [11.8s], collections [1]/[21.1s], total [11.8s]/[1.1h], memory [1.4gb]->[3.9gb]/[7.1gb], all_pools {[young] [392.8mb]->[319.1mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [1gb]->[3.6gb]/[3.7gb]}
[2015-08-01 02:43:00,874][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81287][991] duration [8.7s], collections [1]/[21.1s], total [8.7s]/[1.2h], memory [1.4gb]->[3.9gb]/[7.1gb], all_pools {[young] [392.8mb]->[319.1mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [1gb]->[3.6gb]/[3.7gb]}
[2015-08-01 02:43:56,145][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] create failure (index:[.marvel-2015.08.01] type: [node_stats]): RemoteTransportException[[metrics-datastore-3-es-tune-2][inet[/172.31.36.68:9300]][indices:data/write/bulk[s]]]; nested: EsRejectedExecutionException[rejected execution (queue capacity 100) on org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$PrimaryPhase$1@2ad7e54a];
[2015-08-01 02:44:15,028][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81355][5076] duration [3.2s], collections [1]/[3.6s], total [3.2s]/[1.1h], memory [2.4gb]->[2.2gb]/[7.1gb], all_pools {[young] [1.7gb]->[2.5mb]/[3gb]}{[survivor] [49mb]->[384mb]/[384mb]}{[old] [653.7mb]->[1.8gb]/[3.7gb]}
[2015-08-01 02:44:47,740][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81379][5078] duration [4s], collections [1]/[4.2s], total [4s]/[1.1h], memory [4.1gb]->[3.1gb]/[7.1gb], all_pools {[young] [2.9gb]->[955.3kb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [1.1gb]->[2.7gb]/[3.7gb]}
[2015-08-01 02:45:26,023][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81397][5081] duration [2.2s], collections [1]/[2.5s], total [2.2s]/[1.1h], memory [4.4gb]->[2.7gb]/[7.1gb], all_pools {[young] [2.9gb]->[46.8kb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [1.4gb]->[2.3gb]/[3.7gb]}
[2015-08-01 02:45:35,655][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81401][1000] duration [6.2s], collections [1]/[6.6s], total [6.2s]/[1.2h], memory [4.6gb]->[2.6gb]/[7.1gb], all_pools {[young] [1.9gb]->[1.5mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [2.3gb]->[2.6gb]/[3.7gb]}
[2015-08-01 02:45:50,825][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81408][5084] duration [2s], collections [1]/[5.6s], total [2s]/[1.1h], memory [3.5gb]->[2.8gb]/[7.1gb], all_pools {[young] [2.2gb]->[201mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [1.3gb]->[2.2gb]/[3.7gb]}
[2015-08-01 02:45:56,591][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81410][5085] duration [2.5s], collections [1]/[3s], total [2.5s]/[1.1h], memory [4.2gb]->[2.8gb]/[7.1gb], all_pools {[young] [1.7gb]->[42.5mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [2gb]->[2.4gb]/[3.7gb]}
[2015-08-01 02:46:49,717][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] error sending data to [http://0.0.0.0:9200/.marvel-2015.08.01/_bulk]: SocketTimeoutException[Read timed out]
[2015-08-01 02:46:59,757][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81471][5087] duration [2.5s], collections [1]/[3s], total [2.5s]/[1.1h], memory [4.7gb]->[2.9gb]/[7.1gb], all_pools {[young] [2.5gb]->[19.6mb]/[3gb]}{[survivor] [383mb]->[384mb]/[384mb]}{[old] [1.7gb]->[2.5gb]/[3.7gb]}
[2015-08-01 02:47:14,667][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81481][5088] duration [4.8s], collections [1]/[5.4s], total [4.8s]/[1.1h], memory [2.8gb]->[4.2gb]/[7.1gb], all_pools {[young] [933.1mb]->[384.8mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.5gb]->[3.5gb]/[3.7gb]}
[2015-08-01 02:47:49,289][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81507][5091] duration [2.4s], collections [1]/[3.4s], total [2.4s]/[1.1h], memory [3.6gb]->[2.9gb]/[7.1gb], all_pools {[young] [2gb]->[10.8mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [1.5gb]->[2.5gb]/[3.7gb]}
[2015-08-01 02:48:23,687][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81540][5092] duration [1.7s], collections [1]/[2.1s], total [1.7s]/[1.1h], memory [4.6gb]->[2gb]/[7.1gb], all_pools {[young] [2.9gb]->[5.9kb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.3gb]->[1.7gb]/[3.7gb]}
[2015-08-01 02:48:50,423][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81549][5093] duration [12.3s], collections [1]/[18.7s], total [12.3s]/[1.1h], memory [3.9gb]->[2.2gb]/[7.1gb], all_pools {[young] [1.8gb]->[24.2mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [1.7gb]->[2.2gb]/[3.7gb]}
[2015-08-01 02:48:50,424][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81549][1007] duration [5.9s], collections [1]/[18.7s], total [5.9s]/[1.2h], memory [3.9gb]->[2.2gb]/[7.1gb], all_pools {[young] [1.8gb]->[24.2mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [1.7gb]->[2.2gb]/[3.7gb]}
[2015-08-01 02:49:02,498][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81551][1008] duration [10.3s], collections [1]/[11s], total [10.3s]/[1.2h], memory [4.8gb]->[5.5gb]/[7.1gb], all_pools {[young] [2.6gb]->[1.7gb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2.2gb]->[3.7gb]/[3.7gb]}
[2015-08-01 02:49:17,723][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81560][5095] duration [2.1s], collections [1]/[2.7s], total [2.1s]/[1.1h], memory [3.6gb]->[1.9gb]/[7.1gb], all_pools {[young] [2.8gb]->[3.3mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [844.4mb]->[1.5gb]/[3.7gb]}
[2015-08-01 02:49:27,483][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81569][5096] duration [972ms], collections [1]/[1.7s], total [972ms]/[1.1h], memory [4.8gb]->[2.3gb]/[7.1gb], all_pools {[young] [2.9gb]->[6.2kb]/[3gb]}{[survivor] [384mb]->[383.3mb]/[384mb]}{[old] [1.5gb]->[1.9gb]/[3.7gb]}
[2015-08-01 02:49:42,497][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81579][5097] duration [4.6s], collections [1]/[6s], total [4.6s]/[1.1h], memory [1.7gb]->[3gb]/[7.1gb], all_pools {[young] [866.4mb]->[49.1mb]/[3gb]}{[survivor] [383.3mb]->[384mb]/[384mb]}{[old] [533.1mb]->[2.6gb]/[3.7gb]}
[2015-08-01 02:50:13,113][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81601][5100] duration [2.6s], collections [1]/[3.6s], total [2.6s]/[1.1h], memory [2.4gb]->[1.9gb]/[7.1gb], all_pools {[young] [1.8gb]->[240.8mb]/[3gb]}{[survivor] [150.3mb]->[384mb]/[384mb]}{[old] [381.1mb]->[1.3gb]/[3.7gb]}
[2015-08-01 02:50:30,883][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81602][5101] duration [9.5s], collections [1]/[17.7s], total [9.5s]/[1.1h], memory [1.9gb]->[3gb]/[7.1gb], all_pools {[young] [240.8mb]->[72.9mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [1.3gb]->[2.9gb]/[3.7gb]}
[2015-08-01 02:50:30,884][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81602][1013] duration [7.6s], collections [1]/[17.7s], total [7.6s]/[1.2h], memory [1.9gb]->[3gb]/[7.1gb], all_pools {[young] [240.8mb]->[72.9mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [1.3gb]->[2.9gb]/[3.7gb]}
[2015-08-01 02:51:03,337][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81625][5103] duration [4.7s], collections [1]/[4.8s], total [4.7s]/[1.1h], memory [3.4gb]->[3.3gb]/[7.1gb], all_pools {[young] [2.8gb]->[234.3mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [643.4mb]->[2.7gb]/[3.7gb]}
[2015-08-01 02:51:11,630][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81627][1017] duration [7s], collections [1]/[7.2s], total [7s]/[1.2h], memory [5.5gb]->[2.8gb]/[7.1gb], all_pools {[young] [2.3gb]->[30.5mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [2.7gb]->[2.7gb]/[3.7gb]}
[2015-08-01 02:51:23,469][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81629][1018] duration [10.6s], collections [1]/[10.8s], total [10.6s]/[1.2h], memory [5.2gb]->[4.2gb]/[7.1gb], all_pools {[young] [2.5gb]->[484.7mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2.7gb]->[3.7gb]/[3.7gb]}
[2015-08-01 02:52:45,821][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81699][5109] duration [1.4s], collections [1]/[2.1s], total [1.4s]/[1.1h], memory [3.3gb]->[1.5gb]/[7.1gb], all_pools {[young] [2.3gb]->[12.7mb]/[3gb]}{[survivor] [302.3mb]->[384mb]/[384mb]}{[old] [772mb]->[1.2gb]/[3.7gb]}
[2015-08-01 02:53:07,256][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81719][5111] duration [2s], collections [1]/[2.4s], total [2s]/[1.1h], memory [3.5gb]->[2.2gb]/[7.1gb], all_pools {[young] [2.1gb]->[14.7mb]/[3gb]}{[survivor] [139.3mb]->[384mb]/[384mb]}{[old] [1.2gb]->[1.8gb]/[3.7gb]}
[2015-08-01 02:53:31,974][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81740][5113] duration [2.9s], collections [1]/[4.1s], total [2.9s]/[1.1h], memory [3.7gb]->[3.1gb]/[7.1gb], all_pools {[young] [1.5gb]->[21.7mb]/[3gb]}{[survivor] [384mb]->[384mb]/[384mb]}{[old] [1.8gb]->[2.7gb]/[3.7gb]}
[2015-08-01 02:53:40,294][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81741][1022] duration [7.5s], collections [1]/[8.3s], total [7.5s]/[1.2h], memory [3.1gb]->[3gb]/[7.1gb], all_pools {[young] [21.7mb]->[157.7mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [2.7gb]->[2.8gb]/[3.7gb]}
[2015-08-01 02:53:50,291][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81746][5115] duration [2.9s], collections [1]/[3.7s], total [2.9s]/[1.1h], memory [3.8gb]->[3.1gb]/[7.1gb], all_pools {[young] [1.7gb]->[21.1mb]/[3gb]}{[survivor] [0b]->[384mb]/[384mb]}{[old] [2gb]->[2.7gb]/[3.7gb]}
[2015-08-01 02:54:26,958][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81782][5117] duration [1.3s], collections [1]/[1.6s], total [1.3s]/[1.1h], memory [4.5gb]->[2.4gb]/[7.1gb], all_pools {[young] [2.8gb]->[201.2mb]/[3gb]}{[survivor] [165.8mb]->[384mb]/[384mb]}{[old] [1.6gb]->[1.9gb]/[3.7gb]}
[2015-08-01 02:55:05,248][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81814][5119] duration [3.6s], collections [1]/[4.7s], total [3.6s]/[1.1h], memory [2.7gb]->[2.8gb]/[7.1gb], all_pools {[young] [1.6gb]->[25.9mb]/[3gb]}{[survivor] [170mb]->[384mb]/[384mb]}{[old] [923.7mb]->[2.4gb]/[3.7gb]}
[2015-08-01 02:55:13,101][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81817][1025] duration [5.4s], collections [1]/[5.8s], total [5.4s]/[1.2h], memory [4.7gb]->[2.2gb]/[7.1gb], all_pools {[young] [1.9gb]->[8.2mb]/[3gb]}{[survivor] [384mb]->[0b]/[384mb]}{[old] [2.4gb]->[2.2gb]/[3.7gb]}
[2015-08-01 02:55:23,093][INFO ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][old][81819][1026] duration [7.1s], collections [1]/[7.8s], total [7.1s]/[1.2h], memory [4.3gb]->[2.6gb]/[7.1gb], all_pools {[young] [2.1gb]->[28mb]/[3gb]}{[survivor] [0b]->[0b]/[384mb]}{[old] [2.2gb]->[2.6gb]/[3.7gb]}
[2015-08-01 02:56:15,856][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81867][5125] duration [3.8s], collections [1]/[4.4s], total [3.8s]/[1.1h], memory [2.7gb]->[2.3gb]/[7.1gb], all_pools {[young] [2.2gb]->[23.1mb]/[3gb]}{[survivor] [101.2mb]->[384mb]/[384mb]}{[old] [420.6mb]->[1.9gb]/[3.7gb]}
[2015-08-01 02:57:10,805][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81918][5128] duration [2.9s], collections [1]/[3.4s], total [2.9s]/[1.1h], memory [2.5gb]->[1.9gb]/[7.1gb], all_pools {[young] [1.7gb]->[194.5kb]/[3gb]}{[survivor] [62.2mb]->[384mb]/[384mb]}{[old] [797.2mb]->[1.6gb]/[3.7gb]}
[2015-08-01 02:57:38,576][WARN ][monitor.jvm ] [metrics-datastore-5-es-tune-2] [gc][young][81944][5130] duration [1.7s], collections [1]/[2.7s], total [1.7s]/[1.1h], memory [4gb]->[2.4gb]/[7.1gb], all_pools {[young] [2.2gb]->[32.1mb]/[3gb]}{[survivor] [162.4mb]->[384mb]/[384mb]}{[old] [1.6gb]->[2gb]/[3.7gb]}
[2015-08-01 02:59:01,952][ERROR][marvel.agent.exporter ] [metrics-datastore-5-es-tune-2] error sending data to [http://0.0.0.0:9200/.marvel-2015.08.01/_bulk]: SocketTimeoutException[Read timed out]
[2015-08-01 03:00:38,962][INFO ][cluster.service ] [metrics-datastore-5-es-tune-2] removed {[metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false},}, reason: zen-disco-receive(from master [[metrics-datastore-1-es-tune-2][7PdAs7byRHONpe-0LfUcJg][ip-172-31-37-67.us-west-2.compute.internal][inet[/172.31.37.67:9300]]{data=false, max_local_storage_nodes=1, master=true}])
[2015-08-01 03:00:39,479][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[unclassified1840151916][1]
org.elasticsearch.transport.NodeDisconnectedException: [metrics-datastore-4-es-tune-2][inet[/172.31.32.33:9300]][indices:data/write/bulk[s][r]] disconnected
[2015-08-01 03:00:39,479][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[perf2106556009][1]
org.elasticsearch.transport.NodeDisconnectedException: [metrics-datastore-4-es-tune-2][inet[/172.31.32.33:9300]][indices:data/write/bulk[s][r]] disconnected
[2015-08-01 03:00:39,480][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[perf2106556009][1]
org.elasticsearch.transport.NodeDisconnectedException: [metrics-datastore-4-es-tune-2][inet[/172.31.32.33:9300]][indices:data/write/bulk[s][r]] disconnected
[2015-08-01 03:00:39,481][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[perf2106556009][1]
org.elasticsearch.transport.NodeDisconnectedException: [metrics-datastore-4-es-tune-2][inet[/172.31.32.33:9300]][indices:data/write/bulk[s][r]] disconnected
[2015-08-01 03:00:39,481][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[perf2106556009][1]
org.elasticsearch.transport.NodeDisconnectedException: [metrics-datastore-4-es-tune-2][inet[/172.31.32.33:9300]][indices:data/write/bulk[s][r]] disconnected
[2015-08-01 03:00:39,482][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[perf2106556009][1]
org.elasticsearch.transport.NodeDisconnectedException: [metrics-datastore-4-es-tune-2][inet[/172.31.32.33:9300]][indices:data/write/bulk[s][r]] disconnected
[2015-08-01 03:00:39,482][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[perf2106556009][1]
org.elasticsearch.transport.NodeDisconnectedException: [metrics-datastore-4-es-tune-2][inet[/172.31.32.33:9300]][indices:data/write/bulk[s][r]] disconnected
[2015-08-01 03:00:39,483][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[perf2106556009][1]
org.elasticsearch.transport.NodeDisconnectedException: [metrics-datastore-4-es-tune-2][inet[/172.31.32.33:9300]][indices:data/write/bulk[s][r]] disconnected
[2015-08-01 03:00:39,483][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[perf2106556009][1]
org.elasticsearch.transport.NodeDisconnectedException: [metrics-datastore-4-es-tune-2][inet[/172.31.32.33:9300]][indices:data/write/bulk[s][r]] disconnected
[2015-08-01 03:00:39,484][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[perf2106556009][1]
org.elasticsearch.transport.NodeDisconnectedException: [metrics-datastore-4-es-tune-2][inet[/172.31.32.33:9300]][indices:data/write/bulk[s][r]] disconnected
[2015-08-01 03:00:39,484][WARN ][action.bulk ] [metrics-datastore-5-es-tune-2] failed to perform indices:data/write/bulk[s] on remote replica [metrics-datastore-4-es-tune-2][u85w8NrJQfG1COUl5zsc3A][ip-172-31-32-33.us-west-2.compute.internal][inet[/172.31.32.33:9300]]{max_local_storage_nodes=1, master=false}[perf2106556009][1]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment