-
-
Save anonymous/1a35b911a5a9163693bf659b701e2f02 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name | |
| ip.ip.ip.1 25 98 3 0.03 0.02 0.05 mdi * host1 | |
| ip.ip.ip.2 54 99 1 0.05 0.03 0.05 mdi - host2 | |
| ip.ip.ip.3 32 98 2 0.01 0.02 0.05 mdi - host3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| health status index uuid pri rep docs.count docs.deleted store.size pri.store.size | |
| yellow open index1 6n2fqIziSe-vMYvrI_HXYQ 5 2 9957799 0 24.2gb 9.3gb | |
| yellow open index2 A_VHoPMxRj-OnRCa4tqA9g 5 2 9957799 0 24.2gb 9.3gb |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| index shard prirep state docs store ip node | |
| index2 0 p STARTED 1990082 1.8gb ip.ip.ip.1 host1 | |
| index2 0 r STARTED 1990082 1.8gb ip.ip.ip.2 host2 | |
| index2 0 r STARTED 1990082 1.8gb ip.ip.ip.3 host3 | |
| index2 1 p STARTED 1990050 1.8gb ip.ip.ip.3 host3 | |
| index2 1 r STARTED 1990050 1.8gb ip.ip.ip.2 host2 | |
| index2 1 r STARTED 1990050 1.8gb ip.ip.ip.1 host1 | |
| index2 2 p STARTED 1996938 1.8gb ip.ip.ip.2 host2 | |
| index2 2 r UNASSIGNED | |
| index2 2 r UNASSIGNED | |
| index2 3 p STARTED 1989843 1.8gb ip.ip.ip.1 host1 | |
| index2 3 r STARTED 1989843 1.8gb ip.ip.ip.2 host2 | |
| index2 3 r STARTED 1989843 1.8gb ip.ip.ip.3 host3 | |
| index2 4 p STARTED 1990886 1.8gb ip.ip.ip.3 host3 | |
| index2 4 r STARTED 1990886 1.8gb ip.ip.ip.2 host2 | |
| index2 4 r STARTED 1990886 1.8gb ip.ip.ip.1 host1 | |
| index1 0 p STARTED 1990082 1.8gb ip.ip.ip.1 host1 | |
| index1 0 r STARTED 1990082 1.8gb ip.ip.ip.2 host2 | |
| index1 0 r STARTED 1990082 1.8gb ip.ip.ip.3 host3 | |
| index1 1 p STARTED 1990050 1.8gb ip.ip.ip.3 host3 | |
| index1 1 r STARTED 1990050 1.8gb ip.ip.ip.2 host2 | |
| index1 1 r STARTED 1990050 1.8gb ip.ip.ip.1 host1 | |
| index1 2 p STARTED 1996938 1.8gb ip.ip.ip.2 host2 | |
| index1 2 r UNASSIGNED | |
| index1 2 r UNASSIGNED | |
| index1 3 p STARTED 1989843 1.8gb ip.ip.ip.1 host1 | |
| index1 3 r STARTED 1989843 1.8gb ip.ip.ip.2 host2 | |
| index1 3 r STARTED 1989843 1.8gb ip.ip.ip.3 host3 | |
| index1 4 p STARTED 1990886 1.8gb ip.ip.ip.3 host3 | |
| index1 4 r STARTED 1990886 1.8gb ip.ip.ip.2 host2 | |
| index1 4 r STARTED 1990886 1.8gb ip.ip.ip.1 host1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2017-03-17T22:46:15,179][INFO ][o.e.c.m.MetaDataCreateIndexService] [host1] [index1] creating index, cause [api], templates [], shards [5]/[0], mappings [mytype] | |
| [2017-03-17T22:46:40,787][INFO ][o.e.c.m.MetaDataMappingService] [host1] [index1/6n2fqIziSe-vMYvrI_HXYQ] update_mapping [mytype] | |
| [2017-03-17T22:46:40,811][INFO ][o.e.c.m.MetaDataMappingService] [host1] [index1/6n2fqIziSe-vMYvrI_HXYQ] update_mapping [mytype] | |
| [2017-03-17T23:36:29,173][INFO ][o.e.c.m.MetaDataUpdateSettingsService] [host1] updating number_of_replicas to [2] for indices [index1] | |
| [2017-03-17T23:41:47,413][WARN ][o.e.g.GatewayAllocator$InternalReplicaShardAllocator] [host1] [index1][1]: failed to list shard for shard_store on node [Cn8Hm5-tQcm7QobDcNvzCw] | |
| org.elasticsearch.action.FailedNodeException: Failed node [Cn8Hm5-tQcm7QobDcNvzCw] | |
| at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.onFailure(TransportNodesAction.java:247) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.access$300(TransportNodesAction.java:160) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1.handleException(TransportNodesAction.java:219) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1024) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.transport.TcpTransport.lambda$handleException$17(TcpTransport.java:1411) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:109) [elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:1409) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:1401) [elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1345) [elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) [transport-netty4-5.2.0.jar:5.2.0] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:349) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:341) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293) [netty-codec-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:280) [netty-codec-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:396) [netty-codec-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:248) [netty-codec-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:349) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:341) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:349) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:129) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:642) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:527) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:481) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:441) [netty-transport-4.1.7.Final.jar:4.1.7.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.7.Final.jar:4.1.7.Final] | |
| at java.lang.Thread.run(Thread.java:745) [?:1.8.0_121] | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: [host2][ip.ip.ip.ip2:9300][internal:cluster/nodes/indices/shard/store[n]] | |
| Caused by: org.elasticsearch.ElasticsearchException: Failed to list store metadata for shard [[index1][1]] | |
| at org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:114) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:64) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:145) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:270) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:266) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1488) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:596) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.2.0.jar:5.2.0] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_121] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_121] | |
| ... 1 more | |
| Caused by: java.io.FileNotFoundException: no segments* file found in store(mmapfs(/elasticsearchdata/nodes/0/indices/6n2fqIziSe-vMYvrI_HXYQ/1/index)): files: [recovery.AVrepWc_SlfGaVklv4AX._0.cfe, recovery.AVrepWc_SlfGaVklv4AX._0.cfs, recovery.AVrepWc_SlfGaVklv4AX._0.si, recovery.AVrepWc_SlfGaVklv4AX._1.cfe, recovery.AVrepWc_SlfGaVklv4AX._1.si, recovery.AVrepWc_SlfGaVklv4AX._2.cfe, recovery.AVrepWc_SlfGaVklv4AX._2.cfs, recovery.AVrepWc_SlfGaVklv4AX._2.si, recovery.AVrepWc_SlfGaVklv4AX._3.cfe, recovery.AVrepWc_SlfGaVklv4AX._3.cfs, recovery.AVrepWc_SlfGaVklv4AX._3.si, recovery.AVrepWc_SlfGaVklv4AX._4.cfe, recovery.AVrepWc_SlfGaVklv4AX._4.cfs, recovery.AVrepWc_SlfGaVklv4AX._4.si, recovery.AVrepWc_SlfGaVklv4AX._5.cfe, recovery.AVrepWc_SlfGaVklv4AX._5.cfs, recovery.AVrepWc_SlfGaVklv4AX._5.si, recovery.AVrepWc_SlfGaVklv4AX._6.cfe, recovery.AVrepWc_SlfGaVklv4AX._6.cfs, recovery.AVrepWc_SlfGaVklv4AX._6.si, recovery.AVrepWc_SlfGaVklv4AX._7.cfe, recovery.AVrepWc_SlfGaVklv4AX._7.cfs, recovery.AVrepWc_SlfGaVklv4AX._7.si, recovery.AVrepWc_SlfGaVklv4AX._8.cfe, recovery.AVrepWc_SlfGaVklv4AX._8.cfs, recovery.AVrepWc_SlfGaVklv4AX._8.si, recovery.AVrepWc_SlfGaVklv4AX._9.cfe, recovery.AVrepWc_SlfGaVklv4AX._9.cfs, recovery.AVrepWc_SlfGaVklv4AX._9.si, recovery.AVrepWc_SlfGaVklv4AX._a.cfe, recovery.AVrepWc_SlfGaVklv4AX._a.cfs, recovery.AVrepWc_SlfGaVklv4AX._a.si, recovery.AVrepWc_SlfGaVklv4AX._b.cfe, recovery.AVrepWc_SlfGaVklv4AX._b.cfs, recovery.AVrepWc_SlfGaVklv4AX._b.si, recovery.AVrepWc_SlfGaVklv4AX._c.cfe, recovery.AVrepWc_SlfGaVklv4AX._c.si, recovery.AVrepWc_SlfGaVklv4AX._d.cfe, recovery.AVrepWc_SlfGaVklv4AX._d.cfs, recovery.AVrepWc_SlfGaVklv4AX._d.si, recovery.AVrepWc_SlfGaVklv4AX._e.cfe, recovery.AVrepWc_SlfGaVklv4AX._e.cfs, recovery.AVrepWc_SlfGaVklv4AX._e.si, recovery.AVrepWc_SlfGaVklv4AX.segments_4, write.lock] | |
| at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:685) ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 2017-01-17 15:57:29] | |
| at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:642) ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 2017-01-17 15:57:29] | |
| at org.apache.lucene.index.SegmentInfos.readLatestCommit(SegmentInfos.java:448) ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 2017-01-17 15:57:29] | |
| at org.elasticsearch.common.lucene.Lucene.readSegmentInfos(Lucene.java:126) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.index.store.Store.readSegmentsInfo(Store.java:198) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.index.store.Store.access$200(Store.java:126) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:788) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.index.store.Store$MetadataSnapshot.<init>(Store.java:721) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.index.store.Store.getMetadata(Store.java:240) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:871) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:128) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:112) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:64) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:145) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:270) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:266) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1488) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:596) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-5.2.0.jar:5.2.0] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_121] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_121] | |
| ... 1 more | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment