Created
August 21, 2014 04:33
-
-
Save qiaohaijun/1d35d786a69a3aa0e1e6 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2014-08-21 12:31:20,972][WARN ][cluster.action.shard ] [cloud101415282_dp_nm_ss_nop_sogou-op_org] [sogouquery.pc1401811200][4] sending failed shard for [sogouquery.pc1401811200][4], node[qOMEqm_eRxaK8OnUZp49RQ], [R], s[INITIALIZING], indexUUID [wKxSjNt2QMCwg_-HWRWOiw], reason [Failed to start shard, message [RecoveryFailedException[[sogouquery.pc1401811200][4]: Recovery failed from [cloud101414990_dp_nm_ss_nop_sogou-op_org][l4DTLj4jSNueFWNG1MYrSg][cloud101414990.dp.nm.ss.nop.sogou-op.org][inet[/10.141.49.90:9300]]{max_local_storage_nodes=1, master=false} into [cloud101415282_dp_nm_ss_nop_sogou-op_org][qOMEqm_eRxaK8OnUZp49RQ][cloud101415282.dp.nm.ss.nop.sogou-op.org][inet[/10.141.52.82:9300]]{max_local_storage_nodes=1, master=false}]; nested: RemoteTransportException[[cloud101414990_dp_nm_ss_nop_sogou-op_org][inet[/10.141.49.90:9300]][index/shard/recovery/startRecovery]]; nested: RecoveryEngineException[[sogouquery.pc1401811200][4] Phase[1] Execution failed]; nested: RecoverFilesRecoveryException[[sogouquery.pc1401811200][4] Failed to transfer [154] files with total size of [341.1mb]]; nested: RemoteTransportException[[cloud101415282_dp_nm_ss_nop_sogou-op_org][inet[/10.141.52.82:9300]][index/shard/recovery/fileChunk]]; nested: FileNotFoundException[/search/es-data/SogouQuery/nodes/0/indices/sogouquery.pc1401811200/4/index/_10c3.fdt (Too many open files)]; ]] | |
| [2014-08-21 12:31:21,026][WARN ][indices.cluster ] [cloud101415282_dp_nm_ss_nop_sogou-op_org] [sogouquery.pc1408464000][8] failed to start shard | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [sogouquery.pc1408464000][8]: Recovery failed from [cloud101414990_dp_nm_ss_nop_sogou-op_org][l4DTLj4jSNueFWNG1MYrSg][cloud101414990.dp.nm.ss.nop.sogou-op.org][inet[/10.141.49.90:9300]]{max_local_storage_nodes=1, master=false} into [cloud101415282_dp_nm_ss_nop_sogou-op_org][qOMEqm_eRxaK8OnUZp49RQ][cloud101415282.dp.nm.ss.nop.sogou-op.org][inet[/10.141.52.82:9300]]{max_local_storage_nodes=1, master=false} | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.doRecovery(RecoveryTarget.java:307) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget.access$300(RecoveryTarget.java:65) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$2.run(RecoveryTarget.java:175) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:745) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: [cloud101414990_dp_nm_ss_nop_sogou-op_org][inet[/10.141.49.90:9300]][index/shard/recovery/startRecovery] | |
| Caused by: org.elasticsearch.index.engine.RecoveryEngineException: [sogouquery.pc1408464000][8] Phase[2] Execution failed | |
| at org.elasticsearch.index.engine.internal.InternalEngine.recover(InternalEngine.java:1098) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.recover(InternalIndexShard.java:627) | |
| at org.elasticsearch.indices.recovery.RecoverySource.recover(RecoverySource.java:117) | |
| at org.elasticsearch.indices.recovery.RecoverySource.access$1600(RecoverySource.java:61) | |
| at org.elasticsearch.indices.recovery.RecoverySource$StartRecoveryTransportRequestHandler.messageReceived(RecoverySource.java:337) | |
| at org.elasticsearch.indices.recovery.RecoverySource$StartRecoveryTransportRequestHandler.messageReceived(RecoverySource.java:323) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:270) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:745) | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: [cloud101415282_dp_nm_ss_nop_sogou-op_org][inet[/10.141.52.82:9300]][index/shard/recovery/prepareTranslog] | |
| Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: [sogouquery.pc1408464000][8] failed to open reader on writer | |
| at org.elasticsearch.index.engine.internal.InternalEngine.start(InternalEngine.java:289) | |
| at org.elasticsearch.index.shard.service.InternalIndexShard.performRecoveryPrepareForTranslog(InternalIndexShard.java:699) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$PrepareForTranslogOperationsRequestHandler.messageReceived(RecoveryTarget.java:391) | |
| at org.elasticsearch.indices.recovery.RecoveryTarget$PrepareForTranslogOperationsRequestHandler.messageReceived(RecoveryTarget.java:367) | |
| at org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.run(MessageChannelHandler.java:270) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:745) | |
| Caused by: java.io.FileNotFoundException: /search/es-data/SogouQuery/nodes/0/indices/sogouquery.pc1408464000/8/index/segments_1 (Too many open files) | |
| at java.io.RandomAccessFile.open(Native Method) | |
| at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241) | |
| at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:193) | |
| at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:80) | |
| at org.elasticsearch.index.store.Store$StoreDirectory.openInput(Store.java:473) | |
| at org.apache.lucene.index.SegmentInfos.read(SegmentInfos.java:324) | |
| at org.apache.lucene.index.SegmentInfos$1.doBody(SegmentInfos.java:404) | |
| at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:843) | |
| at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:694) | |
| at org.apache.lucene.index.SegmentInfos.read(SegmentInfos.java:400) | |
| at org.elasticsearch.index.engine.internal.InternalEngine.readLastCommittedSegmentsInfo(InternalEngine.java:298) | |
| at org.elasticsearch.index.engine.internal.InternalEngine.start(InternalEngine.java:280) | |
| ... 7 more | |
| [2014-08-21 12:31:21,027][WARN ][cluster.action.shard ] [cloud101415282_dp_nm_ss_nop_sogou-op_org] [sogouquery.pc1408464000][8] sending failed shard for [sogouquery.pc1408464000][8], node[qOMEqm_eRxaK8OnUZp49RQ], [R], s[INITIALIZING], indexUUID [Zsm9KwMAS82GVOTU34QXOA], reason [Failed to start shard, message [RecoveryFailedException[[sogouquery.pc1408464000][8]: Recovery failed from [cloud101414990_dp_nm_ss_nop_sogou-op_org][l4DTLj4jSNueFWNG1MYrSg][cloud101414990.dp.nm.ss.nop.sogou-op.org][inet[/10.141.49.90:9300]]{max_local_storage_nodes=1, master=false} into [cloud101415282_dp_nm_ss_nop_sogou-op_org][qOMEqm_eRxaK8OnUZp49RQ][cloud101415282.dp.nm.ss.nop.sogou-op.org][inet[/10.141.52.82:9300]]{max_local_storage_nodes=1, master=false}]; nested: RemoteTransportException[[cloud101414990_dp_nm_ss_nop_sogou-op_org][inet[/10.141.49.90:9300]][index/shard/recovery/startRecovery]]; nested: RecoveryEngineException[[sogouquery.pc1408464000][8] Phase[2] Execution failed]; nested: RemoteTransportException[[cloud101415282_dp_nm_ss_nop_sogou-op_org][inet[/10.141.52.82:9300]][index/shard/recovery/prepareTranslog]]; nested: EngineCreationFailureException[[sogouquery.pc1408464000][8] failed to open reader on writer]; nested: FileNotFoundException[/search/es-data/SogouQuery/nodes/0/indices/sogouquery.pc1408464000/8/index/segments_1 (Too many open files)]; ]] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment