Created
April 21, 2019 14:48
-
-
Save wangxiangyu/199d7171ef2f9f3d68ba27dfed94ba80 to your computer and use it in GitHub Desktop.
OOM
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.app-svr.app-feed-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-main.app-svr.app-feed-@2019.04.21-jssz01-0] | |
| [0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=8cojHrnlQ6WQj54m16pLOg] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.app-svr.app-feed-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-main.app-svr.app-feed-@2019.04.21-jssz01-0] | |
| [0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=8cojHrnlQ6WQj54m16pLOg] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.account.account-interface-@2019.04.21-jssz01-0][2]] failed to perform indices:data/write/bulk[s] on replica [billions-main.account.account-interface-@20 | |
| 19.04.21-jssz01-0][2], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=p86QZ3L5QqKr4RkrXmLU2w] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.account.account-interface-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-main.account.account-interface-@20 | |
| 19.04.21-jssz01-0][0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=P1GzdyySTWOF67wHW3JcRQ] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.common-arch.discovery-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-main.common-arch.discovery-@2019.04.21-jssz01-0][0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=C6GVgKseRfWfijrfQ2ecXw] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-adx-server-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-adx-server-@2019.04.21-jssz01-0][0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=ahAjW79wSDaUK_WGQU043w] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-link.im.bim-notify-svr-@2019.04.21-jssz01-0][1]] failed to perform indices:data/write/bulk[s] on replica [billions-link.im.bim-notify-svr-@2019.04.21-jssz01- at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_162] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_162] | |
| at java.lang.Thread.run(Thread.java:748) [?:1.8.0_162] | |
| [2019-04-21T22:02:54,641][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.app-svr.app-wall-job-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-main.app-svr.app-wall-job-@2019.04.21-j | |
| ssz01-0][0], node[8d3ej4YDT0WbQSiKp5yfdQ], [R], s[STARTED], a[id=EhCsVJUbR4qKJhbPl6eIcA] | |
| org.elasticsearch.transport.NodeNotConnectedException: [jssz-billions-es-04-datanode_hot][10.69.23.26:9300] Node not connected | |
| at org.elasticsearch.transport.TcpTransport.getConnection(TcpTransport.java:655) ~[elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.transport.TcpTransport.getConnection(TcpTransport.java:117) ~[elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:513) ~[elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:489) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:962) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.performOnReplica(ReplicationOperation.java:186) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.performOnReplicas(ReplicationOperation.java:170) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:130) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:322) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:264) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:888) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:885) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationLock(IndexShard.java:1656) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:897) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction.access$400(TransportReplicationAction.java:93) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:281) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:260) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:252) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:627) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638) [elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.4.3.jar:5.4.3] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_162] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_162] | |
| at java.lang.Thread.run(Thread.java:748) [?:1.8.0_162] | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-live.live.user-service-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-live.live.user-service-@2019.04.21-jssz01- | |
| 0][0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=ja1zuifSS2eCWFdrLbKAyQ] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-live.live.live-dm-@2019.04.21-jssz01-0][3]] failed to perform indices:data/write/bulk[s] on replica [billions-live.live.live-dm-@2019.04.21-jssz01-0][3], nod | |
| e[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=pifAFdJiSs-3sdYs_kNVRg] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.app-svr.app-feed-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-main.app-svr.app-feed-@2019.04.21-jssz01-0] | |
| [0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=8cojHrnlQ6WQj54m16pLOg] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.app-svr.app-feed-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-main.app-svr.app-feed-@2019.04.21-jssz01-0] | |
| [0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=8cojHrnlQ6WQj54m16pLOg] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.account.account-interface-@2019.04.21-jssz01-0][2]] failed to perform indices:data/write/bulk[s] on replica [billions-main.account.account-interface-@20 | |
| 19.04.21-jssz01-0][2], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=p86QZ3L5QqKr4RkrXmLU2w] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.account.account-interface-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-main.account.account-interface-@20 | |
| 19.04.21-jssz01-0][0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=P1GzdyySTWOF67wHW3JcRQ] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-main.common-arch.discovery-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-main.common-arch.discovery-@2019.04.21 | |
| -jssz01-0][0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=C6GVgKseRfWfijrfQ2ecXw] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-adx-server-@2019.04.21-jssz01-0][0]] failed to perform indices:data/write/bulk[s] on replica [billions-adx-server-@2019.04.21-jssz01-0][0], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=ahAjW79wSDaUK_WGQU043w] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,642][WARN ][o.e.a.b.TransportShardBulkAction] [jssz-billions-es-02-datanode_hot] [[billions-link.im.bim-notify-svr-@2019.04.21-jssz01-0][1]] failed to perform indices:data/write/bulk[s] on replica [billions-link.im.bim-notify-svr-@2019.04.21-jssz01-0][1], node[rrGzzIqxTGS8-7UcmBCWVQ], [R], s[STARTED], a[id=EZ83lpIbTESDCw6XH_jPPQ] | |
| org.elasticsearch.transport.NodeDisconnectedException: [jssz-billions-es-01-datanode_hot][10.69.23.23:9300][indices:data/write/bulk[s][r]] disconnected | |
| [2019-04-21T22:02:54,626][WARN ][i.n.u.c.DefaultPromise ] An exception was thrown by org.elasticsearch.transport.netty4.Netty4Transport$$Lambda$1514/107868687.operationComplete() | |
| java.lang.OutOfMemoryError: unable to create new native thread | |
| at java.lang.Thread.start0(Native Method) ~[?:1.8.0_162] | |
| at java.lang.Thread.start(Thread.java:717) ~[?:1.8.0_162] | |
| at java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:957) ~[?:1.8.0_162] | |
| at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1378) ~[?:1.8.0_162] | |
| at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.doExecute(EsThreadPoolExecutor.java:94) ~[elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.execute(EsThreadPoolExecutor.java:89) ~[elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.transport.TcpTransport.disconnectFromNodeChannel(TcpTransport.java:617) ~[elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.transport.TcpTransport.lambda$openConnection$1(TcpTransport.java:545) ~[elasticsearch-5.4.3.jar:5.4.3] | |
| at org.elasticsearch.transport.netty4.Netty4Transport.lambda$connectToChannels$2(Netty4Transport.java:354) ~[transport-netty4-5.4.3.jar:5.4.3] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507) [netty-common-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:481) [netty-common-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420) [netty-common-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104) [netty-common-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:82) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1058) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:686) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:664) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannel$AbstractUnsafe.flush0(AbstractChannel.java:867) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.flush0(AbstractNioChannel.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannel$AbstractUnsafe.flush(AbstractChannel.java:823) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.flush(DefaultChannelPipeline.java:1296) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.access$1500(AbstractChannelHandlerContext.java:38) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext$WriteAndFlushTask.write(AbstractChannelHandlerContext.java:1137) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext$AbstractWriteTask.run(AbstractChannelHandlerContext.java:1078) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163) [netty-common-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:403) [netty-common-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:462) [netty-transport-4.1.11.Final.jar:4.1.11.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final] | |
| at java.lang.Thread.run(Thread.java:748) [?:1.8.0_162] | |
| [2019-04-21T22:02:54,643][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [jssz-billions-es-02-datanode_hot] fatal error in thread [elasticsearch[jssz-billions-es-02-datanode_hot][generic][T#51]], exiting | |
| java.lang.OutOfMemoryError: unable to create new native thread | |
| at java.lang.Thread.start0(Native Method) ~[?:1.8.0_162] | |
| at java.lang.Thread.start(Thread.java:717) ~[?:1.8.0_162] | |
| at java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:957) ~[?:1.8.0_162] | |
| at java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:1025) ~[?:1.8.0_162] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167) ~[?:1.8.0_162] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_162] | |
| at java.lang.Thread.run(Thread.java:748) [?:1.8.0_162] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment