-
-
Save crispygoth/512693cdffd83b144ee908bf5e8e1f35 to your computer and use it in GitHub Desktop.
Elasticsearch logs / jstack
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2019-06-25 16:45:02 | |
| Full thread dump OpenJDK 64-Bit Server VM (12.0.1+12 mixed mode, sharing): | |
| Threads class SMR info: | |
| _java_thread_list=0x00007f3808002770, length=207, elements={ | |
| 0x00007f3854797000, 0x00007f3854799000, 0x00007f385479d800, 0x00007f385479f800, | |
| 0x00007f38547aa000, 0x00007f38547ac000, 0x00007f38547da800, 0x00007f38547df000, | |
| 0x00007f3854f3c800, 0x00007f3855a94000, 0x00007f38561f2000, 0x00007f3856375000, | |
| 0x00007f38563f6000, 0x00007f385642b800, 0x00007f37dc22f000, 0x00007f37dc001800, | |
| 0x00007f3856cc5800, 0x00007f3856e2a800, 0x00007f3710008000, 0x00007f3856e36000, | |
| 0x00007f36fc007000, 0x00007f36fc01c000, 0x00007f36fc01e000, 0x00007f36fc01f800, | |
| 0x00007f370c003800, 0x00007f36fc021000, 0x00007f36fc023000, 0x00007f36fc025800, | |
| 0x00007f36fc027000, 0x00007f36fc029000, 0x00007f36fc02b000, 0x00007f36fc02d000, | |
| 0x00007f36fc02f000, 0x00007f36fc030800, 0x00007f36fc032800, 0x00007f3710011000, | |
| 0x00007f370c005000, 0x00007f370c007000, 0x00007f370c009000, 0x00007f370c00b000, | |
| 0x00007f370c00c800, 0x00007f370c00e800, 0x00007f370c010800, 0x00007f370c012000, | |
| 0x00007f370c014000, 0x00007f370c016000, 0x00007f370c017800, 0x00007f370c019800, | |
| 0x00007f370c01b800, 0x00007f3700011800, 0x00007f3700013000, 0x00007f3700015000, | |
| 0x00007f3678002000, 0x00007f3856efb000, 0x00007f3856efd800, 0x00007f3854012800, | |
| 0x00007f36b8068000, 0x00007f370012f000, 0x00007f37dc67a000, 0x00007f3670001000, | |
| 0x00007f3670003000, 0x00007f3670005000, 0x00007f3670006800, 0x00007f3670008800, | |
| 0x00007f367000a800, 0x00007f367000c000, 0x00007f367000e000, 0x00007f3670010000, | |
| 0x00007f3670012000, 0x00007f3670013800, 0x00007f3714041000, 0x00007f37240d9800, | |
| 0x00007f37100c8800, 0x00007f3624020000, 0x00007f370c01d000, 0x00007f370c01f000, | |
| 0x00007f370c021000, 0x00007f370c022800, 0x00007f370c024800, 0x00007f370c026800, | |
| 0x00007f370c028800, 0x00007f370c02a000, 0x00007f370c02c000, 0x00007f370c036000, | |
| 0x00007f370c038000, 0x00007f370c03a000, 0x00007f370c03c000, 0x00007f370c03e000, | |
| 0x00007f37100ca000, 0x00007f37100ba000, 0x00007f37100bb800, 0x00007f37100bd800, | |
| 0x00007f37100bf000, 0x00007f37100cb800, 0x00007f37100cd800, 0x00007f37100cf000, | |
| 0x00007f37100d1000, 0x00007f37100d2800, 0x00007f3710115800, 0x00007f3710118000, | |
| 0x00007f3710119800, 0x00007f36b8048800, 0x00007f367c00d000, 0x00007f35f8002800, | |
| 0x00007f3670015000, 0x00007f3670016800, 0x00007f3724061000, 0x00007f3670017800, | |
| 0x00007f3670019800, 0x00007f356c016000, 0x00007f3570013800, 0x00007f356c01c800, | |
| 0x00007f3570021800, 0x00007f356c01e800, 0x00007f3570023000, 0x00007f35e8011000, | |
| 0x00007f35e4011000, 0x00007f35f0011000, 0x00007f35e8013000, 0x00007f35e4013000, | |
| 0x00007f35f0013000, 0x00007f35e8015000, 0x00007f35e4015000, 0x00007f35f0015000, | |
| 0x00007f35e4017000, 0x00007f35f0017000, 0x00007f35e8017000, 0x00007f35e4018800, | |
| 0x00007f35e8018800, 0x00007f35f0018800, 0x00007f35e801a800, 0x00007f35e401a000, | |
| 0x00007f35f001a800, 0x00007f35e401c000, 0x00007f35e801c800, 0x00007f35f001c000, | |
| 0x00007f35e401e000, 0x00007f35e801e800, 0x00007f367001b800, 0x00007f367001c800, | |
| 0x00007f367001e000, 0x00007f367001f000, 0x00007f3670020000, 0x00007f3670021000, | |
| 0x00007f3670022000, 0x00007f3670023800, 0x00007f3670025800, 0x00007f3670026800, | |
| 0x00007f3670027800, 0x00007f3670028000, 0x00007f3670029800, 0x00007f367002a800, | |
| 0x00007f367002c000, 0x00007f367002c800, 0x00007f367002e000, 0x00007f367002f000, | |
| 0x00007f3670038800, 0x00007f3670041800, 0x00007f3670030800, 0x00007f3670031800, | |
| 0x00007f3670033000, 0x00007f3540001000, 0x00007f3670034000, 0x00007f3670035000, | |
| 0x00007f3670035800, 0x00007f3670037000, 0x00007f3670039800, 0x00007f367003a000, | |
| 0x00007f367003b000, 0x00007f367003c800, 0x00007f367003d800, 0x00007f367003e800, | |
| 0x00007f367003f800, 0x00007f3670042800, 0x00007f3670043800, 0x00007f3670044800, | |
| 0x00007f3670045800, 0x00007f3670047000, 0x00007f35bc015000, 0x00007f35cc012800, | |
| 0x00007f35c8012800, 0x00007f35c4013800, 0x00007f3670048000, 0x00007f3670048800, | |
| 0x00007f367004a000, 0x00007f367004b800, 0x00007f367004c000, 0x00007f367004d800, | |
| 0x00007f367004f800, 0x00007f3670050000, 0x00007f3670051000, 0x00007f3638088800, | |
| 0x00007f36380a2000, 0x00007f37e0186000, 0x00007f3428011000, 0x00007f37e0181800, | |
| 0x00007f343c001800, 0x00007f342c006800, 0x00007f3420004000, 0x00007f342000a000, | |
| 0x00007f342000c800, 0x00007f3584012000, 0x00007f358c01c000, 0x00007f3598011800, | |
| 0x00007f3408001000, 0x00007f3408003800, 0x00007f3808001000 | |
| } | |
| "Reference Handler" #2 daemon prio=10 os_prio=0 cpu=105.78ms elapsed=4038.07s tid=0x00007f3854797000 nid=0x5cb0 waiting on condition [0x00007f3823efc000] | |
| java.lang.Thread.State: RUNNABLE | |
| at java.lang.ref.Reference.waitForReferencePendingList(java.base@12.0.1/Native Method) | |
| at java.lang.ref.Reference.processPendingReferences(java.base@12.0.1/Reference.java:241) | |
| at java.lang.ref.Reference$ReferenceHandler.run(java.base@12.0.1/Reference.java:213) | |
| Locked ownable synchronizers: | |
| - None | |
| "Finalizer" #3 daemon prio=8 os_prio=0 cpu=1.88ms elapsed=4038.07s tid=0x00007f3854799000 nid=0x5cb1 in Object.wait() [0x00007f3823dfb000] | |
| java.lang.Thread.State: WAITING (on object monitor) | |
| at java.lang.Object.wait(java.base@12.0.1/Native Method) | |
| - waiting on <0x0000000080dbbb40> (a java.lang.ref.ReferenceQueue$Lock) | |
| at java.lang.ref.ReferenceQueue.remove(java.base@12.0.1/ReferenceQueue.java:155) | |
| - locked <0x0000000080dbbb40> (a java.lang.ref.ReferenceQueue$Lock) | |
| at java.lang.ref.ReferenceQueue.remove(java.base@12.0.1/ReferenceQueue.java:176) | |
| at java.lang.ref.Finalizer$FinalizerThread.run(java.base@12.0.1/Finalizer.java:170) | |
| Locked ownable synchronizers: | |
| - None | |
| "Signal Dispatcher" #4 daemon prio=9 os_prio=0 cpu=0.71ms elapsed=4038.06s tid=0x00007f385479d800 nid=0x5cb2 runnable [0x0000000000000000] | |
| java.lang.Thread.State: RUNNABLE | |
| Locked ownable synchronizers: | |
| - None | |
| "C2 CompilerThread0" #5 daemon prio=9 os_prio=0 cpu=145069.67ms elapsed=4038.06s tid=0x00007f385479f800 nid=0x5cb3 waiting on condition [0x0000000000000000] | |
| java.lang.Thread.State: RUNNABLE | |
| No compile task | |
| Locked ownable synchronizers: | |
| - None | |
| "C1 CompilerThread0" #15 daemon prio=9 os_prio=0 cpu=7951.44ms elapsed=4038.06s tid=0x00007f38547aa000 nid=0x5cb4 waiting on condition [0x0000000000000000] | |
| java.lang.Thread.State: RUNNABLE | |
| No compile task | |
| Locked ownable synchronizers: | |
| - None | |
| "Sweeper thread" #20 daemon prio=9 os_prio=0 cpu=4482.35ms elapsed=4038.06s tid=0x00007f38547ac000 nid=0x5cb5 runnable [0x0000000000000000] | |
| java.lang.Thread.State: RUNNABLE | |
| Locked ownable synchronizers: | |
| - None | |
| "Service Thread" #21 daemon prio=9 os_prio=0 cpu=644.86ms elapsed=4038.04s tid=0x00007f38547da800 nid=0x5cb7 runnable [0x0000000000000000] | |
| java.lang.Thread.State: RUNNABLE | |
| Locked ownable synchronizers: | |
| - None | |
| "Common-Cleaner" #22 daemon prio=8 os_prio=0 cpu=11.96ms elapsed=4038.03s tid=0x00007f38547df000 nid=0x5cb9 in Object.wait() [0x00007f38235f2000] | |
| java.lang.Thread.State: TIMED_WAITING (on object monitor) | |
| at java.lang.Object.wait(java.base@12.0.1/Native Method) | |
| - waiting on <no object reference available> | |
| at java.lang.ref.ReferenceQueue.remove(java.base@12.0.1/ReferenceQueue.java:155) | |
| - locked <0x0000000080dd0488> (a java.lang.ref.ReferenceQueue$Lock) | |
| at jdk.internal.ref.CleanerImpl.run(java.base@12.0.1/CleanerImpl.java:148) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| at jdk.internal.misc.InnocuousThread.run(java.base@12.0.1/InnocuousThread.java:134) | |
| Locked ownable synchronizers: | |
| - None | |
| "process reaper" #27 daemon prio=10 os_prio=0 cpu=0.22ms elapsed=4036.39s tid=0x00007f3854f3c800 nid=0x5cbf runnable [0x00007f385d0ea000] | |
| java.lang.Thread.State: RUNNABLE | |
| at java.lang.ProcessHandleImpl.waitForProcessExit0(java.base@12.0.1/Native Method) | |
| at java.lang.ProcessHandleImpl$1.run(java.base@12.0.1/ProcessHandleImpl.java:138) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1128) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - <0x0000000080de4858> (a java.util.concurrent.ThreadPoolExecutor$Worker) | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][[timer]]" #31 daemon prio=5 os_prio=0 cpu=800.19ms elapsed=4034.11s tid=0x00007f3855a94000 nid=0x5cdf sleeping [0x00007f377f9f6000] | |
| java.lang.Thread.State: TIMED_WAITING (sleeping) | |
| at java.lang.Thread.sleep(java.base@12.0.1/Native Method) | |
| at org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:592) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][scheduler][T#1]" #32 daemon prio=5 os_prio=0 cpu=4668.93ms elapsed=4030.02s tid=0x00007f38561f2000 nid=0x5cef runnable [0x00007f3822fc6000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e59d30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@12.0.1/AbstractQueuedSynchronizer.java:2123) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@12.0.1/ScheduledThreadPoolExecutor.java:1182) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@12.0.1/ScheduledThreadPoolExecutor.java:899) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "Connection evictor" #33 daemon prio=5 os_prio=0 cpu=34.36ms elapsed=4028.58s tid=0x00007f3856375000 nid=0x5cf3 waiting on condition [0x00007f3822dc4000] | |
| java.lang.Thread.State: TIMED_WAITING (sleeping) | |
| at java.lang.Thread.sleep(java.base@12.0.1/Native Method) | |
| at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "ticker-schedule-trigger-engine" #34 daemon prio=5 os_prio=0 cpu=516.81ms elapsed=4028.45s tid=0x00007f38563f6000 nid=0x5cf8 sleeping [0x00007f377d4d9000] | |
| java.lang.Thread.State: TIMED_WAITING (sleeping) | |
| at java.lang.Thread.sleep(java.base@12.0.1/Native Method) | |
| at org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:193) | |
| Locked ownable synchronizers: | |
| - None | |
| "ml-cpp-log-tail-thread" #35 daemon prio=5 os_prio=0 cpu=8.22ms elapsed=4028.38s tid=0x00007f385642b800 nid=0x5cfa runnable [0x00007f377d3d8000] | |
| java.lang.Thread.State: RUNNABLE | |
| at java.io.FileInputStream.readBytes(java.base@12.0.1/Native Method) | |
| at java.io.FileInputStream.read(java.base@12.0.1/FileInputStream.java:251) | |
| at org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:105) | |
| at org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:70) | |
| at org.elasticsearch.xpack.ml.process.NativeController$$Lambda$1978/0x000000080188d440.run(Unknown Source) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][generic][T#1]" #36 daemon prio=5 os_prio=0 cpu=2992.03ms elapsed=4027.46s tid=0x00007f37dc22f000 nid=0x5cfb waiting on condition [0x00007f377cad7000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54190> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][generic][T#2]" #37 daemon prio=5 os_prio=0 cpu=2317.31ms elapsed=4026.45s tid=0x00007f37dc001800 nid=0x5cfd waiting on condition [0x00007f371fffe000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54190> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#1]" #38 daemon prio=5 os_prio=0 cpu=189.73ms elapsed=4026.45s tid=0x00007f3856cc5800 nid=0x5cfe runnable [0x00007f371fefd000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836f9788> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836f96a0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][generic][T#3]" #39 daemon prio=5 os_prio=0 cpu=2994.68ms elapsed=4026.36s tid=0x00007f3856e2a800 nid=0x5cff waiting on condition [0x00007f371fdfc000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54190> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][generic][T#4]" #42 daemon prio=5 os_prio=0 cpu=2540.52ms elapsed=4026.36s tid=0x00007f3710008000 nid=0x5d02 waiting on condition [0x00007f371faf9000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54190> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][clusterApplierService#updateTask][T#1]" #43 daemon prio=5 os_prio=0 cpu=1050.08ms elapsed=4026.35s tid=0x00007f3856e36000 nid=0x5d03 waiting on condition [0x00007f371f9f8000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x00000000836fc098> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@12.0.1/AbstractQueuedSynchronizer.java:2081) | |
| at java.util.concurrent.PriorityBlockingQueue.take(java.base@12.0.1/PriorityBlockingQueue.java:547) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#2]" #44 daemon prio=5 os_prio=0 cpu=239.19ms elapsed=4026.35s tid=0x00007f36fc007000 nid=0x5d04 runnable [0x00007f371f8f7000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fab80> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836faa98> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#3]" #45 daemon prio=5 os_prio=0 cpu=142.84ms elapsed=4026.25s tid=0x00007f36fc01c000 nid=0x5d05 runnable [0x00007f371e7f5000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836f90c8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836f8fe0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#4]" #46 daemon prio=5 os_prio=0 cpu=135.59ms elapsed=4026.25s tid=0x00007f36fc01e000 nid=0x5d06 runnable [0x00007f371e6f4000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fb7f8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fb710> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#5]" #47 daemon prio=5 os_prio=0 cpu=136.92ms elapsed=4026.25s tid=0x00007f36fc01f800 nid=0x5d07 runnable [0x00007f371e5f3000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836f8058> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836f7f70> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#6]" #48 daemon prio=5 os_prio=0 cpu=200.04ms elapsed=4026.25s tid=0x00007f370c003800 nid=0x5d08 runnable [0x00007f371e4f2000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fee38> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fed50> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#7]" #49 daemon prio=5 os_prio=0 cpu=195.07ms elapsed=4026.24s tid=0x00007f36fc021000 nid=0x5d09 runnable [0x00007f371e3f1000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fdca0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fdbb8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#8]" #50 daemon prio=5 os_prio=0 cpu=186.71ms elapsed=4026.24s tid=0x00007f36fc023000 nid=0x5d0a runnable [0x00007f371e2f0000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fe4b0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fe3c8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#9]" #51 daemon prio=5 os_prio=0 cpu=154.31ms elapsed=4026.24s tid=0x00007f36fc025800 nid=0x5d0b runnable [0x00007f371e1ef000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98000e8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9800000> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#10]" #52 daemon prio=5 os_prio=0 cpu=150.68ms elapsed=4026.24s tid=0x00007f36fc027000 nid=0x5d0c runnable [0x00007f371e0ee000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9822ba8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9822ac0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#11]" #53 daemon prio=5 os_prio=0 cpu=184.28ms elapsed=4026.24s tid=0x00007f36fc029000 nid=0x5d0d runnable [0x00007f371dfed000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fd490> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fd3a8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#12]" #54 daemon prio=5 os_prio=0 cpu=159.31ms elapsed=4026.24s tid=0x00007f36fc02b000 nid=0x5d0e runnable [0x00007f371deec000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b980a0a0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9809fb8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#13]" #55 daemon prio=5 os_prio=0 cpu=174.02ms elapsed=4026.24s tid=0x00007f36fc02d000 nid=0x5d0f runnable [0x00007f371ddeb000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98008f8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9800810> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#14]" #56 daemon prio=5 os_prio=0 cpu=189.88ms elapsed=4026.23s tid=0x00007f36fc02f000 nid=0x5d10 runnable [0x00007f371dcea000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98233b8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b98232d0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#15]" #57 daemon prio=5 os_prio=0 cpu=154.15ms elapsed=4026.23s tid=0x00007f36fc030800 nid=0x5d11 runnable [0x00007f371dbe9000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fccb0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fcbc8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#16]" #58 daemon prio=5 os_prio=0 cpu=149.92ms elapsed=4026.23s tid=0x00007f36fc032800 nid=0x5d12 runnable [0x00007f371dae8000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836f9a08> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836f9920> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][masterService#updateTask][T#1]" #59 daemon prio=5 os_prio=0 cpu=399.44ms elapsed=4026.22s tid=0x00007f3710011000 nid=0x5d13 waiting on condition [0x00007f371c9e6000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x00000000b9837ba0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@12.0.1/AbstractQueuedSynchronizer.java:2081) | |
| at java.util.concurrent.PriorityBlockingQueue.take(java.base@12.0.1/PriorityBlockingQueue.java:547) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#17]" #60 daemon prio=5 os_prio=0 cpu=143.14ms elapsed=4026.21s tid=0x00007f370c005000 nid=0x5d14 runnable [0x00007f371c8e5000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fba78> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fb990> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#18]" #61 daemon prio=5 os_prio=0 cpu=161.99ms elapsed=4026.21s tid=0x00007f370c007000 nid=0x5d15 runnable [0x00007f371c7e4000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fd710> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fd628> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#19]" #62 daemon prio=5 os_prio=0 cpu=138.74ms elapsed=4026.20s tid=0x00007f370c009000 nid=0x5d16 runnable [0x00007f371c6e3000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836faee8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fae00> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#20]" #63 daemon prio=5 os_prio=0 cpu=135.21ms elapsed=4026.20s tid=0x00007f370c00b000 nid=0x5d17 runnable [0x00007f371c5e2000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98018c0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fff18> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#21]" #64 daemon prio=5 os_prio=0 cpu=138.83ms elapsed=4026.19s tid=0x00007f370c00c800 nid=0x5d18 runnable [0x00007f371c4e1000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836f82d8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836f81f0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#22]" #65 daemon prio=5 os_prio=0 cpu=132.50ms elapsed=4026.19s tid=0x00007f370c00e800 nid=0x5d19 runnable [0x00007f371c3e0000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fdf20> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fde38> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#23]" #66 daemon prio=5 os_prio=0 cpu=156.70ms elapsed=4026.19s tid=0x00007f370c010800 nid=0x5d1a runnable [0x00007f371c2df000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836ff0b8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836ff060> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#24]" #67 daemon prio=5 os_prio=0 cpu=144.01ms elapsed=4026.19s tid=0x00007f370c012000 nid=0x5d1b runnable [0x00007f371c1de000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fe730> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fe648> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#25]" #68 daemon prio=5 os_prio=0 cpu=147.39ms elapsed=4026.19s tid=0x00007f370c014000 nid=0x5d1c runnable [0x00007f36c0ffb000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fc460> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fc378> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#26]" #69 daemon prio=5 os_prio=0 cpu=142.95ms elapsed=4026.19s tid=0x00007f370c016000 nid=0x5d1d runnable [0x00007f36c0efa000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fdb28> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fd8a8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#27]" #70 daemon prio=5 os_prio=0 cpu=143.68ms elapsed=4026.19s tid=0x00007f370c017800 nid=0x5d1e runnable [0x00007f36c0df9000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fe1a0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fe0b8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#28]" #71 daemon prio=5 os_prio=0 cpu=171.84ms elapsed=4026.18s tid=0x00007f370c019800 nid=0x5d1f runnable [0x00007f36c0cf8000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836f9c88> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836f9ba0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#29]" #72 daemon prio=5 os_prio=0 cpu=159.76ms elapsed=4026.18s tid=0x00007f370c01b800 nid=0x5d20 runnable [0x00007f36c0bf7000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fbcf8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fbc10> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][management][T#1]" #73 daemon prio=5 os_prio=0 cpu=5101.22ms elapsed=4025.99s tid=0x00007f3700011800 nid=0x5d21 waiting on condition [0x00007f36c08f6000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e52ab8> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][ml_utility][T#1]" #74 daemon prio=5 os_prio=0 cpu=0.45ms elapsed=4025.99s tid=0x00007f3700013000 nid=0x5d22 waiting on condition [0x00007f36c07f5000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e52dd0> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][management][T#2]" #75 daemon prio=5 os_prio=0 cpu=4880.34ms elapsed=4025.99s tid=0x00007f3700015000 nid=0x5d23 waiting on condition [0x00007f36c06f4000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e52ab8> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][trigger_engine_scheduler][T#1]" #76 daemon prio=5 os_prio=0 cpu=1.59ms elapsed=4025.98s tid=0x00007f3678002000 nid=0x5d24 waiting on condition [0x00007f36c05f3000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x00000000836f9278> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@12.0.1/AbstractQueuedSynchronizer.java:2123) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@12.0.1/ScheduledThreadPoolExecutor.java:1182) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@12.0.1/ScheduledThreadPoolExecutor.java:899) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#1]" #77 daemon prio=5 os_prio=0 cpu=344.63ms elapsed=4025.95s tid=0x00007f3856efb000 nid=0x5d25 runnable [0x00007f36c04f2000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9801b40> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9801a58> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[keepAlive/7.1.1]" #24 prio=5 os_prio=0 cpu=0.28ms elapsed=4025.93s tid=0x00007f3856efd800 nid=0x5d26 waiting on condition [0x00007f36c03f1000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000081111308> (a java.util.concurrent.CountDownLatch$Sync) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@12.0.1/AbstractQueuedSynchronizer.java:885) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(java.base@12.0.1/AbstractQueuedSynchronizer.java:1039) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@12.0.1/AbstractQueuedSynchronizer.java:1345) | |
| at java.util.concurrent.CountDownLatch.await(java.base@12.0.1/CountDownLatch.java:232) | |
| at org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:80) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "DestroyJavaVM" #78 prio=5 os_prio=0 cpu=11397.40ms elapsed=4025.93s tid=0x00007f3854012800 nid=0x5ca9 waiting on condition [0x0000000000000000] | |
| java.lang.Thread.State: RUNNABLE | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][fetch_shard_started][T#1]" #79 daemon prio=5 os_prio=0 cpu=44.58ms elapsed=4025.75s tid=0x00007f36b8068000 nid=0x5d27 waiting on condition [0x00007f36c02f0000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e55170> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][trigger_engine_scheduler][T#1]" #85 daemon prio=5 os_prio=0 cpu=0.25ms elapsed=4025.39s tid=0x00007f370012f000 nid=0x5d2d waiting on condition [0x00007f36849f7000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x00000000836ff768> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@12.0.1/AbstractQueuedSynchronizer.java:2123) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@12.0.1/ScheduledThreadPoolExecutor.java:1182) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@12.0.1/ScheduledThreadPoolExecutor.java:899) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][refresh][T#1]" #86 daemon prio=5 os_prio=0 cpu=386239.10ms elapsed=4023.77s tid=0x00007f37dc67a000 nid=0x5d2e waiting on condition [0x00007f3822ec5000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54820> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#2]" #92 daemon prio=5 os_prio=0 cpu=283.87ms elapsed=4013.50s tid=0x00007f3670001000 nid=0x5d34 runnable [0x00007f3658e4f000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b99e0410> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9808f98> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#3]" #93 daemon prio=5 os_prio=0 cpu=394.67ms elapsed=4013.50s tid=0x00007f3670003000 nid=0x5d35 runnable [0x00007f3658d4e000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9804cc8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fc7c0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#4]" #94 daemon prio=5 os_prio=0 cpu=240.72ms elapsed=4013.50s tid=0x00007f3670005000 nid=0x5d36 runnable [0x00007f3658c4d000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000ba4e9a88> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9804428> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#5]" #95 daemon prio=5 os_prio=0 cpu=308.14ms elapsed=4013.50s tid=0x00007f3670006800 nid=0x5d37 runnable [0x00007f3658b4c000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9d84658> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836f9cd8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#6]" #96 daemon prio=5 os_prio=0 cpu=227.57ms elapsed=4013.50s tid=0x00007f3670008800 nid=0x5d38 runnable [0x00007f3658a4b000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836ffab0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836ff9c8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#7]" #97 daemon prio=5 os_prio=0 cpu=326.77ms elapsed=4013.50s tid=0x00007f367000a800 nid=0x5d39 runnable [0x00007f365894a000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000ba1c7d40> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000ba1b92f0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#8]" #98 daemon prio=5 os_prio=0 cpu=272.50ms elapsed=4013.50s tid=0x00007f367000c000 nid=0x5d3a runnable [0x00007f3658849000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836f95c0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836f94d8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#9]" #99 daemon prio=5 os_prio=0 cpu=350.55ms elapsed=4013.49s tid=0x00007f367000e000 nid=0x5d3b runnable [0x00007f3658748000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9885380> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836f84a0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#10]" #100 daemon prio=5 os_prio=0 cpu=270.02ms elapsed=4013.49s tid=0x00007f3670010000 nid=0x5d3c runnable [0x00007f3658647000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836ff500> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836ff418> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#11]" #101 daemon prio=5 os_prio=0 cpu=266.42ms elapsed=4013.49s tid=0x00007f3670012000 nid=0x5d3d runnable [0x00007f3658546000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836feb98> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836feab0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#12]" #102 daemon prio=5 os_prio=0 cpu=342.61ms elapsed=4013.49s tid=0x00007f3670013800 nid=0x5d3e runnable [0x00007f3658445000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98014d8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b98013f0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#3]" #103 daemon prio=5 os_prio=0 cpu=16480.69ms elapsed=4008.66s tid=0x00007f3714041000 nid=0x5d3f waiting on condition [0x00007f38230c7000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#2]" #104 daemon prio=5 os_prio=0 cpu=16448.49ms elapsed=4008.66s tid=0x00007f37240d9800 nid=0x5d40 waiting on condition [0x00007f38238f6000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#1]" #105 daemon prio=5 os_prio=0 cpu=16559.78ms elapsed=4008.66s tid=0x00007f37100c8800 nid=0x5d41 waiting on condition [0x00007f3658144000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][management][T#3]" #106 daemon prio=5 os_prio=0 cpu=6162.61ms elapsed=3998.50s tid=0x00007f3624020000 nid=0x5d42 waiting on condition [0x00007f363cffb000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e52ab8> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#30]" #107 daemon prio=5 os_prio=0 cpu=173.46ms elapsed=3995.43s tid=0x00007f370c01d000 nid=0x5d43 runnable [0x00007f363cefa000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836ffd30> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836ffc48> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#31]" #108 daemon prio=5 os_prio=0 cpu=11014.60ms elapsed=3995.34s tid=0x00007f370c01f000 nid=0x5d44 runnable [0x00007f363cdf9000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000ba866a48> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9804e60> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#32]" #109 daemon prio=5 os_prio=0 cpu=11205.46ms elapsed=3995.34s tid=0x00007f370c021000 nid=0x5d45 runnable [0x00007f363ccf8000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98bbea8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b988e318> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#33]" #110 daemon prio=5 os_prio=0 cpu=10772.31ms elapsed=3995.34s tid=0x00007f370c022800 nid=0x5d46 runnable [0x00007f363cbf7000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b995d350> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b980b2c8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#34]" #111 daemon prio=5 os_prio=0 cpu=1508.95ms elapsed=3995.33s tid=0x00007f370c024800 nid=0x5d47 runnable [0x00007f363caf6000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9870240> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b980b138> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#35]" #112 daemon prio=5 os_prio=0 cpu=148.87ms elapsed=3995.33s tid=0x00007f370c026800 nid=0x5d48 runnable [0x00007f363c9f5000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9801758> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9801670> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#36]" #113 daemon prio=5 os_prio=0 cpu=320.18ms elapsed=3995.33s tid=0x00007f370c028800 nid=0x5d49 runnable [0x00007f363c8f4000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9856e10> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b980cde0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#37]" #114 daemon prio=5 os_prio=0 cpu=302.57ms elapsed=3995.33s tid=0x00007f370c02a000 nid=0x5d4a runnable [0x00007f363c7f3000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98006f8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9800610> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#38]" #115 daemon prio=5 os_prio=0 cpu=1064.65ms elapsed=3995.33s tid=0x00007f370c02c000 nid=0x5d4b runnable [0x00007f363c6f2000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000836fa750> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fa668> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#39]" #116 daemon prio=5 os_prio=0 cpu=1069.61ms elapsed=3995.33s tid=0x00007f370c036000 nid=0x5d4c runnable [0x00007f363c5f1000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b988c0f0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9801ef0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#40]" #117 daemon prio=5 os_prio=0 cpu=1049.22ms elapsed=3995.33s tid=0x00007f370c038000 nid=0x5d4d runnable [0x00007f363c4f0000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b980d680> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000836fd1b8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#41]" #118 daemon prio=5 os_prio=0 cpu=1057.93ms elapsed=3995.33s tid=0x00007f370c03a000 nid=0x5d4e runnable [0x00007f363c3ef000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b980e038> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9800718> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#42]" #119 daemon prio=5 os_prio=0 cpu=1049.86ms elapsed=3995.32s tid=0x00007f370c03c000 nid=0x5d4f runnable [0x00007f363c2ee000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b999ccb8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9801778> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#43]" #120 daemon prio=5 os_prio=0 cpu=1049.12ms elapsed=3995.32s tid=0x00007f370c03e000 nid=0x5d50 runnable [0x00007f363c1ed000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b982e678> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b982e590> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#44]" #121 daemon prio=5 os_prio=0 cpu=12969.30ms elapsed=3995.27s tid=0x00007f37100ca000 nid=0x5d51 runnable [0x00007f362effd000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b980a320> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b980a238> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#45]" #122 daemon prio=5 os_prio=0 cpu=11864.17ms elapsed=3995.27s tid=0x00007f37100ba000 nid=0x5d52 runnable [0x00007f362eefc000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b980a5a0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b980a4b8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#46]" #123 daemon prio=5 os_prio=0 cpu=12038.22ms elapsed=3995.27s tid=0x00007f37100bb800 nid=0x5d53 runnable [0x00007f362edfb000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9fed9b8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b980d818> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#47]" #124 daemon prio=5 os_prio=0 cpu=1281.08ms elapsed=3995.27s tid=0x00007f37100bd800 nid=0x5d54 runnable [0x00007f362ecfa000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a87c90> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b980a738> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#48]" #125 daemon prio=5 os_prio=0 cpu=184.45ms elapsed=3995.27s tid=0x00007f37100bf000 nid=0x5d55 runnable [0x00007f362ebf9000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9822e28> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9822d40> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#49]" #126 daemon prio=5 os_prio=0 cpu=170.78ms elapsed=3995.27s tid=0x00007f37100cb800 nid=0x5d56 runnable [0x00007f362eaf8000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9823638> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9823550> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#50]" #127 daemon prio=5 os_prio=0 cpu=185.41ms elapsed=3995.26s tid=0x00007f37100cd800 nid=0x5d57 runnable [0x00007f362e9f7000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98230a8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9822fc0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#51]" #128 daemon prio=5 os_prio=0 cpu=1658.36ms elapsed=3995.26s tid=0x00007f37100cf000 nid=0x5d58 runnable [0x00007f362e8f6000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98238b8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b98237d0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#52]" #129 daemon prio=5 os_prio=0 cpu=1652.18ms elapsed=3995.26s tid=0x00007f37100d1000 nid=0x5d59 runnable [0x00007f362e7f5000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b990e230> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9823240> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#53]" #130 daemon prio=5 os_prio=0 cpu=1644.89ms elapsed=3995.26s tid=0x00007f37100d2800 nid=0x5d5a runnable [0x00007f362e6f4000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b982e6f0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b982e698> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#54]" #131 daemon prio=5 os_prio=0 cpu=1719.29ms elapsed=3995.26s tid=0x00007f3710115800 nid=0x5d5b runnable [0x00007f362e5f3000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98a6098> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9823a50> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#55]" #132 daemon prio=5 os_prio=0 cpu=1655.51ms elapsed=3995.26s tid=0x00007f3710118000 nid=0x5d5c runnable [0x00007f362e4f2000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98cfe70> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b982e878> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][transport_worker][T#56]" #133 daemon prio=5 os_prio=0 cpu=1700.73ms elapsed=3995.26s tid=0x00007f3710119800 nid=0x5d5d runnable [0x00007f362e3f1000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b987bbd0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b987bae8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][fetch_shard_store][T#6]" #134 daemon prio=5 os_prio=0 cpu=10.70ms elapsed=3995.02s tid=0x00007f36b8048800 nid=0x5d5e waiting on condition [0x00007f362df28000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e533f8> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][management][T#4]" #135 daemon prio=5 os_prio=0 cpu=4977.57ms elapsed=3994.77s tid=0x00007f367c00d000 nid=0x5d5f waiting on condition [0x00007f362e2f0000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e52ab8> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][management][T#5]" #136 daemon prio=5 os_prio=0 cpu=5592.16ms elapsed=3964.76s tid=0x00007f35f8002800 nid=0x5d69 waiting on condition [0x00007f371fcfb000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e52ab8> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#13]" #137 daemon prio=5 os_prio=0 cpu=200.15ms elapsed=3934.76s tid=0x00007f3670015000 nid=0x5d6c runnable [0x00007f371fbfa000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a4f390> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b987c108> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#14]" #138 daemon prio=5 os_prio=0 cpu=206.41ms elapsed=3928.41s tid=0x00007f3670016800 nid=0x5d6d runnable [0x00007f36848f6000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b998faa0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b998f9b8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][generic][T#5]" #139 daemon prio=5 os_prio=0 cpu=1780.93ms elapsed=3928.16s tid=0x00007f3724061000 nid=0x5d70 waiting on condition [0x00007f3668efa000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54190> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#15]" #140 daemon prio=5 os_prio=0 cpu=11350.52ms elapsed=3927.41s tid=0x00007f3670017800 nid=0x5d71 runnable [0x00007f3668bf9000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b99c9470> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b999ce50> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#16]" #141 daemon prio=5 os_prio=0 cpu=5568.79ms elapsed=3927.41s tid=0x00007f3670019800 nid=0x5d72 runnable [0x00007f3668af8000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b998fec8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b998fde0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#5]" #142 daemon prio=5 os_prio=0 cpu=16874.22ms elapsed=3927.36s tid=0x00007f356c016000 nid=0x5d73 waiting on condition [0x00007f36689f7000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#4]" #143 daemon prio=5 os_prio=0 cpu=16552.34ms elapsed=3927.36s tid=0x00007f3570013800 nid=0x5d74 waiting on condition [0x00007f366866c000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#6]" #144 daemon prio=5 os_prio=0 cpu=16891.78ms elapsed=3927.36s tid=0x00007f356c01c800 nid=0x5d75 waiting on condition [0x00007f366856b000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#7]" #145 daemon prio=5 os_prio=0 cpu=16677.54ms elapsed=3927.36s tid=0x00007f3570021800 nid=0x5d76 waiting on condition [0x00007f366846a000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#8]" #146 daemon prio=5 os_prio=0 cpu=16763.07ms elapsed=3927.36s tid=0x00007f356c01e800 nid=0x5d77 waiting on condition [0x00007f3668369000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#9]" #147 daemon prio=5 os_prio=0 cpu=16790.47ms elapsed=3927.36s tid=0x00007f3570023000 nid=0x5d78 waiting on condition [0x00007f3660ffb000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#11]" #149 daemon prio=5 os_prio=0 cpu=16639.20ms elapsed=3927.35s tid=0x00007f35e8011000 nid=0x5d79 waiting on condition [0x00007f3660efa000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#12]" #150 daemon prio=5 os_prio=0 cpu=16801.87ms elapsed=3927.35s tid=0x00007f35e4011000 nid=0x5d7a waiting on condition [0x00007f3660df9000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#10]" #148 daemon prio=5 os_prio=0 cpu=16524.87ms elapsed=3927.35s tid=0x00007f35f0011000 nid=0x5d7b waiting on condition [0x00007f3660cf8000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#13]" #151 daemon prio=5 os_prio=0 cpu=16759.23ms elapsed=3927.35s tid=0x00007f35e8013000 nid=0x5d7c waiting on condition [0x00007f3660bf7000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#14]" #152 daemon prio=5 os_prio=0 cpu=16497.34ms elapsed=3927.35s tid=0x00007f35e4013000 nid=0x5d7d waiting on condition [0x00007f3660af6000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#15]" #153 daemon prio=5 os_prio=0 cpu=16745.38ms elapsed=3927.35s tid=0x00007f35f0013000 nid=0x5d7e waiting on condition [0x00007f36609f5000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#16]" #154 daemon prio=5 os_prio=0 cpu=16813.70ms elapsed=3927.34s tid=0x00007f35e8015000 nid=0x5d7f waiting on condition [0x00007f36608f4000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#17]" #155 daemon prio=5 os_prio=0 cpu=16719.85ms elapsed=3927.34s tid=0x00007f35e4015000 nid=0x5d80 waiting on condition [0x00007f36607f3000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#18]" #156 daemon prio=5 os_prio=0 cpu=16827.93ms elapsed=3927.34s tid=0x00007f35f0015000 nid=0x5d81 waiting on condition [0x00007f36606f2000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#19]" #157 daemon prio=5 os_prio=0 cpu=16585.45ms elapsed=3927.19s tid=0x00007f35e4017000 nid=0x5d82 waiting on condition [0x00007f366039a000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#21]" #159 daemon prio=5 os_prio=0 cpu=16731.69ms elapsed=3927.19s tid=0x00007f35f0017000 nid=0x5d83 waiting on condition [0x00007f3660299000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#20]" #158 daemon prio=5 os_prio=0 cpu=16748.17ms elapsed=3927.18s tid=0x00007f35e8017000 nid=0x5d84 waiting on condition [0x00007f3660198000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#22]" #160 daemon prio=5 os_prio=0 cpu=16797.37ms elapsed=3927.18s tid=0x00007f35e4018800 nid=0x5d85 waiting on condition [0x00007f3658ffb000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#23]" #161 daemon prio=5 os_prio=0 cpu=16808.72ms elapsed=3927.18s tid=0x00007f35e8018800 nid=0x5d86 waiting on condition [0x00007f362e1ef000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#24]" #162 daemon prio=5 os_prio=0 cpu=16735.44ms elapsed=3927.17s tid=0x00007f35f0018800 nid=0x5d87 waiting on condition [0x00007f362e0ee000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#25]" #163 daemon prio=5 os_prio=0 cpu=16702.76ms elapsed=3927.17s tid=0x00007f35e801a800 nid=0x5d88 waiting on condition [0x00007f362de27000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#27]" #165 daemon prio=5 os_prio=0 cpu=16676.87ms elapsed=3927.17s tid=0x00007f35e401a000 nid=0x5d89 waiting on condition [0x00007f362dd26000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#26]" #164 daemon prio=5 os_prio=0 cpu=16820.32ms elapsed=3927.17s tid=0x00007f35f001a800 nid=0x5d8a waiting on condition [0x00007f362dc25000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#28]" #166 daemon prio=5 os_prio=0 cpu=16932.03ms elapsed=3927.16s tid=0x00007f35e401c000 nid=0x5d8b waiting on condition [0x00007f362db24000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#29]" #167 daemon prio=5 os_prio=0 cpu=16657.64ms elapsed=3927.16s tid=0x00007f35e801c800 nid=0x5d8c waiting on condition [0x00007f362da23000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#30]" #168 daemon prio=5 os_prio=0 cpu=16816.39ms elapsed=3927.09s tid=0x00007f35f001c000 nid=0x5d8d waiting on condition [0x00007f362d922000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#31]" #169 daemon prio=5 os_prio=0 cpu=16516.93ms elapsed=3927.08s tid=0x00007f35e401e000 nid=0x5d8e waiting on condition [0x00007f362d821000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][write][T#32]" #170 daemon prio=5 os_prio=0 cpu=16719.59ms elapsed=3927.07s tid=0x00007f35e801e800 nid=0x5d8f waiting on condition [0x00007f362d720000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e520f8> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#17]" #456 daemon prio=5 os_prio=0 cpu=197.08ms elapsed=3730.91s tid=0x00007f367001b800 nid=0x5ed9 runnable [0x00007f362d61f000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000ba15db50> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000ba15da00> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#18]" #460 daemon prio=5 os_prio=0 cpu=182.08ms elapsed=3728.05s tid=0x00007f367001c800 nid=0x5edd runnable [0x00007f3668ffb000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000ba150768> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000ba150618> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#19]" #461 daemon prio=5 os_prio=0 cpu=187.87ms elapsed=3728.05s tid=0x00007f367001e000 nid=0x5ede runnable [0x00007f362d31e000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000ba134660> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000ba134510> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#20]" #462 daemon prio=5 os_prio=0 cpu=204.72ms elapsed=3713.05s tid=0x00007f367001f000 nid=0x5edf runnable [0x00007f3684efa000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9fa01b0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9fa0060> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#21]" #463 daemon prio=5 os_prio=0 cpu=248.94ms elapsed=3713.05s tid=0x00007f3670020000 nid=0x5ee0 runnable [0x00007f3684df9000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9f9fbe0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9f9fa90> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#22]" #465 daemon prio=5 os_prio=0 cpu=187.71ms elapsed=3680.78s tid=0x00007f3670021000 nid=0x5ee2 runnable [0x00007f36c01ef000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9f99500> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9f993b0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#23]" #502 daemon prio=5 os_prio=0 cpu=260.68ms elapsed=3398.53s tid=0x00007f3670022000 nid=0x5f12 runnable [0x00007f36843f9000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9f70d58> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9f70c08> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#24]" #503 daemon prio=5 os_prio=0 cpu=235.23ms elapsed=3398.53s tid=0x00007f3670023800 nid=0x5f13 runnable [0x00007f3668188000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9f65f40> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9f5b858> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#25]" #504 daemon prio=5 os_prio=0 cpu=192.23ms elapsed=3398.53s tid=0x00007f3670025800 nid=0x5f14 runnable [0x00007f36842f8000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9f5b1a0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9f4f468> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#26]" #505 daemon prio=5 os_prio=0 cpu=154.99ms elapsed=3380.07s tid=0x00007f3670026800 nid=0x5f16 runnable [0x00007f3684ffb000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9f4edb0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9f40808> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#27]" #519 daemon prio=5 os_prio=0 cpu=130.04ms elapsed=3279.82s tid=0x00007f3670027800 nid=0x5f28 runnable [0x00007f36841f7000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9f40150> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9f29430> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#28]" #526 daemon prio=5 os_prio=0 cpu=138.76ms elapsed=3236.62s tid=0x00007f3670028000 nid=0x5f42 runnable [0x00007f377c9d6000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9f28d78> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9eb7020> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#29]" #527 daemon prio=5 os_prio=0 cpu=129.52ms elapsed=3236.60s tid=0x00007f3670029800 nid=0x5f44 runnable [0x00007f3684cf8000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9eb6968> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9eaefb0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#30]" #533 daemon prio=5 os_prio=0 cpu=138.42ms elapsed=3179.57s tid=0x00007f367002a800 nid=0x617e runnable [0x00007f36847f5000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9eae8f8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9ea4bf0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#31]" #540 daemon prio=5 os_prio=0 cpu=144.57ms elapsed=3129.45s tid=0x00007f367002c000 nid=0x6186 runnable [0x00007f362cef5000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9ea4538> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9de69d8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#32]" #548 daemon prio=5 os_prio=0 cpu=147.65ms elapsed=3079.34s tid=0x00007f367002c800 nid=0x618e runnable [0x00007f362c96d000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9de6320> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9ddda68> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#33]" #553 daemon prio=5 os_prio=0 cpu=133.13ms elapsed=3029.23s tid=0x00007f367002e000 nid=0x61b6 runnable [0x00007f34f3a42000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9d9eda0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9d9ec50> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#34]" #939 daemon prio=5 os_prio=0 cpu=124.18ms elapsed=2855.52s tid=0x00007f367002f000 nid=0x6378 runnable [0x00007f34f3941000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9d92800> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9d926b0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#35]" #940 daemon prio=5 os_prio=0 cpu=171.93ms elapsed=2855.52s tid=0x00007f3670038800 nid=0x6379 runnable [0x00007f362cdf4000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9d84258> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9d84108> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#36]" #941 daemon prio=5 os_prio=0 cpu=117.35ms elapsed=2855.52s tid=0x00007f3670041800 nid=0x637a runnable [0x00007f362c86c000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9d7aca0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9d7ab50> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#37]" #1453 daemon prio=5 os_prio=0 cpu=134.77ms elapsed=2647.52s tid=0x00007f3670030800 nid=0x6580 runnable [0x00007f362d05b000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9c7d178> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9c7d028> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#38]" #1454 daemon prio=5 os_prio=0 cpu=137.25ms elapsed=2647.52s tid=0x00007f3670031800 nid=0x6581 runnable [0x00007f362c76b000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9c7cba8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9c7ca58> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#39]" #1455 daemon prio=5 os_prio=0 cpu=176.10ms elapsed=2647.52s tid=0x00007f3670033000 nid=0x6582 runnable [0x00007f34f3edb000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9c75e40> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9c75cf0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][flush][T#1]" #2008 daemon prio=5 os_prio=0 cpu=39.20ms elapsed=2420.78s tid=0x00007f3540001000 nid=0x67ad waiting on condition [0x00007f34f2e76000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53108> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#40]" #2208 daemon prio=5 os_prio=0 cpu=150.40ms elapsed=2338.52s tid=0x00007f3670034000 nid=0x687a runnable [0x00007f34f315e000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9c68678> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9c68528> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#41]" #2287 daemon prio=5 os_prio=0 cpu=107.90ms elapsed=2306.53s tid=0x00007f3670035000 nid=0x68ca runnable [0x00007f362d21d000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9be22b8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9ba8100> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#42]" #3035 daemon prio=5 os_prio=0 cpu=104.42ms elapsed=1986.52s tid=0x00007f3670035800 nid=0x6bbc runnable [0x00007f34f3ffe000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9ba7a48> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9b9adb0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#43]" #3036 daemon prio=5 os_prio=0 cpu=155.79ms elapsed=1986.52s tid=0x00007f3670037000 nid=0x6bbd runnable [0x00007f34f3615000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9b9a6f8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9b90e20> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#44]" #3183 daemon prio=5 os_prio=0 cpu=93.97ms elapsed=1922.52s tid=0x00007f3670039800 nid=0x6c51 runnable [0x00007f362c4d4000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9b90768> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9b870c8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#45]" #3295 daemon prio=5 os_prio=0 cpu=142.17ms elapsed=1876.52s tid=0x00007f367003a000 nid=0x6cc1 runnable [0x00007f34f2b4e000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9b86a10> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9b79f20> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#46]" #3372 daemon prio=5 os_prio=0 cpu=93.91ms elapsed=1844.53s tid=0x00007f367003b000 nid=0x6d0e runnable [0x00007f34f2d75000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9b79868> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9b6e560> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#47]" #3722 daemon prio=5 os_prio=0 cpu=132.19ms elapsed=1700.52s tid=0x00007f367003c800 nid=0x6e78 runnable [0x00007f34f0b81000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9b6dea8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9b44fc8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#48]" #3840 daemon prio=5 os_prio=0 cpu=132.24ms elapsed=1652.53s tid=0x00007f367003d800 nid=0x6ef1 runnable [0x00007f362c5de000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9b44910> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9a9a238> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#49]" #4201 daemon prio=5 os_prio=0 cpu=138.78ms elapsed=1492.52s tid=0x00007f367003e800 nid=0x705d runnable [0x00007f34f0cdc000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a99b80> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9a8f668> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#50]" #4311 daemon prio=5 os_prio=0 cpu=110.62ms elapsed=1444.52s tid=0x00007f367003f800 nid=0x70cc runnable [0x00007f34f3b6b000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a86858> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9a86708> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#51]" #4312 daemon prio=5 os_prio=0 cpu=112.21ms elapsed=1444.52s tid=0x00007f3670042800 nid=0x70cd runnable [0x00007f34f3840000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a7dd08> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9a7dbb8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#52]" #4334 daemon prio=5 os_prio=0 cpu=73.97ms elapsed=1435.28s tid=0x00007f3670043800 nid=0x70f5 runnable [0x00007f34f373f000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a76fe0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9a76e90> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#53]" #4335 daemon prio=5 os_prio=0 cpu=81.36ms elapsed=1435.28s tid=0x00007f3670044800 nid=0x70f7 runnable [0x00007f34f2c74000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a6de48> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9a6dcf8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#54]" #5069 daemon prio=5 os_prio=0 cpu=88.80ms elapsed=1127.52s tid=0x00007f3670045800 nid=0x761a runnable [0x00007f34f338e000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a5d7b0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9a5d660> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#55]" #5070 daemon prio=5 os_prio=0 cpu=83.96ms elapsed=1127.52s tid=0x00007f3670047000 nid=0x761b runnable [0x00007f34f2449000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a5d1e0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9a5d090> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][refresh][T#2]" #5427 daemon prio=5 os_prio=0 cpu=14754.41ms elapsed=981.55s tid=0x00007f35bc015000 nid=0x77a9 waiting on condition [0x00007f362ca88000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54820> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][refresh][T#4]" #5429 daemon prio=5 os_prio=0 cpu=14799.82ms elapsed=981.55s tid=0x00007f35cc012800 nid=0x77aa waiting on condition [0x00007f34f328d000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54820> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][refresh][T#3]" #5428 daemon prio=5 os_prio=0 cpu=14556.27ms elapsed=981.55s tid=0x00007f35c8012800 nid=0x77ab waiting on condition [0x00007f34f18a9000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54820> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][refresh][T#5]" #5430 daemon prio=5 os_prio=0 cpu=14824.29ms elapsed=981.45s tid=0x00007f35c4013800 nid=0x77ac waiting on condition [0x00007f34f2760000] | |
| java.lang.Thread.State: TIMED_WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54820> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) | |
| at java.util.concurrent.locks.LockSupport.parkNanos(java.base@12.0.1/LockSupport.java:235) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:740) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.poll(java.base@12.0.1/LinkedTransferQueue.java:1374) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1053) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#56]" #5434 daemon prio=5 os_prio=0 cpu=63.47ms elapsed=979.54s tid=0x00007f3670048000 nid=0x77b0 runnable [0x00007f34f1a62000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a58f48> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9a58df8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#57]" #5441 daemon prio=5 os_prio=0 cpu=51.51ms elapsed=977.53s tid=0x00007f3670048800 nid=0x77b7 runnable [0x00007f34f17a8000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9a4b1c8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9a4b078> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#58]" #5442 daemon prio=5 os_prio=0 cpu=64.37ms elapsed=976.93s tid=0x00007f367004a000 nid=0x77b8 runnable [0x00007f34f2919000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b988abd0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b987f158> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#59]" #5446 daemon prio=5 os_prio=0 cpu=74.17ms elapsed=976.25s tid=0x00007f367004b800 nid=0x77bc runnable [0x00007f34f16a7000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b987eaa0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b986e5a0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#60]" #5448 daemon prio=5 os_prio=0 cpu=73.90ms elapsed=974.95s tid=0x00007f367004c000 nid=0x77be runnable [0x00007f34f1472000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b986dee8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b98666e0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#61]" #5449 daemon prio=5 os_prio=0 cpu=88.45ms elapsed=974.37s tid=0x00007f367004d800 nid=0x77bf runnable [0x00007f34f0e97000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b9866028> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9857ca8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#62]" #5456 daemon prio=5 os_prio=0 cpu=97.78ms elapsed=972.40s tid=0x00007f367004f800 nid=0x77c6 runnable [0x00007f34f0881000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b98575f0> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b983c7e8> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#63]" #5464 daemon prio=5 os_prio=0 cpu=108.90ms elapsed=970.07s tid=0x00007f3670050000 nid=0x77ce runnable [0x00007f34bfd8a000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b983c130> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b98304b0> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][http_server_worker][T#64]" #5469 daemon prio=5 os_prio=0 cpu=103.78ms elapsed=967.71s tid=0x00007f3670051000 nid=0x77d3 runnable [0x00007f34f01b0000] | |
| java.lang.Thread.State: RUNNABLE | |
| at sun.nio.ch.EPoll.wait(java.base@12.0.1/Native Method) | |
| at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@12.0.1/EPollSelectorImpl.java:120) | |
| at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@12.0.1/SelectorImpl.java:124) | |
| - locked <0x00000000b982fdf8> (a sun.nio.ch.Util$2) | |
| - locked <0x00000000b9805b18> (a sun.nio.ch.EPollSelectorImpl) | |
| at sun.nio.ch.SelectorImpl.select(java.base@12.0.1/SelectorImpl.java:136) | |
| at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765) | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413) | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#1]" #6238 daemon prio=5 os_prio=0 cpu=36.98ms elapsed=298.39s tid=0x00007f3638088800 nid=0x7af7 waiting on condition [0x00007f34be5b6000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#2]" #6239 daemon prio=5 os_prio=0 cpu=44.62ms elapsed=298.39s tid=0x00007f36380a2000 nid=0x7af8 waiting on condition [0x00007f34bf664000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][listener][T#1]" #6241 daemon prio=5 os_prio=0 cpu=0.61ms elapsed=297.38s tid=0x00007f37e0186000 nid=0x7afa waiting on condition [0x00007f34bd8bb000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54b10> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#3]" #6242 daemon prio=5 os_prio=0 cpu=13.57ms elapsed=297.38s tid=0x00007f3428011000 nid=0x7afb waiting on condition [0x00007f34bdb2f000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][listener][T#2]" #6243 daemon prio=5 os_prio=0 cpu=0.44ms elapsed=297.35s tid=0x00007f37e0181800 nid=0x7afc waiting on condition [0x00007f34bec7f000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e54b10> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#4]" #6244 daemon prio=5 os_prio=0 cpu=11.26ms elapsed=297.35s tid=0x00007f343c001800 nid=0x7afd waiting on condition [0x00007f34bd3bd000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#5]" #6245 daemon prio=5 os_prio=0 cpu=6.89ms elapsed=297.33s tid=0x00007f342c006800 nid=0x7afe waiting on condition [0x00007f34bc156000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#6]" #6246 daemon prio=5 os_prio=0 cpu=18.12ms elapsed=297.32s tid=0x00007f3420004000 nid=0x7aff waiting on condition [0x00007f34a0186000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#7]" #6247 daemon prio=5 os_prio=0 cpu=4.89ms elapsed=297.32s tid=0x00007f342000a000 nid=0x7b00 waiting on condition [0x00007f3482aee000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#8]" #6248 daemon prio=5 os_prio=0 cpu=5.46ms elapsed=297.32s tid=0x00007f342000c800 nid=0x7b01 waiting on condition [0x00007f34829ed000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#10]" #6250 daemon prio=5 os_prio=0 cpu=4.77ms elapsed=186.29s tid=0x00007f3584012000 nid=0x7b07 waiting on condition [0x00007f34828ec000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#9]" #6249 daemon prio=5 os_prio=0 cpu=4.38ms elapsed=186.29s tid=0x00007f358c01c000 nid=0x7b08 waiting on condition [0x00007f34827eb000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#11]" #6251 daemon prio=5 os_prio=0 cpu=1.31ms elapsed=185.46s tid=0x00007f3598011800 nid=0x7b09 waiting on condition [0x00007f34826ea000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#12]" #6252 daemon prio=5 os_prio=0 cpu=0.62ms elapsed=185.46s tid=0x00007f3408001000 nid=0x7b0a waiting on condition [0x00007f34825e9000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "elasticsearch[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][search][T#13]" #6253 daemon prio=5 os_prio=0 cpu=0.76ms elapsed=185.46s tid=0x00007f3408003800 nid=0x7b0b waiting on condition [0x00007f34812a4000] | |
| java.lang.Thread.State: WAITING (parking) | |
| at jdk.internal.misc.Unsafe.park(java.base@12.0.1/Native Method) | |
| - parking to wait for <0x0000000080e53b20> (a java.util.concurrent.LinkedTransferQueue) | |
| at java.util.concurrent.locks.LockSupport.park(java.base@12.0.1/LockSupport.java:194) | |
| at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@12.0.1/LinkedTransferQueue.java:743) | |
| at java.util.concurrent.LinkedTransferQueue.xfer(java.base@12.0.1/LinkedTransferQueue.java:684) | |
| at java.util.concurrent.LinkedTransferQueue.take(java.base@12.0.1/LinkedTransferQueue.java:1366) | |
| at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165) | |
| at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@12.0.1/ThreadPoolExecutor.java:1054) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@12.0.1/ThreadPoolExecutor.java:1114) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@12.0.1/ThreadPoolExecutor.java:628) | |
| at java.lang.Thread.run(java.base@12.0.1/Thread.java:835) | |
| Locked ownable synchronizers: | |
| - None | |
| "Attach Listener" #6254 daemon prio=9 os_prio=0 cpu=0.89ms elapsed=11.04s tid=0x00007f3808001000 nid=0x7b70 waiting on condition [0x0000000000000000] | |
| java.lang.Thread.State: RUNNABLE | |
| Locked ownable synchronizers: | |
| - None | |
| "VM Thread" os_prio=0 cpu=10375.86ms elapsed=4038.07s tid=0x00007f3854794000 nid=0x5caf runnable | |
| "GC Thread#0" os_prio=0 cpu=6897.76ms elapsed=4038.56s tid=0x00007f3854067000 nid=0x5caa runnable | |
| "GC Thread#1" os_prio=0 cpu=6431.64ms elapsed=4035.87s tid=0x00007f3818001000 nid=0x5cc4 runnable | |
| "GC Thread#2" os_prio=0 cpu=6410.29ms elapsed=4035.86s tid=0x00007f3818002800 nid=0x5cc5 runnable | |
| "GC Thread#3" os_prio=0 cpu=6516.36ms elapsed=4035.86s tid=0x00007f3818004000 nid=0x5cc6 runnable | |
| "GC Thread#4" os_prio=0 cpu=6421.30ms elapsed=4035.86s tid=0x00007f3818005800 nid=0x5cc7 runnable | |
| "GC Thread#5" os_prio=0 cpu=6470.13ms elapsed=4035.86s tid=0x00007f3818007000 nid=0x5cc8 runnable | |
| "GC Thread#6" os_prio=0 cpu=6471.18ms elapsed=4035.86s tid=0x00007f3818008800 nid=0x5cc9 runnable | |
| "GC Thread#7" os_prio=0 cpu=6496.36ms elapsed=4035.86s tid=0x00007f381800a000 nid=0x5cca runnable | |
| "GC Thread#8" os_prio=0 cpu=6553.81ms elapsed=4035.86s tid=0x00007f381800c000 nid=0x5ccb runnable | |
| "GC Thread#9" os_prio=0 cpu=6450.00ms elapsed=4035.86s tid=0x00007f381800d800 nid=0x5ccc runnable | |
| "GC Thread#10" os_prio=0 cpu=6380.98ms elapsed=4035.86s tid=0x00007f381800f800 nid=0x5ccd runnable | |
| "GC Thread#11" os_prio=0 cpu=6451.06ms elapsed=4035.86s tid=0x00007f3818011000 nid=0x5cce runnable | |
| "GC Thread#12" os_prio=0 cpu=6408.25ms elapsed=4035.86s tid=0x00007f3818013000 nid=0x5ccf runnable | |
| "GC Thread#13" os_prio=0 cpu=6418.31ms elapsed=4035.86s tid=0x00007f3818015000 nid=0x5cd0 runnable | |
| "GC Thread#14" os_prio=0 cpu=6451.71ms elapsed=4035.86s tid=0x00007f3818016800 nid=0x5cd1 runnable | |
| "GC Thread#15" os_prio=0 cpu=6363.63ms elapsed=4035.86s tid=0x00007f3818018800 nid=0x5cd2 runnable | |
| "GC Thread#16" os_prio=0 cpu=6388.47ms elapsed=4035.86s tid=0x00007f381801a000 nid=0x5cd3 runnable | |
| "GC Thread#17" os_prio=0 cpu=6403.40ms elapsed=4035.86s tid=0x00007f381801c000 nid=0x5cd4 runnable | |
| "GC Thread#18" os_prio=0 cpu=6468.75ms elapsed=4035.86s tid=0x00007f381801d800 nid=0x5cd5 runnable | |
| "GC Thread#19" os_prio=0 cpu=6400.05ms elapsed=4035.86s tid=0x00007f381801f800 nid=0x5cd6 runnable | |
| "GC Thread#20" os_prio=0 cpu=6419.80ms elapsed=4035.86s tid=0x00007f3818021800 nid=0x5cd7 runnable | |
| "GC Thread#21" os_prio=0 cpu=6455.84ms elapsed=4035.86s tid=0x00007f3818023000 nid=0x5cd8 runnable | |
| "GC Thread#22" os_prio=0 cpu=6439.05ms elapsed=4035.86s tid=0x00007f3818025000 nid=0x5cd9 runnable | |
| "G1 Main Marker" os_prio=0 cpu=1742.34ms elapsed=4038.56s tid=0x00007f3854075000 nid=0x5cab runnable | |
| "G1 Conc#0" os_prio=0 cpu=94496.94ms elapsed=4038.56s tid=0x00007f3854077000 nid=0x5cac runnable | |
| "G1 Conc#1" os_prio=0 cpu=94590.70ms elapsed=4034.66s tid=0x00007f382c001000 nid=0x5cda runnable | |
| "G1 Conc#2" os_prio=0 cpu=94635.08ms elapsed=4034.66s tid=0x00007f382c002800 nid=0x5cdb runnable | |
| "G1 Conc#3" os_prio=0 cpu=94537.30ms elapsed=4034.66s tid=0x00007f382c004000 nid=0x5cdc runnable | |
| "G1 Conc#4" os_prio=0 cpu=94549.92ms elapsed=4034.66s tid=0x00007f382c005800 nid=0x5cdd runnable | |
| "G1 Conc#5" os_prio=0 cpu=94530.35ms elapsed=4034.66s tid=0x00007f382c007000 nid=0x5cde runnable | |
| "G1 Refine#0" os_prio=0 cpu=9.93ms elapsed=4038.08s tid=0x00007f3854762000 nid=0x5cad runnable | |
| "G1 Refine#1" os_prio=0 cpu=7.70ms elapsed=4031.75s tid=0x00007f3824001000 nid=0x5ce0 runnable | |
| "G1 Refine#2" os_prio=0 cpu=7.14ms elapsed=4031.75s tid=0x00007f3764001000 nid=0x5ce1 runnable | |
| "G1 Refine#3" os_prio=0 cpu=6.35ms elapsed=4031.75s tid=0x00007f3758001000 nid=0x5ce2 runnable | |
| "G1 Refine#4" os_prio=0 cpu=5.70ms elapsed=4031.75s tid=0x00007f375c001000 nid=0x5ce3 runnable | |
| "G1 Refine#5" os_prio=0 cpu=5.53ms elapsed=4031.75s tid=0x00007f3750001000 nid=0x5ce4 runnable | |
| "G1 Refine#6" os_prio=0 cpu=5.38ms elapsed=4031.75s tid=0x00007f3754001000 nid=0x5ce5 runnable | |
| "G1 Refine#7" os_prio=0 cpu=4.97ms elapsed=4031.75s tid=0x00007f3748001000 nid=0x5ce6 runnable | |
| "G1 Refine#8" os_prio=0 cpu=4.80ms elapsed=4031.75s tid=0x00007f374c001000 nid=0x5ce7 runnable | |
| "G1 Refine#9" os_prio=0 cpu=4.25ms elapsed=4031.75s tid=0x00007f3740001000 nid=0x5ce8 runnable | |
| "G1 Refine#10" os_prio=0 cpu=4.30ms elapsed=4031.75s tid=0x00007f3744001000 nid=0x5ce9 runnable | |
| "G1 Refine#11" os_prio=0 cpu=3.35ms elapsed=4031.75s tid=0x00007f3738001000 nid=0x5cea runnable | |
| "G1 Refine#12" os_prio=0 cpu=3.41ms elapsed=4031.75s tid=0x00007f373c001000 nid=0x5ceb runnable | |
| "G1 Refine#13" os_prio=0 cpu=0.20ms elapsed=4031.75s tid=0x00007f3730001000 nid=0x5cec runnable | |
| "G1 Refine#14" os_prio=0 cpu=0.10ms elapsed=4031.75s tid=0x00007f3734001000 nid=0x5ced runnable | |
| "G1 Young RemSet Sampling" os_prio=0 cpu=2745.79ms elapsed=4038.08s tid=0x00007f3854764000 nid=0x5cae runnable | |
| "VM Periodic Task Thread" os_prio=0 cpu=2584.48ms elapsed=4038.04s tid=0x00007f38547dc800 nid=0x5cb8 waiting on condition | |
| JNI global refs: 43, weak refs: 44 | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2019-06-25T15:39:34,625][INFO ][o.e.c.m.MetaDataCreateIndexService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] [pafentries_2019-01] creating index, cause [api], templates [], shards [6]/[1], mappings [_doc] | |
| [2019-06-25T15:39:36,144][INFO ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[pafentries_2019-01][5]] ...]). | |
| [2019-06-25T15:42:24,490][WARN ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failing shard [failed shard, shard [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=JCySt1ObSSma-wxB_EcdUA], message [failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=JCySt1ObSSma-wxB_EcdUA]], failure [RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2044911890/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2044895168/1.9gb], new bytes reserved: [16722/16.3kb]]; ], markAsStale [true]] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2044911890/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2044895168/1.9gb], new bytes reserved: [16722/16.3kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:24,503][WARN ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failing shard [failed shard, shard [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=AF1iz2jfShOja4qkHWXnJg], message [failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=AF1iz2jfShOja4qkHWXnJg]], failure [RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2041776134/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2041749440/1.9gb], new bytes reserved: [26694/26kb]]; ], markAsStale [true]] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2041776134/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2041749440/1.9gb], new bytes reserved: [26694/26kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:24,510][INFO ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] Cluster health status changed from [GREEN] to [YELLOW] (reason: [shards failed [[pafentries_2019-01][0], [pafentries_2019-01][2]] ...]). | |
| [2019-06-25T15:42:27,938][WARN ][o.e.i.c.IndicesClusterStateService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] [[pafentries_2019-01][2]] marking and sending shard failed due to [failed recovery] | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [pafentries_2019-01][2]: Recovery failed from {postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net}{4TKl6givSLuc9EvlhPesHQ}{88uxsGgHTHazQIc1vdOOGw}{192.168.0.115}{192.168.0.115:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true} into {postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20} | |
| at org.elasticsearch.indices.recovery.PeerRecoveryTargetService.lambda$doRecovery$2(PeerRecoveryTargetService.java:253) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.indices.recovery.PeerRecoveryTargetService$1.handleException(PeerRecoveryTargetService.java:298) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.PlainTransportFuture.handleException(PlainTransportFuture.java:97) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1124) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.1.1.jar:7.1.1] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][internal:index/shard/recovery/start_recovery] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2065434846/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2064362728/1.9gb], new bytes reserved: [1072118/1mb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) ~[?:?] | |
| [2019-06-25T15:42:27,945][WARN ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failing shard [failed shard, shard [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], recovery_source[peer recovery], s[INITIALIZING], a[id=zWBfOvgcTO2YwkD8pNHJ3g], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-06-25T14:42:24.503Z], failed_attempts[1], delayed=false, details[failed shard on node [MuM6g5L-SNCvEiybTbw3sQ]: failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=AF1iz2jfShOja4qkHWXnJg], failure RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2041776134/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2041749440/1.9gb], new bytes reserved: [26694/26kb]]; ], allocation_status[no_attempt]], expected_shard_size[4715159], message [failed recovery], failure [RecoveryFailedException[[pafentries_2019-01][2]: Recovery failed from {postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net}{4TKl6givSLuc9EvlhPesHQ}{88uxsGgHTHazQIc1vdOOGw}{192.168.0.115}{192.168.0.115:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true} into {postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: RemoteTransportException[[postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][internal:index/shard/recovery/start_recovery]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2065434846/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2064362728/1.9gb], new bytes reserved: [1072118/1mb]]; ], markAsStale [true]] | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [pafentries_2019-01][2]: Recovery failed from {postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net}{4TKl6givSLuc9EvlhPesHQ}{88uxsGgHTHazQIc1vdOOGw}{192.168.0.115}{192.168.0.115:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true} into {postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20} | |
| at org.elasticsearch.indices.recovery.PeerRecoveryTargetService.lambda$doRecovery$2(PeerRecoveryTargetService.java:253) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.indices.recovery.PeerRecoveryTargetService$1.handleException(PeerRecoveryTargetService.java:298) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.PlainTransportFuture.handleException(PlainTransportFuture.java:97) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1124) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][internal:index/shard/recovery/start_recovery] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2065434846/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2064362728/1.9gb], new bytes reserved: [1072118/1mb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) ~[?:?] | |
| [2019-06-25T15:42:28,049][WARN ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failing shard [failed shard, shard [pafentries_2019-01][2], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=bGRByYCLRwGAn_p9_eFV4Q], message [shard failure, reason [primary shard [[pafentries_2019-01][2], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=bGRByYCLRwGAn_p9_eFV4Q]] was demoted while failing replica shard]], failure [CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2098974442/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8706/8.5kb]]], markAsStale [true]] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098974442/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8706/8.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:28,054][WARN ][o.e.i.c.IndicesClusterStateService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] [[pafentries_2019-01][0]] marking and sending shard failed due to [failed recovery] | |
| org.elasticsearch.indices.recovery.RecoveryFailedException: [pafentries_2019-01][0]: Recovery failed from {postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net}{4TKl6givSLuc9EvlhPesHQ}{88uxsGgHTHazQIc1vdOOGw}{192.168.0.115}{192.168.0.115:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true} into {postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20} | |
| at org.elasticsearch.indices.recovery.PeerRecoveryTargetService.lambda$doRecovery$2(PeerRecoveryTargetService.java:253) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.indices.recovery.PeerRecoveryTargetService$1.handleException(PeerRecoveryTargetService.java:298) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.PlainTransportFuture.handleException(PlainTransportFuture.java:97) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1124) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.1.1.jar:7.1.1] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| Caused by: org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][internal:index/shard/recovery/start_recovery] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2136737808/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2135665896/1.9gb], new bytes reserved: [1071912/1mb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) ~[?:?] | |
| [2019-06-25T15:42:28,056][WARN ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failing shard [failed shard, shard [pafentries_2019-01][0], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=oclPiozGS9a-baNadpPnqQ], message [shard failure, reason [primary shard [[pafentries_2019-01][0], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=oclPiozGS9a-baNadpPnqQ]] was demoted while failing replica shard]], failure [CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2098974100/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8364/8.1kb]]], markAsStale [true]] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098974100/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8364/8.1kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| Suppressed: org.apache.lucene.store.AlreadyClosedException: engine is closed | |
| at org.elasticsearch.index.shard.IndexShard.getEngine(IndexShard.java:2300) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.index.shard.IndexShard.failShard(IndexShard.java:1206) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.failShard(TransportReplicationAction.java:1034) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.onNoLongerPrimary(ReplicationOperation.java:224) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.access$700(ReplicationOperation.java:50) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation$1.lambda$onFailure$2(ReplicationOperation.java:195) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.ActionListener$1.onFailure(ActionListener.java:69) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.ActionListener.onFailure(ActionListener.java:133) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportRequestDeduplicator$CompositeListener.onCompleted(TransportRequestDeduplicator.java:97) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportRequestDeduplicator$CompositeListener.onFailure(TransportRequestDeduplicator.java:111) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.cluster.action.shard.ShardStateAction$1.handleException(ShardStateAction.java:132) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1124) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:999) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:991) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:940) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:28,070][WARN ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failing shard [failed shard, shard [pafentries_2019-01][4], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=74Za3PCKRe-CwZo_ghWMig], message [shard failure, reason [primary shard [[pafentries_2019-01][4], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=74Za3PCKRe-CwZo_ghWMig]] was demoted while failing replica shard]], failure [CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2097923872/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2097917160/1.9gb], new bytes reserved: [6712/6.5kb]]], markAsStale [true]] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2097923872/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2097917160/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:28,079][INFO ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] Cluster health status changed from [YELLOW] to [RED] (reason: [shards failed [[pafentries_2019-01][2], [pafentries_2019-01][0], [pafentries_2019-01][4]] ...]). | |
| [2019-06-25T15:42:28,172][INFO ][o.e.i.s.IndexShard ] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] [pafentries_2019-01][4] primary-replica resync completed with 0 operations | |
| [2019-06-25T15:42:31,819][INFO ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] Cluster health status changed from [RED] to [YELLOW] (reason: [shards started [[pafentries_2019-01][0]] ...]). | |
| [2019-06-25T15:42:36,132][INFO ][o.e.c.r.a.AllocationService] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[pafentries_2019-01][0]] ...]). | |
| [2019-06-25T16:57:11,378][DEBUG][o.e.a.a.i.s.TransportIndicesStatsAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute [indices:monitor/stats] on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][indices:monitor/stats[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2125286270/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2125284872/1.9gb], new bytes reserved: [1398/1.3kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:11,378][DEBUG][o.e.a.a.c.n.s.TransportNodesStatsAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][cluster:monitor/nodes/stats[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2125286144/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2125284872/1.9gb], new bytes reserved: [1272/1.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:11,378][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][cluster:monitor/nodes/info[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2125286122/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2125284872/1.9gb], new bytes reserved: [1250/1.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:11,378][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][cluster:monitor/nodes/info[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2125286122/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2125284872/1.9gb], new bytes reserved: [1250/1.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:27,379][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][cluster:monitor/nodes/info[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2129480426/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2129479176/1.9gb], new bytes reserved: [1250/1.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:27,383][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][cluster:monitor/nodes/info[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2129480426/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2129479176/1.9gb], new bytes reserved: [1250/1.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:27,383][DEBUG][o.e.a.a.i.s.TransportIndicesStatsAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute [indices:monitor/stats] on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][indices:monitor/stats[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2129480574/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2129479176/1.9gb], new bytes reserved: [1398/1.3kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:27,383][DEBUG][o.e.a.a.c.n.s.TransportNodesStatsAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][cluster:monitor/nodes/stats[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2129480448/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2129479176/1.9gb], new bytes reserved: [1272/1.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:29,226][DEBUG][o.e.a.a.c.n.s.TransportNodesStatsAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][cluster:monitor/nodes/stats[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2129480472/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2129479176/1.9gb], new bytes reserved: [1296/1.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:29,226][DEBUG][o.e.a.a.i.s.TransportIndicesStatsAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute [indices:monitor/stats] on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][indices:monitor/stats[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2129480574/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2129479176/1.9gb], new bytes reserved: [1398/1.3kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:43,379][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][cluster:monitor/nodes/info[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2133674730/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2133673480/1.9gb], new bytes reserved: [1250/1.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:43,384][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][cluster:monitor/nodes/info[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2133674730/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2133673480/1.9gb], new bytes reserved: [1250/1.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:43,384][DEBUG][o.e.a.a.c.n.s.TransportNodesStatsAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][cluster:monitor/nodes/stats[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2133674752/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2133673480/1.9gb], new bytes reserved: [1272/1.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:57:43,384][DEBUG][o.e.a.a.i.s.TransportIndicesStatsAction] [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net] failed to execute [indices:monitor/stats] on node [4TKl6givSLuc9EvlhPesHQ] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net][192.168.0.115:9300][indices:monitor/stats[n]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2133674878/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2133673480/1.9gb], new bytes reserved: [1398/1.3kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2019-06-25T15:42:24,442][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][2]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=AF1iz2jfShOja4qkHWXnJg] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2041776134/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2041749440/1.9gb], new bytes reserved: [26694/26kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:24,443][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][2]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=AF1iz2jfShOja4qkHWXnJg] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2041776560/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2041749440/1.9gb], new bytes reserved: [27120/26.4kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:24,442][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][0]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=JCySt1ObSSma-wxB_EcdUA] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2044911890/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2044895168/1.9gb], new bytes reserved: [16722/16.3kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,917][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][4]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][4], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=J7x3lv1pQKC1qKjho02tiA] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2087450738/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2087431400/1.9gb], new bytes reserved: [19338/18.8kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,923][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][0]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], recovery_source[peer recovery], s[INITIALIZING], a[id=C246iB75TSiqAbxsHq3HJw], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-06-25T14:42:24.488Z], failed_attempts[1], delayed=false, details[failed shard on node [MuM6g5L-SNCvEiybTbw3sQ]: failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=JCySt1ObSSma-wxB_EcdUA], failure RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2044911890/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2044895168/1.9gb], new bytes reserved: [16722/16.3kb]]; ], allocation_status[no_attempt]], expected_shard_size[4698081] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2096887894/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2096868584/1.9gb], new bytes reserved: [19310/18.8kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,921][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][2]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], recovery_source[peer recovery], s[INITIALIZING], a[id=zWBfOvgcTO2YwkD8pNHJ3g], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-06-25T14:42:24.503Z], failed_attempts[1], delayed=false, details[failed shard on node [MuM6g5L-SNCvEiybTbw3sQ]: failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=AF1iz2jfShOja4qkHWXnJg], failure RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2041776134/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2041749440/1.9gb], new bytes reserved: [26694/26kb]]; ], allocation_status[no_attempt]], expected_shard_size[4715159] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2087449356/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2087431400/1.9gb], new bytes reserved: [17956/17.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:426) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,926][WARN ][o.e.c.a.s.ShardStateAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] unexpected failure while sending request [internal:cluster/shard/failure] to [{postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true}] for shard entry [shard id [[pafentries_2019-01][4]], allocation id [J7x3lv1pQKC1qKjho02tiA], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][4], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=J7x3lv1pQKC1qKjho02tiA]], failure [RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2087450738/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2087431400/1.9gb], new bytes reserved: [19338/18.8kb]]; ], markAsStale [true]] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][internal:cluster/shard/failure] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2097923872/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2097917160/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,922][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][4]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][4], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=J7x3lv1pQKC1qKjho02tiA] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2087459468/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2087431400/1.9gb], new bytes reserved: [28068/27.4kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,927][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][2]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], recovery_source[peer recovery], s[INITIALIZING], a[id=zWBfOvgcTO2YwkD8pNHJ3g], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-06-25T14:42:24.503Z], failed_attempts[1], delayed=false, details[failed shard on node [MuM6g5L-SNCvEiybTbw3sQ]: failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=AF1iz2jfShOja4qkHWXnJg], failure RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2041776134/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2041749440/1.9gb], new bytes reserved: [26694/26kb]]; ], allocation_status[no_attempt]], expected_shard_size[4715159] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2087455652/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2087431400/1.9gb], new bytes reserved: [24252/23.6kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,927][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][0]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], recovery_source[peer recovery], s[INITIALIZING], a[id=C246iB75TSiqAbxsHq3HJw], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-06-25T14:42:24.488Z], failed_attempts[1], delayed=false, details[failed shard on node [MuM6g5L-SNCvEiybTbw3sQ]: failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=JCySt1ObSSma-wxB_EcdUA], failure RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2044911890/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2044895168/1.9gb], new bytes reserved: [16722/16.3kb]]; ], allocation_status[no_attempt]], expected_shard_size[4698081] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2096884268/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2096868584/1.9gb], new bytes reserved: [15684/15.3kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,933][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][0]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], recovery_source[peer recovery], s[INITIALIZING], a[id=C246iB75TSiqAbxsHq3HJw], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-06-25T14:42:24.488Z], failed_attempts[1], delayed=false, details[failed shard on node [MuM6g5L-SNCvEiybTbw3sQ]: failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=JCySt1ObSSma-wxB_EcdUA], failure RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2044911890/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2044895168/1.9gb], new bytes reserved: [16722/16.3kb]]; ], allocation_status[no_attempt]], expected_shard_size[4698081] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2096890412/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2096868584/1.9gb], new bytes reserved: [21828/21.3kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,934][WARN ][o.e.c.a.s.ShardStateAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] unexpected failure while sending request [internal:cluster/shard/failure] to [{postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true}] for shard entry [shard id [[pafentries_2019-01][0]], allocation id [C246iB75TSiqAbxsHq3HJw], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], recovery_source[peer recovery], s[INITIALIZING], a[id=C246iB75TSiqAbxsHq3HJw], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-06-25T14:42:24.488Z], failed_attempts[1], delayed=false, details[failed shard on node [MuM6g5L-SNCvEiybTbw3sQ]: failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=JCySt1ObSSma-wxB_EcdUA], failure RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2044911890/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2044895168/1.9gb], new bytes reserved: [16722/16.3kb]]; ], allocation_status[no_attempt]], expected_shard_size[4698081]], failure [RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2096887894/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2096868584/1.9gb], new bytes reserved: [19310/18.8kb]]; ], markAsStale [true]] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][internal:cluster/shard/failure] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098974100/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8364/8.1kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,935][WARN ][o.e.c.a.s.ShardStateAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] unexpected failure while sending request [internal:cluster/shard/failure] to [{postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true}] for shard entry [shard id [[pafentries_2019-01][2]], allocation id [zWBfOvgcTO2YwkD8pNHJ3g], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], recovery_source[peer recovery], s[INITIALIZING], a[id=zWBfOvgcTO2YwkD8pNHJ3g], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-06-25T14:42:24.503Z], failed_attempts[1], delayed=false, details[failed shard on node [MuM6g5L-SNCvEiybTbw3sQ]: failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=AF1iz2jfShOja4qkHWXnJg], failure RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2041776134/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2041749440/1.9gb], new bytes reserved: [26694/26kb]]; ], allocation_status[no_attempt]], expected_shard_size[4715159]], failure [RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2087449356/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2087431400/1.9gb], new bytes reserved: [17956/17.5kb]]; ], markAsStale [true]] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][internal:cluster/shard/failure] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098974442/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8706/8.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,949][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][2]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], recovery_source[peer recovery], s[INITIALIZING], a[id=zWBfOvgcTO2YwkD8pNHJ3g], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-06-25T14:42:24.503Z], failed_attempts[1], delayed=false, details[failed shard on node [MuM6g5L-SNCvEiybTbw3sQ]: failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=AF1iz2jfShOja4qkHWXnJg], failure RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2041776134/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2041749440/1.9gb], new bytes reserved: [26694/26kb]]; ], allocation_status[no_attempt]], expected_shard_size[4715159] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098999092/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [33356/32.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,955][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][4]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][4], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=J7x3lv1pQKC1qKjho02tiA] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2087451006/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2087431400/1.9gb], new bytes reserved: [19606/19.1kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,971][WARN ][o.e.i.e.Engine ] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [pafentries_2019-01][0] failed engine [primary shard [[pafentries_2019-01][0], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=oclPiozGS9a-baNadpPnqQ]] was demoted while failing replica shard] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098974100/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8364/8.1kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,975][WARN ][o.e.i.e.Engine ] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [pafentries_2019-01][0] tried to fail engine but engine is already failed. ignoring. [primary shard [[pafentries_2019-01][0], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=oclPiozGS9a-baNadpPnqQ]] was demoted while failing replica shard] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098974100/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8364/8.1kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,981][WARN ][o.e.i.e.Engine ] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [pafentries_2019-01][4] failed engine [primary shard [[pafentries_2019-01][4], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=74Za3PCKRe-CwZo_ghWMig]] was demoted while failing replica shard] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2097923872/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2097917160/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,984][WARN ][o.e.i.e.Engine ] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [pafentries_2019-01][2] failed engine [primary shard [[pafentries_2019-01][2], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=bGRByYCLRwGAn_p9_eFV4Q]] was demoted while failing replica shard] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098974442/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8706/8.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,986][WARN ][o.e.i.e.Engine ] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [pafentries_2019-01][2] tried to fail engine but engine is already failed. ignoring. [primary shard [[pafentries_2019-01][2], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=bGRByYCLRwGAn_p9_eFV4Q]] was demoted while failing replica shard] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098974442/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8706/8.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,977][WARN ][o.e.i.c.IndicesClusterStateService] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][0]] marking and sending shard failed due to [shard failure, reason [primary shard [[pafentries_2019-01][0], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=oclPiozGS9a-baNadpPnqQ]] was demoted while failing replica shard]] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098974100/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8364/8.1kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| Suppressed: org.apache.lucene.store.AlreadyClosedException: engine is closed | |
| at org.elasticsearch.index.shard.IndexShard.getEngine(IndexShard.java:2300) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.index.shard.IndexShard.failShard(IndexShard.java:1206) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.failShard(TransportReplicationAction.java:1034) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.onNoLongerPrimary(ReplicationOperation.java:224) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.access$700(ReplicationOperation.java:50) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation$1.lambda$onFailure$2(ReplicationOperation.java:195) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.ActionListener$1.onFailure(ActionListener.java:69) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.ActionListener.onFailure(ActionListener.java:133) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportRequestDeduplicator$CompositeListener.onCompleted(TransportRequestDeduplicator.java:97) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportRequestDeduplicator$CompositeListener.onFailure(TransportRequestDeduplicator.java:111) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.cluster.action.shard.ShardStateAction$1.handleException(ShardStateAction.java:132) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1124) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:999) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:991) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:940) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,989][WARN ][o.e.i.c.IndicesClusterStateService] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][2]] marking and sending shard failed due to [shard failure, reason [primary shard [[pafentries_2019-01][2], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=bGRByYCLRwGAn_p9_eFV4Q]] was demoted while failing replica shard]] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2098974442/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2098965736/1.9gb], new bytes reserved: [8706/8.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T15:42:27,992][WARN ][o.e.i.c.IndicesClusterStateService] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][4]] marking and sending shard failed due to [shard failure, reason [primary shard [[pafentries_2019-01][4], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=74Za3PCKRe-CwZo_ghWMig]] was demoted while failing replica shard]] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2097923872/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2097917160/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,147][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][2]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=wQI6rGbZQiG_3Bpp5iZK9A] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2090718528/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2090701328/1.9gb], new bytes reserved: [17200/16.7kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,147][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][0]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=GbLr0sVBQ0eNdbkBebJmcw] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2090718602/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2090701328/1.9gb], new bytes reserved: [17274/16.8kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,147][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][2]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=wQI6rGbZQiG_3Bpp5iZK9A] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2090722088/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2090701328/1.9gb], new bytes reserved: [20760/20.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,153][WARN ][o.e.a.b.TransportShardBulkAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][0]] failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=GbLr0sVBQ0eNdbkBebJmcw] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2092816886/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2092798480/1.9gb], new bytes reserved: [18406/17.9kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,154][WARN ][o.e.c.a.s.ShardStateAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] unexpected failure while sending request [internal:cluster/shard/failure] to [{postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true}] for shard entry [shard id [[pafentries_2019-01][0]], allocation id [GbLr0sVBQ0eNdbkBebJmcw], primary term [3], message [failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][0], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=GbLr0sVBQ0eNdbkBebJmcw]], failure [RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2090718602/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2090701328/1.9gb], new bytes reserved: [17274/16.8kb]]; ], markAsStale [true]] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][internal:cluster/shard/failure] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2092805192/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2092798480/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,154][WARN ][o.e.c.a.s.ShardStateAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] unexpected failure while sending request [internal:cluster/shard/failure] to [{postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true}] for shard entry [shard id [[pafentries_2019-01][2]], allocation id [wQI6rGbZQiG_3Bpp5iZK9A], primary term [3], message [failed to perform indices:data/write/bulk[s] on replica [pafentries_2019-01][2], node[MuM6g5L-SNCvEiybTbw3sQ], [R], s[STARTED], a[id=wQI6rGbZQiG_3Bpp5iZK9A]], failure [RemoteTransportException[[postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][indices:data/write/bulk[s][r]]]; nested: CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2090718528/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2090701328/1.9gb], new bytes reserved: [17200/16.7kb]]; ], markAsStale [true]] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][internal:cluster/shard/failure] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2092805192/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2092798480/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,191][WARN ][o.e.i.e.Engine ] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [pafentries_2019-01][2] failed engine [primary shard [[pafentries_2019-01][2], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=bGRByYCLRwGAn_p9_eFV4Q]] was demoted while failing replica shard] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2092805192/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2092798480/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,193][WARN ][o.e.i.e.Engine ] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [pafentries_2019-01][2] tried to fail engine but engine is already failed. ignoring. [primary shard [[pafentries_2019-01][2], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=bGRByYCLRwGAn_p9_eFV4Q]] was demoted while failing replica shard] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2092805192/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2092798480/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,193][WARN ][o.e.i.c.IndicesClusterStateService] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][2]] marking and sending shard failed due to [shard failure, reason [primary shard [[pafentries_2019-01][2], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=bGRByYCLRwGAn_p9_eFV4Q]] was demoted while failing replica shard]] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2092805192/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2092798480/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,199][WARN ][o.e.c.a.s.ShardStateAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] unexpected failure while sending request [internal:cluster/shard/failure] to [{postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true}] for shard entry [shard id [[pafentries_2019-01][2]], allocation id [bGRByYCLRwGAn_p9_eFV4Q], primary term [0], message [shard failure, reason [primary shard [[pafentries_2019-01][2], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=bGRByYCLRwGAn_p9_eFV4Q]] was demoted while failing replica shard]], failure [CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2092805192/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2092798480/1.9gb], new bytes reserved: [6712/6.5kb]]], markAsStale [true]] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][internal:cluster/shard/failure] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2100144890/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2100138512/1.9gb], new bytes reserved: [6378/6.2kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,200][WARN ][o.e.i.e.Engine ] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [pafentries_2019-01][0] failed engine [primary shard [[pafentries_2019-01][0], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=oclPiozGS9a-baNadpPnqQ]] was demoted while failing replica shard] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2092805192/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2092798480/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,202][WARN ][o.e.i.c.IndicesClusterStateService] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] [[pafentries_2019-01][0]] marking and sending shard failed due to [shard failure, reason [primary shard [[pafentries_2019-01][0], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=oclPiozGS9a-baNadpPnqQ]] was demoted while failing replica shard]] | |
| org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2092805192/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2092798480/1.9gb], new bytes reserved: [6712/6.5kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| Suppressed: org.apache.lucene.store.AlreadyClosedException: engine is closed | |
| at org.elasticsearch.index.shard.IndexShard.getEngine(IndexShard.java:2300) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.index.shard.IndexShard.failShard(IndexShard.java:1206) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.failShard(TransportReplicationAction.java:1034) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.onNoLongerPrimary(ReplicationOperation.java:224) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation.access$700(ReplicationOperation.java:50) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.support.replication.ReplicationOperation$1.lambda$onFailure$2(ReplicationOperation.java:195) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.ActionListener$1.onFailure(ActionListener.java:69) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.action.ActionListener.onFailure(ActionListener.java:133) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportRequestDeduplicator$CompositeListener.onCompleted(TransportRequestDeduplicator.java:97) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportRequestDeduplicator$CompositeListener.onFailure(TransportRequestDeduplicator.java:111) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.cluster.action.shard.ShardStateAction$1.handleException(ShardStateAction.java:132) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1124) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:999) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:991) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:940) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
| [2019-06-25T16:34:23,209][WARN ][o.e.c.a.s.ShardStateAction] [postcodes-elasticsearch-2-postcodes-elasticsearch-2.dev-chrisb.zcode.net] unexpected failure while sending request [internal:cluster/shard/failure] to [{postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net}{MuM6g5L-SNCvEiybTbw3sQ}{GLiJP4HKQHOPnVVJLIM5nQ}{192.168.0.114}{192.168.0.114:9300}{ml.machine_memory=4294967296, ml.max_open_jobs=20, xpack.installed=true}] for shard entry [shard id [[pafentries_2019-01][0]], allocation id [oclPiozGS9a-baNadpPnqQ], primary term [0], message [shard failure, reason [primary shard [[pafentries_2019-01][0], node[4TKl6givSLuc9EvlhPesHQ], [P], s[STARTED], a[id=oclPiozGS9a-baNadpPnqQ]] was demoted while failing replica shard]], failure [CircuitBreakingException[[parent] Data too large, data for [<transport_request>] would be [2092805192/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2092798480/1.9gb], new bytes reserved: [6712/6.5kb]]], markAsStale [true]] | |
| org.elasticsearch.transport.RemoteTransportException: [postcodes-elasticsearch-1-postcodes-elasticsearch-1.dev-chrisb.zcode.net][192.168.0.114:9300][internal:cluster/shard/failure] | |
| Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [<transport_request>] would be [2100152742/1.9gb], which is larger than the limit of [2040109465/1.8gb], real usage: [2100138512/1.9gb], new bytes reserved: [14230/13.8kb] | |
| at org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService.checkParentLimit(HierarchyCircuitBreakerService.java:343) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker.addEstimateBytesAndMaybeBreak(ChildMemoryCircuitBreaker.java:128) ~[elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1026) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:922) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:753) [elasticsearch-7.1.1.jar:7.1.1] | |
| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:53) [transport-netty4-client-7.1.1.jar:7.1.1] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) [netty-handler-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) [netty-transport-4.1.32.Final.jar:4.1.32.Final] | |
| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) [netty-common-4.1.32.Final.jar:4.1.32.Final] | |
| at java.lang.Thread.run(Thread.java:835) [?:?] | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment