gistfile1.txt

Full thread dump OpenJDK 64-Bit Server VM (11.0.2+9-Ubuntu-3ubuntu118.04.3 mixed mode, sharing):

Threads class SMR info:
_java_thread_list=0x00007fc8e8047b30, length=17, elements={
0x00007fca28011800, 0x00007fca2847a000, 0x00007fca2847c000, 0x00007fca28484000,
0x00007fca28486000, 0x00007fca28488000, 0x00007fca2848a000, 0x00007fca284b9800,
0x00007fca284be000, 0x00007fca2872c000, 0x00007fca28788800, 0x00007fca28a33000,
0x00007fc9181e7800, 0x00007fc918104800, 0x00007fc9b80ea800, 0x00007fc9b80eb800,
0x00007fca28c08000
}

"main" #1 prio=5 os_prio=0 cpu=19967.69ms elapsed=22.32s tid=0x00007fca28011800 nid=0x166b runnable  [0x00007fca321ad000]
   java.lang.Thread.State: RUNNABLE
        at org.nd4j.nativeblas.Nd4jCpu$NativeOps.execCustomOp(Native Method)
        at org.nd4j.linalg.cpu.nativecpu.ops.NativeOpExecutioner.exec(NativeOpExecutioner.java:2017)
        at org.nd4j.linalg.factory.Nd4j.exec(Nd4j.java:7268)
        at org.deeplearning4j.nn.layers.mkldnn.MKLDNNConvHelper.backpropGradient(MKLDNNConvHelper.java:102)
        at org.deeplearning4j.nn.layers.convolution.ConvolutionLayer.backpropGradient(ConvolutionLayer.java:167)
        at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.calcBackpropGradients(MultiLayerNetwork.java:1898)
        at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.computeGradientAndScore(MultiLayerNetwork.java:2684)
        at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.computeGradientAndScore(MultiLayerNetwork.java:2627)
        at org.deeplearning4j.optimize.solvers.BaseOptimizer.gradientAndScore(BaseOptimizer.java:160)
        at org.deeplearning4j.optimize.solvers.StochasticGradientDescent.optimize(StochasticGradientDescent.java:63)
        at org.deeplearning4j.optimize.Solver.optimize(Solver.java:52)
        at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fitHelper(MultiLayerNetwork.java:2231)
        at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:2189)
        - locked <0x000000048f6eec00> (a org.deeplearning4j.nn.multilayer.MultiLayerNetwork)
        at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:2252)
        at org.deeplearning4j.earlystopping.trainer.EarlyStoppingTrainer.fit(EarlyStoppingTrainer.java:60)
        at org.deeplearning4j.earlystopping.trainer.BaseEarlyStoppingTrainer.fit(BaseEarlyStoppingTrainer.java:151)
        at org.deeplearning4j.earlystopping.trainer.BaseEarlyStoppingTrainer.fit(BaseEarlyStoppingTrainer.java:94)
        at com.luiwammes.pc.trainer.cnn.agegroup.AgeGroupClassifierCNN.main(AgeGroupClassifierCNN.java:65)

"Reference Handler" #2 daemon prio=10 os_prio=0 cpu=32.02ms elapsed=22.30s tid=0x00007fca2847a000 nid=0x1672 waiting on condition  [0x00007fc9e17fe000]
   java.lang.Thread.State: RUNNABLE
        at java.lang.ref.Reference.waitForReferencePendingList(java.base@11.0.2/Native Method)
        at java.lang.ref.Reference.processPendingReferences(java.base@11.0.2/Reference.java:241)
        at java.lang.ref.Reference$ReferenceHandler.run(java.base@11.0.2/Reference.java:213)

"Finalizer" #3 daemon prio=8 os_prio=0 cpu=1.37ms elapsed=22.30s tid=0x00007fca2847c000 nid=0x1673 in Object.wait()  [0x00007fc9e16fd000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(java.base@11.0.2/Native Method)
        - waiting on <0x00000004512dc680> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:155)
        - waiting to re-lock in wait() <0x00000004512dc680> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:176)
        at java.lang.ref.Finalizer$FinalizerThread.run(java.base@11.0.2/Finalizer.java:170)

"Signal Dispatcher" #4 daemon prio=9 os_prio=0 cpu=0.31ms elapsed=22.30s tid=0x00007fca28484000 nid=0x1674 waiting on condition  [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"C2 CompilerThread0" #5 daemon prio=9 os_prio=0 cpu=4370.43ms elapsed=22.30s tid=0x00007fca28486000 nid=0x1675 waiting on condition  [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE
   No compile task

"C1 CompilerThread0" #15 daemon prio=9 os_prio=0 cpu=756.81ms elapsed=22.30s tid=0x00007fca28488000 nid=0x1676 waiting on condition  [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE
   No compile task

"Sweeper thread" #20 daemon prio=9 os_prio=0 cpu=124.59ms elapsed=22.30s tid=0x00007fca2848a000 nid=0x1677 runnable  [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Service Thread" #21 daemon prio=9 os_prio=0 cpu=0.23ms elapsed=22.28s tid=0x00007fca284b9800 nid=0x1679 runnable  [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE

"Common-Cleaner" #22 daemon prio=8 os_prio=0 cpu=1.53ms elapsed=22.27s tid=0x00007fca284be000 nid=0x167b in Object.wait()  [0x00007fc9e0be6000]
   java.lang.Thread.State: TIMED_WAITING (on object monitor)
        at java.lang.Object.wait(java.base@11.0.2/Native Method)
        - waiting on <0x00000004512de5e8> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:155)
        - waiting to re-lock in wait() <0x00000004512de5e8> (a java.lang.ref.ReferenceQueue$Lock)
        at jdk.internal.ref.CleanerImpl.run(java.base@11.0.2/CleanerImpl.java:148)
        at java.lang.Thread.run(java.base@11.0.2/Thread.java:834)
        at jdk.internal.misc.InnocuousThread.run(java.base@11.0.2/InnocuousThread.java:134)
"cluster-ClusterId{value='5cc6d6c93445121669242aef', description='null'}-ds159782.mlab.com:59782" #24 daemon prio=5 os_prio=0 cpu=123.93ms elapsed=21.57s tid=0x00007fca2872c000 nid=0x1681 waiting on condition  [0x00007fc997282000]
   java.lang.Thread.State: TIMED_WAITING (parking)
        at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method)
        - parking to wait for  <0x00000004512dfdf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123)
        at com.mongodb.connection.DefaultServerMonitor$ServerMonitorRunnable.waitForSignalOrTimeout(DefaultServerMonitor.java:224)
        at com.mongodb.connection.DefaultServerMonitor$ServerMonitorRunnable.waitForNext(DefaultServerMonitor.java:205)
        at com.mongodb.connection.DefaultServerMonitor$ServerMonitorRunnable.run(DefaultServerMonitor.java:153)
        - locked <0x00000004512dff18> (a com.mongodb.connection.DefaultServerMonitor$ServerMonitorRunnable)
        at java.lang.Thread.run(java.base@11.0.2/Thread.java:834)

"pool-2-thread-1" #25 daemon prio=5 os_prio=0 cpu=5.48ms elapsed=21.56s tid=0x00007fca28788800 nid=0x1682 waiting on condition  [0x00007fc9974d0000]
   java.lang.Thread.State: TIMED_WAITING (parking)
        at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method)
        - parking to wait for  <0x00000004512dcb00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1182)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899)
        at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628)
        at java.lang.Thread.run(java.base@11.0.2/Thread.java:834)

"JavaCPP Deallocator" #26 daemon prio=10 os_prio=0 cpu=65.16ms elapsed=17.38s tid=0x00007fca28a33000 nid=0x16aa in Object.wait()  [0x00007fc996d7e000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(java.base@11.0.2/Native Method)
        - waiting on <0x000000046cb1b468> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:155)
        - waiting to re-lock in wait() <0x000000046cb1b468> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:176)
        at org.bytedeco.javacpp.Pointer$DeallocatorThread.run(Pointer.java:303)

"DeallocatorServiceThread_0" #28 daemon prio=5 os_prio=0 cpu=0.31ms elapsed=10.02s tid=0x00007fc9181e7800 nid=0x16d4 in Object.wait()  [0x00007fc996b7c000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(java.base@11.0.2/Native Method)
        - waiting on <0x000000048f2249b0> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:155)
        - waiting to re-lock in wait() <0x000000048f2249b0> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:176)
        at org.nd4j.linalg.memory.deallocation.DeallocatorService$DeallocatorServiceThread.run(DeallocatorService.java:117)

"DeallocatorServiceThread_1" #29 daemon prio=5 os_prio=0 cpu=0.25ms elapsed=10.02s tid=0x00007fc918104800 nid=0x16d5 in Object.wait()  [0x00007fc9e06bd000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(java.base@11.0.2/Native Method)
        - waiting on <0x000000048f228ac0> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:155)
        - waiting to re-lock in wait() <0x000000048f228ac0> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:176)
        at org.nd4j.linalg.memory.deallocation.DeallocatorService$DeallocatorServiceThread.run(DeallocatorService.java:117)

"C1 CompilerThread1" #16 daemon prio=9 os_prio=0 cpu=92.38ms elapsed=9.98s tid=0x00007fc9b80ea800 nid=0x16d8 waiting on condition  [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE
   No compile task

"C1 CompilerThread2" #17 daemon prio=9 os_prio=0 cpu=72.68ms elapsed=9.98s tid=0x00007fc9b80eb800 nid=0x16d9 waiting on condition  [0x0000000000000000]
   java.lang.Thread.State: RUNNABLE
   No compile task

"ADSI prefetch thread" #30 daemon prio=5 os_prio=0 cpu=953.12ms elapsed=9.79s tid=0x00007fca28c08000 nid=0x16e1 waiting on condition  [0x00007fc996c7d000]
   java.lang.Thread.State: WAITING (parking)
        at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method)
        - parking to wait for  <0x000000048f4fe2e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
        at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081)
        at java.util.concurrent.LinkedBlockingQueue.put(java.base@11.0.2/LinkedBlockingQueue.java:341)
        at org.deeplearning4j.datasets.iterator.AsyncDataSetIterator$AsyncPrefetchThread.run(AsyncDataSetIterator.java:435)

"VM Thread" os_prio=0 cpu=86.30ms elapsed=22.30s tid=0x00007fca28477000 nid=0x1671 runnable  

"GC Thread#0" os_prio=0 cpu=134.46ms elapsed=22.32s tid=0x00007fca28029000 nid=0x166c runnable  

"GC Thread#1" os_prio=0 cpu=127.01ms elapsed=20.65s tid=0x00007fc9d4071000 nid=0x1688 runnable  

"GC Thread#2" os_prio=0 cpu=129.67ms elapsed=20.65s tid=0x00007fc9d4072000 nid=0x1689 runnable  

"GC Thread#3" os_prio=0 cpu=127.47ms elapsed=20.65s tid=0x00007fc9d4073000 nid=0x168a runnable  

"GC Thread#4" os_prio=0 cpu=130.37ms elapsed=20.65s tid=0x00007fc9d4074000 nid=0x168b runnable  

"GC Thread#5" os_prio=0 cpu=130.43ms elapsed=20.65s tid=0x00007fc9d4075000 nid=0x168c runnable  

"GC Thread#6" os_prio=0 cpu=129.00ms elapsed=20.65s tid=0x00007fc9d4076800 nid=0x168d runnable  

"GC Thread#7" os_prio=0 cpu=125.80ms elapsed=20.65s tid=0x00007fc9d4078000 nid=0x168e runnable  

"GC Thread#8" os_prio=0 cpu=128.77ms elapsed=20.65s tid=0x00007fc9d4079800 nid=0x168f runnable  

"GC Thread#9" os_prio=0 cpu=128.67ms elapsed=20.65s tid=0x00007fc9d407b800 nid=0x1690 runnable  

"GC Thread#10" os_prio=0 cpu=128.70ms elapsed=20.65s tid=0x00007fc9d407d000 nid=0x1691 runnable  

"GC Thread#11" os_prio=0 cpu=127.76ms elapsed=20.65s tid=0x00007fc9d407e800 nid=0x1692 runnable  

"GC Thread#12" os_prio=0 cpu=127.64ms elapsed=20.65s tid=0x00007fc9d4080000 nid=0x1693 runnable  

"GC Thread#13" os_prio=0 cpu=129.21ms elapsed=20.65s tid=0x00007fc9d4081800 nid=0x1694 runnable  

"GC Thread#14" os_prio=0 cpu=128.29ms elapsed=20.65s tid=0x00007fc9d4083000 nid=0x1695 runnable  

"GC Thread#15" os_prio=0 cpu=127.99ms elapsed=20.65s tid=0x00007fc9d4084800 nid=0x1696 runnable  

"GC Thread#16" os_prio=0 cpu=129.19ms elapsed=20.65s tid=0x00007fc9d4086000 nid=0x1697 runnable  
"GC Thread#17" os_prio=0 cpu=127.74ms elapsed=20.65s tid=0x00007fc9d4087800 nid=0x1698 runnable  

"GC Thread#18" os_prio=0 cpu=128.91ms elapsed=20.65s tid=0x00007fc9d4089000 nid=0x1699 runnable  

"GC Thread#19" os_prio=0 cpu=125.51ms elapsed=20.65s tid=0x00007fc9d408a800 nid=0x169a runnable  

"GC Thread#20" os_prio=0 cpu=128.75ms elapsed=20.65s tid=0x00007fc9d408c000 nid=0x169b runnable  

"GC Thread#21" os_prio=0 cpu=130.56ms elapsed=20.65s tid=0x00007fc9d408d800 nid=0x169c runnable  

"GC Thread#22" os_prio=0 cpu=73.24ms elapsed=12.27s tid=0x00007fc9d40e1000 nid=0x16cf runnable  

"G1 Main Marker" os_prio=0 cpu=3.14ms elapsed=22.32s tid=0x00007fca28093000 nid=0x166d runnable  

"G1 Conc#0" os_prio=0 cpu=39.29ms elapsed=22.32s tid=0x00007fca28095000 nid=0x166e runnable  

"G1 Conc#1" os_prio=0 cpu=37.98ms elapsed=16.24s tid=0x00007fc9e4001000 nid=0x16ae runnable  

"G1 Conc#2" os_prio=0 cpu=38.11ms elapsed=16.24s tid=0x00007fc9e4002000 nid=0x16af runnable  

"G1 Conc#3" os_prio=0 cpu=37.57ms elapsed=16.24s tid=0x00007fc9e4003000 nid=0x16b0 runnable  

"G1 Conc#4" os_prio=0 cpu=38.05ms elapsed=16.24s tid=0x00007fc9e4004000 nid=0x16b1 runnable  

"G1 Conc#5" os_prio=0 cpu=38.65ms elapsed=16.24s tid=0x00007fc9e4005800 nid=0x16b2 runnable  

"G1 Refine#0" os_prio=0 cpu=3.35ms elapsed=22.32s tid=0x00007fca283bf800 nid=0x166f runnable  

"G1 Refine#1" os_prio=0 cpu=1.04ms elapsed=19.63s tid=0x00007fc9d8001000 nid=0x16a6 runnable  

"G1 Young RemSet Sampling" os_prio=0 cpu=20.23ms elapsed=22.32s tid=0x00007fca283c1000 nid=0x1670 runnable  
"VM Periodic Task Thread" os_prio=0 cpu=21.65ms elapsed=22.28s tid=0x00007fca284bb800 nid=0x167a waiting on condition  

JNI global refs: 32, weak refs: 557

Heap
 garbage-first heap   total 1396736K, used 895217K [0x0000000450600000, 0x0000000800000000)
  region size 2048K, 404 young (827392K), 24 survivors (49152K)
 Metaspace       used 27735K, capacity 28267K, committed 28800K, reserved 1075200K
  class space    used 3221K, capacity 3385K, committed 3456K, reserved 1048576K