public
Last active

ZK leader killed, app leadership moves to different cluster member

  • Download Gist
gistfile1.txt
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
Original leader (localhost:14181):
 
16:11:18.002 [ForkJoinPool-1-worker-9] DEBUG c.n.s.n.DeliveryService$DeliverySerializer - received SerializedDelivery: none
16:11:18.002 [ForkJoinPool-1-worker-9] DEBUG c.n.s.n.DeliveryServiceImpl - delivering noDigest events
 
16:11:18.128 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected
16:11:18.139 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 isLeader=true, Cluster participants: localhost:14181, localhost:14185, leader=localhost:14181
16:11:18.250 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=None state=Disconnected
16:11:24.498 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 isLeader=true, Cluster participants: localhost:14181, localhost:14185, leader=localhost:14181
16:11:24.500 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=None state=SyncConnected
 
16:11:54.526 [main-EventThread] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (30025)
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na]
at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) [ZookeeperServiceImpl$$anon$2.class:na]
at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) [curator-framework-1.2.3.jar:na]
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507]
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507]
 
16:11:59.025 [ThreadPoolTaskScheduler-1] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (34524)
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na]
at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na]
at com.nyx.services.notification.DeliveryServiceImpl.scheduledNoDigest(DeliveryService.scala:264) [DeliveryServiceImpl.class:na]
at com.nyx.services.notification.DeliveryServiceImpl$$anonfun$start$2.apply$mcV$sp(DeliveryService.scala:220) [DeliveryServiceImpl$$anonfun$start$2.class:na]
at com.nyx.services.SchedulerServiceImpl$$anon$1.run(SchedulerService.scala:134) [SchedulerServiceImpl$$anon$1.class:na]
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:51) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE]
at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439) [na:1.6.0_37]
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [na:1.6.0_37]
at java.util.concurrent.FutureTask.run(FutureTask.java:138) [na:1.6.0_37]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:98) [na:1.6.0_37]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:206) [na:1.6.0_37]
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [na:1.6.0_37]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [na:1.6.0_37]
at java.lang.Thread.run(Thread.java:680) [na:1.6.0_37]
16:12:09.541 [main-EventThread] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (45041)
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na]
at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) [ZookeeperServiceImpl$$anon$2.class:na]
at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) [curator-framework-1.2.3.jar:na]
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507]
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507]
16:12:14.040 [ThreadPoolTaskScheduler-1] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (49539)
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na]
at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na]
at com.nyx.services.notification.DeliveryServiceImpl.scheduledNoDigest(DeliveryService.scala:264) [DeliveryServiceImpl.class:na]
at com.nyx.services.notification.DeliveryServiceImpl$$anonfun$start$2.apply$mcV$sp(DeliveryService.scala:220) [DeliveryServiceImpl$$anonfun$start$2.class:na]
at com.nyx.services.SchedulerServiceImpl$$anon$1.run(SchedulerService.scala:134) [SchedulerServiceImpl$$anon$1.class:na]
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:51) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE]
at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439) [na:1.6.0_37]
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [na:1.6.0_37]
at java.util.concurrent.FutureTask.run(FutureTask.java:138) [na:1.6.0_37]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:98) [na:1.6.0_37]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:206) [na:1.6.0_37]
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [na:1.6.0_37]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [na:1.6.0_37]
at java.lang.Thread.run(Thread.java:680) [na:1.6.0_37]
16:12:39.566 [main-EventThread] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (75065)
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na]
at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) [ZookeeperServiceImpl$$anon$2.class:na]
at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) [curator-framework-1.2.3.jar:na]
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507]
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507]
16:12:44.067 [ThreadPoolTaskScheduler-1] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (79566)
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na]
at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na]
at com.nyx.services.notification.DeliveryServiceImpl.scheduledNoDigest(DeliveryService.scala:264) [DeliveryServiceImpl.class:na]
at com.nyx.services.notification.DeliveryServiceImpl$$anonfun$start$2.apply$mcV$sp(DeliveryService.scala:220) [DeliveryServiceImpl$$anonfun$start$2.class:na]
at com.nyx.services.SchedulerServiceImpl$$anon$1.run(SchedulerService.scala:134) [SchedulerServiceImpl$$anon$1.class:na]
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:51) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE]
at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439) [na:1.6.0_37]
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [na:1.6.0_37]
at java.util.concurrent.FutureTask.run(FutureTask.java:138) [na:1.6.0_37]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:98) [na:1.6.0_37]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:206) [na:1.6.0_37]
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [na:1.6.0_37]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [na:1.6.0_37]
at java.lang.Thread.run(Thread.java:680) [na:1.6.0_37]
16:12:54.579 [main-EventThread] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (90078)
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na]
at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) [ZookeeperServiceImpl$$anon$2.class:na]
at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) [curator-framework-1.2.3.jar:na]
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507]
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507]
16:12:54.579 [main-EventThread] ERROR c.n.c.f.imps.CuratorFrameworkImpl - Watcher exception
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na]
at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) ~[curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) ~[curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) ~[curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) ~[curator-framework-1.2.3.jar:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) ~[curator-client-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) ~[curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) ~[curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) ~[curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) ~[curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) ~[curator-recipes-1.2.3.jar:na]
at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) ~[curator-recipes-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) ~[ZookeeperServiceImpl$$anon$3.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) ~[ZookeeperServiceImpl$$anon$3.class:na]
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) ~[curator-client-1.2.3.jar:na]
at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) ~[ZookeeperServiceImpl.class:na]
at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) ~[ZookeeperServiceImpl$$anon$2.class:na]
at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) ~[curator-framework-1.2.3.jar:na]
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507]
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507]
16:12:54.581 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected
16:12:54.596 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 isLeader=false, Cluster participants: localhost:14181, localhost:14185, leader=localhost:14185
16:12:54.596 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected
16:12:54.605 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 isLeader=false, Cluster participants: localhost:14185, localhost:14181, leader=localhost:14185
 
 
 
New leader (localhost:14185):
 
16:11:18.136 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected
16:11:18.165 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14185 isLeader=false, Cluster participants: localhost:14181, localhost:14185, leader=localhost:14181
 
16:11:18.991 [main-EventThread] ERROR c.n.c.f.imps.CuratorFrameworkImpl - Background operation retry gave up
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at org.apache.zookeeper.KeeperException.create(KeeperException.java:99) ~[zookeeper-3.4.4.jar:3.4.4-1386507]
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:448) ~[curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:165) [curator-framework-1.2.3.jar:na]
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590) [zookeeper-3.4.4.jar:3.4.4-1386507]
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507]
16:11:19.132 [main-EventThread] ERROR c.n.c.f.imps.CuratorFrameworkImpl - Background operation retry gave up
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at org.apache.zookeeper.KeeperException.create(KeeperException.java:99) ~[zookeeper-3.4.4.jar:3.4.4-1386507]
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:448) ~[curator-framework-1.2.3.jar:na]
at com.netflix.curator.framework.imps.BackgroundSyncImpl$1.processResult(BackgroundSyncImpl.java:49) [curator-framework-1.2.3.jar:na]
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:606) [zookeeper-3.4.4.jar:3.4.4-1386507]
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507]
 
16:12:59.047 [ForkJoinPool-1-worker-4] DEBUG c.n.s.n.DeliveryService$DeliverySerializer - received SerializedDelivery: none
16:12:59.047 [ForkJoinPool-1-worker-4] DEBUG c.n.s.n.DeliveryServiceImpl - delivering noDigest events

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.