Created
December 9, 2012 21:18
-
-
Save ericacm/4247058 to your computer and use it in GitHub Desktop.
ZK leader killed, app leadership moves to different cluster member
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original leader (localhost:14181): | |
| 16:11:18.002 [ForkJoinPool-1-worker-9] DEBUG c.n.s.n.DeliveryService$DeliverySerializer - received SerializedDelivery: none | |
| 16:11:18.002 [ForkJoinPool-1-worker-9] DEBUG c.n.s.n.DeliveryServiceImpl - delivering noDigest events | |
| 16:11:18.128 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected | |
| 16:11:18.139 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 isLeader=true, Cluster participants: localhost:14181, localhost:14185, leader=localhost:14181 | |
| 16:11:18.250 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=None state=Disconnected | |
| 16:11:24.498 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 isLeader=true, Cluster participants: localhost:14181, localhost:14185, leader=localhost:14181 | |
| 16:11:24.500 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=None state=SyncConnected | |
| 16:11:54.526 [main-EventThread] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (30025) | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) [ZookeeperServiceImpl$$anon$2.class:na] | |
| at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) [curator-framework-1.2.3.jar:na] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| 16:11:59.025 [ThreadPoolTaskScheduler-1] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (34524) | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.notification.DeliveryServiceImpl.scheduledNoDigest(DeliveryService.scala:264) [DeliveryServiceImpl.class:na] | |
| at com.nyx.services.notification.DeliveryServiceImpl$$anonfun$start$2.apply$mcV$sp(DeliveryService.scala:220) [DeliveryServiceImpl$$anonfun$start$2.class:na] | |
| at com.nyx.services.SchedulerServiceImpl$$anon$1.run(SchedulerService.scala:134) [SchedulerServiceImpl$$anon$1.class:na] | |
| at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:51) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE] | |
| at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE] | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439) [na:1.6.0_37] | |
| at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [na:1.6.0_37] | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:138) [na:1.6.0_37] | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:98) [na:1.6.0_37] | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:206) [na:1.6.0_37] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [na:1.6.0_37] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [na:1.6.0_37] | |
| at java.lang.Thread.run(Thread.java:680) [na:1.6.0_37] | |
| 16:12:09.541 [main-EventThread] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (45041) | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) [ZookeeperServiceImpl$$anon$2.class:na] | |
| at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) [curator-framework-1.2.3.jar:na] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| 16:12:14.040 [ThreadPoolTaskScheduler-1] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (49539) | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.notification.DeliveryServiceImpl.scheduledNoDigest(DeliveryService.scala:264) [DeliveryServiceImpl.class:na] | |
| at com.nyx.services.notification.DeliveryServiceImpl$$anonfun$start$2.apply$mcV$sp(DeliveryService.scala:220) [DeliveryServiceImpl$$anonfun$start$2.class:na] | |
| at com.nyx.services.SchedulerServiceImpl$$anon$1.run(SchedulerService.scala:134) [SchedulerServiceImpl$$anon$1.class:na] | |
| at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:51) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE] | |
| at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE] | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439) [na:1.6.0_37] | |
| at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [na:1.6.0_37] | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:138) [na:1.6.0_37] | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:98) [na:1.6.0_37] | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:206) [na:1.6.0_37] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [na:1.6.0_37] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [na:1.6.0_37] | |
| at java.lang.Thread.run(Thread.java:680) [na:1.6.0_37] | |
| 16:12:39.566 [main-EventThread] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (75065) | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) [ZookeeperServiceImpl$$anon$2.class:na] | |
| at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) [curator-framework-1.2.3.jar:na] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| 16:12:44.067 [ThreadPoolTaskScheduler-1] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (79566) | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.notification.DeliveryServiceImpl.scheduledNoDigest(DeliveryService.scala:264) [DeliveryServiceImpl.class:na] | |
| at com.nyx.services.notification.DeliveryServiceImpl$$anonfun$start$2.apply$mcV$sp(DeliveryService.scala:220) [DeliveryServiceImpl$$anonfun$start$2.class:na] | |
| at com.nyx.services.SchedulerServiceImpl$$anon$1.run(SchedulerService.scala:134) [SchedulerServiceImpl$$anon$1.class:na] | |
| at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:51) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE] | |
| at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:81) [spring-context-3.0.5.RELEASE.jar:3.0.5.RELEASE] | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439) [na:1.6.0_37] | |
| at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) [na:1.6.0_37] | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:138) [na:1.6.0_37] | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:98) [na:1.6.0_37] | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:206) [na:1.6.0_37] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) [na:1.6.0_37] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) [na:1.6.0_37] | |
| at java.lang.Thread.run(Thread.java:680) [na:1.6.0_37] | |
| 16:12:54.579 [main-EventThread] ERROR com.netflix.curator.ConnectionState - Connection timed out for connection string (localhost:14181,localhost:14185,localhost:14189) and timeout (15000) / elapsed (90078) | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) [curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) [curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) [curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) [ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) [curator-client-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) [ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) [ZookeeperServiceImpl$$anon$2.class:na] | |
| at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) [curator-framework-1.2.3.jar:na] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| 16:12:54.579 [main-EventThread] ERROR c.n.c.f.imps.CuratorFrameworkImpl - Watcher exception | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:94) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:105) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:410) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:213) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:202) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) ~[curator-client-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:198) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:190) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:37) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:149) ~[curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:131) ~[curator-recipes-1.2.3.jar:na] | |
| at com.netflix.curator.framework.recipes.leader.LeaderLatch.getParticipants(LeaderLatch.java:276) ~[curator-recipes-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:247) ~[ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$3.call(ZookeeperService.scala:244) ~[ZookeeperServiceImpl$$anon$3.class:na] | |
| at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:106) ~[curator-client-1.2.3.jar:na] | |
| at com.nyx.services.ZookeeperServiceImpl.clusterStatus(ZookeeperService.scala:244) ~[ZookeeperServiceImpl.class:na] | |
| at com.nyx.services.ZookeeperServiceImpl$$anon$2.process(ZookeeperService.scala:234) ~[ZookeeperServiceImpl$$anon$2.class:na] | |
| at com.netflix.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:56) ~[curator-framework-1.2.3.jar:na] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| 16:12:54.581 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected | |
| 16:12:54.596 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 isLeader=false, Cluster participants: localhost:14181, localhost:14185, leader=localhost:14185 | |
| 16:12:54.596 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected | |
| 16:12:54.605 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14181 isLeader=false, Cluster participants: localhost:14185, localhost:14181, leader=localhost:14185 | |
| New leader (localhost:14185): | |
| 16:11:18.136 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - watchedEvent: eventType=NodeChildrenChanged state=SyncConnected | |
| 16:11:18.165 [main-EventThread] INFO c.nyx.services.ZookeeperServiceImpl - nodeId=localhost:14185 isLeader=false, Cluster participants: localhost:14181, localhost:14185, leader=localhost:14181 | |
| 16:11:18.991 [main-EventThread] ERROR c.n.c.f.imps.CuratorFrameworkImpl - Background operation retry gave up | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at org.apache.zookeeper.KeeperException.create(KeeperException.java:99) ~[zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:448) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:165) [curator-framework-1.2.3.jar:na] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| 16:11:19.132 [main-EventThread] ERROR c.n.c.f.imps.CuratorFrameworkImpl - Background operation retry gave up | |
| org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss | |
| at org.apache.zookeeper.KeeperException.create(KeeperException.java:99) ~[zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at com.netflix.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:448) ~[curator-framework-1.2.3.jar:na] | |
| at com.netflix.curator.framework.imps.BackgroundSyncImpl$1.processResult(BackgroundSyncImpl.java:49) [curator-framework-1.2.3.jar:na] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:606) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495) [zookeeper-3.4.4.jar:3.4.4-1386507] | |
| 16:12:59.047 [ForkJoinPool-1-worker-4] DEBUG c.n.s.n.DeliveryService$DeliverySerializer - received SerializedDelivery: none | |
| 16:12:59.047 [ForkJoinPool-1-worker-4] DEBUG c.n.s.n.DeliveryServiceImpl - delivering noDigest events |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment