broker-failure

broker-failure

07:23:49.246 [main-SendThread(<ip:host>)] WARN  org.apache.zookeeper.ClientCnxn      - Exception while seeking for r/w server <zk-host-address>
java.net.UnknownHostException: <zk-host-address>
        at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:184) ~[na:1.8.0_112]
        at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[na:1.8.0_112]
        at java.net.Socket.connect(Socket.java:589) ~[na:1.8.0_112]
        at java.net.Socket.connect(Socket.java:538) ~[na:1.8.0_112]
        at java.net.Socket.<init>(Socket.java:434) ~[na:1.8.0_112]
        at java.net.Socket.<init>(Socket.java:211) ~[na:1.8.0_112]
        at org.apache.zookeeper.ClientCnxn$SendThread.pingRwServer(ClientCnxn.java:1141) [zookeeper-3.4.6.jar:3.4.6-1569965]
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1076) [zookeeper-3.4.6.jar:3.4.6-1569965]

07:23:49.348 [main-EventThread] INFO  c.y.p.zookeeper.ZooKeeperDataCache   - [State:CONNECTEDREADONLY Timeout:40000 sessionid:0x85af9a5e32c0022 local:null remoteserver:null lastZxid:163208768973 xid:240412 sent:255307 recv:255379 queuedpkts:0 pendingresp:0 queuedevents:0] Received ZooKeeper watch event: WatchedEvent state:Disconnected type:None path:null
:
07:23:50.147 [main-SendThread<zk-thread>] INFO  org.apache.zookeeper.ClientCnxn      - Unable to reconnect to ZooKeeper service, session 0x85af9a5e32c0022 has expired, closing socket connection
07:23:50.147 [main-EventThread] WARN  c.y.p.zookeeper.GlobalZooKeeperCache - Global ZK session lost. Triggering reconnection State:CLOSED sessionid:0x85af9a5e32c0022 local:/<ip:port> remoteserver:<ip:port> lastZxid:163208768973 xid:240412 sent:255308 recv:255380 queuedpkts:1 pendingresp:0 queuedevents:1
:
07:23:50.147 [main-EventThread] INFO  c.y.p.zookeeper.GlobalZooKeeperCache - Re-starting global ZK session in the background...
:
07:23:50.148 [main-EventThread] INFO  org.apache.zookeeper.ZooKeeper       - Initiating client connection, connectString=<zk-connect-string>
:
07:24:02.444 [pulsar-backlog-quota-checker-65-1] ERROR c.y.p.b.service.BacklogQuotaManager  - Failed to read policies data, will apply the default backlog quota: namespace=<namespace>
java.lang.NullPointerException: null
        at com.yahoo.pulsar.zookeeper.ZooKeeperCache.lambda$getDataAsync$6(ZooKeeperCache.java:264) ~[pulsar-zookeeper-utils-1.17.2.jar:na]
        at com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache.lambda$get$2(LocalAsyncLoadingCache.java:126) ~[caffeine-2.3.3.jar:na]
        at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$16(BoundedLocalCache.java:1973) ~[caffeine-2.3.3.jar:na]
        at java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1853) ~[na:1.8.0_112]
        at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:1971) ~[caffeine-2.3.3.jar:na]
        at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:1954) ~[caffeine-2.3.3.jar:na]
        at com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache.get(LocalAsyncLoadingCache.java:125) ~[caffeine-2.3.3.jar:na]
        at com.github.benmanes.caffeine.cache.LocalAsyncLoadingCache.get(LocalAsyncLoadingCache.java:117) ~[caffeine-2.3.3.jar:na]
        at com.yahoo.pulsar.zookeeper.ZooKeeperCache.getDataAsync(ZooKeeperCache.java:260) ~[pulsar-zookeeper-utils-1.17.2.jar:na]
        at com.yahoo.pulsar.zookeeper.ZooKeeperCache.getData(ZooKeeperCache.java:238) ~[pulsar-zookeeper-utils-1.17.2.jar:na]
        at com.yahoo.pulsar.zookeeper.ZooKeeperDataCache.getWithStat(ZooKeeperDataCache.java:76) ~[pulsar-zookeeper-utils-1.17.2.jar:na]
        at com.yahoo.pulsar.zookeeper.ZooKeeperDataCache.get(ZooKeeperDataCache.java:72) ~[pulsar-zookeeper-utils-1.17.2.jar:na]
        at com.yahoo.pulsar.broker.service.BacklogQuotaManager.getBacklogQuota(BacklogQuotaManager.java:59) [pulsar-broker-1.17.2.jar:na]
        at com.yahoo.pulsar.broker.service.BacklogQuotaManager.getBacklogQuotaLimit(BacklogQuotaManager.java:75) [pulsar-broker-1.17.2.jar:na]
        at com.yahoo.pulsar.broker.service.BrokerService.isBacklogExceeded(BrokerService.java:706) [pulsar-broker-1.17.2.jar:na]
        at com.yahoo.pulsar.broker.service.BrokerService.lambda$monitorBacklogQuota$15(BrokerService.java:724) [pulsar-broker-1.17.2.jar:na]
        at com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap$Section.forEach(ConcurrentOpenHashMap.java:383) ~[pulsar-common-1.17.2.jar:na]
        at com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap.forEach(ConcurrentOpenHashMap.java:157) ~[pulsar-common-1.17.2.jar:na]
        at com.yahoo.pulsar.broker.service.BrokerService.monitorBacklogQuota(BrokerService.java:721) [pulsar-broker-1.17.2.jar:na]
        at org.apache.bookkeeper.mledger.util.SafeRun$1.safeRun(SafeRun.java:27) ~[managed-ledger-1.17.2.jar:na]
        at org.apache.bookkeeper.util.SafeRunnable.run(SafeRunnable.java:31) ~[bookkeeper-server-4.3.1.56-yahoo.jar:4.3.1.56-yahoo]
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_112]
        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) ~[na:1.8.0_112]
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) ~[na:1.8.0_112]
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) ~[na:1.8.0_112]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[na:1.8.0_112]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[na:1.8.0_112]
        at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) ~[netty-common-4.0.42.Final.jar:4.0.42.Final]
        at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_112]