Created
December 25, 2018 15:24
-
-
Save plaflamme/25a47dce6edd920653a33e9fc612428a to your computer and use it in GitHub Desktop.
Exception on Tephra shutdown
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 17:25:26.040 [tx-snapshot] INFO org.apache.tephra.TransactionManager - Writing final snapshot prior to shutdown | |
| 17:25:26.041 [tx-snapshot] DEBUG org.apache.tephra.TransactionManager - Starting snapshot of transaction state with timestamp 1545672326041 | |
| 17:25:26.041 [tx-snapshot] DEBUG org.apache.tephra.TransactionManager - Returning snapshot of state: TransactionSnapshot{timestamp=1545672326041, readPointer=1545669311067000088, writePointer=1545669311067000088, invalidSize=117, inProgressSize=0, committingSize=0, committedSize=31} | |
| 17:25:26.041 [tx-snapshot] DEBUG o.a.t.p.HDFSTransactionStateStorage - Writing snapshot to temporary file /system/tephra/.in-progress.snapshot.1545672326041 | |
| 17:25:26.071 [TransactionPruningService STOPPING] INFO o.a.t.t.TransactionPruningService - Stopped TransactionPruningService | |
| 2018-12-24 17:25:26,083 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over master/192.168.50.4:9200. Trying to fail over immediately. | |
| java.io.IOException: Failed on local exception: java.io.InterruptedIOException: Interruped while waiting for IO on channel java.nio.channels.SocketChannel[connected local=/192.168.50.4:47928 remote=master/192.168.50.4:9200]. 59982 millis timeout left.; Host Details : local host is: "master/192.168.50.4"; destination host is: "master":9200; | |
| at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:772) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1476) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| Caused by: java.io.InterruptedIOException: Interruped while waiting for IO on channel java.nio.channels.SocketChannel[connected local=/192.168.50.4:47928 remote=master/192.168.50.4:9200]. 59982 millis timeout left. | |
| at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:352) | |
| at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) | |
| at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) | |
| at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) | |
| at java.io.FilterInputStream.read(FilterInputStream.java:133) | |
| at java.io.FilterInputStream.read(FilterInputStream.java:133) | |
| at org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:521) | |
| at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) | |
| at java.io.BufferedInputStream.read(BufferedInputStream.java:265) | |
| at java.io.DataInputStream.readInt(DataInputStream.java:387) | |
| at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1081) | |
| at org.apache.hadoop.ipc.Client$Connection.run(Client.java:976) | |
| 2018-12-24 17:25:26,097 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over secondary/192.168.50.5:9200 after 1 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,097 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over master/192.168.50.4:9200 after 2 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,098 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over secondary/192.168.50.5:9200 after 3 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,123 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over master/192.168.50.4:9200 after 4 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,124 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over secondary/192.168.50.5:9200 after 5 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,125 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over master/192.168.50.4:9200 after 6 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,126 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over secondary/192.168.50.5:9200 after 7 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,127 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over master/192.168.50.4:9200 after 8 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,127 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over secondary/192.168.50.5:9200 after 9 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,128 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over master/192.168.50.4:9200 after 10 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,129 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over secondary/192.168.50.5:9200 after 11 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,130 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over master/192.168.50.4:9200 after 12 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,131 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over secondary/192.168.50.5:9200 after 13 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,133 INFO [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking create of class ClientNamenodeProtocolTranslatorPB over master/192.168.50.4:9200 after 14 fail over attempts. Trying to fail over immediately. | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 2018-12-24 17:25:26,134 WARN [tx-snapshot] retry.RetryInvocationHandler: Exception while invoking class org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create over secondary/192.168.50.5:9200. Not retrying because failovers (15) exceeded maximum allowed (15) | |
| java.io.IOException: The client is stopped | |
| at org.apache.hadoop.ipc.Client.getConnection(Client.java:1504) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1448) | |
| at org.apache.hadoop.ipc.Client.call(Client.java:1409) | |
| at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) | |
| at com.sun.proxy.$Proxy29.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:301) | |
| at sun.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256) | |
| at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) | |
| at com.sun.proxy.$Proxy30.create(Unknown Source) | |
| at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:2047) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1759) | |
| at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1683) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:425) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:421) | |
| at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:362) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:925) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906) | |
| at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:868) | |
| at org.apache.tephra.persist.HDFSTransactionStateStorage.writeSnapshot(HDFSTransactionStateStorage.java:123) | |
| at org.apache.tephra.TransactionManager.doSnapshot(TransactionManager.java:480) | |
| at org.apache.tephra.TransactionManager.access$400(TransactionManager.java:110) | |
| at org.apache.tephra.TransactionManager$2.onShutdown(TransactionManager.java:322) | |
| at org.apache.tephra.TransactionManager$DaemonThreadExecutor.run(TransactionManager.java:1427) | |
| 17:25:26.134 [tx-snapshot] INFO org.apache.tephra.TransactionManager - Exiting thread tx-snapshot | |
| 17:25:26.184 [ThriftRPCServer] INFO org.apache.tephra.TransactionManager - Took 141.0 ms to stop |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment