gistfile1.txt

@400000005f75e50221cdbd6c [01 Oct 2020 19:47:28] ERROR jobs.ServerJob - Error in job for server: Server [host=gap-usl-spark-worker-1309500, port=22223, url=service:jmx:rmi:///jndi/rmi://gap-usl-spark-worker-1309500:22223/jmxrmi, cronExpression=null, numQueryThreads=1]
@400000005f75e50221cdd0f4 java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: gap-usl-spark-worker-1309500; nested exception is: 
@400000005f75e50221ce78ec 	java.net.ConnectException: Connection refused]
@400000005f75e50221ce80bc 	at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369)
@400000005f75e50221ce84a4 	at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
@400000005f75e50221cea3e4 	at com.googlecode.jmxtrans.util.JmxUtils.getServerConnection(JmxUtils.java:291)
@400000005f75e50221cea7cc 	at com.googlecode.jmxtrans.util.JmxConnectionFactory.makeObject(JmxConnectionFactory.java:27)
@400000005f75e50221ceaf9c 	at org.apache.commons.pool.impl.GenericKeyedObjectPool.borrowObject(GenericKeyedObjectPool.java:1212)
@400000005f75e50221ceb76c 	at com.googlecode.jmxtrans.jobs.ServerJob.execute(ServerJob.java:37)
@400000005f75e50221cee264 	at org.quartz.core.JobRunShell.run(JobRunShell.java:216)
@400000005f75e50221ceea34 	at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:549)
@400000005f75e50221ceee1c Caused by: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: gap-usl-spark-worker-1309500; nested exception is: 
@400000005f75e50221cef9d4 	java.net.ConnectException: Connection refused]
@400000005f75e50221cf1144 	at com.sun.jndi.rmi.registry.RegistryContext.lookup(RegistryContext.java:122)
@400000005f75e50221cf1914 	at com.sun.jndi.toolkit.url.GenericURLContext.lookup(GenericURLContext.java:205)
@400000005f75e50221cf1cfc 	at javax.naming.InitialContext.lookup(InitialContext.java:417)
@400000005f75e50221cf24cc 	at javax.management.remote.rmi.RMIConnector.findRMIServerJNDI(RMIConnector.java:1929)
@400000005f75e50221cf28b4 	at javax.management.remote.rmi.RMIConnector.findRMIServer(RMIConnector.java:1896)
@400000005f75e50221cf440c 	at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:287)
@400000005f75e50221cf4bdc 	... 7 more
@400000005f75e50221cf4fc4 Caused by: java.rmi.ConnectException: Connection refused to host: gap-usl-spark-worker-1309500; nested exception is: 
@400000005f75e50221cf5794 	java.net.ConnectException: Connection refused
@400000005f75e50221cf5b7c 	at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:619)
@400000005f75e50221cf7ea4 	at sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:216)
@400000005f75e50221cf8674 	at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:202)
@400000005f75e50221cf8e44 	at sun.rmi.server.UnicastRef.newCall(UnicastRef.java:342)
@400000005f75e50221cf922c 	at sun.rmi.registry.RegistryImpl_Stub.lookup(Unknown Source)
@400000005f75e50221cf9614 	at com.sun.jndi.rmi.registry.RegistryContext.lookup(RegistryContext.java:118)
@400000005f75e50221cfb16c 	... 12 more
@400000005f75e50221cfb554 Caused by: java.net.ConnectException: Connection refused
@400000005f75e50221cfb93c 	at java.net.PlainSocketImpl.socketConnect(Native Method)
@400000005f75e50221cfbd24 	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:345)
@400000005f75e50221cfc4f4 	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
@400000005f75e50221cfe04c 	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
@400000005f75e50221cfe434 	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
@400000005f75e50221cfec04 	at java.net.Socket.connect(Socket.java:589)
@400000005f75e50221cfefec 	at java.net.Socket.connect(Socket.java:538)
@400000005f75e50221cff3d4 	at java.net.Socket.<init>(Socket.java:434)
@400000005f75e50221d00b44 	at java.net.Socket.<init>(Socket.java:211)
@400000005f75e50221d02e6c 	at sun.rmi.transport.proxy.RMIDirectSocketFactory.createSocket(RMIDirectSocketFactory.java:40)
@400000005f75e50221d0363c 	at sun.rmi.transport.proxy.RMIMasterSocketFactory.createSocket(RMIMasterSocketFactory.java:148)
@400000005f75e50221d03e0c 	at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:613)
@400000005f75e50221d041f4 	... 17 more
@400000005f75e5041353da64 [01 Oct 2020 19:47:30] ERROR util.JmxUtils - Error executing query: Query [obj=*, resultAlias=null, attr=null]
@400000005f75e5041353ea04 javax.management.InstanceNotFoundException: kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-131892
@400000005f75e5041353f1d4 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1095)
@400000005f75e5041353f9a4 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBeanInfo(DefaultMBeanServerInterceptor.java:1375)
@400000005f75e50413545f34 	at com.sun.jmx.mbeanserver.JmxMBeanServer.getMBeanInfo(JmxMBeanServer.java:920)
@400000005f75e50413546704 	at javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1460)
@400000005f75e50413546aec 	at javax.management.remote.rmi.RMIConnectionImpl.access$300(RMIConnectionImpl.java:76)
@400000005f75e50413546ed4 	at javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1307)
@400000005f75e50413548a2c 	at javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1399)
@400000005f75e50413548e14 	at javax.management.remote.rmi.RMIConnectionImpl.getMBeanInfo(RMIConnectionImpl.java:904)
@400000005f75e504135495e4 	at sun.reflect.GeneratedMethodAccessor230.invoke(Unknown Source)
@400000005f75e5041354b90c 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
@400000005f75e5041354bcf4 	at java.lang.reflect.Method.invoke(Method.java:483)
@400000005f75e5041354c0dc 	at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
@400000005f75e5041354c8ac 	at sun.rmi.transport.Transport$1.run(Transport.java:178)
@400000005f75e5041354cc94 	at sun.rmi.transport.Transport$1.run(Transport.java:175)
@400000005f75e5041354e01c 	at java.security.AccessController.doPrivileged(Native Method)
@400000005f75e5041354e404 	at sun.rmi.transport.Transport.serviceCall(Transport.java:174)
@400000005f75e5041354e7ec 	at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:557)
@400000005f75e5041354efbc 	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:812)
@400000005f75e5041354f3a4 	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:671)
@400000005f75e50413550b14 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
@400000005f75e50413550efc 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
@400000005f75e504135512e4 	at java.lang.Thread.run(Thread.java:745)
@400000005f75e504135516cc 	at sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
@400000005f75e50413553ddc 	at sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
@400000005f75e504135541c4 	at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
@400000005f75e504135545ac 	at com.sun.jmx.remote.internal.PRef.invoke(Unknown Source)
@400000005f75e50413554d7c 	at javax.management.remote.rmi.RMIConnectionImpl_Stub.getMBeanInfo(Unknown Source)
@400000005f75e50413555164 	at javax.management.remote.rmi.RMIConnector$RemoteMBeanServerConnection.getMBeanInfo(RMIConnector.java:1081)
@400000005f75e504135568d4 	at com.googlecode.jmxtrans.util.JmxUtils.processQuery(JmxUtils.java:226)
@400000005f75e504135570a4 	at com.googlecode.jmxtrans.util.JmxUtils$ProcessQueryThread.run(JmxUtils.java:165)
@400000005f75e5041355748c 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
@400000005f75e50413557874 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
@400000005f75e50413558fe4 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
@400000005f75e504135593cc 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
@400000005f75e50413559b9c 	at java.lang.Thread.run(Thread.java:745)
@400000005f75e511051ead34 Java HotSpot(TM) 64-Bit Server VM warning: ignoring option PermSize=384m; support was removed in 8.0
@400000005f75e511052106c4 Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=384m; support was removed in 8.0
@400000005f75e51105257b64 Java HotSpot(TM) 64-Bit Server VM warning: Using incremental CMS is deprecated and will likely be removed in a future release
@400000005f75e5112b835204 [01 Oct 2020 19:47:43] INFO  jmxtrans.JmxTransformer - Starting Jmxtrans on : /etc/cosmos-jmx
@400000005f75e5120365aac4 [01 Oct 2020 19:47:44] ERROR jobs.ServerJob - Error in job for server: Server [host=gap-usl-spark-worker-1309500, port=22223, url=service:jmx:rmi:///jndi/rmi://gap-usl-spark-worker-1309500:22223/jmxrmi, cronExpression=null, numQueryThreads=1]
@400000005f75e5120365b67c java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: gap-usl-spark-worker-1309500; nested exception is: 
@400000005f75e51203661824 	java.net.ConnectException: Connection refused]
@400000005f75e51203661c0c 	at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369)
@400000005f75e51203661c0c 	at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
@400000005f75e512036627c4 	at com.googlecode.jmxtrans.util.JmxUtils.getServerConnection(JmxUtils.java:291)
@400000005f75e51203662bac 	at com.googlecode.jmxtrans.util.JmxConnectionFactory.makeObject(JmxConnectionFactory.java:27)
@400000005f75e51203662bac 	at org.apache.commons.pool.impl.GenericKeyedObjectPool.borrowObject(GenericKeyedObjectPool.java:1212)
@400000005f75e51203662f94 	at com.googlecode.jmxtrans.jobs.ServerJob.execute(ServerJob.java:37)
@400000005f75e51203663f34 	at org.quartz.core.JobRunShell.run(JobRunShell.java:216)
@400000005f75e5120366431c 	at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:549)
@400000005f75e5120366431c Caused by: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: gap-usl-spark-worker-1309500; nested exception is: 
@400000005f75e51203664704 	java.net.ConnectException: Connection refused]
@400000005f75e512036652bc 	at com.sun.jndi.rmi.registry.RegistryContext.lookup(RegistryContext.java:122)
@400000005f75e512036656a4 	at com.sun.jndi.toolkit.url.GenericURLContext.lookup(GenericURLContext.java:205)
@400000005f75e512036656a4 	at javax.naming.InitialContext.lookup(InitialContext.java:417)
@400000005f75e51203665a8c 	at javax.management.remote.rmi.RMIConnector.findRMIServerJNDI(RMIConnector.java:1929)
@400000005f75e51203665a8c 	at javax.management.remote.rmi.RMIConnector.findRMIServer(RMIConnector.java:1896)
@400000005f75e51203666644 	at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:287)
@400000005f75e51203666644 	... 7 more
@400000005f75e51203666a2c Caused by: java.rmi.ConnectException: Connection refused to host: gap-usl-spark-worker-1309500; nested exception is: 
@400000005f75e51203666a2c 	java.net.ConnectException: Connection refused
@400000005f75e51203666e14 	at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:619)
@400000005f75e512036679cc 	at sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:216)
@400000005f75e51203667db4 	at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:202)
@400000005f75e51203667db4 	at sun.rmi.server.UnicastRef.newCall(UnicastRef.java:342)
@400000005f75e5120366819c 	at sun.rmi.registry.RegistryImpl_Stub.lookup(Unknown Source)
@400000005f75e5120366819c 	at com.sun.jndi.rmi.registry.RegistryContext.lookup(RegistryContext.java:118)
@400000005f75e51203668d54 	... 12 more
@400000005f75e51203668d54 Caused by: java.net.ConnectException: Connection refused
@400000005f75e5120366913c 	at java.net.PlainSocketImpl.socketConnect(Native Method)
@400000005f75e5120366913c 	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:345)
@400000005f75e5120366913c 	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
@400000005f75e5120366db74 	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
@400000005f75e5120366df5c 	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
@400000005f75e5120366df5c 	at java.net.Socket.connect(Socket.java:589)
@400000005f75e5120366e344 	at java.net.Socket.connect(Socket.java:538)
@400000005f75e5120366e344 	at java.net.Socket.<init>(Socket.java:434)
@400000005f75e5120366eb14 	at java.net.Socket.<init>(Socket.java:211)
@400000005f75e5120366f6cc 	at sun.rmi.transport.proxy.RMIDirectSocketFactory.createSocket(RMIDirectSocketFactory.java:40)
@400000005f75e5120366fab4 	at sun.rmi.transport.proxy.RMIMasterSocketFactory.createSocket(RMIMasterSocketFactory.java:148)
@400000005f75e5120366fab4 	at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:613)
@400000005f75e5120366fe9c 	... 17 more
@400000005f75e5130cc923d4 [01 Oct 2020 19:47:45] ERROR util.JmxUtils - Error executing query: Query [obj=*, resultAlias=null, attr=null]
@400000005f75e5130cc92f8c javax.management.InstanceNotFoundException: kafka.consumer:type=consumer-node-metrics,client-id=consumer-132179,node-id=node-11
@400000005f75e5130cc9375c 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1095)
@400000005f75e5130cc93b44 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBeanInfo(DefaultMBeanServerInterceptor.java:1375)
@400000005f75e5130cc9c014 	at com.sun.jmx.mbeanserver.JmxMBeanServer.getMBeanInfo(JmxMBeanServer.java:920)
@400000005f75e5130cc9c3fc 	at javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1460)
@400000005f75e5130cc9c7e4 	at javax.management.remote.rmi.RMIConnectionImpl.access$300(RMIConnectionImpl.java:76)
@400000005f75e5130cc9cfb4 	at javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1307)
@400000005f75e5130cc9e724 	at javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1399)
@400000005f75e5130cc9eb0c 	at javax.management.remote.rmi.RMIConnectionImpl.getMBeanInfo(RMIConnectionImpl.java:904)
@400000005f75e5130cc9eef4 	at sun.reflect.GeneratedMethodAccessor230.invoke(Unknown Source)
@400000005f75e5130cca0e34 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
@400000005f75e5130cca121c 	at java.lang.reflect.Method.invoke(Method.java:483)
@400000005f75e5130cca1604 	at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
@400000005f75e5130cca19ec 	at sun.rmi.transport.Transport$1.run(Transport.java:178)
@400000005f75e5130cca1dd4 	at sun.rmi.transport.Transport$1.run(Transport.java:175)
@400000005f75e5130cca2d74 	at java.security.AccessController.doPrivileged(Native Method)
@400000005f75e5130cca315c 	at sun.rmi.transport.Transport.serviceCall(Transport.java:174)
@400000005f75e5130cca3544 	at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:557)
@400000005f75e5130cca392c 	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:812)
@400000005f75e5130cca3d14 	at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:671)
@400000005f75e5130cca4cb4 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
@400000005f75e5130cca5484 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
@400000005f75e5130cca586c 	at java.lang.Thread.run(Thread.java:745)
@400000005f75e5130cca586c 	at sun.rmi.transport.StreamRemoteCall.exceptionReceivedFromServer(StreamRemoteCall.java:276)
@400000005f75e5130cca7b94 	at sun.rmi.transport.StreamRemoteCall.executeCall(StreamRemoteCall.java:253)
@400000005f75e5130cca7f7c 	at sun.rmi.server.UnicastRef.invoke(UnicastRef.java:162)
@400000005f75e5130cca8364 	at com.sun.jmx.remote.internal.PRef.invoke(Unknown Source)
@400000005f75e5130cca8364 	at javax.management.remote.rmi.RMIConnectionImpl_Stub.getMBeanInfo(Unknown Source)
@400000005f75e5130cca874c 	at javax.management.remote.rmi.RMIConnector$RemoteMBeanServerConnection.getMBeanInfo(RMIConnector.java:1081)
@400000005f75e5130ccad184 	at com.googlecode.jmxtrans.util.JmxUtils.processQuery(JmxUtils.java:226)
@400000005f75e5130ccad56c 	at com.googlecode.jmxtrans.util.JmxUtils$ProcessQueryThread.run(JmxUtils.java:165)
@400000005f75e5130ccad954 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
@400000005f75e5130ccadd3c 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
@400000005f75e5130ccae8f4 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
@400000005f75e5130ccaecdc 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
@400000005f75e5130ccaf0c4 	at java.lang.Thread.run(Thread.java:745)