Created
February 7, 2018 10:57
-
-
Save fredsted/e37d60948d9f765eef615b21eebc6bfd to your computer and use it in GitHub Desktop.
EC2 Discovery not working in ES 6.1/6.2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "Version": "2012-10-17", | |
| "Statement": [ | |
| { | |
| "Sid": "Stmt1493803477000", | |
| "Effect": "Allow", | |
| "Action": [ | |
| "ec2:DescribeAvailabilityZones", | |
| "ec2:DescribeInstances", | |
| "ec2:DescribeRegions", | |
| "ec2:DescribeSecurityGroups", | |
| "ec2:DescribeTags" | |
| ], | |
| "Resource": [ | |
| "*" | |
| ] | |
| } | |
| ] | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2018-02-07T10:46:48,499][INFO ][o.e.n.Node ] [esnode01] initializing ... | |
| [2018-02-07T10:46:48,555][INFO ][o.e.e.NodeEnvironment ] [esnode01] using [1] data paths, mounts [[/ (/dev/xvda1)]], net usable_space [289.3gb], net total_space [290.7gb], types [ext4] | |
| [2018-02-07T10:46:48,555][INFO ][o.e.e.NodeEnvironment ] [esnode01] heap size [5.9gb], compressed ordinary object pointers [true] | |
| [2018-02-07T10:46:48,556][INFO ][o.e.n.Node ] [esnode01] node name [esnode01], node ID [0nJL8fJaT4Orl3oaiKMtdw] | |
| [2018-02-07T10:46:48,556][INFO ][o.e.n.Node ] [esnode01] version[6.2.0], pid[18049], build[37cdac1/2018-02-01T17:31:12.527918Z], OS[Linux/4.4.0-1013-aws/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/1.8.0_151/25.151-b12] | |
| [2018-02-07T10:46:48,557][INFO ][o.e.n.Node ] [esnode01] JVM arguments [-Xms6g, -Xmx6g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+DisableExplicitGC, -XX:+AlwaysPreTouch, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j.skipJansi=true, -XX:+HeapDumpOnOutOfMemoryError, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/etc/elasticsearch/es-node] | |
| [2018-02-07T10:46:49,449][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [aggs-matrix-stats] | |
| [2018-02-07T10:46:49,449][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [analysis-common] | |
| [2018-02-07T10:46:49,449][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [ingest-common] | |
| [2018-02-07T10:46:49,449][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [lang-expression] | |
| [2018-02-07T10:46:49,450][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [lang-mustache] | |
| [2018-02-07T10:46:49,450][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [lang-painless] | |
| [2018-02-07T10:46:49,450][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [mapper-extras] | |
| [2018-02-07T10:46:49,450][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [parent-join] | |
| [2018-02-07T10:46:49,450][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [percolator] | |
| [2018-02-07T10:46:49,450][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [rank-eval] | |
| [2018-02-07T10:46:49,450][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [reindex] | |
| [2018-02-07T10:46:49,450][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [repository-url] | |
| [2018-02-07T10:46:49,450][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [transport-netty4] | |
| [2018-02-07T10:46:49,450][INFO ][o.e.p.PluginsService ] [esnode01] loaded module [tribe] | |
| [2018-02-07T10:46:49,451][INFO ][o.e.p.PluginsService ] [esnode01] loaded plugin [discovery-ec2] | |
| [2018-02-07T10:46:52,316][INFO ][o.e.d.DiscoveryModule ] [esnode01] using discovery type [zen] | |
| [2018-02-07T10:46:53,032][INFO ][o.e.n.Node ] [esnode01] initialized | |
| [2018-02-07T10:46:53,041][INFO ][o.e.n.Node ] [esnode01] starting ... | |
| [2018-02-07T10:46:53,307][INFO ][o.e.t.TransportService ] [esnode01] publish_address {10.10.10.11:9300}, bound_addresses {10.10.10.11:9300} | |
| [2018-02-07T10:46:53,316][INFO ][o.e.b.BootstrapChecks ] [esnode01] bound or publishing to a non-loopback address, enforcing bootstrap checks | |
| [2018-02-07T10:46:53,410][WARN ][c.a.j.SdkMBeanRegistrySupport] | |
| java.security.AccessControlException: access denied ("javax.management.MBeanServerPermission" "findMBeanServer") | |
| at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472) ~[?:1.8.0_151] | |
| at java.security.AccessController.checkPermission(AccessController.java:884) ~[?:1.8.0_151] | |
| at java.lang.SecurityManager.checkPermission(SecurityManager.java:549) ~[?:1.8.0_151] | |
| at javax.management.MBeanServerFactory.checkPermission(MBeanServerFactory.java:413) ~[?:1.8.0_151] | |
| at javax.management.MBeanServerFactory.findMBeanServer(MBeanServerFactory.java:361) ~[?:1.8.0_151] | |
| at com.amazonaws.jmx.MBeans.getMBeanServer(MBeans.java:111) ~[?:?] | |
| at com.amazonaws.jmx.MBeans.registerMBean(MBeans.java:50) ~[?:?] | |
| at com.amazonaws.jmx.SdkMBeanRegistrySupport.registerMetricAdminMBean(SdkMBeanRegistrySupport.java:27) ~[?:?] | |
| at com.amazonaws.metrics.AwsSdkMetrics.registerMetricAdminMBean(AwsSdkMetrics.java:398) ~[?:?] | |
| at com.amazonaws.metrics.AwsSdkMetrics.<clinit>(AwsSdkMetrics.java:359) ~[?:?] | |
| at com.amazonaws.AmazonWebServiceClient.requestMetricCollector(AmazonWebServiceClient.java:635) ~[?:?] | |
| at com.amazonaws.AmazonWebServiceClient.isRMCEnabledAtClientOrSdkLevel(AmazonWebServiceClient.java:574) ~[?:?] | |
| at com.amazonaws.AmazonWebServiceClient.isRequestMetricsEnabled(AmazonWebServiceClient.java:566) ~[?:?] | |
| at com.amazonaws.AmazonWebServiceClient.createExecutionContext(AmazonWebServiceClient.java:536) ~[?:?] | |
| at com.amazonaws.AmazonWebServiceClient.createExecutionContext(AmazonWebServiceClient.java:531) ~[?:?] | |
| at com.amazonaws.services.ec2.AmazonEC2Client.executeDescribeInstances(AmazonEC2Client.java:6860) ~[?:?] | |
| at com.amazonaws.services.ec2.AmazonEC2Client.describeInstances(AmazonEC2Client.java:6854) ~[?:?] | |
| at org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider.lambda$fetchDynamicNodes$0(AwsEc2UnicastHostsProvider.java:115) ~[?:?] | |
| at java.security.AccessController.doPrivileged(Native Method) [?:1.8.0_151] | |
| at org.elasticsearch.discovery.ec2.SocketAccess.doPrivileged(SocketAccess.java:42) [discovery-ec2-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider.fetchDynamicNodes(AwsEc2UnicastHostsProvider.java:115) [discovery-ec2-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider$DiscoNodesCache.refresh(AwsEc2UnicastHostsProvider.java:241) [discovery-ec2-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider$DiscoNodesCache.refresh(AwsEc2UnicastHostsProvider.java:226) [discovery-ec2-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider.buildDynamicNodes(AwsEc2UnicastHostsProvider.java:101) [discovery-ec2-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.UnicastZenPing.ping(UnicastZenPing.java:309) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.UnicastZenPing.ping(UnicastZenPing.java:286) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.ZenDiscovery.pingAndWait(ZenDiscovery.java:1044) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.ZenDiscovery.findMaster(ZenDiscovery.java:894) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.ZenDiscovery.innerJoinCluster(ZenDiscovery.java:448) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.ZenDiscovery.access$2500(ZenDiscovery.java:90) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.ZenDiscovery$JoinThreadControl$1.run(ZenDiscovery.java:1253) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:566) [elasticsearch-6.2.0.jar:6.2.0] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_151] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_151] | |
| at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151] | |
| [2018-02-07T10:46:53,421][WARN ][c.a.m.AwsSdkMetrics ] | |
| java.security.AccessControlException: access denied ("javax.management.MBeanServerPermission" "findMBeanServer") | |
| at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472) ~[?:1.8.0_151] | |
| at java.security.AccessController.checkPermission(AccessController.java:884) ~[?:1.8.0_151] | |
| at java.lang.SecurityManager.checkPermission(SecurityManager.java:549) ~[?:1.8.0_151] | |
| at javax.management.MBeanServerFactory.checkPermission(MBeanServerFactory.java:413) ~[?:1.8.0_151] | |
| at javax.management.MBeanServerFactory.findMBeanServer(MBeanServerFactory.java:361) ~[?:1.8.0_151] | |
| at com.amazonaws.jmx.MBeans.getMBeanServer(MBeans.java:111) ~[?:?] | |
| at com.amazonaws.jmx.MBeans.isRegistered(MBeans.java:98) ~[?:?] | |
| at com.amazonaws.jmx.SdkMBeanRegistrySupport.isMBeanRegistered(SdkMBeanRegistrySupport.java:46) ~[?:?] | |
| at com.amazonaws.metrics.AwsSdkMetrics.registerMetricAdminMBean(AwsSdkMetrics.java:404) ~[?:?] | |
| at com.amazonaws.metrics.AwsSdkMetrics.<clinit>(AwsSdkMetrics.java:359) ~[?:?] | |
| at com.amazonaws.AmazonWebServiceClient.requestMetricCollector(AmazonWebServiceClient.java:635) ~[?:?] | |
| at com.amazonaws.AmazonWebServiceClient.isRMCEnabledAtClientOrSdkLevel(AmazonWebServiceClient.java:574) ~[?:?] | |
| at com.amazonaws.AmazonWebServiceClient.isRequestMetricsEnabled(AmazonWebServiceClient.java:566) ~[?:?] | |
| at com.amazonaws.AmazonWebServiceClient.createExecutionContext(AmazonWebServiceClient.java:536) ~[?:?] | |
| at com.amazonaws.AmazonWebServiceClient.createExecutionContext(AmazonWebServiceClient.java:531) ~[?:?] | |
| at com.amazonaws.services.ec2.AmazonEC2Client.executeDescribeInstances(AmazonEC2Client.java:6860) ~[?:?] | |
| at com.amazonaws.services.ec2.AmazonEC2Client.describeInstances(AmazonEC2Client.java:6854) ~[?:?] | |
| at org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider.lambda$fetchDynamicNodes$0(AwsEc2UnicastHostsProvider.java:115) ~[?:?] | |
| at java.security.AccessController.doPrivileged(Native Method) [?:1.8.0_151] | |
| at org.elasticsearch.discovery.ec2.SocketAccess.doPrivileged(SocketAccess.java:42) [discovery-ec2-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider.fetchDynamicNodes(AwsEc2UnicastHostsProvider.java:115) [discovery-ec2-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider$DiscoNodesCache.refresh(AwsEc2UnicastHostsProvider.java:241) [discovery-ec2-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider$DiscoNodesCache.refresh(AwsEc2UnicastHostsProvider.java:226) [discovery-ec2-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider.buildDynamicNodes(AwsEc2UnicastHostsProvider.java:101) [discovery-ec2-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.UnicastZenPing.ping(UnicastZenPing.java:309) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.UnicastZenPing.ping(UnicastZenPing.java:286) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.ZenDiscovery.pingAndWait(ZenDiscovery.java:1044) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.ZenDiscovery.findMaster(ZenDiscovery.java:894) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.ZenDiscovery.innerJoinCluster(ZenDiscovery.java:448) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.ZenDiscovery.access$2500(ZenDiscovery.java:90) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.discovery.zen.ZenDiscovery$JoinThreadControl$1.run(ZenDiscovery.java:1253) [elasticsearch-6.2.0.jar:6.2.0] | |
| at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:566) [elasticsearch-6.2.0.jar:6.2.0] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_151] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_151] | |
| at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151] | |
| [2018-02-07T10:46:57,320][INFO ][o.e.c.s.MasterService ] [esnode01] zen-disco-elected-as-master ([0] nodes joined), reason: new_master {esnode01}{0nJL8fJaT4Orl3oaiKMtdw}{BxRBt0-VSqq1uXsZhKp7lw}{10.10.10.11}{10.10.10.11:9300} | |
| [2018-02-07T10:46:57,324][INFO ][o.e.c.s.ClusterApplierService] [esnode01] new_master {esnode01}{0nJL8fJaT4Orl3oaiKMtdw}{BxRBt0-VSqq1uXsZhKp7lw}{10.10.10.11}{10.10.10.11:9300}, reason: apply cluster state (from master [master {esnode01}{0nJL8fJaT4Orl3oaiKMtdw}{BxRBt0-VSqq1uXsZhKp7lw}{10.10.10.11}{10.10.10.11:9300} committed version [1] source [zen-disco-elected-as-master ([0] nodes joined)]]) | |
| [2018-02-07T10:46:57,347][INFO ][o.e.g.GatewayService ] [esnode01] recovered [0] indices into cluster_state | |
| [2018-02-07T10:46:57,351][INFO ][o.e.h.n.Netty4HttpServerTransport] [esnode01] publish_address {10.10.10.11:9200}, bound_addresses {10.10.10.11:9200} | |
| [2018-02-07T10:46:57,352][INFO ][o.e.n.Node ] [esnode01] started |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #################################### General #################################### | |
| cluster.name: es | |
| discovery.ec2.tag.Group: es-node | |
| discovery.zen.hosts_provider: ec2 | |
| http.port: 9200 | |
| network.host: _ec2:privateIpv4_ | |
| node.name: es-node01 | |
| path.repo: | |
| - /var/lib/elasticsearch/snapshots/es-node | |
| transport.tcp.port: 9300 | |
| #################################### Paths #################################### | |
| path.data: /var/lib/elasticsearch/es-node01-es-node | |
| path.logs: /var/log/elasticsearch/es-node01-es-node |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment